diff --git a/.forgejo/patches/ghc-9.8.patch b/.forgejo/patches/ghc-9.8.patch deleted file mode 100644 index 85796d787d..0000000000 --- a/.forgejo/patches/ghc-9.8.patch +++ /dev/null @@ -1,18 +0,0 @@ -Support ghc-9.8 by widening a lot of constraints. - -This patch can be removed once upstream supports ghc 9.8 offically. - -diff -uprN git-annex-10.20240227.orig/cabal.project git-annex-10.20240227/cabal.project ---- git-annex-10.20240227.orig/cabal.project 1970-01-01 01:00:00.000000000 +0100 -+++ git-annex-10.20240227/cabal.project 2024-04-28 13:30:14.061706299 +0200 -@@ -0,0 +1,10 @@ -+packages: *.cabal -+ -+allow-newer: dav -+allow-newer: haskeline:filepath -+allow-newer: haskeline:directory -+allow-newer: xml-hamlet -+allow-newer: aws:filepath -+allow-newer: dbus:network -+allow-newer: dbus:filepath -+allow-newer: microstache:filepath diff --git a/.forgejo/workflows/generate-lockfile.yml b/.forgejo/workflows/generate-lockfile.yml deleted file mode 100644 index 8dbb579e67..0000000000 --- a/.forgejo/workflows/generate-lockfile.yml +++ /dev/null @@ -1,89 +0,0 @@ -on: - workflow_dispatch: - inputs: - ref_name: - description: 'Tag or commit' - required: true - type: string - - push: - tags: - - '*' - -jobs: - cabal-config-edge: - name: Generate cabal config for edge - runs-on: x86_64 - container: - image: alpine:edge - env: - CI_ALPINE_TARGET_RELEASE: edge - steps: - - name: Environment setup - run: | - apk upgrade -a - apk add nodejs git cabal patch - - name: Repo pull - uses: actions/checkout@v4 - with: - fetch-depth: 1 - ref: ${{ inputs.ref_name }} - - name: Config generation - run: | - patch -p1 -i .forgejo/patches/ghc-9.8.patch - HOME="${{ github.workspace}}"/cabal_cache cabal update - HOME="${{ github.workspace}}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted" - mv cabal.project.freeze git-annex.config - - name: Package upload - uses: forgejo/upload-artifact@v3 - with: - name: cabalconfigedge - path: git-annex*.config - cabal-config-v322: - name: Generate cabal config for v3.22 - runs-on: x86_64 - container: - image: alpine:3.22 - env: - CI_ALPINE_TARGET_RELEASE: v3.22 - steps: - - name: Environment setup - run: | - apk upgrade -a - apk add nodejs git cabal patch - - name: Repo pull - uses: actions/checkout@v4 - with: - fetch-depth: 1 - ref: ${{ inputs.ref_name }} - - name: Config generation - run: | - patch -p1 -i .forgejo/patches/ghc-9.8.patch - HOME="${{ github.workspace }}"/cabal_cache cabal update - HOME="${{ github.workspace }}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted" - mv cabal.project.freeze git-annex.config - - name: Package upload - uses: forgejo/upload-artifact@v3 - with: - name: cabalconfig322 - path: git-annex*.config - upload-tarball: - name: Upload to generic repo - runs-on: x86_64 - needs: [cabal-config-edge,cabal-config-v322] - container: - image: alpine:latest - steps: - - name: Environment setup - run: apk add nodejs curl findutils - - name: Package download - uses: forgejo/download-artifact@v3 - - name: Package deployment - run: | - if test $GITHUB_REF_NAME == "ci" ; then - CI_REF_NAME=${{ inputs.ref_name }} - else - CI_REF_NAME=$GITHUB_REF_NAME - fi - curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-edge.cabal - curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig322/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-v322.cabal diff --git a/.forgejo/workflows/mirror-repository.yml b/.forgejo/workflows/mirror-repository.yml deleted file mode 100644 index f44c4668cf..0000000000 --- a/.forgejo/workflows/mirror-repository.yml +++ /dev/null @@ -1,50 +0,0 @@ -on: - workflow_dispatch: - - schedule: - - cron: '@hourly' - -jobs: - mirror: - name: Pull from upstream - runs-on: x86_64 - container: - image: alpine:latest - env: - upstream: https://git.joeyh.name/git/git-annex.git - tags: '10.2025*' - steps: - - name: Environment setup - run: apk add grep git sed coreutils bash nodejs - - name: Fetch destination - uses: actions/checkout@v4 - with: - fetch_depth: 1 - ref: ci - token: ${{ secrets.CODE_FORGEJO_TOKEN }} - - name: Missing tag detecting - run: | - git ls-remote $upstream "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' | sort > upstream_tags - git ls-remote ${{ github.server_url}}/${{ github.repository }} "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' | sort > destination_tags - comm -23 upstream_tags destination_tags > missing_tags - echo "Missing tags:" - cat missing_tags - - name: Missing tag fetch - run: | - git remote add upstream $upstream - while read tag; do - git fetch upstream tag $tag --no-tags - done < missing_tags - - name: Packaging workflow injection - run: | - while read tag; do - git checkout $tag - git tag -d $tag - git checkout ci -- ./.forgejo - git config user.name "forgejo-actions[bot]" - git config user.email "dev@ayakael.net" - git commit -m 'Inject custom workflow' - git tag -a $tag -m $tag - done < missing_tags - - name: Push to destination - run: git push --force origin refs/tags/*:refs/tags/* --tags diff --git a/.ghci b/.ghci new file mode 100644 index 0000000000..c5550cee6e --- /dev/null +++ b/.ghci @@ -0,0 +1 @@ +:load Common diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..5d425843f2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +debian/changelog merge=dpkg-mergechangelogs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..69650caa1a --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +tmp +test +build-stamp +Build/SysConfig.hs +git-annex +git-annex.1 +git-annex-shell.1 +git-union-merge.1 +doc/.ikiwiki +html +*.tix +.hpc +dist +# Sandboxed builds +cabal-dev +# Project-local emacs configuration +.dir-locals.el +# OSX related +.DS_Store +.virthualenv +tags diff --git a/Annex.hs b/Annex.hs new file mode 100644 index 0000000000..2a17fffe17 --- /dev/null +++ b/Annex.hs @@ -0,0 +1,218 @@ +{- git-annex monad + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE PackageImports, GeneralizedNewtypeDeriving, TypeFamilies, MultiParamTypeClasses #-} + +module Annex ( + Annex, + AnnexState(..), + FileInfo(..), + PreferredContentMap, + new, + newState, + run, + eval, + exec, + getState, + changeState, + setFlag, + setField, + setOutput, + getFlag, + getField, + addCleanup, + gitRepo, + inRepo, + fromRepo, + getGitConfig, + changeGitConfig, + changeGitRepo, +) where + +import "mtl" Control.Monad.State.Strict +import Control.Monad.Trans.Control (StM, MonadBaseControl, liftBaseWith, restoreM) +import Control.Monad.Base (liftBase, MonadBase) +import System.Posix.Types (Fd) + +import Common +import qualified Git +import qualified Git.Config +import Git.CatFile +import Git.CheckAttr +import Git.SharedRepository +import qualified Git.Queue +import Types.Backend +import Types.GitConfig +import qualified Types.Remote +import Types.Crypto +import Types.BranchState +import Types.TrustLevel +import Types.Group +import Types.Messages +import Types.UUID +import Utility.State +import qualified Utility.Matcher +import qualified Data.Map as M +import qualified Data.Set as S + +-- git-annex's monad +newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a } + deriving ( + Monad, + MonadIO, + MonadState AnnexState, + Functor, + Applicative + ) + +instance MonadBase IO Annex where + liftBase = Annex . liftBase + +instance MonadBaseControl IO Annex where + newtype StM Annex a = StAnnex (StM (StateT AnnexState IO) a) + liftBaseWith f = Annex $ liftBaseWith $ \runInIO -> + f $ liftM StAnnex . runInIO . runAnnex + restoreM = Annex . restoreM . unStAnnex + where + unStAnnex (StAnnex st) = st + +type Matcher a = Either [Utility.Matcher.Token a] (Utility.Matcher.Matcher a) + +data FileInfo = FileInfo + { relFile :: FilePath -- may be relative to cwd + , matchFile :: FilePath -- filepath to match on; may be relative to top + } + +type PreferredContentMap = M.Map UUID (Utility.Matcher.Matcher (S.Set UUID -> FileInfo -> Annex Bool)) + +-- internal state storage +data AnnexState = AnnexState + { repo :: Git.Repo + , gitconfig :: GitConfig + , backends :: [BackendA Annex] + , remotes :: [Types.Remote.RemoteA Annex] + , output :: MessageState + , force :: Bool + , fast :: Bool + , auto :: Bool + , branchstate :: BranchState + , repoqueue :: Maybe Git.Queue.Queue + , catfilehandle :: Maybe CatFileHandle + , checkattrhandle :: Maybe CheckAttrHandle + , forcebackend :: Maybe String + , limit :: Matcher (FileInfo -> Annex Bool) + , uuidmap :: Maybe UUIDMap + , preferredcontentmap :: Maybe PreferredContentMap + , shared :: Maybe SharedRepository + , forcetrust :: TrustMap + , trustmap :: Maybe TrustMap + , groupmap :: Maybe GroupMap + , ciphers :: M.Map StorableCipher Cipher + , lockpool :: M.Map FilePath Fd + , flags :: M.Map String Bool + , fields :: M.Map String String + , cleanup :: M.Map String (Annex ()) + , inodeschanged :: Maybe Bool + } + +newState :: Git.Repo -> AnnexState +newState gitrepo = AnnexState + { repo = gitrepo + , gitconfig = extractGitConfig gitrepo + , backends = [] + , remotes = [] + , output = defaultMessageState + , force = False + , fast = False + , auto = False + , branchstate = startBranchState + , repoqueue = Nothing + , catfilehandle = Nothing + , checkattrhandle = Nothing + , forcebackend = Nothing + , limit = Left [] + , uuidmap = Nothing + , preferredcontentmap = Nothing + , shared = Nothing + , forcetrust = M.empty + , trustmap = Nothing + , groupmap = Nothing + , ciphers = M.empty + , lockpool = M.empty + , flags = M.empty + , fields = M.empty + , cleanup = M.empty + , inodeschanged = Nothing + } + +{- Makes an Annex state object for the specified git repo. + - Ensures the config is read, if it was not already. -} +new :: Git.Repo -> IO AnnexState +new = newState <$$> Git.Config.read + +{- performs an action in the Annex monad -} +run :: AnnexState -> Annex a -> IO (a, AnnexState) +run s a = runStateT (runAnnex a) s +eval :: AnnexState -> Annex a -> IO a +eval s a = evalStateT (runAnnex a) s +exec :: AnnexState -> Annex a -> IO AnnexState +exec s a = execStateT (runAnnex a) s + +{- Sets a flag to True -} +setFlag :: String -> Annex () +setFlag flag = changeState $ \s -> + s { flags = M.insertWith' const flag True $ flags s } + +{- Sets a field to a value -} +setField :: String -> String -> Annex () +setField field value = changeState $ \s -> + s { fields = M.insertWith' const field value $ fields s } + +{- Adds a cleanup action to perform. -} +addCleanup :: String -> Annex () -> Annex () +addCleanup uid a = changeState $ \s -> + s { cleanup = M.insertWith' const uid a $ cleanup s } + +{- Sets the type of output to emit. -} +setOutput :: OutputType -> Annex () +setOutput o = changeState $ \s -> + s { output = (output s) { outputType = o } } + +{- Checks if a flag was set. -} +getFlag :: String -> Annex Bool +getFlag flag = fromMaybe False . M.lookup flag <$> getState flags + +{- Gets the value of a field. -} +getField :: String -> Annex (Maybe String) +getField field = M.lookup field <$> getState fields + +{- Returns the annex's git repository. -} +gitRepo :: Annex Git.Repo +gitRepo = getState repo + +{- Runs an IO action in the annex's git repository. -} +inRepo :: (Git.Repo -> IO a) -> Annex a +inRepo a = liftIO . a =<< gitRepo + +{- Extracts a value from the annex's git repisitory. -} +fromRepo :: (Git.Repo -> a) -> Annex a +fromRepo a = a <$> gitRepo + +{- Gets the GitConfig settings. -} +getGitConfig :: Annex GitConfig +getGitConfig = getState gitconfig + +{- Modifies a GitConfig setting. -} +changeGitConfig :: (GitConfig -> GitConfig) -> Annex () +changeGitConfig a = changeState $ \s -> s { gitconfig = a (gitconfig s) } + +{- Changing the git Repo data also involves re-extracting its GitConfig. -} +changeGitRepo :: Git.Repo -> Annex () +changeGitRepo r = changeState $ \s -> s + { repo = r + , gitconfig = extractGitConfig r + } diff --git a/Annex/Branch.hs b/Annex/Branch.hs new file mode 100644 index 0000000000..4a36de66aa --- /dev/null +++ b/Annex/Branch.hs @@ -0,0 +1,364 @@ +{- management of the git-annex branch + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Branch ( + fullname, + name, + hasOrigin, + hasSibling, + siblingBranches, + create, + update, + forceUpdate, + updateTo, + get, + change, + commit, + files, +) where + +import qualified Data.ByteString.Lazy.Char8 as L +import System.Posix.Env + +import Common.Annex +import Annex.BranchState +import Annex.Journal +import qualified Git +import qualified Git.Command +import qualified Git.Ref +import qualified Git.Branch +import qualified Git.UnionMerge +import qualified Git.UpdateIndex +import Git.HashObject +import Git.Types +import Git.FilePath +import Annex.CatFile +import Annex.Perms +import qualified Annex + +{- Name of the branch that is used to store git-annex's information. -} +name :: Git.Ref +name = Git.Ref "git-annex" + +{- Fully qualified name of the branch. -} +fullname :: Git.Ref +fullname = Git.Ref $ "refs/heads/" ++ show name + +{- Branch's name in origin. -} +originname :: Git.Ref +originname = Git.Ref $ "origin/" ++ show name + +{- Does origin/git-annex exist? -} +hasOrigin :: Annex Bool +hasOrigin = inRepo $ Git.Ref.exists originname + +{- Does the git-annex branch or a sibling foo/git-annex branch exist? -} +hasSibling :: Annex Bool +hasSibling = not . null <$> siblingBranches + +{- List of git-annex (refs, branches), including the main one and any + - from remotes. Duplicate refs are filtered out. -} +siblingBranches :: Annex [(Git.Ref, Git.Branch)] +siblingBranches = inRepo $ Git.Ref.matchingUniq name + +{- Creates the branch, if it does not already exist. -} +create :: Annex () +create = void getBranch + +{- Returns the ref of the branch, creating it first if necessary. -} +getBranch :: Annex Git.Ref +getBranch = maybe (hasOrigin >>= go >>= use) return =<< branchsha + where + go True = do + inRepo $ Git.Command.run + [Param "branch", Param $ show name, Param $ show originname] + fromMaybe (error $ "failed to create " ++ show name) + <$> branchsha + go False = withIndex' True $ + inRepo $ Git.Branch.commit "branch created" fullname [] + use sha = do + setIndexSha sha + return sha + branchsha = inRepo $ Git.Ref.sha fullname + +{- Ensures that the branch and index are up-to-date; should be + - called before data is read from it. Runs only once per git-annex run. -} +update :: Annex () +update = runUpdateOnce $ void $ updateTo =<< siblingBranches + +{- Forces an update even if one has already been run. -} +forceUpdate :: Annex Bool +forceUpdate = updateTo =<< siblingBranches + +{- Merges the specified Refs into the index, if they have any changes not + - already in it. The Branch names are only used in the commit message; + - it's even possible that the provided Branches have not been updated to + - point to the Refs yet. + - + - The branch is fast-forwarded if possible, otherwise a merge commit is + - made. + - + - Before Refs are merged into the index, it's important to first stage the + - journal into the index. Otherwise, any changes in the journal would + - later get staged, and might overwrite changes made during the merge. + - This is only done if some of the Refs do need to be merged. + - + - Returns True if any refs were merged in, False otherwise. + -} +updateTo :: [(Git.Ref, Git.Branch)] -> Annex Bool +updateTo pairs = do + -- ensure branch exists, and get its current ref + branchref <- getBranch + dirty <- journalDirty + (refs, branches) <- unzip <$> filterM isnewer pairs + if null refs + {- Even when no refs need to be merged, the index + - may still be updated if the branch has gotten ahead + - of the index. -} + then whenM (needUpdateIndex branchref) $ lockJournal $ do + forceUpdateIndex branchref + {- When there are journalled changes + - as well as the branch being updated, + - a commit needs to be done. -} + when dirty $ + go branchref True [] [] + else lockJournal $ go branchref dirty refs branches + return $ not $ null refs + where + isnewer (r, _) = inRepo $ Git.Branch.changed fullname r + go branchref dirty refs branches = withIndex $ do + cleanjournal <- if dirty then stageJournal else return noop + let merge_desc = if null branches + then "update" + else "merging " ++ + unwords (map Git.Ref.describe branches) ++ + " into " ++ show name + unless (null branches) $ do + showSideAction merge_desc + mergeIndex refs + ff <- if dirty + then return False + else inRepo $ Git.Branch.fastForward fullname refs + if ff + then updateIndex branchref + else commitBranch branchref merge_desc + (nub $ fullname:refs) + liftIO cleanjournal + +{- Gets the content of a file, which may be in the journal, or committed + - to the branch. Due to limitatons of git cat-file, does *not* get content + - that has only been staged to the index. + - + - Updates the branch if necessary, to ensure the most up-to-date available + - content is available. + - + - Returns an empty string if the file doesn't exist yet. -} +get :: FilePath -> Annex String +get = get' False + +{- Like get, but does not merge the branch, so the info returned may not + - reflect changes in remotes. (Changing the value this returns, and then + - merging is always the same as using get, and then changing its value.) -} +getStale :: FilePath -> Annex String +getStale = get' True + +get' :: Bool -> FilePath -> Annex String +get' staleok file = fromjournal =<< getJournalFile file + where + fromjournal (Just content) = return content + fromjournal Nothing + | staleok = withIndex frombranch + | otherwise = do + update + frombranch + frombranch = withIndex $ L.unpack <$> catFile fullname file + +{- Applies a function to modifiy the content of a file. + - + - Note that this does not cause the branch to be merged, it only + - modifes the current content of the file on the branch. + -} +change :: FilePath -> (String -> String) -> Annex () +change file a = lockJournal $ a <$> getStale file >>= set file + +{- Records new content of a file into the journal -} +set :: FilePath -> String -> Annex () +set file content = setJournalFile file content + +{- Stages the journal, and commits staged changes to the branch. -} +commit :: String -> Annex () +commit message = whenM journalDirty $ lockJournal $ do + cleanjournal <- stageJournal + ref <- getBranch + withIndex $ commitBranch ref message [fullname] + liftIO $ cleanjournal + +{- Commits the staged changes in the index to the branch. + - + - Ensures that the branch's index file is first updated to the state + - of the branch at branchref, before running the commit action. This + - is needed because the branch may have had changes pushed to it, that + - are not yet reflected in the index. + - + - Also safely handles a race that can occur if a change is being pushed + - into the branch at the same time. When the race happens, the commit will + - be made on top of the newly pushed change, but without the index file + - being updated to include it. The result is that the newly pushed + - change is reverted. This race is detected and another commit made + - to fix it. + - + - The branchref value can have been obtained using getBranch at any + - previous point, though getting it a long time ago makes the race + - more likely to occur. + -} +commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex () +commitBranch branchref message parents = do + showStoringStateAction + commitBranch' branchref message parents +commitBranch' :: Git.Ref -> String -> [Git.Ref] -> Annex () +commitBranch' branchref message parents = do + updateIndex branchref + committedref <- inRepo $ Git.Branch.commit message fullname parents + setIndexSha committedref + parentrefs <- commitparents <$> catObject committedref + when (racedetected branchref parentrefs) $ + fixrace committedref parentrefs + where + -- look for "parent ref" lines and return the refs + commitparents = map (Git.Ref . snd) . filter isparent . + map (toassoc . L.unpack) . L.lines + toassoc = separate (== ' ') + isparent (k,_) = k == "parent" + + {- The race can be detected by checking the commit's + - parent, which will be the newly pushed branch, + - instead of the expected ref that the index was updated to. -} + racedetected expectedref parentrefs + | expectedref `elem` parentrefs = False -- good parent + | otherwise = True -- race! + + {- To recover from the race, union merge the lost refs + - into the index, and recommit on top of the bad commit. -} + fixrace committedref lostrefs = do + mergeIndex lostrefs + commitBranch committedref racemessage [committedref] + + racemessage = message ++ " (recovery from race)" + +{- Lists all files on the branch. There may be duplicates in the list. -} +files :: Annex [FilePath] +files = do + update + withIndex $ do + bfiles <- inRepo $ Git.Command.pipeNullSplitZombie + [ Params "ls-tree --name-only -r -z" + , Param $ show fullname + ] + jfiles <- getJournalledFiles + return $ jfiles ++ bfiles + +{- Populates the branch's index file with the current branch contents. + - + - This is only done when the index doesn't yet exist, and the index + - is used to build up changes to be commited to the branch, and merge + - in changes from other branches. + -} +genIndex :: Git.Repo -> IO () +genIndex g = Git.UpdateIndex.streamUpdateIndex g + [Git.UpdateIndex.lsTree fullname g] + +{- Merges the specified refs into the index. + - Any changes staged in the index will be preserved. -} +mergeIndex :: [Git.Ref] -> Annex () +mergeIndex branches = do + h <- catFileHandle + inRepo $ \g -> Git.UnionMerge.mergeIndex h g branches + +{- Runs an action using the branch's index file. -} +withIndex :: Annex a -> Annex a +withIndex = withIndex' False +withIndex' :: Bool -> Annex a -> Annex a +withIndex' bootstrapping a = do + f <- fromRepo gitAnnexIndex + g <- gitRepo +#ifdef __ANDROID__ + {- Work around for weird getEnvironment breakage on Android. See + - https://github.com/neurocyte/ghc-android/issues/7 + - Instead, use getEnv to get some key environment variables that + - git expects to have. -} + let keyenv = words "USER PATH GIT_EXEC_PATH HOSTNAME HOME" + let getEnvPair k = maybe Nothing (\v -> Just (k, v)) <$> getEnv k + e <- liftIO $ catMaybes <$> forM keyenv getEnvPair +#else + e <- liftIO getEnvironment +#endif + let g' = g { gitEnv = Just $ ("GIT_INDEX_FILE", f):e } + + Annex.changeState $ \s -> s { Annex.repo = g' } + checkIndexOnce $ unlessM (liftIO $ doesFileExist f) $ do + unless bootstrapping create + liftIO $ createDirectoryIfMissing True $ takeDirectory f + unless bootstrapping $ inRepo genIndex + r <- a + Annex.changeState $ \s -> s { Annex.repo = (Annex.repo s) { gitEnv = gitEnv g} } + + return r + +{- Updates the branch's index to reflect the current contents of the branch. + - Any changes staged in the index will be preserved. + - + - Compares the ref stored in the lock file with the current + - ref of the branch to see if an update is needed. + -} +updateIndex :: Git.Ref -> Annex () +updateIndex branchref = whenM (needUpdateIndex branchref) $ + forceUpdateIndex branchref + +forceUpdateIndex :: Git.Ref -> Annex () +forceUpdateIndex branchref = do + withIndex $ mergeIndex [fullname] + setIndexSha branchref + +{- Checks if the index needs to be updated. -} +needUpdateIndex :: Git.Ref -> Annex Bool +needUpdateIndex branchref = do + lock <- fromRepo gitAnnexIndexLock + lockref <- Git.Ref . firstLine <$> + liftIO (catchDefaultIO "" $ readFileStrict lock) + return (lockref /= branchref) + +{- Record that the branch's index has been updated to correspond to a + - given ref of the branch. -} +setIndexSha :: Git.Ref -> Annex () +setIndexSha ref = do + lock <- fromRepo gitAnnexIndexLock + liftIO $ writeFile lock $ show ref ++ "\n" + setAnnexPerm lock + +{- Stages the journal into the index and returns an action that will + - clean up the staged journal files, which should only be run once + - the index has been committed to the branch. Should be run within + - lockJournal, to prevent others from modifying the journal. -} +stageJournal :: Annex (IO ()) +stageJournal = withIndex $ do + g <- gitRepo + let dir = gitAnnexJournalDir g + fs <- getJournalFiles + liftIO $ do + h <- hashObjectStart g + Git.UpdateIndex.streamUpdateIndex g + [genstream dir h fs] + hashObjectStop h + return $ liftIO $ mapM_ removeFile $ map (dir ) fs + where + genstream dir h fs streamer = forM_ fs $ \file -> do + let path = dir file + sha <- hashFile h path + streamer $ Git.UpdateIndex.updateIndexLine + sha FileBlob (asTopFilePath $ fileJournal file) diff --git a/Annex/BranchState.hs b/Annex/BranchState.hs new file mode 100644 index 0000000000..9b2f9a04c5 --- /dev/null +++ b/Annex/BranchState.hs @@ -0,0 +1,43 @@ +{- git-annex branch state management + - + - Runtime state about the git-annex branch. + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.BranchState where + +import Common.Annex +import Types.BranchState +import qualified Annex + +getState :: Annex BranchState +getState = Annex.getState Annex.branchstate + +setState :: BranchState -> Annex () +setState state = Annex.changeState $ \s -> s { Annex.branchstate = state } + +changeState :: (BranchState -> BranchState) -> Annex () +changeState changer = setState =<< changer <$> getState + +{- Runs an action to check that the index file exists, if it's not been + - checked before in this run of git-annex. -} +checkIndexOnce :: Annex () -> Annex () +checkIndexOnce a = unlessM (indexChecked <$> getState) $ do + a + changeState $ \s -> s { indexChecked = True } + +{- Runs an action to update the branch, if it's not been updated before + - in this run of git-annex. -} +runUpdateOnce :: Annex () -> Annex () +runUpdateOnce a = unlessM (branchUpdated <$> getState) $ do + a + disableUpdate + +{- Avoids updating the branch. A useful optimisation when the branch + - is known to have not changed, or git-annex won't be relying on info + - from it. -} +disableUpdate :: Annex () +disableUpdate = changeState $ \s -> s { branchUpdated = True } diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs new file mode 100644 index 0000000000..292d624603 --- /dev/null +++ b/Annex/CatFile.hs @@ -0,0 +1,62 @@ +{- git cat-file interface, with handle automatically stored in the Annex monad + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.CatFile ( + catFile, + catObject, + catObjectDetails, + catFileHandle, + catKey, + catKeyFile, +) where + +import qualified Data.ByteString.Lazy as L + +import Common.Annex +import qualified Git +import qualified Git.CatFile +import qualified Annex +import Git.Types + +catFile :: Git.Branch -> FilePath -> Annex L.ByteString +catFile branch file = do + h <- catFileHandle + liftIO $ Git.CatFile.catFile h branch file + +catObject :: Git.Ref -> Annex L.ByteString +catObject ref = do + h <- catFileHandle + liftIO $ Git.CatFile.catObject h ref + +catObjectDetails :: Git.Ref -> Annex (Maybe (L.ByteString, Sha)) +catObjectDetails ref = do + h <- catFileHandle + liftIO $ Git.CatFile.catObjectDetails h ref + +catFileHandle :: Annex Git.CatFile.CatFileHandle +catFileHandle = maybe startup return =<< Annex.getState Annex.catfilehandle + where + startup = do + h <- inRepo Git.CatFile.catFileStart + Annex.changeState $ \s -> s { Annex.catfilehandle = Just h } + return h + +{- From the Sha or Ref of a symlink back to the key. -} +catKey :: Ref -> Annex (Maybe Key) +catKey ref = do + l <- encodeW8 . L.unpack <$> catObject ref + return $ if isLinkToAnnex l + then fileKey $ takeFileName l + else Nothing + +{- From a file in git back to the key. + - + - Prefixing the file with ./ makes this work even if in a subdirectory + - of a repo. For some reason, HEAD is sometimes needed. + -} +catKeyFile :: FilePath -> Annex (Maybe Key) +catKeyFile f = catKey $ Ref $ "HEAD:./" ++ f diff --git a/Annex/CheckAttr.hs b/Annex/CheckAttr.hs new file mode 100644 index 0000000000..8eed9e804c --- /dev/null +++ b/Annex/CheckAttr.hs @@ -0,0 +1,35 @@ +{- git check-attr interface, with handle automatically stored in the Annex monad + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.CheckAttr ( + checkAttr, + checkAttrHandle +) where + +import Common.Annex +import qualified Git.CheckAttr as Git +import qualified Annex + +{- All gitattributes used by git-annex. -} +annexAttrs :: [Git.Attr] +annexAttrs = + [ "annex.backend" + , "annex.numcopies" + ] + +checkAttr :: Git.Attr -> FilePath -> Annex String +checkAttr attr file = do + h <- checkAttrHandle + liftIO $ Git.checkAttr h attr file + +checkAttrHandle :: Annex Git.CheckAttrHandle +checkAttrHandle = maybe startup return =<< Annex.getState Annex.checkattrhandle + where + startup = do + h <- inRepo $ Git.checkAttrStart annexAttrs + Annex.changeState $ \s -> s { Annex.checkattrhandle = Just h } + return h diff --git a/Annex/Content.hs b/Annex/Content.hs new file mode 100644 index 0000000000..0439cb3679 --- /dev/null +++ b/Annex/Content.hs @@ -0,0 +1,482 @@ +{- git-annex file content managing + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Content ( + inAnnex, + inAnnexSafe, + lockContent, + calcGitLink, + getViaTmp, + getViaTmpChecked, + getViaTmpUnchecked, + withTmp, + checkDiskSpace, + moveAnnex, + sendAnnex, + prepSendAnnex, + removeAnnex, + fromAnnex, + moveBad, + getKeysPresent, + saveState, + downloadUrl, + preseedTmp, + freezeContent, + thawContent, + replaceFile, + cleanObjectLoc, +) where + +import System.IO.Unsafe (unsafeInterleaveIO) + +import Common.Annex +import Logs.Location +import qualified Git +import qualified Annex +import qualified Annex.Queue +import qualified Annex.Branch +import Utility.DiskFree +import Utility.FileMode +import qualified Utility.Url as Url +import Types.Key +import Utility.DataUnits +import Utility.CopyFile +import Config +import Annex.Exception +import Git.SharedRepository +import Annex.Perms +import Annex.Content.Direct +import Backend + +{- Checks if a given key's content is currently present. -} +inAnnex :: Key -> Annex Bool +inAnnex = inAnnex' id False $ liftIO . doesFileExist + +{- Generic inAnnex, handling both indirect and direct mode. + - + - In direct mode, at least one of the associated files must pass the + - check. Additionally, the file must be unmodified. + -} +inAnnex' :: (a -> Bool) -> a -> (FilePath -> Annex a) -> Key -> Annex a +inAnnex' isgood bad check key = withObjectLoc key checkindirect checkdirect + where + checkindirect loc = do + whenM (fromRepo Git.repoIsUrl) $ + error "inAnnex cannot check remote repo" + check loc + checkdirect [] = return bad + checkdirect (loc:locs) = do + r <- check loc + if isgood r + then ifM (goodContent key loc) + ( return r + , checkdirect locs + ) + else checkdirect locs + +{- A safer check; the key's content must not only be present, but + - is not in the process of being removed. -} +inAnnexSafe :: Key -> Annex (Maybe Bool) +inAnnexSafe = inAnnex' (fromMaybe False) (Just False) go + where + go f = liftIO $ openforlock f >>= check + openforlock f = catchMaybeIO $ + openFd f ReadOnly Nothing defaultFileFlags + check Nothing = return is_missing + check (Just h) = do + v <- getLock h (ReadLock, AbsoluteSeek, 0, 0) + closeFd h + return $ case v of + Just _ -> is_locked + Nothing -> is_unlocked + is_locked = Nothing + is_unlocked = Just True + is_missing = Just False + +{- Content is exclusively locked while running an action that might remove + - it. (If the content is not present, no locking is done.) -} +lockContent :: Key -> Annex a -> Annex a +lockContent key a = do + file <- inRepo $ gitAnnexLocation key + bracketIO (openforlock file >>= lock) unlock a + where + {- Since files are stored with the write bit disabled, have + - to fiddle with permissions to open for an exclusive lock. -} + openforlock f = catchMaybeIO $ ifM (doesFileExist f) + ( withModifiedFileMode f + (`unionFileModes` ownerWriteMode) + open + , open + ) + where + open = openFd f ReadWrite Nothing defaultFileFlags + lock Nothing = return Nothing + lock (Just fd) = do + v <- tryIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0) + case v of + Left _ -> error "content is locked" + Right _ -> return $ Just fd + unlock Nothing = noop + unlock (Just l) = closeFd l + +{- Calculates the relative path to use to link a file to a key. -} +calcGitLink :: FilePath -> Key -> Annex FilePath +calcGitLink file key = do + cwd <- liftIO getCurrentDirectory + let absfile = fromMaybe whoops $ absNormPath cwd file + loc <- inRepo $ gitAnnexLocation key + return $ relPathDirToFile (parentDir absfile) loc + where + whoops = error $ "unable to normalize " ++ file + +{- Runs an action, passing it a temporary filename to get, + - and if the action succeeds, moves the temp file into + - the annex as a key's content. -} +getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool +getViaTmp = getViaTmpChecked (return True) + +{- Like getViaTmp, but does not check that there is enough disk space + - for the incoming key. For use when the key content is already on disk + - and not being copied into place. -} +getViaTmpUnchecked :: Key -> (FilePath -> Annex Bool) -> Annex Bool +getViaTmpUnchecked = finishGetViaTmp (return True) + +getViaTmpChecked :: Annex Bool -> Key -> (FilePath -> Annex Bool) -> Annex Bool +getViaTmpChecked check key action = do + tmp <- fromRepo $ gitAnnexTmpLocation key + + -- Check that there is enough free disk space. + -- When the temp file already exists, count the space + -- it is using as free. + e <- liftIO $ doesFileExist tmp + alreadythere <- if e + then fromIntegral . fileSize <$> liftIO (getFileStatus tmp) + else return 0 + ifM (checkDiskSpace Nothing key alreadythere) + ( do + when e $ thawContent tmp + finishGetViaTmp check key action + , return False + ) + +finishGetViaTmp :: Annex Bool -> Key -> (FilePath -> Annex Bool) -> Annex Bool +finishGetViaTmp check key action = do + tmpfile <- prepTmp key + ifM (action tmpfile <&&> check) + ( do + moveAnnex key tmpfile + logStatus key InfoPresent + return True + , do + -- the tmp file is left behind, in case caller wants + -- to resume its transfer + return False + ) + +prepTmp :: Key -> Annex FilePath +prepTmp key = do + tmp <- fromRepo $ gitAnnexTmpLocation key + createAnnexDirectory (parentDir tmp) + return tmp + +{- Creates a temp file, runs an action on it, and cleans up the temp file. -} +withTmp :: Key -> (FilePath -> Annex a) -> Annex a +withTmp key action = do + tmp <- prepTmp key + res <- action tmp + liftIO $ nukeFile tmp + return res + +{- Checks that there is disk space available to store a given key, + - in a destination (or the annex) printing a warning if not. -} +checkDiskSpace :: Maybe FilePath -> Key -> Integer -> Annex Bool +checkDiskSpace destination key alreadythere = do + reserve <- annexDiskReserve <$> Annex.getGitConfig + free <- liftIO . getDiskFree =<< dir + force <- Annex.getState Annex.force + case (free, keySize key) of + (Just have, Just need) -> do + let ok = (need + reserve <= have + alreadythere) || force + unless ok $ do + liftIO $ print (need, reserve, have, alreadythere) + needmorespace (need + reserve - have - alreadythere) + return ok + _ -> return True + where + dir = maybe (fromRepo gitAnnexDir) return destination + needmorespace n = + warning $ "not enough free space, need " ++ + roughSize storageUnits True n ++ + " more" ++ forcemsg + forcemsg = " (use --force to override this check or adjust annex.diskreserve)" + +{- Moves a key's content into .git/annex/objects/ + - + - In direct mode, moves it to the associated file, or files. + - + - What if the key there already has content? This could happen for + - various reasons; perhaps the same content is being annexed again. + - Perhaps there has been a hash collision generating the keys. + - + - The current strategy is to assume that in this case it's safe to delete + - one of the two copies of the content; and the one already in the annex + - is left there, assuming it's the original, canonical copy. + - + - I considered being more paranoid, and checking that both files had + - the same content. Decided against it because A) users explicitly choose + - a backend based on its hashing properties and so if they're dealing + - with colliding files it's their own fault and B) adding such a check + - would not catch all cases of colliding keys. For example, perhaps + - a remote has a key; if it's then added again with different content then + - the overall system now has two different peices of content for that + - key, and one of them will probably get deleted later. So, adding the + - check here would only raise expectations that git-annex cannot truely + - meet. + -} +moveAnnex :: Key -> FilePath -> Annex () +moveAnnex key src = withObjectLoc key storeobject storedirect + where + storeobject dest = ifM (liftIO $ doesFileExist dest) + ( liftIO $ removeFile src + , do + createContentDir dest + liftIO $ moveFile src dest + freezeContent dest + freezeContentDir dest + ) + storedirect fs = storedirect' =<< filterM validsymlink fs + validsymlink f = (==) (Just key) <$> isAnnexLink f + + storedirect' [] = storeobject =<< inRepo (gitAnnexLocation key) + storedirect' (dest:fs) = do + updateInodeCache key src + thawContent src + replaceFile dest $ liftIO . moveFile src + forM_ fs $ \f -> replaceFile f $ + void . liftIO . copyFileExternal dest + +{- Replaces any existing file with a new version, by running an action. + - First, makes sure the file is deleted. Or, if it didn't already exist, + - makes sure the parent directory exists. -} +replaceFile :: FilePath -> (FilePath -> Annex ()) -> Annex () +replaceFile file a = do + liftIO $ do + r <- tryIO $ removeFile file + case r of + Left _ -> createDirectoryIfMissing True $ parentDir file + _ -> noop + a file + +{- Runs an action to transfer an object's content. + - + - In direct mode, it's possible for the file to change as it's being sent. + - If this happens, runs the rollback action and returns False. The + - rollback action should remove the data that was transferred. + -} +sendAnnex :: Key -> Annex () -> (FilePath -> Annex Bool) -> Annex Bool +sendAnnex key rollback sendobject = go =<< prepSendAnnex key + where + go Nothing = return False + go (Just (f, checksuccess)) = do + r <- sendobject f + ifM checksuccess + ( return r + , do + rollback + return False + ) + +{- Returns a file that contains an object's content, + - and an check to run after the transfer is complete. + - + - In direct mode, it's possible for the file to change as it's being sent, + - and the check detects this case and returns False. + -} +prepSendAnnex :: Key -> Annex (Maybe (FilePath, Annex Bool)) +prepSendAnnex key = withObjectLoc key indirect direct + where + indirect f = return $ Just (f, return True) + direct [] = return Nothing + direct (f:fs) = do + cache <- recordedInodeCache key + -- check that we have a good file + ifM (sameInodeCache f cache) + ( return $ Just (f, sameInodeCache f cache) + , direct fs + ) + +{- Performs an action, passing it the location to use for a key's content. + - + - In direct mode, the associated files will be passed. But, if there are + - no associated files for a key, the indirect mode action will be + - performed instead. -} +withObjectLoc :: Key -> (FilePath -> Annex a) -> ([FilePath] -> Annex a) -> Annex a +withObjectLoc key indirect direct = ifM isDirect + ( do + fs <- associatedFiles key + if null fs + then goindirect + else direct fs + , goindirect + ) + where + goindirect = indirect =<< inRepo (gitAnnexLocation key) + +cleanObjectLoc :: Key -> Annex () +cleanObjectLoc key = do + file <- inRepo $ gitAnnexLocation key + unlessM crippledFileSystem $ + void $ liftIO $ catchMaybeIO $ allowWrite $ parentDir file + liftIO $ removeparents file (3 :: Int) + where + removeparents _ 0 = noop + removeparents file n = do + let dir = parentDir file + maybe noop (const $ removeparents dir (n-1)) + <=< catchMaybeIO $ removeDirectory dir + +{- Removes a key's file from .git/annex/objects/ + - + - In direct mode, deletes the associated files or files, and replaces + - them with symlinks. -} +removeAnnex :: Key -> Annex () +removeAnnex key = withObjectLoc key remove removedirect + where + remove file = do + unlessM crippledFileSystem $ + liftIO $ allowWrite $ parentDir file + liftIO $ nukeFile file + removeInodeCache key + cleanObjectLoc key + removedirect fs = do + cache <- recordedInodeCache key + removeInodeCache key + mapM_ (resetfile cache) fs + resetfile cache f = whenM (sameInodeCache f cache) $ do + l <- calcGitLink f key + top <- fromRepo Git.repoPath + cwd <- liftIO getCurrentDirectory + let top' = fromMaybe top $ absNormPath cwd top + let l' = relPathDirToFile top' (fromMaybe l $ absNormPath top' l) + replaceFile f $ const $ + makeAnnexLink l' f + +{- Moves a key's file out of .git/annex/objects/ -} +fromAnnex :: Key -> FilePath -> Annex () +fromAnnex key dest = do + file <- inRepo $ gitAnnexLocation key + unlessM crippledFileSystem $ + liftIO $ allowWrite $ parentDir file + thawContent file + liftIO $ moveFile file dest + cleanObjectLoc key + +{- Moves a key out of .git/annex/objects/ into .git/annex/bad, and + - returns the file it was moved to. -} +moveBad :: Key -> Annex FilePath +moveBad key = do + src <- inRepo $ gitAnnexLocation key + bad <- fromRepo gitAnnexBadDir + let dest = bad takeFileName src + createAnnexDirectory (parentDir dest) + unlessM crippledFileSystem $ + liftIO $ allowWrite (parentDir src) + liftIO $ moveFile src dest + cleanObjectLoc key + logStatus key InfoMissing + return dest + +{- List of keys whose content exists in the annex. -} +getKeysPresent :: Annex [Key] +getKeysPresent = do + direct <- isDirect + dir <- fromRepo gitAnnexObjectDir + liftIO $ traverse direct (2 :: Int) dir + where + traverse direct depth dir = do + contents <- catchDefaultIO [] (dirContents dir) + if depth == 0 + then do + contents' <- filterM (present direct) contents + let keys = mapMaybe (fileKey . takeFileName) contents' + continue keys [] + else do + let deeper = traverse direct (depth - 1) + continue [] (map deeper contents) + continue keys [] = return keys + continue keys (a:as) = do + {- Force lazy traversal with unsafeInterleaveIO. -} + morekeys <- unsafeInterleaveIO a + continue (morekeys++keys) as + + {- In indirect mode, look for the key. In direct mode, + - the inode cache file is only present when a key's content + - is present. -} + present False d = doesFileExist $ contentfile d + present True d = doesFileExist $ contentfile d ++ ".cache" + contentfile d = d takeFileName d + +{- Things to do to record changes to content when shutting down. + - + - It's acceptable to avoid committing changes to the branch, + - especially if performing a short-lived action. + -} +saveState :: Bool -> Annex () +saveState nocommit = doSideAction $ do + Annex.Queue.flush + unless nocommit $ + whenM (annexAlwaysCommit <$> Annex.getGitConfig) $ + Annex.Branch.commit "update" + +{- Downloads content from any of a list of urls. -} +downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool +downloadUrl urls file = do + o <- map Param . annexWebOptions <$> Annex.getGitConfig + headers <- getHttpHeaders + liftIO $ anyM (\u -> Url.download u headers o file) urls + +{- Copies a key's content, when present, to a temp file. + - This is used to speed up some rsyncs. -} +preseedTmp :: Key -> FilePath -> Annex Bool +preseedTmp key file = go =<< inAnnex key + where + go False = return False + go True = do + ok <- copy + when ok $ thawContent file + return ok + copy = ifM (liftIO $ doesFileExist file) + ( return True + , do + s <- inRepo $ gitAnnexLocation key + liftIO $ copyFileExternal s file + ) + +{- Blocks writing to an annexed file. The file is made unwritable + - to avoid accidental edits. core.sharedRepository may change + - who can read it. -} +freezeContent :: FilePath -> Annex () +freezeContent file = unlessM crippledFileSystem $ + liftIO . go =<< fromRepo getSharedRepository + where + go GroupShared = modifyFileMode file $ + removeModes writeModes . + addModes [ownerReadMode, groupReadMode] + go AllShared = modifyFileMode file $ + removeModes writeModes . + addModes readModes + go _ = preventWrite file + +{- Allows writing to an annexed file that freezeContent was called on + - before. -} +thawContent :: FilePath -> Annex () +thawContent file = unlessM crippledFileSystem $ + liftIO . go =<< fromRepo getSharedRepository + where + go GroupShared = groupWriteRead file + go AllShared = groupWriteRead file + go _ = allowWrite file diff --git a/Annex/Content/Direct.hs b/Annex/Content/Direct.hs new file mode 100644 index 0000000000..b0d549fc69 --- /dev/null +++ b/Annex/Content/Direct.hs @@ -0,0 +1,198 @@ +{- git-annex file content managing for direct mode + - + - Copyright 2012-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Content.Direct ( + associatedFiles, + removeAssociatedFile, + addAssociatedFile, + goodContent, + recordedInodeCache, + updateInodeCache, + writeInodeCache, + sameInodeCache, + sameFileStatus, + removeInodeCache, + toInodeCache, + inodesChanged, + createInodeSentinalFile, +) where + +import Common.Annex +import qualified Annex +import Annex.Perms +import qualified Git +import Utility.TempFile +import Logs.Location +import Utility.InodeCache + +{- Absolute FilePaths of Files in the tree that are associated with a key. -} +associatedFiles :: Key -> Annex [FilePath] +associatedFiles key = do + files <- associatedFilesRelative key + top <- fromRepo Git.repoPath + return $ map (top ) files + +{- List of files in the tree that are associated with a key, relative to + - the top of the repo. -} +associatedFilesRelative :: Key -> Annex [FilePath] +associatedFilesRelative key = do + mapping <- inRepo $ gitAnnexMapping key + liftIO $ catchDefaultIO [] $ do + h <- openFile mapping ReadMode + fileEncoding h + lines <$> hGetContents h + +{- Changes the associated files information for a key, applying a + - transformation to the list. Returns new associatedFiles value. -} +changeAssociatedFiles :: Key -> ([FilePath] -> [FilePath]) -> Annex [FilePath] +changeAssociatedFiles key transform = do + mapping <- inRepo $ gitAnnexMapping key + files <- associatedFilesRelative key + let files' = transform files + when (files /= files') $ do + createContentDir mapping + liftIO $ viaTmp write mapping $ unlines files' + top <- fromRepo Git.repoPath + return $ map (top ) files' + where + write file content = do + h <- openFile file WriteMode + fileEncoding h + hPutStr h content + hClose h + +{- Removes an associated file. Returns new associatedFiles value. -} +removeAssociatedFile :: Key -> FilePath -> Annex [FilePath] +removeAssociatedFile key file = do + file' <- normaliseAssociatedFile file + fs <- changeAssociatedFiles key $ filter (/= file') + when (null fs) $ + logStatus key InfoMissing + return fs + +{- Adds an associated file. Returns new associatedFiles value. -} +addAssociatedFile :: Key -> FilePath -> Annex [FilePath] +addAssociatedFile key file = do + file' <- normaliseAssociatedFile file + changeAssociatedFiles key $ \files -> + if file' `elem` files + then files + else file':files + +{- Associated files are always stored relative to the top of the repository. + - The input FilePath is relative to the CWD. -} +normaliseAssociatedFile :: FilePath -> Annex FilePath +normaliseAssociatedFile file = do + top <- fromRepo Git.repoPath + liftIO $ relPathDirToFile top <$> absPath file + +{- Checks if a file in the tree, associated with a key, has not been modified. + - + - To avoid needing to fsck the file's content, which can involve an + - expensive checksum, this relies on a cache that contains the file's + - expected mtime and inode. + -} +goodContent :: Key -> FilePath -> Annex Bool +goodContent key file = sameInodeCache file =<< recordedInodeCache key + +{- Gets the recorded inode cache for a key. -} +recordedInodeCache :: Key -> Annex (Maybe InodeCache) +recordedInodeCache key = withInodeCacheFile key $ \f -> + liftIO $ catchDefaultIO Nothing $ readInodeCache <$> readFile f + +{- Stores a cache of attributes for a file that is associated with a key. -} +updateInodeCache :: Key -> FilePath -> Annex () +updateInodeCache key file = maybe noop (writeInodeCache key) + =<< liftIO (genInodeCache file) + +{- Writes a cache for a key. -} +writeInodeCache :: Key -> InodeCache -> Annex () +writeInodeCache key cache = withInodeCacheFile key $ \f -> do + createContentDir f + liftIO $ writeFile f $ showInodeCache cache + +{- Removes an inode cache. -} +removeInodeCache :: Key -> Annex () +removeInodeCache key = withInodeCacheFile key $ \f -> do + createContentDir f -- also thaws directory + liftIO $ nukeFile f + +withInodeCacheFile :: Key -> (FilePath -> Annex a) -> Annex a +withInodeCacheFile key a = a =<< inRepo (gitAnnexInodeCache key) + +{- Checks if a InodeCache matches the current version of a file. -} +sameInodeCache :: FilePath -> Maybe InodeCache -> Annex Bool +sameInodeCache _ Nothing = return False +sameInodeCache file (Just old) = go =<< liftIO (genInodeCache file) + where + go Nothing = return False + go (Just curr) = compareInodeCaches curr old + +{- Checks if a FileStatus matches the recorded InodeCache of a file. -} +sameFileStatus :: Key -> FileStatus -> Annex Bool +sameFileStatus key status = do + old <- recordedInodeCache key + let curr = toInodeCache status + r <- case (old, curr) of + (Just o, Just c) -> compareInodeCaches o c + (Nothing, Nothing) -> return True + _ -> return False + return r + +{- If the inodes have changed, only the size and mtime are compared. -} +compareInodeCaches :: InodeCache -> InodeCache -> Annex Bool +compareInodeCaches x y + | x == y = return True + | otherwise = ifM inodesChanged + ( return $ compareWeak x y + , return False + ) + +{- Some filesystems get new inodes each time they are mounted. + - In order to work on such a filesystem, a sentinal file is used to detect + - when the inodes have changed. + - + - If the sentinal file does not exist, we have to assume that the + - inodes have changed. + -} +inodesChanged :: Annex Bool +inodesChanged = maybe calc return =<< Annex.getState Annex.inodeschanged + where + calc = do + scache <- liftIO . genInodeCache + =<< fromRepo gitAnnexInodeSentinal + scached <- readInodeSentinalFile + let changed = case (scache, scached) of + (Just c1, Just c2) -> c1 /= c2 + _ -> True + Annex.changeState $ \s -> s { Annex.inodeschanged = Just changed } + return changed + +readInodeSentinalFile :: Annex (Maybe InodeCache) +readInodeSentinalFile = do + sentinalcachefile <- fromRepo gitAnnexInodeSentinalCache + liftIO $ catchDefaultIO Nothing $ + readInodeCache <$> readFile sentinalcachefile + +writeInodeSentinalFile :: Annex () +writeInodeSentinalFile = do + sentinalfile <- fromRepo gitAnnexInodeSentinal + sentinalcachefile <- fromRepo gitAnnexInodeSentinalCache + liftIO $ writeFile sentinalfile "" + liftIO $ maybe noop (writeFile sentinalcachefile . showInodeCache) + =<< genInodeCache sentinalfile + +{- The sentinal file is only created when first initializing a repository. + - If there are any annexed objects in the repository already, creating + - the file would invalidate their inode caches. -} +createInodeSentinalFile :: Annex () +createInodeSentinalFile = + unlessM (alreadyexists <||> hasobjects) + writeInodeSentinalFile + where + alreadyexists = isJust <$> readInodeSentinalFile + hasobjects = liftIO . doesDirectoryExist =<< fromRepo gitAnnexObjectDir diff --git a/Annex/Direct.hs b/Annex/Direct.hs new file mode 100644 index 0000000000..c6f12a7b82 --- /dev/null +++ b/Annex/Direct.hs @@ -0,0 +1,209 @@ +{- git-annex direct mode + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Direct where + +import Common.Annex +import qualified Git +import qualified Git.LsFiles +import qualified Git.Merge +import qualified Git.DiffTree as DiffTree +import Git.Sha +import Git.Types +import Annex.CatFile +import Utility.FileMode +import qualified Annex.Queue +import Logs.Location +import Backend +import Types.KeySource +import Annex.Content +import Annex.Content.Direct +import Annex.Link +import Utility.InodeCache +import Utility.CopyFile + +{- Uses git ls-files to find files that need to be committed, and stages + - them into the index. Returns True if some changes were staged. -} +stageDirect :: Annex Bool +stageDirect = do + Annex.Queue.flush + top <- fromRepo Git.repoPath + (l, cleanup) <- inRepo $ Git.LsFiles.stagedDetails [top] + forM_ l go + void $ liftIO cleanup + staged <- Annex.Queue.size + Annex.Queue.flush + return $ staged /= 0 + where + {- Determine what kind of modified or deleted file this is, as + - efficiently as we can, by getting any key that's associated + - with it in git, as well as its stat info. -} + go (file, Just sha) = do + mkey <- catKey sha + mstat <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file + case (mkey, mstat, toInodeCache =<< mstat) of + (Just key, _, Just cache) -> do + {- All direct mode files will show as + - modified, so compare the cache to see if + - it really was. -} + oldcache <- recordedInodeCache key + when (oldcache /= Just cache) $ + modifiedannexed file key cache + (Just key, Nothing, _) -> deletedannexed file key + (Nothing, Nothing, _) -> deletegit file + (_, Just _, _) -> addgit file + go _ = noop + + modifiedannexed file oldkey cache = do + void $ removeAssociatedFile oldkey file + void $ addDirect file cache + + deletedannexed file key = do + void $ removeAssociatedFile key file + deletegit file + + addgit file = Annex.Queue.addCommand "add" [Param "-f"] [file] + + deletegit file = Annex.Queue.addCommand "rm" [Param "-f"] [file] + +{- Adds a file to the annex in direct mode. Can fail, if the file is + - modified or deleted while it's being added. -} +addDirect :: FilePath -> InodeCache -> Annex Bool +addDirect file cache = do + showStart "add" file + let source = KeySource + { keyFilename = file + , contentLocation = file + , inodeCache = Just cache + } + got =<< genKey source =<< chooseBackend file + where + got Nothing = do + showEndFail + return False + got (Just (key, _)) = ifM (sameInodeCache file $ Just cache) + ( do + stageSymlink file =<< hashSymlink =<< calcGitLink file key + writeInodeCache key cache + void $ addAssociatedFile key file + logStatus key InfoPresent + showEndOk + return True + , do + showEndFail + return False + ) + +{- In direct mode, git merge would usually refuse to do anything, since it + - sees present direct mode files as type changed files. To avoid this, + - merge is run with the work tree set to a temp directory. + - + - This should only be used once any changes to the real working tree have + - already been committed, because it overwrites files in the working tree. + -} +mergeDirect :: FilePath -> Git.Ref -> Git.Repo -> IO Bool +mergeDirect d branch g = do + createDirectoryIfMissing True d + let g' = g { location = Local { gitdir = Git.localGitDir g, worktree = Just d } } + Git.Merge.mergeNonInteractive branch g' + +{- Cleans up after a direct mode merge. The merge must have been committed, + - and the commit sha passed in, along with the old sha of the tree + - before the merge. Uses git diff-tree to find files that changed between + - the two shas, and applies those changes to the work tree. + -} +mergeDirectCleanup :: FilePath -> Git.Ref -> Git.Ref -> Annex () +mergeDirectCleanup d oldsha newsha = do + (items, cleanup) <- inRepo $ DiffTree.diffTreeRecursive oldsha newsha + forM_ items updated + void $ liftIO $ cleanup + liftIO $ removeDirectoryRecursive d + where + updated item = do + go DiffTree.srcsha DiffTree.srcmode moveout moveout_raw + go DiffTree.dstsha DiffTree.dstmode movein movein_raw + where + go getsha getmode a araw + | getsha item == nullSha = noop + | isSymLink (getmode item) = + maybe (araw f) (\k -> void $ a k f) + =<< catKey (getsha item) + | otherwise = araw f + f = DiffTree.file item + + moveout = removeDirect + + {- Files deleted by the merge are removed from the work tree. + - Empty work tree directories are removed, per git behavior. -} + moveout_raw f = liftIO $ do + nukeFile f + void $ tryIO $ removeDirectory $ parentDir f + + {- The symlink is created from the key, rather than moving in the + - symlink created in the temp directory by the merge. This because + - a conflicted merge will write to some other file in the temp + - directory. + - + - Symlinks are replaced with their content, if it's available. -} + movein k f = do + l <- calcGitLink f k + replaceFile f $ + makeAnnexLink l + toDirect k f + + {- Any new, modified, or renamed files were written to the temp + - directory by the merge, and are moved to the real work tree. -} + movein_raw f = liftIO $ do + createDirectoryIfMissing True $ parentDir f + void $ tryIO $ rename (d f) f + +{- If possible, converts a symlink in the working tree into a direct + - mode file. -} +toDirect :: Key -> FilePath -> Annex () +toDirect k f = fromMaybe noop =<< toDirectGen k f + +toDirectGen :: Key -> FilePath -> Annex (Maybe (Annex ())) +toDirectGen k f = do + loc <- inRepo $ gitAnnexLocation k + absf <- liftIO $ absPath f + locs <- filter (/= absf) <$> addAssociatedFile k f + case locs of + [] -> ifM (liftIO $ doesFileExist loc) + ( return $ Just $ do + {- Move content from annex to direct file. -} + updateInodeCache k loc + thawContent loc + replaceFile f $ + liftIO . moveFile loc + , return Nothing + ) + (loc':_) -> ifM (isNothing <$> getAnnexLinkTarget loc') + {- Another direct file has the content; copy it. -} + ( return $ Just $ + replaceFile f $ + void . liftIO . copyFileExternal loc' + , return Nothing + ) + +{- Removes a direct mode file, while retaining its content. -} +removeDirect :: Key -> FilePath -> Annex () +removeDirect k f = do + locs <- removeAssociatedFile k f + when (null locs) $ + whenM (isNothing <$> getAnnexLinkTarget f) $ + moveAnnex k f + liftIO $ do + nukeFile f + void $ tryIO $ removeDirectory $ parentDir f + +{- Called when a direct mode file has been changed. Its old content may be + - lost. -} +changedDirect :: Key -> FilePath -> Annex () +changedDirect oldk f = do + locs <- removeAssociatedFile oldk f + whenM (pure (null locs) <&&> not <$> inAnnex oldk) $ + logStatus oldk InfoMissing diff --git a/Annex/Exception.hs b/Annex/Exception.hs new file mode 100644 index 0000000000..4d21297b10 --- /dev/null +++ b/Annex/Exception.hs @@ -0,0 +1,32 @@ +{- exception handling in the git-annex monad + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Exception ( + bracketIO, + handle, + tryAnnex, + throw, +) where + +import Control.Exception.Lifted (handle, try) +import Control.Monad.Trans.Control (liftBaseOp) +import Control.Exception hiding (handle, try, throw) + +import Common.Annex + +{- Runs an Annex action, with setup and cleanup both in the IO monad. -} +bracketIO :: IO c -> (c -> IO b) -> Annex a -> Annex a +bracketIO setup cleanup go = + liftBaseOp (Control.Exception.bracket setup cleanup) (const go) + +{- try in the Annex monad -} +tryAnnex :: Annex a -> Annex (Either SomeException a) +tryAnnex = try + +{- Throws an exception in the Annex monad. -} +throw :: Control.Exception.Exception e => e -> Annex a +throw = liftIO . throwIO diff --git a/Annex/Journal.hs b/Annex/Journal.hs new file mode 100644 index 0000000000..2df5294ee8 --- /dev/null +++ b/Annex/Journal.hs @@ -0,0 +1,89 @@ +{- management of the git-annex journal + - + - The journal is used to queue up changes before they are committed to the + - git-annex branch. Amoung other things, it ensures that if git-annex is + - interrupted, its recorded data is not lost. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Journal where + +import System.IO.Binary + +import Common.Annex +import Annex.Exception +import qualified Git +import Annex.Perms + +{- Records content for a file in the branch to the journal. + - + - Using the journal, rather than immediatly staging content to the index + - avoids git needing to rewrite the index after every change. -} +setJournalFile :: FilePath -> String -> Annex () +setJournalFile file content = do + createAnnexDirectory =<< fromRepo gitAnnexJournalDir + createAnnexDirectory =<< fromRepo gitAnnexTmpDir + -- journal file is written atomically + jfile <- fromRepo $ journalFile file + tmp <- fromRepo gitAnnexTmpDir + let tmpfile = tmp takeFileName jfile + liftIO $ do + writeBinaryFile tmpfile content + moveFile tmpfile jfile + +{- Gets any journalled content for a file in the branch. -} +getJournalFile :: FilePath -> Annex (Maybe String) +getJournalFile file = inRepo $ \g -> catchMaybeIO $ + readFileStrict $ journalFile file g + +{- List of files that have updated content in the journal. -} +getJournalledFiles :: Annex [FilePath] +getJournalledFiles = map fileJournal <$> getJournalFiles + +{- List of existing journal files. -} +getJournalFiles :: Annex [FilePath] +getJournalFiles = do + g <- gitRepo + fs <- liftIO $ catchDefaultIO [] $ + getDirectoryContents $ gitAnnexJournalDir g + return $ filter (`notElem` [".", ".."]) fs + +{- Checks if there are changes in the journal. -} +journalDirty :: Annex Bool +journalDirty = not . null <$> getJournalFiles + +{- Produces a filename to use in the journal for a file on the branch. + - + - The journal typically won't have a lot of files in it, so the hashing + - used in the branch is not necessary, and all the files are put directly + - in the journal directory. + -} +journalFile :: FilePath -> Git.Repo -> FilePath +journalFile file repo = gitAnnexJournalDir repo concatMap mangle file + where + mangle '/' = "_" + mangle '_' = "__" + mangle c = [c] + +{- Converts a journal file (relative to the journal dir) back to the + - filename on the branch. -} +fileJournal :: FilePath -> FilePath +fileJournal = replace "//" "_" . replace "_" "/" + +{- Runs an action that modifies the journal, using locking to avoid + - contention with other git-annex processes. -} +lockJournal :: Annex a -> Annex a +lockJournal a = do + file <- fromRepo gitAnnexJournalLock + createAnnexDirectory $ takeDirectory file + mode <- annexFileMode + bracketIO (lock file mode) unlock a + where + lock file mode = do + l <- noUmask mode $ createFile file mode + waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0) + return l + unlock = closeFd diff --git a/Annex/Link.hs b/Annex/Link.hs new file mode 100644 index 0000000000..650fc19a1c --- /dev/null +++ b/Annex/Link.hs @@ -0,0 +1,81 @@ +{- git-annex links to content + - + - On file systems that support them, symlinks are used. + - + - On other filesystems, git instead stores the symlink target in a regular + - file. + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Link where + +import Common.Annex +import qualified Annex +import qualified Git.HashObject +import qualified Git.UpdateIndex +import qualified Annex.Queue +import Git.Types + +type LinkTarget = String + +{- Checks if a file is a link to a key. -} +isAnnexLink :: FilePath -> Annex (Maybe Key) +isAnnexLink file = maybe Nothing (fileKey . takeFileName) <$> getAnnexLinkTarget file + +{- Gets the link target of a symlink. + - + - On a filesystem that does not support symlinks, get the link + - target by looking inside the file. (Only return at first 8k of the file, + - more than enough for any symlink target.) + - + - Returns Nothing if the file is not a symlink, or not a link to annex + - content. + -} +getAnnexLinkTarget :: FilePath -> Annex (Maybe LinkTarget) +getAnnexLinkTarget file = do + v <- ifM (coreSymlinks <$> Annex.getGitConfig) + ( liftIO $ catchMaybeIO $ readSymbolicLink file + , liftIO $ catchMaybeIO $ readfilestart file + ) + case v of + Nothing -> return Nothing + Just l + | isLinkToAnnex l -> return v + | otherwise -> return Nothing + where + readfilestart f = do + h <- openFile f ReadMode + fileEncoding h + take 8192 <$> hGetContents h + +{- Creates a link on disk. + - + - On a filesystem that does not support symlinks, writes the link target + - to a file. Note that git will only treat the file as a symlink if + - it's staged as such, so use addAnnexLink when adding a new file or + - modified link to git. + -} +makeAnnexLink :: LinkTarget -> FilePath -> Annex () +makeAnnexLink linktarget file = ifM (coreSymlinks <$> Annex.getGitConfig) + ( liftIO $ createSymbolicLink linktarget file + , liftIO $ writeFile file linktarget + ) + +{- Creates a link on disk, and additionally stages it in git. -} +addAnnexLink :: LinkTarget -> FilePath -> Annex () +addAnnexLink linktarget file = do + makeAnnexLink linktarget file + stageSymlink file =<< hashSymlink linktarget + +{- Injects a symlink target into git, returning its Sha. -} +hashSymlink :: LinkTarget -> Annex Sha +hashSymlink linktarget = inRepo $ Git.HashObject.hashObject BlobObject linktarget + +{- Stages a symlink to the annex, using a Sha of its target. -} +stageSymlink :: FilePath -> Sha -> Annex () +stageSymlink file sha = + Annex.Queue.addUpdateIndex =<< + inRepo (Git.UpdateIndex.stageSymlink file sha) diff --git a/Annex/LockPool.hs b/Annex/LockPool.hs new file mode 100644 index 0000000000..45fc55b3c4 --- /dev/null +++ b/Annex/LockPool.hs @@ -0,0 +1,45 @@ +{- git-annex lock pool + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.LockPool where + +import qualified Data.Map as M +import System.Posix.Types (Fd) + +import Common.Annex +import Annex +import Annex.Perms + +{- Create a specified lock file, and takes a shared lock. -} +lockFile :: FilePath -> Annex () +lockFile file = go =<< fromPool file + where + go (Just _) = noop -- already locked + go Nothing = do + mode <- annexFileMode + fd <- liftIO $ noUmask mode $ + openFd file ReadOnly (Just mode) defaultFileFlags + liftIO $ waitToSetLock fd (ReadLock, AbsoluteSeek, 0, 0) + changePool $ M.insert file fd + +unlockFile :: FilePath -> Annex () +unlockFile file = maybe noop go =<< fromPool file + where + go fd = do + liftIO $ closeFd fd + changePool $ M.delete file + +getPool :: Annex (M.Map FilePath Fd) +getPool = getState lockpool + +fromPool :: FilePath -> Annex (Maybe Fd) +fromPool file = M.lookup file <$> getPool + +changePool :: (M.Map FilePath Fd -> M.Map FilePath Fd) -> Annex () +changePool a = do + m <- getPool + changeState $ \s -> s { lockpool = a m } diff --git a/Annex/Perms.hs b/Annex/Perms.hs new file mode 100644 index 0000000000..dc1cb2f8b1 --- /dev/null +++ b/Annex/Perms.hs @@ -0,0 +1,100 @@ +{- git-annex file permissions + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Perms ( + setAnnexPerm, + annexFileMode, + createAnnexDirectory, + noUmask, + createContentDir, + freezeContentDir, +) where + +import Common.Annex +import Utility.FileMode +import Git.SharedRepository +import qualified Annex +import Config + +import System.Posix.Types + +withShared :: (SharedRepository -> Annex a) -> Annex a +withShared a = maybe startup a =<< Annex.getState Annex.shared + where + startup = do + shared <- fromRepo getSharedRepository + Annex.changeState $ \s -> s { Annex.shared = Just shared } + a shared + +{- Sets appropriate file mode for a file or directory in the annex, + - other than the content files and content directory. Normally, + - use the default mode, but with core.sharedRepository set, + - allow the group to write, etc. -} +setAnnexPerm :: FilePath -> Annex () +setAnnexPerm file = unlessM crippledFileSystem $ + withShared $ liftIO . go + where + go GroupShared = groupWriteRead file + go AllShared = modifyFileMode file $ addModes $ + [ ownerWriteMode, groupWriteMode ] ++ readModes + go _ = noop + +{- Gets the appropriate mode to use for creating a file in the annex + - (other than content files, which are locked down more). -} +annexFileMode :: Annex FileMode +annexFileMode = withShared $ return . go + where + go GroupShared = sharedmode + go AllShared = combineModes (sharedmode:readModes) + go _ = stdFileMode + sharedmode = combineModes + [ ownerWriteMode, groupWriteMode + , ownerReadMode, groupReadMode + ] + +{- Creates a directory inside the gitAnnexDir, including any parent + - directories. Makes directories with appropriate permissions. -} +createAnnexDirectory :: FilePath -> Annex () +createAnnexDirectory dir = traverse dir [] =<< top + where + top = parentDir <$> fromRepo gitAnnexDir + traverse d below stop + | d `equalFilePath` stop = done + | otherwise = ifM (liftIO $ doesDirectoryExist d) + ( done + , traverse (parentDir d) (d:below) stop + ) + where + done = forM_ below $ \p -> do + liftIO $ createDirectoryIfMissing True p + setAnnexPerm p + +{- Blocks writing to the directory an annexed file is in, to prevent the + - file accidentially being deleted. However, if core.sharedRepository + - is set, this is not done, since the group must be allowed to delete the + - file. + -} +freezeContentDir :: FilePath -> Annex () +freezeContentDir file = unlessM crippledFileSystem $ + liftIO . go =<< fromRepo getSharedRepository + where + dir = parentDir file + go GroupShared = groupWriteRead dir + go AllShared = groupWriteRead dir + go _ = preventWrite dir + +{- Makes the directory tree to store an annexed file's content, + - with appropriate permissions on each level. -} +createContentDir :: FilePath -> Annex () +createContentDir dest = do + unlessM (liftIO $ doesDirectoryExist dir) $ + createAnnexDirectory dir + -- might have already existed with restricted perms + unlessM crippledFileSystem $ + liftIO $ allowWrite dir + where + dir = parentDir dest diff --git a/Annex/Queue.hs b/Annex/Queue.hs new file mode 100644 index 0000000000..a5ef600379 --- /dev/null +++ b/Annex/Queue.hs @@ -0,0 +1,62 @@ +{- git-annex command queue + - + - Copyright 2011, 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Queue ( + addCommand, + addUpdateIndex, + flush, + flushWhenFull, + size +) where + +import Common.Annex +import Annex hiding (new) +import qualified Git.Queue +import qualified Git.UpdateIndex + +{- Adds a git command to the queue. -} +addCommand :: String -> [CommandParam] -> [FilePath] -> Annex () +addCommand command params files = do + q <- get + store <=< inRepo $ Git.Queue.addCommand command params files q + +{- Adds an update-index stream to the queue. -} +addUpdateIndex :: Git.UpdateIndex.Streamer -> Annex () +addUpdateIndex streamer = do + q <- get + store <=< inRepo $ Git.Queue.addUpdateIndex streamer q + +{- Runs the queue if it is full. Should be called periodically. -} +flushWhenFull :: Annex () +flushWhenFull = do + q <- get + when (Git.Queue.full q) flush + +{- Runs (and empties) the queue. -} +flush :: Annex () +flush = do + q <- get + unless (0 == Git.Queue.size q) $ do + showStoringStateAction + q' <- inRepo $ Git.Queue.flush q + store q' + +{- Gets the size of the queue. -} +size :: Annex Int +size = Git.Queue.size <$> get + +get :: Annex Git.Queue.Queue +get = maybe new return =<< getState repoqueue + +new :: Annex Git.Queue.Queue +new = do + q <- Git.Queue.new . annexQueueSize <$> getGitConfig + store q + return q + +store :: Git.Queue.Queue -> Annex () +store q = changeState $ \s -> s { repoqueue = Just q } diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs new file mode 100644 index 0000000000..a8bd1f7b62 --- /dev/null +++ b/Annex/Ssh.hs @@ -0,0 +1,156 @@ +{- git-annex ssh interface, with connection caching + - + - Copyright 2012,2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Annex.Ssh ( + sshParams, + sshCleanup, +) where + +import qualified Data.Map as M +import System.Posix.Env + +import Common.Annex +import Annex.LockPool +import Annex.Perms +import qualified Build.SysConfig as SysConfig +import qualified Annex +import Config + +{- Generates parameters to ssh to a given host (or user@host) on a given + - port, with connection caching. -} +sshParams :: (String, Maybe Integer) -> [CommandParam] -> Annex [CommandParam] +sshParams (host, port) opts = go =<< sshInfo (host, port) + where + go (Nothing, params) = ret params + go (Just socketfile, params) = do + cleanstale + liftIO $ createDirectoryIfMissing True $ parentDir socketfile + lockFile $ socket2lock socketfile + ret params + ret ps = return $ ps ++ opts ++ portParams port ++ + [Param "-T", Param host] + -- If the lock pool is empty, this is the first ssh of this + -- run. There could be stale ssh connections hanging around + -- from a previous git-annex run that was interrupted. + cleanstale = whenM (not . any isLock . M.keys <$> getPool) $ + sshCleanup + +{- Returns a filename to use for a ssh connection caching socket, and + - parameters to enable ssh connection caching. -} +sshInfo :: (String, Maybe Integer) -> Annex (Maybe FilePath, [CommandParam]) +sshInfo (host, port) = go =<< sshCacheDir + where + go Nothing = return (Nothing, []) + go (Just dir) = do + let socketfile = dir hostport2socket host port + if valid_unix_socket_path socketfile + then return (Just socketfile, cacheparams socketfile) + else do + socketfile' <- liftIO $ relPathCwdToFile socketfile + if valid_unix_socket_path socketfile' + then return (Just socketfile', cacheparams socketfile') + else return (Nothing, []) + cacheparams :: FilePath -> [CommandParam] + cacheparams socketfile = + [ Param "-S", Param socketfile + , Params "-o ControlMaster=auto -o ControlPersist=yes" + ] + +{- ssh connection caching creates sockets, so will not work on a + - crippled filesystem. A GIT_ANNEX_TMP_DIR can be provided to use + - a different filesystem. -} +sshCacheDir :: Annex (Maybe FilePath) +sshCacheDir + | SysConfig.sshconnectioncaching = ifM crippledFileSystem + ( maybe (return Nothing) usetmpdir =<< gettmpdir + , ifM (fromMaybe True . annexSshCaching <$> Annex.getGitConfig) + ( Just <$> fromRepo gitAnnexSshDir + , return Nothing + ) + ) + | otherwise = return Nothing + where + gettmpdir = liftIO $ getEnv "GIT_ANNEX_TMP_DIR" + usetmpdir tmpdir = liftIO $ catchMaybeIO $ do + createDirectoryIfMissing True tmpdir + return $ tmpdir + +portParams :: Maybe Integer -> [CommandParam] +portParams Nothing = [] +portParams (Just port) = [Param "-p", Param $ show port] + +{- Stop any unused ssh processes. -} +sshCleanup :: Annex () +sshCleanup = go =<< sshCacheDir + where + go Nothing = noop + go (Just dir) = do + sockets <- filter (not . isLock) <$> + liftIO (catchDefaultIO [] $ dirContents dir) + forM_ sockets cleanup + cleanup socketfile = do + -- Drop any shared lock we have, and take an + -- exclusive lock, without blocking. If the lock + -- succeeds, nothing is using this ssh, and it can + -- be stopped. + let lockfile = socket2lock socketfile + unlockFile lockfile + mode <- annexFileMode + fd <- liftIO $ noUmask mode $ + openFd lockfile ReadWrite (Just mode) defaultFileFlags + v <- liftIO $ tryIO $ + setLock fd (WriteLock, AbsoluteSeek, 0, 0) + case v of + Left _ -> noop + Right _ -> stopssh socketfile + liftIO $ closeFd fd + stopssh socketfile = do + let (host, port) = socket2hostport socketfile + (_, params) <- sshInfo (host, port) + -- "ssh -O stop" is noisy on stderr even with -q + void $ liftIO $ catchMaybeIO $ + withQuietOutput createProcessSuccess $ + proc "ssh" $ toCommand $ + [ Params "-O stop" + ] ++ params ++ [Param host] + -- Cannot remove the lock file; other processes may + -- be waiting on our exclusive lock to use it. + +hostport2socket :: String -> Maybe Integer -> FilePath +hostport2socket host Nothing = host +hostport2socket host (Just port) = host ++ "!" ++ show port + +socket2hostport :: FilePath -> (String, Maybe Integer) +socket2hostport socket + | null p = (h, Nothing) + | otherwise = (h, readish p) + where + (h, p) = separate (== '!') $ takeFileName socket + +socket2lock :: FilePath -> FilePath +socket2lock socket = socket ++ lockExt + +isLock :: FilePath -> Bool +isLock f = lockExt `isSuffixOf` f + +lockExt :: String +lockExt = ".lock" + +{- This is the size of the sun_path component of sockaddr_un, which + - is the limit to the total length of the filename of a unix socket. + - + - On Linux, this is 108. On OSX, 104. TODO: Probe + -} +sizeof_sockaddr_un_sun_path :: Int +sizeof_sockaddr_un_sun_path = 100 + +{- Note that this looks at the true length of the path in bytes, as it will + - appear on disk. -} +valid_unix_socket_path :: FilePath -> Bool +valid_unix_socket_path f = length (decodeW8 f) < sizeof_sockaddr_un_sun_path diff --git a/Annex/UUID.hs b/Annex/UUID.hs new file mode 100644 index 0000000000..8b84aba0bf --- /dev/null +++ b/Annex/UUID.hs @@ -0,0 +1,73 @@ +{- git-annex uuids + - + - Each git repository used by git-annex has an annex.uuid setting that + - uniquely identifies that repository. + - + - UUIDs of remotes are cached in git config, using keys named + - remote..annex-uuid + - + - Copyright 2010-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.UUID ( + getUUID, + getRepoUUID, + getUncachedUUID, + ensureUUID, + genUUID, + removeRepoUUID, + storeUUID, +) where + +import Common.Annex +import qualified Git +import qualified Git.Config +import Config +import Types.UUID + +configkey :: ConfigKey +configkey = annexConfig "uuid" + +{- Get current repository's UUID. -} +getUUID :: Annex (Maybe UUID) +getUUID = getRepoUUID =<< gitRepo + +{- Looks up a repo's UUID, caching it in .git/config if it's not already. -} +getRepoUUID :: Git.Repo -> Annex (Maybe UUID) +getRepoUUID r = do + c <- toUUID <$> getConfig cachekey "" + case getUncachedUUID r of + v@(Just u) + | c /= v -> do + updatecache u + return v + _ -> return c + where + updatecache u = do + g <- gitRepo + when (g /= r) $ storeUUID cachekey u + cachekey = remoteConfig r "uuid" + +removeRepoUUID :: Annex () +removeRepoUUID = unsetConfig configkey + +getUncachedUUID :: Git.Repo -> Maybe UUID +getUncachedUUID = toUUID . Git.Config.get key "" + where + (ConfigKey key) = configkey + +{- Make sure that the repo has an annex.uuid setting. -} +ensureUUID :: Annex UUID +ensureUUID = do + mu <- getUUID + case mu of + Just u -> return u + Nothing -> do + u <- liftIO genUUID + storeUUID configkey u + return u + +storeUUID :: ConfigKey -> UUID -> Annex () +storeUUID configfield = setConfig configfield . fromUUID diff --git a/Annex/Version.hs b/Annex/Version.hs new file mode 100644 index 0000000000..31c6501bef --- /dev/null +++ b/Annex/Version.hs @@ -0,0 +1,47 @@ +{- git-annex repository versioning + - + - Copyright 2010,2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Version where + +import Common.Annex +import Config +import qualified Annex + +type Version = String + +defaultVersion :: Version +defaultVersion = "3" + +directModeVersion :: Version +directModeVersion = "4" + +supportedVersions :: [Version] +supportedVersions = [defaultVersion, directModeVersion] + +upgradableVersions :: [Version] +upgradableVersions = ["0", "1", "2"] + +versionField :: ConfigKey +versionField = annexConfig "version" + +getVersion :: Annex (Maybe Version) +getVersion = annexVersion <$> Annex.getGitConfig + +setVersion :: Version -> Annex () +setVersion = setConfig versionField + +removeVersion :: Annex () +removeVersion = unsetConfig versionField + +checkVersion :: Version -> Annex () +checkVersion v + | v `elem` supportedVersions = noop + | v `elem` upgradableVersions = err "Upgrade this repository: git-annex upgrade" + | otherwise = err "Upgrade git-annex." + where + err msg = error $ "Repository version " ++ v ++ + " is not supported. " ++ msg diff --git a/Annex/Wanted.hs b/Annex/Wanted.hs new file mode 100644 index 0000000000..2500f80d13 --- /dev/null +++ b/Annex/Wanted.hs @@ -0,0 +1,33 @@ +{- git-annex control over whether content is wanted + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Annex.Wanted where + +import Common.Annex +import Logs.PreferredContent +import Annex.UUID +import Types.Remote + +import qualified Data.Set as S + +{- Check if a file is preferred content for the local repository. -} +wantGet :: Bool -> AssociatedFile -> Annex Bool +wantGet def Nothing = return def +wantGet def (Just file) = isPreferredContent Nothing S.empty file def + +{- Check if a file is preferred content for a remote. -} +wantSend :: Bool -> AssociatedFile -> UUID -> Annex Bool +wantSend def Nothing _ = return def +wantSend def (Just file) to = isPreferredContent (Just to) S.empty file def + +{- Check if a file can be dropped, maybe from a remote. + - Don't drop files that are preferred content. -} +wantDrop :: Bool -> Maybe UUID -> AssociatedFile -> Annex Bool +wantDrop def _ Nothing = return $ not def +wantDrop def from (Just file) = do + u <- maybe getUUID (return . id) from + not <$> isPreferredContent (Just u) (S.singleton u) file def diff --git a/Assistant.hs b/Assistant.hs new file mode 100644 index 0000000000..63a5dafae1 --- /dev/null +++ b/Assistant.hs @@ -0,0 +1,237 @@ +{- git-annex assistant daemon + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + - + - Overview of threads and MVars, etc: + - + - Thread 1: parent + - The initial thread run, double forks to background, starts other + - threads, and then stops, waiting for them to terminate, + - or for a ctrl-c. + - Thread 2: Watcher + - Notices new files, and calls handlers for events, queuing changes. + - Thread 3: inotify internal + - Used by haskell inotify library to ensure inotify event buffer is + - kept drained. + - Thread 4: inotify startup scanner + - Scans the tree and registers inotify watches for each directory. + - A MVar lock is used to prevent other inotify handlers from running + - until this is complete. + - Thread 5: Committer + - Waits for changes to occur, and runs the git queue to update its + - index, then commits. Also queues Transfer events to send added + - files to other remotes. + - Thread 6: Pusher + - Waits for commits to be made, and pushes updated branches to remotes, + - in parallel. (Forks a process for each git push.) + - Thread 7: PushRetryer + - Runs every 30 minutes when there are failed pushes, and retries + - them. + - Thread 8: Merger + - Waits for pushes to be received from remotes, and merges the + - updated branches into the current branch. + - (This uses inotify on .git/refs/heads, so there are additional + - inotify threads associated with it, too.) + - Thread 9: TransferWatcher + - Watches for transfer information files being created and removed, + - and maintains the DaemonStatus currentTransfers map. + - (This uses inotify on .git/annex/transfer/, so there are + - additional inotify threads associated with it, too.) + - Thread 10: TransferPoller + - Polls to determine how much of each ongoing transfer is complete. + - Thread 11: Transferrer + - Waits for Transfers to be queued and does them. + - Thread 12: StatusLogger + - Wakes up periodically and records the daemon's status to disk. + - Thread 13: SanityChecker + - Wakes up periodically (rarely) and does sanity checks. + - Thread 14: MountWatcher + - Either uses dbus to watch for drive mount events, or, when + - there's no dbus, polls to find newly mounted filesystems. + - Once a filesystem that contains a remote is mounted, updates + - state about that remote, pulls from it, and queues a push to it, + - as well as an update, and queues it onto the + - ConnectedRemoteChan + - Thread 15: NetWatcher + - Deals with network connection interruptions, which would cause + - transfers to fail, and can be recovered from by waiting for a + - network connection, and syncing with all network remotes. + - Uses dbus to watch for network connections, or when dbus + - cannot be used, assumes there's been one every 30 minutes. + - Thread 16: TransferScanner + - Does potentially expensive checks to find data that needs to be + - transferred from or to remotes, and queues Transfers. + - Uses the ScanRemotes map.a + - Thread 17: PairListener + - Listens for incoming pairing traffic, and takes action. + - Thread 18: ConfigMonitor + - Triggered by changes to the git-annex branch, checks for changed + - config files, and reloads configs. + - Thread 19: XMPPClient + - Built-in XMPP client. + - Thread 20: WebApp + - Spawns more threads as necessary to handle clients. + - Displays the DaemonStatus. + - Thread 21: Glacier + - Deals with retrieving files from Amazon Glacier. + - + - ThreadState: (MVar) + - The Annex state is stored here, which allows resuscitating the + - Annex monad in IO actions run by the watcher and committer + - threads. Thus, a single state is shared amoung the threads, and + - only one at a time can access it. + - DaemonStatusHandle: (STM TMVar) + - The daemon's current status. + - ChangeChan: (STM TChan) + - Changes are indicated by writing to this channel. The committer + - reads from it. + - CommitChan: (STM TChan) + - Commits are indicated by writing to this channel. The pusher reads + - from it. + - FailedPushMap (STM TMVar) + - Failed pushes are indicated by writing to this TMVar. The push + - retrier blocks until they're available. + - TransferQueue (STM TChan) + - Transfers to make are indicated by writing to this channel. + - TransferSlots (QSemN) + - Count of the number of currently available transfer slots. + - Updated by the transfer watcher, this allows other threads + - to block until a slot is available. + - This MVar should only be manipulated from inside the Annex monad, + - which ensures it's accessed only after the ThreadState MVar. + - ScanRemotes (STM TMVar) + - Remotes that have been disconnected, and should be scanned + - are indicated by writing to this TMVar. + - BranchChanged (STM SampleVar) + - Changes to the git-annex branch are indicated by updating this + - SampleVar. + - NetMessager (STM TChan, TMVar, SampleVar) + - Used to feed messages to the built-in XMPP client, handle + - pushes, and signal it when it needs to restart due to configuration + - or networking changes. + - UrlRenderer (MVar) + - A Yesod route rendering function is stored here. This allows + - things that need to render Yesod routes to block until the webapp + - has started up and such rendering is possible. + -} + +{-# LANGUAGE CPP #-} + +module Assistant where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.NamedThread +import Assistant.Types.ThreadedMonad +import Assistant.Threads.DaemonStatus +import Assistant.Threads.Watcher +import Assistant.Threads.Committer +import Assistant.Threads.Pusher +import Assistant.Threads.Merger +import Assistant.Threads.TransferWatcher +import Assistant.Threads.Transferrer +import Assistant.Threads.SanityChecker +import Assistant.Threads.MountWatcher +import Assistant.Threads.NetWatcher +import Assistant.Threads.TransferScanner +import Assistant.Threads.TransferPoller +import Assistant.Threads.ConfigMonitor +import Assistant.Threads.Glacier +#ifdef WITH_WEBAPP +import Assistant.WebApp +import Assistant.Threads.WebApp +#ifdef WITH_PAIRING +import Assistant.Threads.PairListener +#endif +#ifdef WITH_XMPP +import Assistant.Threads.XMPPClient +#endif +#else +#warning Building without the webapp. You probably need to install Yesod.. +#endif +import Assistant.Environment +import qualified Utility.Daemon +import Utility.LogFile +import Utility.ThreadScheduler + +stopDaemon :: Annex () +stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile + +{- Starts the daemon. If the daemon is run in the foreground, once it's + - running, can start the browser. + - + - startbrowser is passed the url and html shim file, as well as the original + - stdout and stderr descriptors. -} +startDaemon :: Bool -> Bool -> Maybe (Maybe Handle -> Maybe Handle -> String -> FilePath -> IO ()) -> Annex () +startDaemon assistant foreground startbrowser = do + pidfile <- fromRepo gitAnnexPidFile + logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile + if foreground + then do + liftIO $ Utility.Daemon.lockPidFile pidfile + origout <- liftIO $ catchMaybeIO $ + fdToHandle =<< dup stdOutput + origerr <- liftIO $ catchMaybeIO $ + fdToHandle =<< dup stdError + liftIO $ Utility.LogFile.redirLog logfd + showStart (if assistant then "assistant" else "watch") "." + start id $ + case startbrowser of + Nothing -> Nothing + Just a -> Just $ a origout origerr + else + start (Utility.Daemon.daemonize logfd (Just pidfile) False) Nothing + where + start daemonize webappwaiter = withThreadState $ \st -> do + checkCanWatch + when assistant $ checkEnvironment + dstatus <- startDaemonStatus + liftIO $ daemonize $ + flip runAssistant (go webappwaiter) + =<< newAssistantData st dstatus + + go webappwaiter = do +#ifdef WITH_WEBAPP + d <- getAssistant id + urlrenderer <- liftIO newUrlRenderer + mapM_ (startthread $ Just urlrenderer) +#else + mapM_ (startthread Nothing) +#endif + [ watch $ commitThread +#ifdef WITH_WEBAPP + , assist $ webAppThread d urlrenderer False Nothing webappwaiter +#ifdef WITH_PAIRING + , assist $ pairListenerThread urlrenderer +#endif +#ifdef WITH_XMPP + , assist $ xmppClientThread urlrenderer +#endif +#endif + , assist $ pushThread + , assist $ pushRetryThread + , assist $ mergeThread + , assist $ transferWatcherThread + , assist $ transferPollerThread + , assist $ transfererThread + , assist $ daemonStatusThread + , assist $ sanityCheckerDailyThread + , assist $ sanityCheckerHourlyThread + , assist $ mountWatcherThread + , assist $ netWatcherThread + , assist $ netWatcherFallbackThread + , assist $ transferScannerThread + , assist $ configMonitorThread + , assist $ glacierThread + , watch $ watchThread + ] + + liftIO waitForTermination + + watch a = (True, a) + assist a = (False, a) + startthread urlrenderer (watcher, t) + | watcher || assistant = startNamedThread urlrenderer t + | otherwise = noop diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs new file mode 100644 index 0000000000..ea576c6682 --- /dev/null +++ b/Assistant/Alert.hs @@ -0,0 +1,373 @@ +{- git-annex assistant alerts + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Alert where + +import Common.Annex +import qualified Remote +import Utility.Tense +import Logs.Transfer + +import qualified Data.Text as T +import Data.Text (Text) +import qualified Data.Map as M +import Data.String + +{- Different classes of alerts are displayed differently. -} +data AlertClass = Success | Message | Activity | Warning | Error + deriving (Eq, Ord) + +data AlertPriority = Filler | Low | Medium | High | Pinned + deriving (Eq, Ord) + +{- An alert can have an name, which is used to combine it with other similar + - alerts. -} +data AlertName + = FileAlert TenseChunk + | SanityCheckFixAlert + | WarningAlert String + | PairAlert String + | XMPPNeededAlert + deriving (Eq) + +{- The first alert is the new alert, the second is an old alert. + - Should return a modified version of the old alert. -} +type AlertCombiner = Alert -> Alert -> Maybe Alert + +data Alert = Alert + { alertClass :: AlertClass + , alertHeader :: Maybe TenseText + , alertMessageRender :: [TenseChunk] -> TenseText + , alertData :: [TenseChunk] + , alertBlockDisplay :: Bool + , alertClosable :: Bool + , alertPriority :: AlertPriority + , alertIcon :: Maybe AlertIcon + , alertCombiner :: Maybe AlertCombiner + , alertName :: Maybe AlertName + , alertButton :: Maybe AlertButton + } + +data AlertIcon = ActivityIcon | SuccessIcon | ErrorIcon | InfoIcon | TheCloud + +{- When clicked, a button always redirects to a URL + - It may also run an IO action in the background, which is useful + - to make the button close or otherwise change the alert. -} +data AlertButton = AlertButton + { buttonLabel :: Text + , buttonUrl :: Text + , buttonAction :: Maybe (AlertId -> IO ()) + } + +type AlertPair = (AlertId, Alert) + +type AlertMap = M.Map AlertId Alert + +{- Higher AlertId indicates a more recent alert. -} +newtype AlertId = AlertId Integer + deriving (Read, Show, Eq, Ord) + +firstAlertId :: AlertId +firstAlertId = AlertId 0 + +nextAlertId :: AlertId -> AlertId +nextAlertId (AlertId i) = AlertId $ succ i + +{- This is as many alerts as it makes sense to display at a time. + - A display might be smaller, or larger, the point is to not overwhelm the + - user with a ton of alerts. -} +displayAlerts :: Int +displayAlerts = 6 + +{- This is not a hard maximum, but there's no point in keeping a great + - many filler alerts in an AlertMap, so when there's more than this many, + - they start being pruned, down toward displayAlerts. -} +maxAlerts :: Int +maxAlerts = displayAlerts * 2 + +{- The desired order is the reverse of: + - + - - Pinned alerts + - - High priority alerts, newest first + - - Medium priority Activity, newest first (mostly used for Activity) + - - Low priority alerts, newest first + - - Filler priorty alerts, newest first + - - Ties are broken by the AlertClass, with Errors etc coming first. + -} +compareAlertPairs :: AlertPair -> AlertPair -> Ordering +compareAlertPairs + (aid, Alert { alertClass = aclass, alertPriority = aprio }) + (bid, Alert { alertClass = bclass, alertPriority = bprio }) + = compare aprio bprio + `thenOrd` compare aid bid + `thenOrd` compare aclass bclass + +sortAlertPairs :: [AlertPair] -> [AlertPair] +sortAlertPairs = sortBy compareAlertPairs + +{- Renders an alert's header for display, if it has one. -} +renderAlertHeader :: Alert -> Maybe Text +renderAlertHeader alert = renderTense (alertTense alert) <$> alertHeader alert + +{- Renders an alert's message for display. -} +renderAlertMessage :: Alert -> Text +renderAlertMessage alert = renderTense (alertTense alert) $ + (alertMessageRender alert) (alertData alert) + +showAlert :: Alert -> String +showAlert alert = T.unpack $ T.unwords $ catMaybes + [ renderAlertHeader alert + , Just $ renderAlertMessage alert + ] + +alertTense :: Alert -> Tense +alertTense alert + | alertClass alert == Activity = Present + | otherwise = Past + +{- Checks if two alerts display the same. -} +effectivelySameAlert :: Alert -> Alert -> Bool +effectivelySameAlert x y = all id + [ alertClass x == alertClass y + , alertHeader x == alertHeader y + , alertData x == alertData y + , alertBlockDisplay x == alertBlockDisplay y + , alertClosable x == alertClosable y + , alertPriority x == alertPriority y + ] + +makeAlertFiller :: Bool -> Alert -> Alert +makeAlertFiller success alert + | isFiller alert = alert + | otherwise = alert + { alertClass = if c == Activity then c' else c + , alertPriority = Filler + , alertClosable = True + , alertButton = Nothing + , alertIcon = Just $ if success then SuccessIcon else ErrorIcon + } + where + c = alertClass alert + c' + | success = Success + | otherwise = Error + +isFiller :: Alert -> Bool +isFiller alert = alertPriority alert == Filler + +{- Updates the Alertmap, adding or updating an alert. + - + - Any old filler that looks the same as the alert is removed. + - + - Or, if the alert has an alertCombiner that combines it with + - an old alert, the old alert is replaced with the result, and the + - alert is removed. + - + - Old filler alerts are pruned once maxAlerts is reached. + -} +mergeAlert :: AlertId -> Alert -> AlertMap -> AlertMap +mergeAlert i al m = maybe updatePrune updateCombine (alertCombiner al) + where + pruneSame k al' = k == i || not (effectivelySameAlert al al') + pruneBloat m' + | bloat > 0 = M.fromList $ pruneold $ M.toList m' + | otherwise = m' + where + bloat = M.size m' - maxAlerts + pruneold l = + let (f, rest) = partition (\(_, a) -> isFiller a) l + in drop bloat f ++ rest + updatePrune = pruneBloat $ M.filterWithKey pruneSame $ + M.insertWith' const i al m + updateCombine combiner = + let combined = M.mapMaybe (combiner al) m + in if M.null combined + then updatePrune + else M.delete i $ M.union combined m + +baseActivityAlert :: Alert +baseActivityAlert = Alert + { alertClass = Activity + , alertHeader = Nothing + , alertMessageRender = tenseWords + , alertData = [] + , alertBlockDisplay = False + , alertClosable = False + , alertPriority = Medium + , alertIcon = Just ActivityIcon + , alertCombiner = Nothing + , alertName = Nothing + , alertButton = Nothing + } + +warningAlert :: String -> String -> Alert +warningAlert name msg = Alert + { alertClass = Warning + , alertHeader = Just $ tenseWords ["warning"] + , alertMessageRender = tenseWords + , alertData = [UnTensed $ T.pack msg] + , alertBlockDisplay = True + , alertClosable = True + , alertPriority = High + , alertIcon = Just ErrorIcon + , alertCombiner = Just $ dataCombiner (++) + , alertName = Just $ WarningAlert name + , alertButton = Nothing + } + +activityAlert :: Maybe TenseText -> [TenseChunk] -> Alert +activityAlert header dat = baseActivityAlert + { alertHeader = header + , alertData = dat + } + +startupScanAlert :: Alert +startupScanAlert = activityAlert Nothing + [Tensed "Performing" "Performed", "startup scan"] + +commitAlert :: Alert +commitAlert = activityAlert Nothing + [Tensed "Committing" "Committed", "changes to git"] + +showRemotes :: [Remote] -> TenseChunk +showRemotes = UnTensed . T.unwords . map (T.pack . Remote.name) + +pushAlert :: [Remote] -> Alert +pushAlert rs = activityAlert Nothing + [Tensed "Syncing" "Synced", "with", showRemotes rs] + +pushRetryAlert :: [Remote] -> Alert +pushRetryAlert rs = activityAlert + (Just $ tenseWords [Tensed "Retrying" "Retried", "sync"]) + ["with", showRemotes rs] + +syncAlert :: [Remote] -> Alert +syncAlert rs = baseActivityAlert + { alertHeader = Just $ tenseWords + [Tensed "Syncing" "Synced", "with", showRemotes rs] + , alertData = [] + , alertPriority = Low + } + +scanAlert :: [Remote] -> Alert +scanAlert rs = baseActivityAlert + { alertHeader = Just $ tenseWords + [Tensed "Scanning" "Scanned", showRemotes rs] + , alertBlockDisplay = True + , alertPriority = Low + } + +sanityCheckAlert :: Alert +sanityCheckAlert = activityAlert + (Just $ tenseWords [Tensed "Running" "Ran", "daily sanity check"]) + ["to make sure everything is ok."] + +sanityCheckFixAlert :: String -> Alert +sanityCheckFixAlert msg = Alert + { alertClass = Warning + , alertHeader = Just $ tenseWords ["Fixed a problem"] + , alertMessageRender = render + , alertData = [UnTensed $ T.pack msg] + , alertBlockDisplay = True + , alertPriority = High + , alertClosable = True + , alertIcon = Just ErrorIcon + , alertName = Just SanityCheckFixAlert + , alertCombiner = Just $ dataCombiner (++) + , alertButton = Nothing + } + where + render dta = tenseWords $ alerthead : dta ++ [alertfoot] + alerthead = "The daily sanity check found and fixed a problem:" + alertfoot = "If these problems persist, consider filing a bug report." + +pairingAlert :: AlertButton -> Alert +pairingAlert button = baseActivityAlert + { alertData = [ UnTensed "Pairing in progress" ] + , alertPriority = High + , alertButton = Just button + } + +pairRequestReceivedAlert :: String -> AlertButton -> Alert +pairRequestReceivedAlert who button = Alert + { alertClass = Message + , alertHeader = Nothing + , alertMessageRender = tenseWords + , alertData = [UnTensed $ T.pack $ who ++ " is sending a pair request."] + , alertBlockDisplay = False + , alertPriority = High + , alertClosable = True + , alertIcon = Just InfoIcon + , alertName = Just $ PairAlert who + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertButton = Just button + } + +pairRequestAcknowledgedAlert :: String -> Maybe AlertButton -> Alert +pairRequestAcknowledgedAlert who button = baseActivityAlert + { alertData = ["Pairing with", UnTensed (T.pack who), Tensed "in progress" "complete"] + , alertPriority = High + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertButton = button + } + +xmppNeededAlert :: AlertButton -> Alert +xmppNeededAlert button = Alert + { alertHeader = Just "Share with friends, and keep your devices in sync across the cloud." + , alertIcon = Just TheCloud + , alertPriority = High + , alertButton = Just button + , alertClosable = True + , alertClass = Message + , alertMessageRender = tenseWords + , alertBlockDisplay = True + , alertName = Just $ XMPPNeededAlert + , alertCombiner = Just $ dataCombiner $ \_old new -> new + , alertData = [] + } + +fileAlert :: TenseChunk -> FilePath -> Alert +fileAlert msg file = (activityAlert Nothing [f]) + { alertName = Just $ FileAlert msg + , alertMessageRender = render + , alertCombiner = Just $ dataCombiner combiner + } + where + f = fromString $ shortFile $ takeFileName file + render fs = tenseWords $ msg : fs + combiner new old = take 10 $ new ++ old + +addFileAlert :: FilePath -> Alert +addFileAlert = fileAlert (Tensed "Adding" "Added") + +{- This is only used as a success alert after a transfer, not during it. -} +transferFileAlert :: Direction -> Bool -> FilePath -> Alert +transferFileAlert direction True + | direction == Upload = fileAlert "Uploaded" + | otherwise = fileAlert "Downloaded" +transferFileAlert direction False + | direction == Upload = fileAlert "Upload failed" + | otherwise = fileAlert "Download failed" + +dataCombiner :: ([TenseChunk] -> [TenseChunk] -> [TenseChunk]) -> AlertCombiner +dataCombiner combiner new old + | alertClass new /= alertClass old = Nothing + | alertName new == alertName old = + Just $! old { alertData = alertData new `combiner` alertData old } + | otherwise = Nothing + +shortFile :: FilePath -> String +shortFile f + | len < maxlen = f + | otherwise = take half f ++ ".." ++ drop (len - half) f + where + len = length f + maxlen = 20 + half = (maxlen - 2) `div` 2 + diff --git a/Assistant/BranchChange.hs b/Assistant/BranchChange.hs new file mode 100644 index 0000000000..c9354544a5 --- /dev/null +++ b/Assistant/BranchChange.hs @@ -0,0 +1,19 @@ +{- git-annex assistant git-annex branch change tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.BranchChange where + +import Assistant.Common +import Assistant.Types.BranchChange + +import Control.Concurrent.MSampleVar + +branchChanged :: Assistant () +branchChanged = flip writeSV () <<~ (fromBranchChangeHandle . branchChangeHandle) + +waitBranchChange :: Assistant () +waitBranchChange = readSV <<~ (fromBranchChangeHandle . branchChangeHandle) diff --git a/Assistant/Changes.hs b/Assistant/Changes.hs new file mode 100644 index 0000000000..3d39568995 --- /dev/null +++ b/Assistant/Changes.hs @@ -0,0 +1,39 @@ +{- git-annex assistant change tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Changes where + +import Assistant.Common +import Assistant.Types.Changes +import Utility.TSet + +import Data.Time.Clock + +{- Handlers call this when they made a change that needs to get committed. -} +madeChange :: FilePath -> ChangeType -> Assistant (Maybe Change) +madeChange f t = Just <$> (Change <$> liftIO getCurrentTime <*> pure f <*> pure t) + +noChange :: Assistant (Maybe Change) +noChange = return Nothing + +{- Indicates an add needs to be done, but has not started yet. -} +pendingAddChange :: FilePath -> Assistant (Maybe Change) +pendingAddChange f = Just <$> (PendingAddChange <$> liftIO getCurrentTime <*> pure f) + +{- Gets all unhandled changes. + - Blocks until at least one change is made. -} +getChanges :: Assistant [Change] +getChanges = getTSet <<~ changeChan + +{- Puts unhandled changes back into the channel. + - Note: Original order is not preserved. -} +refillChanges :: [Change] -> Assistant () +refillChanges cs = flip putTSet cs <<~ changeChan + +{- Records a change in the channel. -} +recordChange :: Change -> Assistant () +recordChange c = flip putTSet1 c <<~ changeChan diff --git a/Assistant/Commits.hs b/Assistant/Commits.hs new file mode 100644 index 0000000000..79555fee56 --- /dev/null +++ b/Assistant/Commits.hs @@ -0,0 +1,27 @@ +{- git-annex assistant commit tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Commits where + +import Assistant.Common +import Assistant.Types.Commits + +import Utility.TSet + +{- Gets all unhandled commits. + - Blocks until at least one commit is made. -} +getCommits :: Assistant [Commit] +getCommits = getTSet <<~ commitChan + +{- Puts unhandled commits back into the channel. + - Note: Original order is not preserved. -} +refillCommits :: [Commit] -> Assistant () +refillCommits cs = flip putTSet cs <<~ commitChan + +{- Records a commit in the channel. -} +recordCommit :: Assistant () +recordCommit = flip putTSet1 Commit <<~ commitChan diff --git a/Assistant/Common.hs b/Assistant/Common.hs new file mode 100644 index 0000000000..0be536250b --- /dev/null +++ b/Assistant/Common.hs @@ -0,0 +1,13 @@ +{- Common infrastructure for the git-annex assistant. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Common (module X) where + +import Common.Annex as X +import Assistant.Monad as X +import Assistant.Types.DaemonStatus as X +import Assistant.Types.NamedThread as X diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs new file mode 100644 index 0000000000..f682657c99 --- /dev/null +++ b/Assistant/DaemonStatus.hs @@ -0,0 +1,241 @@ +{- git-annex assistant daemon status + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.DaemonStatus where + +import Assistant.Common +import Assistant.Alert +import Utility.TempFile +import Assistant.Types.NetMessager +import Utility.NotificationBroadcaster +import Logs.Transfer +import Logs.Trust +import qualified Remote +import qualified Types.Remote as Remote +import qualified Git + +import Control.Concurrent.STM +import System.Posix.Types +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale +import qualified Data.Map as M +import qualified Data.Text as T + +getDaemonStatus :: Assistant DaemonStatus +getDaemonStatus = (atomically . readTMVar) <<~ daemonStatusHandle + +modifyDaemonStatus_ :: (DaemonStatus -> DaemonStatus) -> Assistant () +modifyDaemonStatus_ a = modifyDaemonStatus $ \s -> (a s, ()) + +modifyDaemonStatus :: (DaemonStatus -> (DaemonStatus, b)) -> Assistant b +modifyDaemonStatus a = do + dstatus <- getAssistant daemonStatusHandle + liftIO $ do + (s, b) <- atomically $ do + r@(s, _) <- a <$> takeTMVar dstatus + putTMVar dstatus s + return r + sendNotification $ changeNotifier s + return b + + +{- Returns a function that updates the lists of syncable remotes. -} +calcSyncRemotes :: Annex (DaemonStatus -> DaemonStatus) +calcSyncRemotes = do + rs <- filter (remoteAnnexSync . Remote.gitconfig) . + concat . Remote.byCost <$> Remote.enabledRemoteList + alive <- trustExclude DeadTrusted (map Remote.uuid rs) + let good r = Remote.uuid r `elem` alive + let syncable = filter good rs + return $ \dstatus -> dstatus + { syncRemotes = syncable + , syncGitRemotes = filter (not . Remote.specialRemote) syncable + , syncDataRemotes = filter (not . isXMPPRemote) syncable + } + +{- Updates the sycRemotes list from the list of all remotes in Annex state. -} +updateSyncRemotes :: Assistant () +updateSyncRemotes = do + modifyDaemonStatus_ =<< liftAnnex calcSyncRemotes + liftIO . sendNotification =<< syncRemotesNotifier <$> getDaemonStatus + +{- Load any previous daemon status file, and store it in a MVar for this + - process to use as its DaemonStatus. Also gets current transfer status. -} +startDaemonStatus :: Annex DaemonStatusHandle +startDaemonStatus = do + file <- fromRepo gitAnnexDaemonStatusFile + status <- liftIO $ + flip catchDefaultIO (readDaemonStatusFile file) =<< newDaemonStatus + transfers <- M.fromList <$> getTransfers + addsync <- calcSyncRemotes + liftIO $ atomically $ newTMVar $ addsync $ status + { scanComplete = False + , sanityCheckRunning = False + , currentTransfers = transfers + } + +{- Don't just dump out the structure, because it will change over time, + - and parts of it are not relevant. -} +writeDaemonStatusFile :: FilePath -> DaemonStatus -> IO () +writeDaemonStatusFile file status = + viaTmp writeFile file =<< serialized <$> getPOSIXTime + where + serialized now = unlines + [ "lastRunning:" ++ show now + , "scanComplete:" ++ show (scanComplete status) + , "sanityCheckRunning:" ++ show (sanityCheckRunning status) + , "lastSanityCheck:" ++ maybe "" show (lastSanityCheck status) + ] + +readDaemonStatusFile :: FilePath -> IO DaemonStatus +readDaemonStatusFile file = parse <$> newDaemonStatus <*> readFile file + where + parse status = foldr parseline status . lines + parseline line status + | key == "lastRunning" = parseval readtime $ \v -> + status { lastRunning = Just v } + | key == "scanComplete" = parseval readish $ \v -> + status { scanComplete = v } + | key == "sanityCheckRunning" = parseval readish $ \v -> + status { sanityCheckRunning = v } + | key == "lastSanityCheck" = parseval readtime $ \v -> + status { lastSanityCheck = Just v } + | otherwise = status -- unparsable line + where + (key, value) = separate (== ':') line + parseval parser a = maybe status a (parser value) + readtime s = do + d <- parseTime defaultTimeLocale "%s%Qs" s + Just $ utcTimeToPOSIXSeconds d + +{- Checks if a time stamp was made after the daemon was lastRunning. + - + - Some slop is built in; this really checks if the time stamp was made + - at least ten minutes after the daemon was lastRunning. This is to + - ensure the daemon shut down cleanly, and deal with minor clock skew. + - + - If the daemon has never ran before, this always returns False. + -} +afterLastDaemonRun :: EpochTime -> DaemonStatus -> Bool +afterLastDaemonRun timestamp status = maybe False (< t) (lastRunning status) + where + t = realToFrac (timestamp + slop) :: POSIXTime + slop = fromIntegral tenMinutes + +tenMinutes :: Int +tenMinutes = 10 * 60 + +{- Mutates the transfer map. Runs in STM so that the transfer map can + - be modified in the same transaction that modifies the transfer queue. + - Note that this does not send a notification of the change; that's left + - to the caller. -} +adjustTransfersSTM :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> STM () +adjustTransfersSTM dstatus a = do + s <- takeTMVar dstatus + putTMVar dstatus $ s { currentTransfers = a (currentTransfers s) } + +{- Alters a transfer's info, if the transfer is in the map. -} +alterTransferInfo :: Transfer -> (TransferInfo -> TransferInfo) -> Assistant () +alterTransferInfo t a = updateTransferInfo' $ M.adjust a t + +{- Updates a transfer's info. Adds the transfer to the map if necessary, + - or if already present, updates it while preserving the old transferTid, + - transferPaused, and bytesComplete values, which are not written to disk. -} +updateTransferInfo :: Transfer -> TransferInfo -> Assistant () +updateTransferInfo t info = updateTransferInfo' $ M.insertWith' merge t info + where + merge new old = new + { transferTid = maybe (transferTid new) Just (transferTid old) + , transferPaused = transferPaused new || transferPaused old + , bytesComplete = maybe (bytesComplete new) Just (bytesComplete old) + } + +updateTransferInfo' :: (TransferMap -> TransferMap) -> Assistant () +updateTransferInfo' a = notifyTransfer `after` modifyDaemonStatus_ update + where + update s = s { currentTransfers = a (currentTransfers s) } + +{- Removes a transfer from the map, and returns its info. -} +removeTransfer :: Transfer -> Assistant (Maybe TransferInfo) +removeTransfer t = notifyTransfer `after` modifyDaemonStatus remove + where + remove s = + let (info, ts) = M.updateLookupWithKey + (\_k _v -> Nothing) + t (currentTransfers s) + in (s { currentTransfers = ts }, info) + +{- Send a notification when a transfer is changed. -} +notifyTransfer :: Assistant () +notifyTransfer = do + dstatus <- getAssistant daemonStatusHandle + liftIO $ sendNotification + =<< transferNotifier <$> atomically (readTMVar dstatus) + +{- Send a notification when alerts are changed. -} +notifyAlert :: Assistant () +notifyAlert = do + dstatus <- getAssistant daemonStatusHandle + liftIO $ sendNotification + =<< alertNotifier <$> atomically (readTMVar dstatus) + +{- Returns the alert's identifier, which can be used to remove it. -} +addAlert :: Alert -> Assistant AlertId +addAlert alert = do + notice [showAlert alert] + notifyAlert `after` modifyDaemonStatus add + where + add s = (s { lastAlertId = i, alertMap = m }, i) + where + i = nextAlertId $ lastAlertId s + m = mergeAlert i alert (alertMap s) + +removeAlert :: AlertId -> Assistant () +removeAlert i = updateAlert i (const Nothing) + +updateAlert :: AlertId -> (Alert -> Maybe Alert) -> Assistant () +updateAlert i a = updateAlertMap $ \m -> M.update a i m + +updateAlertMap :: (AlertMap -> AlertMap) -> Assistant () +updateAlertMap a = notifyAlert `after` modifyDaemonStatus_ update + where + update s = s { alertMap = a (alertMap s) } + +{- Displays an alert while performing an activity that returns True on + - success. + - + - The alert is left visible afterwards, as filler. + - Old filler is pruned, to prevent the map growing too large. -} +alertWhile :: Alert -> Assistant Bool -> Assistant Bool +alertWhile alert a = alertWhile' alert $ do + r <- a + return (r, r) + +{- Like alertWhile, but allows the activity to return a value too. -} +alertWhile' :: Alert -> Assistant (Bool, a) -> Assistant a +alertWhile' alert a = do + let alert' = alert { alertClass = Activity } + i <- addAlert alert' + (ok, r) <- a + updateAlertMap $ mergeAlert i $ makeAlertFiller ok alert' + return r + +{- Displays an alert while performing an activity, then removes it. -} +alertDuring :: Alert -> Assistant a -> Assistant a +alertDuring alert a = do + i <- addAlert $ alert { alertClass = Activity } + removeAlert i `after` a + +{- Remotes using the XMPP transport have urls like xmpp::user@host -} +isXMPPRemote :: Remote -> Bool +isXMPPRemote remote = Git.repoIsUrl r && "xmpp::" `isPrefixOf` Git.repoLocation r + where + r = Remote.repo remote + +getXMPPClientID :: Remote -> ClientID +getXMPPClientID r = T.pack $ drop (length "xmpp::") (Git.repoLocation (Remote.repo r)) diff --git a/Assistant/Drop.hs b/Assistant/Drop.hs new file mode 100644 index 0000000000..4c060ba31a --- /dev/null +++ b/Assistant/Drop.hs @@ -0,0 +1,87 @@ +{- git-annex assistant dropping of unwanted content + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Drop where + +import Assistant.Common +import Assistant.DaemonStatus +import Logs.Location +import Logs.Trust +import Types.Remote (AssociatedFile, uuid) +import qualified Remote +import qualified Command.Drop +import Command +import Annex.Wanted +import Annex.Exception +import Config + +import qualified Data.Set as S + +type Reason = String + +{- Drop from local and/or remote when allowed by the preferred content and + - numcopies settings. -} +handleDrops :: Reason -> Bool -> Key -> AssociatedFile -> Maybe Remote -> Assistant () +handleDrops _ _ _ Nothing _ = noop +handleDrops reason fromhere key f knownpresentremote = do + syncrs <- syncDataRemotes <$> getDaemonStatus + locs <- liftAnnex $ loggedLocations key + handleDropsFrom locs syncrs reason fromhere key f knownpresentremote + +{- The UUIDs are ones where the content is believed to be present. + - The Remote list can include other remotes that do not have the content; + - only ones that match the UUIDs will be dropped from. + - If allows to drop fromhere, that drop will be tried first. -} +handleDropsFrom :: [UUID] -> [Remote] -> Reason -> Bool -> Key -> AssociatedFile -> Maybe Remote -> Assistant () +handleDropsFrom _ _ _ _ _ Nothing _ = noop +handleDropsFrom locs rs reason fromhere key (Just f) knownpresentremote + | fromhere = do + n <- getcopies + if checkcopies n + then go rs =<< dropl n + else go rs n + | otherwise = go rs =<< getcopies + where + getcopies = liftAnnex $ do + have <- length <$> trustExclude UnTrusted locs + numcopies <- getNumCopies =<< numCopies f + return (have, numcopies) + checkcopies (have, numcopies) = have > numcopies + decrcopies (have, numcopies) = (have - 1, numcopies) + + go [] _ = noop + go (r:rest) n + | uuid r `S.notMember` slocs = go rest n + | checkcopies n = dropr r n >>= go rest + | otherwise = noop + + checkdrop n@(have, numcopies) u a = + ifM (liftAnnex $ wantDrop True u (Just f)) + ( ifM (liftAnnex $ safely $ doCommand $ a (Just numcopies)) + ( do + debug + [ "dropped" + , f + , "(from " ++ maybe "here" show u ++ ")" + , "(copies now " ++ show (have - 1) ++ ")" + , ": " ++ reason + ] + return $ decrcopies n + , return n + ) + , return n + ) + + dropl n = checkdrop n Nothing $ \numcopies -> + Command.Drop.startLocal f numcopies key knownpresentremote + + dropr r n = checkdrop n (Just $ Remote.uuid r) $ \numcopies -> + Command.Drop.startRemote f numcopies key r + + safely a = either (const False) id <$> tryAnnex a + + slocs = S.fromList locs diff --git a/Assistant/Environment.hs b/Assistant/Environment.hs new file mode 100644 index 0000000000..3153fcfe55 --- /dev/null +++ b/Assistant/Environment.hs @@ -0,0 +1,26 @@ +{- git-annex assistant environment + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Environment where + +import Assistant.Common +import Utility.UserInfo +import qualified Git.Config + +import System.Posix.Env + +{- Checks that the system's environment allows git to function. + - Git requires a GECOS username, or suitable git configuration, or + - environment variables. -} +checkEnvironment :: Annex () +checkEnvironment = do + username <- liftIO myUserName + gecos <- liftIO myUserGecos + gitusername <- fromRepo $ Git.Config.getMaybe "user.name" + when (null gecos && (gitusername == Nothing || gitusername == Just "")) $ + -- existing environment is not overwritten + liftIO $ setEnv "GIT_AUTHOR_NAME" username False diff --git a/Assistant/Install.hs b/Assistant/Install.hs new file mode 100644 index 0000000000..0db021ef77 --- /dev/null +++ b/Assistant/Install.hs @@ -0,0 +1,93 @@ +{- Assistant installation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Install where + +import Assistant.Common +import Assistant.Install.AutoStart +import Assistant.Ssh +import Locations.UserConfig +import Utility.FileMode +import Utility.Shell + +#ifdef darwin_HOST_OS +import Utility.OSX +#else +import Utility.FreeDesktop +#endif + +import System.Posix.Env + +standaloneAppBase :: IO (Maybe FilePath) +standaloneAppBase = getEnv "GIT_ANNEX_APP_BASE" + +{- The standalone app does not have an installation process. + - So when it's run, it needs to set up autostarting of the assistant + - daemon, as well as writing the programFile, and putting a + - git-annex-shell wrapper into ~/.ssh + - + - Note that this is done every time it's started, so if the user moves + - it around, the paths this sets up won't break. + -} +ensureInstalled :: IO () +ensureInstalled = go =<< standaloneAppBase + where + go Nothing = noop + go (Just base) = do + let program = base "git-annex" + programfile <- programFile + createDirectoryIfMissing True (parentDir programfile) + writeFile programfile program + +#ifdef darwin_HOST_OS + autostartfile <- userAutoStart osxAutoStartLabel +#else + autostartfile <- autoStartPath "git-annex" <$> userConfigDir +#endif + installAutoStart program autostartfile + + {- This shim is only updated if it doesn't + - already exist with the right content. This + - ensures that there's no race where it would have + - worked, but is unavailable due to being updated. -} + sshdir <- sshDir + let shim = sshdir "git-annex-shell" + let content = unlines + [ shebang + , "set -e" + , "exec", base "runshell" ++ + " git-annex-shell -c \"$SSH_ORIGINAL_COMMAND\"" + ] + curr <- catchDefaultIO "" $ readFileStrict shim + when (curr /= content) $ do + createDirectoryIfMissing True (parentDir shim) + writeFile shim content + modifyFileMode shim $ addModes [ownerExecuteMode] + +{- Returns a cleaned up environment that lacks settings used to make the + - standalone builds use their bundled libraries and programs. + - Useful when calling programs not included in the standalone builds. + - + - For a non-standalone build, returns Nothing. + -} +cleanEnvironment :: IO (Maybe [(String, String)]) +cleanEnvironment = clean <$> getEnvironment + where + clean env + | null vars = Nothing + | otherwise = Just $ catMaybes $ map (restoreorig env) env + | otherwise = Nothing + where + vars = words $ fromMaybe "" $ + lookup "GIT_ANNEX_STANDLONE_ENV" env + restoreorig oldenv p@(k, _v) + | k `elem` vars = case lookup ("ORIG_" ++ k) oldenv of + Nothing -> Nothing + (Just v') -> Just (k, v') + | otherwise = Just p diff --git a/Assistant/Install/AutoStart.hs b/Assistant/Install/AutoStart.hs new file mode 100644 index 0000000000..13b4bc7ab0 --- /dev/null +++ b/Assistant/Install/AutoStart.hs @@ -0,0 +1,38 @@ +{- Assistant autostart file installation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Install.AutoStart where + +import Utility.FreeDesktop +#ifdef darwin_HOST_OS +import Utility.OSX +import Utility.Path +import System.Directory +#endif + +installAutoStart :: FilePath -> FilePath -> IO () +installAutoStart command file = do +#ifdef darwin_HOST_OS + createDirectoryIfMissing True (parentDir file) + writeFile file $ genOSXAutoStartFile osxAutoStartLabel command + ["assistant", "--autostart"] +#else + writeDesktopMenuFile (fdoAutostart command) file +#endif + +osxAutoStartLabel :: String +osxAutoStartLabel = "com.branchable.git-annex.assistant" + +fdoAutostart :: FilePath -> DesktopEntry +fdoAutostart command = genDesktopEntry + "Git Annex Assistant" + "Autostart" + False + (command ++ " assistant --autostart") + [] diff --git a/Assistant/MakeRemote.hs b/Assistant/MakeRemote.hs new file mode 100644 index 0000000000..b01b051f6d --- /dev/null +++ b/Assistant/MakeRemote.hs @@ -0,0 +1,114 @@ +{- git-annex assistant remote creation utilities + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.MakeRemote where + +import Assistant.Common +import Assistant.Ssh +import Assistant.Sync +import qualified Types.Remote as R +import qualified Remote +import Remote.List +import qualified Remote.Rsync as Rsync +import qualified Git +import qualified Git.Command +import qualified Command.InitRemote +import Logs.UUID +import Logs.Remote +import Git.Remote + +import qualified Data.Text as T +import qualified Data.Map as M + +{- Sets up and begins syncing with a new ssh or rsync remote. -} +makeSshRemote :: Bool -> SshData -> Assistant Remote +makeSshRemote forcersync sshdata = do + r <- liftAnnex $ + addRemote $ maker (sshRepoName sshdata) sshurl + syncNewRemote r + return r + where + rsync = forcersync || rsyncOnly sshdata + maker + | rsync = makeRsyncRemote + | otherwise = makeGitRemote + sshurl = T.unpack $ T.concat $ + if rsync + then [u, h, T.pack ":", sshDirectory sshdata, T.pack "/"] + else [T.pack "ssh://", u, h, d, T.pack "/"] + where + u = maybe (T.pack "") (\v -> T.concat [v, T.pack "@"]) $ sshUserName sshdata + h = sshHostName sshdata + d + | T.pack "/" `T.isPrefixOf` sshDirectory sshdata = sshDirectory sshdata + | otherwise = T.concat [T.pack "/~/", sshDirectory sshdata] + +{- Runs an action that returns a name of the remote, and finishes adding it. -} +addRemote :: Annex String -> Annex Remote +addRemote a = do + name <- a + void remoteListRefresh + maybe (error "failed to add remote") return =<< Remote.byName (Just name) + +{- Inits a rsync special remote, and returns its name. -} +makeRsyncRemote :: String -> String -> Annex String +makeRsyncRemote name location = makeRemote name location $ + const $ makeSpecialRemote name Rsync.remote config + where + config = M.fromList + [ ("encryption", "shared") + , ("rsyncurl", location) + , ("type", "rsync") + ] + +{- Inits a special remote. -} +makeSpecialRemote :: String -> RemoteType -> R.RemoteConfig -> Annex () +makeSpecialRemote name remotetype config = do + (u, c) <- Command.InitRemote.findByName name + c' <- R.setup remotetype u $ M.union config c + describeUUID u name + configSet u c' + +{- Returns the name of the git remote it created. If there's already a + - remote at the location, returns its name. -} +makeGitRemote :: String -> String -> Annex String +makeGitRemote basename location = makeRemote basename location $ \name -> + void $ inRepo $ Git.Command.runBool + [Param "remote", Param "add", Param name, Param location] + +{- If there's not already a remote at the location, adds it using the + - action, which is passed the name of the remote to make. + - + - Returns the name of the remote. -} +makeRemote :: String -> String -> (String -> Annex ()) -> Annex String +makeRemote basename location a = do + g <- gitRepo + if not (any samelocation $ Git.remotes g) + then do + + let name = uniqueRemoteName basename 0 g + a name + return name + else return basename + where + samelocation x = Git.repoLocation x == location + +{- Generate an unused name for a remote, adding a number if + - necessary. + - + - Ensures that the returned name is a legal git remote name. -} +uniqueRemoteName :: String -> Int -> Git.Repo -> String +uniqueRemoteName basename n r + | null namecollision = name + | otherwise = uniqueRemoteName legalbasename (succ n) r + where + namecollision = filter samename (Git.remotes r) + samename x = Git.remoteName x == Just name + name + | n == 0 = legalbasename + | otherwise = legalbasename ++ show n + legalbasename = makeLegalName basename diff --git a/Assistant/Monad.hs b/Assistant/Monad.hs new file mode 100644 index 0000000000..140b9f582e --- /dev/null +++ b/Assistant/Monad.hs @@ -0,0 +1,136 @@ +{- git-annex assistant monad + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE PackageImports, GeneralizedNewtypeDeriving, MultiParamTypeClasses #-} + +module Assistant.Monad ( + Assistant, + AssistantData(..), + newAssistantData, + runAssistant, + getAssistant, + liftAnnex, + (<~>), + (<<~), + asIO, + asIO1, + asIO2, + ThreadName, + debug, + notice +) where + +import "mtl" Control.Monad.Reader +import Control.Monad.Base (liftBase, MonadBase) +import System.Log.Logger + +import Common.Annex +import Assistant.Types.ThreadedMonad +import Assistant.Types.DaemonStatus +import Assistant.Types.ScanRemotes +import Assistant.Types.TransferQueue +import Assistant.Types.TransferSlots +import Assistant.Types.Pushes +import Assistant.Types.BranchChange +import Assistant.Types.Commits +import Assistant.Types.Changes +import Assistant.Types.Buddies +import Assistant.Types.NetMessager +import Assistant.Types.ThreadName + +newtype Assistant a = Assistant { mkAssistant :: ReaderT AssistantData IO a } + deriving ( + Monad, + MonadIO, + MonadReader AssistantData, + Functor, + Applicative + ) + +instance MonadBase IO Assistant where + liftBase = Assistant . liftBase + +data AssistantData = AssistantData + { threadName :: ThreadName + , threadState :: ThreadState + , daemonStatusHandle :: DaemonStatusHandle + , scanRemoteMap :: ScanRemoteMap + , transferQueue :: TransferQueue + , transferSlots :: TransferSlots + , failedPushMap :: FailedPushMap + , commitChan :: CommitChan + , changeChan :: ChangeChan + , branchChangeHandle :: BranchChangeHandle + , buddyList :: BuddyList + , netMessager :: NetMessager + } + +newAssistantData :: ThreadState -> DaemonStatusHandle -> IO AssistantData +newAssistantData st dstatus = AssistantData + <$> pure (ThreadName "main") + <*> pure st + <*> pure dstatus + <*> newScanRemoteMap + <*> newTransferQueue + <*> newTransferSlots + <*> newFailedPushMap + <*> newCommitChan + <*> newChangeChan + <*> newBranchChangeHandle + <*> newBuddyList + <*> newNetMessager + +runAssistant :: AssistantData -> Assistant a -> IO a +runAssistant d a = runReaderT (mkAssistant a) d + +getAssistant :: (AssistantData -> a) -> Assistant a +getAssistant = reader + +{- Runs an action in the git-annex monad. Note that the same monad state + - is shared amoung all assistant threads, so only one of these can run at + - a time. Therefore, long-duration actions should be avoided. -} +liftAnnex :: Annex a -> Assistant a +liftAnnex a = do + st <- reader threadState + liftIO $ runThreadState st a + +{- Runs an IO action, passing it an IO action that runs an Assistant action. -} +(<~>) :: (IO a -> IO b) -> Assistant a -> Assistant b +io <~> a = do + d <- reader id + liftIO $ io $ runAssistant d a + +{- Creates an IO action that will run an Assistant action when run. -} +asIO :: Assistant a -> Assistant (IO a) +asIO a = do + d <- reader id + return $ runAssistant d a + +asIO1 :: (a -> Assistant b) -> Assistant (a -> IO b) +asIO1 a = do + d <- reader id + return $ \v -> runAssistant d $ a v + +asIO2 :: (a -> b -> Assistant c) -> Assistant (a -> b -> IO c) +asIO2 a = do + d <- reader id + return $ \v1 v2 -> runAssistant d (a v1 v2) + +{- Runs an IO action on a selected field of the AssistantData. -} +(<<~) :: (a -> IO b) -> (AssistantData -> a) -> Assistant b +io <<~ v = reader v >>= liftIO . io + +debug :: [String] -> Assistant () +debug = logaction debugM + +notice :: [String] -> Assistant () +notice = logaction noticeM + +logaction :: (String -> String -> IO ()) -> [String] -> Assistant () +logaction a ws = do + ThreadName name <- getAssistant threadName + liftIO $ a name $ unwords $ (name ++ ":") : ws diff --git a/Assistant/NamedThread.hs b/Assistant/NamedThread.hs new file mode 100644 index 0000000000..33af2c3040 --- /dev/null +++ b/Assistant/NamedThread.hs @@ -0,0 +1,102 @@ +{- git-annex assistant named threads. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.NamedThread where + +import Common.Annex +import Assistant.Types.NamedThread +import Assistant.Types.ThreadName +import Assistant.Types.DaemonStatus +import Assistant.DaemonStatus +import Assistant.Monad + +import Control.Concurrent +import Control.Concurrent.Async +import qualified Data.Map as M +import qualified Control.Exception as E + +#ifdef WITH_WEBAPP +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.Alert +import qualified Data.Text as T +#endif + +{- Starts a named thread, if it's not already running. + - + - Named threads are run by a management thread, so if they crash + - an alert is displayed, allowing the thread to be restarted. -} +#ifdef WITH_WEBAPP +startNamedThread :: Maybe UrlRenderer -> NamedThread -> Assistant () +startNamedThread urlrenderer namedthread@(NamedThread name a) = do +#else +startNamedThread :: Maybe Bool -> NamedThread -> Assistant () +startNamedThread urlrenderer namedthread@(NamedThread name a) = do +#endif + m <- startedThreads <$> getDaemonStatus + case M.lookup name m of + Nothing -> start + Just (aid, _) -> do + r <- liftIO (E.try (poll aid) :: IO (Either E.SomeException (Maybe (Either E.SomeException ())))) + case r of + Right Nothing -> noop + _ -> start + where + start = do + d <- getAssistant id + aid <- liftIO $ runmanaged $ d { threadName = name } + restart <- asIO $ startNamedThread urlrenderer namedthread + modifyDaemonStatus_ $ \s -> s + { startedThreads = M.insertWith' const name (aid, restart) (startedThreads s) } + runmanaged d = do + aid <- async $ runAssistant d a + void $ forkIO $ manager d aid + return aid + manager d aid = do + r <- E.try (wait aid) :: IO (Either E.SomeException ()) + case r of + Right _ -> noop + Left e -> do + let msg = unwords + [ fromThreadName $ threadName d + , "crashed:", show e + ] + hPutStrLn stderr msg +#ifdef WITH_WEBAPP + button <- runAssistant d $ + case urlrenderer of + Nothing -> return Nothing + Just renderer -> do + close <- asIO1 removeAlert + url <- liftIO $ renderUrl renderer (RestartThreadR name) [] + return $ Just $ AlertButton + { buttonLabel = T.pack "Restart Thread" + , buttonUrl = url + , buttonAction = Just close + } + runAssistant d $ void $ + addAlert $ (warningAlert (fromThreadName name) msg) + { alertButton = button } +#endif + +namedThreadId :: NamedThread -> Assistant (Maybe ThreadId) +namedThreadId (NamedThread name _) = do + m <- startedThreads <$> getDaemonStatus + return $ asyncThreadId . fst <$> M.lookup name m + +{- Waits for all named threads that have been started to finish. + - + - Note that if a named thread crashes, it will probably + - cause this to crash as well. Also, named threads that are started + - after this is called will not be waited on. -} +waitNamedThreads :: Assistant () +waitNamedThreads = do + m <- startedThreads <$> getDaemonStatus + liftIO $ mapM_ (wait . fst) $ M.elems m + diff --git a/Assistant/NetMessager.hs b/Assistant/NetMessager.hs new file mode 100644 index 0000000000..2191e06f2f --- /dev/null +++ b/Assistant/NetMessager.hs @@ -0,0 +1,95 @@ +{- git-annex assistant out of band network messager interface + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.NetMessager where + +import Assistant.Common +import Assistant.Types.NetMessager + +import Control.Concurrent +import Control.Concurrent.STM +import Control.Concurrent.MSampleVar +import Control.Exception as E +import qualified Data.Set as S + +sendNetMessage :: NetMessage -> Assistant () +sendNetMessage m = + (atomically . flip writeTChan m) <<~ (netMessages . netMessager) + +waitNetMessage :: Assistant (NetMessage) +waitNetMessage = (atomically . readTChan) <<~ (netMessages . netMessager) + +notifyNetMessagerRestart :: Assistant () +notifyNetMessagerRestart = + flip writeSV () <<~ (netMessagerRestart . netMessager) + +waitNetMessagerRestart :: Assistant () +waitNetMessagerRestart = readSV <<~ (netMessagerRestart . netMessager) + +{- Runs an action that runs either the send or receive side of a push. + - + - While the push is running, netMessagesPush will get messages put into it + - relating to this push, while any messages relating to other pushes + - on the same side go to netMessagesDeferred. Once the push finishes, + - those deferred messages will be fed to handledeferred for processing. + -} +runPush :: PushSide -> ClientID -> (NetMessage -> Assistant ()) -> Assistant a -> Assistant a +runPush side clientid handledeferred a = do + nm <- getAssistant netMessager + let runningv = getSide side $ netMessagerPushRunning nm + let setup = void $ atomically $ swapTMVar runningv $ Just clientid + let cleanup = atomically $ do + void $ swapTMVar runningv Nothing + emptytchan (getSide side $ netMessagesPush nm) + r <- E.bracket_ setup cleanup <~> a + (void . forkIO) <~> processdeferred nm + return r + where + emptytchan c = maybe noop (const $ emptytchan c) =<< tryReadTChan c + processdeferred nm = do + s <- liftIO $ atomically $ swapTMVar (getSide side $ netMessagesPushDeferred nm) S.empty + mapM_ rundeferred (S.toList s) + rundeferred m = (void . (E.try :: (IO () -> IO (Either SomeException ())))) + <~> handledeferred m + +{- While a push is running, matching push messages are put into + - netMessagesPush, while others that involve the same side go to + - netMessagesDeferredPush. + - + - When no push is running involving the same side, returns False. + - + - To avoid bloating memory, only messages that initiate pushes are + - deferred. + -} +queueNetPushMessage :: NetMessage -> Assistant Bool +queueNetPushMessage m@(Pushing clientid stage) = do + nm <- getAssistant netMessager + liftIO $ atomically $ do + v <- readTMVar (getSide side $ netMessagerPushRunning nm) + case v of + Nothing -> return False + (Just runningclientid) + | runningclientid == clientid -> queue nm + | isPushInitiation stage -> defer nm + | otherwise -> discard + where + side = pushDestinationSide stage + queue nm = do + writeTChan (getSide side $ netMessagesPush nm) m + return True + defer nm = do + let mv = getSide side $ netMessagesPushDeferred nm + s <- takeTMVar mv + putTMVar mv $ S.insert m s + return True + discard = return True +queueNetPushMessage _ = return False + +waitNetPushMessage :: PushSide -> Assistant (NetMessage) +waitNetPushMessage side = (atomically . readTChan) + <<~ (getSide side . netMessagesPush . netMessager) + diff --git a/Assistant/Pairing.hs b/Assistant/Pairing.hs new file mode 100644 index 0000000000..4736c4396c --- /dev/null +++ b/Assistant/Pairing.hs @@ -0,0 +1,92 @@ +{- git-annex assistant repo pairing, core data types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Pairing where + +import Common.Annex +import Utility.Verifiable +import Assistant.Ssh + +import Control.Concurrent +import Network.Socket +import Data.Char +import qualified Data.Text as T + +data PairStage + {- "I'll pair with anybody who shares the secret that can be used + - to verify this request." -} + = PairReq + {- "I've verified your request, and you can verify this to see + - that I know the secret. I set up your ssh key already. + - Here's mine for you to set up." -} + | PairAck + {- "I saw your PairAck; you can stop sending them." -} + | PairDone + deriving (Eq, Read, Show, Ord) + +newtype PairMsg = PairMsg (Verifiable (PairStage, PairData, SomeAddr)) + deriving (Eq, Read, Show) + +verifiedPairMsg :: PairMsg -> PairingInProgress -> Bool +verifiedPairMsg (PairMsg m) pip = verify m $ inProgressSecret pip + +fromPairMsg :: PairMsg -> Verifiable (PairStage, PairData, SomeAddr) +fromPairMsg (PairMsg m) = m + +pairMsgStage :: PairMsg -> PairStage +pairMsgStage (PairMsg (Verifiable (s, _, _) _)) = s + +pairMsgData :: PairMsg -> PairData +pairMsgData (PairMsg (Verifiable (_, d, _) _)) = d + +pairMsgAddr :: PairMsg -> SomeAddr +pairMsgAddr (PairMsg (Verifiable (_, _, a) _)) = a + +data PairData = PairData + -- uname -n output, not a full domain name + { remoteHostName :: Maybe HostName + , remoteUserName :: UserName + , remoteDirectory :: FilePath + , remoteSshPubKey :: SshPubKey + , pairUUID :: UUID + } + deriving (Eq, Read, Show) + +type UserName = String + +{- A pairing that is in progress has a secret, a thread that is + - broadcasting pairing messages, and a SshKeyPair that has not yet been + - set up on disk. -} +data PairingInProgress = PairingInProgress + { inProgressSecret :: Secret + , inProgressThreadId :: Maybe ThreadId + , inProgressSshKeyPair :: SshKeyPair + , inProgressPairData :: PairData + , inProgressPairStage :: PairStage + } + deriving (Show) + +data SomeAddr = IPv4Addr HostAddress +{- My Android build of the Network library does not currently have IPV6 + - support. -} +#ifndef __ANDROID__ + | IPv6Addr HostAddress6 +#endif + deriving (Ord, Eq, Read, Show) + +{- This contains the whole secret, just lightly obfuscated to make it not + - too obvious. It's only displayed in the user's web browser. -} +newtype SecretReminder = SecretReminder [Int] + deriving (Show, Eq, Ord, Read) + +toSecretReminder :: T.Text -> SecretReminder +toSecretReminder = SecretReminder . map ord . T.unpack + +fromSecretReminder :: SecretReminder -> T.Text +fromSecretReminder (SecretReminder s) = T.pack $ map chr s diff --git a/Assistant/Pairing/MakeRemote.hs b/Assistant/Pairing/MakeRemote.hs new file mode 100644 index 0000000000..24a83b43c6 --- /dev/null +++ b/Assistant/Pairing/MakeRemote.hs @@ -0,0 +1,90 @@ +{- git-annex assistant pairing remote creation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Pairing.MakeRemote where + +import Assistant.Common +import Assistant.Ssh +import Assistant.Pairing +import Assistant.Pairing.Network +import Assistant.MakeRemote + +import Network.Socket +import qualified Data.Text as T + +{- Authorized keys are set up before pairing is complete, so that the other + - side can immediately begin syncing. -} +setupAuthorizedKeys :: PairMsg -> FilePath -> IO () +setupAuthorizedKeys msg repodir = do + validateSshPubKey pubkey + unlessM (liftIO $ addAuthorizedKeys False repodir pubkey) $ + error "failed setting up ssh authorized keys" + where + pubkey = remoteSshPubKey $ pairMsgData msg + +{- When local pairing is complete, this is used to set up the remote for + - the host we paired with. -} +finishedLocalPairing :: PairMsg -> SshKeyPair -> Assistant () +finishedLocalPairing msg keypair = do + sshdata <- liftIO $ setupSshKeyPair keypair =<< pairMsgToSshData msg + {- Ensure that we know the ssh host key for the host we paired with. + - If we don't, ssh over to get it. -} + liftIO $ unlessM (knownHost $ sshHostName sshdata) $ + void $ sshTranscript + [ sshOpt "StrictHostKeyChecking" "no" + , sshOpt "NumberOfPasswordPrompts" "0" + , "-n" + , genSshHost (sshHostName sshdata) (sshUserName sshdata) + , "git-annex-shell -c configlist " ++ T.unpack (sshDirectory sshdata) + ] + Nothing + void $ makeSshRemote False sshdata + +{- Mostly a straightforward conversion. Except: + - * Determine the best hostname to use to contact the host. + - * Strip leading ~/ from the directory name. + -} +pairMsgToSshData :: PairMsg -> IO SshData +pairMsgToSshData msg = do + let d = pairMsgData msg + hostname <- liftIO $ bestHostName msg + let dir = case remoteDirectory d of + ('~':'/':v) -> v + v -> v + return SshData + { sshHostName = T.pack hostname + , sshUserName = Just (T.pack $ remoteUserName d) + , sshDirectory = T.pack dir + , sshRepoName = genSshRepoName hostname dir + , sshPort = 22 + , needsPubKey = True + , rsyncOnly = False + } + +{- Finds the best hostname to use for the host that sent the PairMsg. + - + - If remoteHostName is set, tries to use a .local address based on it. + - That's the most robust, if this system supports .local. + - Otherwise, looks up the hostname in the DNS for the remoteAddress, + - if any. May fall back to remoteAddress if there's no DNS. Ugh. -} +bestHostName :: PairMsg -> IO HostName +bestHostName msg = case remoteHostName $ pairMsgData msg of + Just h -> do + let localname = h ++ ".local" + addrs <- catchDefaultIO [] $ + getAddrInfo Nothing (Just localname) Nothing + maybe fallback (const $ return localname) (headMaybe addrs) + Nothing -> fallback + where + fallback = do + let a = pairMsgAddr msg + let sockaddr = case a of + IPv4Addr addr -> SockAddrInet (PortNum 0) addr + IPv6Addr addr -> SockAddrInet6 (PortNum 0) 0 addr 0 + fromMaybe (showAddr a) + <$> catchDefaultIO Nothing + (fst <$> getNameInfo [] True False sockaddr) diff --git a/Assistant/Pairing/Network.hs b/Assistant/Pairing/Network.hs new file mode 100644 index 0000000000..6c625f8814 --- /dev/null +++ b/Assistant/Pairing/Network.hs @@ -0,0 +1,130 @@ +{- git-annex assistant pairing network code + - + - All network traffic is sent over multicast UDP. For reliability, + - each message is repeated until acknowledged. This is done using a + - thread, that gets stopped before the next message is sent. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Pairing.Network where + +import Assistant.Common +import Assistant.Pairing +import Assistant.DaemonStatus +import Utility.ThreadScheduler +import Utility.Verifiable + +import Network.Multicast +import Network.Info +import Network.Socket +import Control.Exception (bracket) +import qualified Data.Map as M +import Control.Concurrent + +{- This is an arbitrary port in the dynamic port range, that could + - conceivably be used for some other broadcast messages. + - If so, hope they ignore the garbage from us; we'll certianly + - ignore garbage from them. Wild wild west. -} +pairingPort :: PortNumber +pairingPort = 55556 + +{- Goal: Reach all hosts on the same network segment. + - Method: Use same address that avahi uses. Other broadcast addresses seem + - to not be let through some routers. -} +multicastAddress :: SomeAddr -> HostName +multicastAddress (IPv4Addr _) = "224.0.0.251" +multicastAddress (IPv6Addr _) = "ff02::fb" + +{- Multicasts a message repeatedly on all interfaces, with a 2 second + - delay between each transmission. The message is repeated forever + - unless a number of repeats is specified. + - + - The remoteHostAddress is set to the interface's IP address. + - + - Note that new sockets are opened each time. This is hardly efficient, + - but it allows new network interfaces to be used as they come up. + - On the other hand, the expensive DNS lookups are cached. + -} +multicastPairMsg :: Maybe Int -> Secret -> PairData -> PairStage -> IO () +multicastPairMsg repeats secret pairdata stage = go M.empty repeats + where + go _ (Just 0) = noop + go cache n = do + addrs <- activeNetworkAddresses + let cache' = updatecache cache addrs + mapM_ (sendinterface cache') addrs + threadDelaySeconds (Seconds 2) + go cache' $ pred <$> n + {- The multicast library currently chokes on ipv6 addresses. -} + sendinterface _ (IPv6Addr _) = noop + sendinterface cache i = void $ tryIO $ + withSocketsDo $ bracket setup cleanup use + where + setup = multicastSender (multicastAddress i) pairingPort + cleanup (sock, _) = sClose sock -- FIXME does not work + use (sock, addr) = do + setInterface sock (showAddr i) + maybe noop (\s -> void $ sendTo sock s addr) + (M.lookup i cache) + updatecache cache [] = cache + updatecache cache (i:is) + | M.member i cache = updatecache cache is + | otherwise = updatecache (M.insert i (show $ mkmsg i) cache) is + mkmsg addr = PairMsg $ + mkVerifiable (stage, pairdata, addr) secret + +startSending :: PairingInProgress -> PairStage -> (PairStage -> IO ()) -> Assistant () +startSending pip stage sender = do + a <- asIO start + void $ liftIO $ forkIO a + where + start = do + tid <- liftIO myThreadId + let pip' = pip { inProgressPairStage = stage, inProgressThreadId = Just tid } + oldpip <- modifyDaemonStatus $ + \s -> (s { pairingInProgress = Just pip' }, pairingInProgress s) + maybe noop stopold oldpip + liftIO $ sender stage + stopold = maybe noop (liftIO . killThread) . inProgressThreadId + +stopSending :: PairingInProgress -> Assistant () +stopSending pip = do + maybe noop (liftIO . killThread) $ inProgressThreadId pip + modifyDaemonStatus_ $ \s -> s { pairingInProgress = Nothing } + +class ToSomeAddr a where + toSomeAddr :: a -> SomeAddr + +instance ToSomeAddr IPv4 where + toSomeAddr (IPv4 a) = IPv4Addr a + +instance ToSomeAddr IPv6 where + toSomeAddr (IPv6 o1 o2 o3 o4) = IPv6Addr (o1, o2, o3, o4) + +showAddr :: SomeAddr -> HostName +showAddr (IPv4Addr a) = show $ IPv4 a +showAddr (IPv6Addr (o1, o2, o3, o4)) = show $ IPv6 o1 o2 o3 o4 + +activeNetworkAddresses :: IO [SomeAddr] +activeNetworkAddresses = filter (not . all (`elem` "0.:") . showAddr) + . concatMap (\ni -> [toSomeAddr $ ipv4 ni, toSomeAddr $ ipv6 ni]) + <$> getNetworkInterfaces + +{- A human-visible description of the repository being paired with. + - Note that the repository's description is not shown to the user, because + - it could be something like "my repo", which is confusing when pairing + - with someone else's repo. However, this has the same format as the + - default decription of a repo. -} +pairRepo :: PairMsg -> String +pairRepo msg = concat + [ remoteUserName d + , "@" + , fromMaybe (showAddr $ pairMsgAddr msg) (remoteHostName d) + , ":" + , remoteDirectory d + ] + where + d = pairMsgData msg diff --git a/Assistant/Pushes.hs b/Assistant/Pushes.hs new file mode 100644 index 0000000000..54f31a84bc --- /dev/null +++ b/Assistant/Pushes.hs @@ -0,0 +1,40 @@ +{- git-annex assistant push tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Pushes where + +import Assistant.Common +import Assistant.Types.Pushes + +import Control.Concurrent.STM +import Data.Time.Clock +import qualified Data.Map as M + +{- Blocks until there are failed pushes. + - Returns Remotes whose pushes failed a given time duration or more ago. + - (This may be an empty list.) -} +getFailedPushesBefore :: NominalDiffTime -> Assistant [Remote] +getFailedPushesBefore duration = do + v <- getAssistant failedPushMap + liftIO $ do + m <- atomically $ readTMVar v + now <- getCurrentTime + return $ M.keys $ M.filter (not . toorecent now) m + where + toorecent now time = now `diffUTCTime` time < duration + +{- Modifies the map. -} +changeFailedPushMap :: (PushMap -> PushMap) -> Assistant () +changeFailedPushMap a = do + v <- getAssistant failedPushMap + liftIO $ atomically $ store v . a . fromMaybe M.empty =<< tryTakeTMVar v + where + {- tryTakeTMVar empties the TMVar; refill it only if + - the modified map is not itself empty -} + store v m + | m == M.empty = noop + | otherwise = putTMVar v $! m diff --git a/Assistant/ScanRemotes.hs b/Assistant/ScanRemotes.hs new file mode 100644 index 0000000000..2743c0f361 --- /dev/null +++ b/Assistant/ScanRemotes.hs @@ -0,0 +1,41 @@ +{- git-annex assistant remotes needing scanning + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.ScanRemotes where + +import Assistant.Common +import Assistant.Types.ScanRemotes +import qualified Types.Remote as Remote + +import Data.Function +import Control.Concurrent.STM +import qualified Data.Map as M + +{- Blocks until there is a remote or remotes that need to be scanned. + - + - The list has higher priority remotes listed first. -} +getScanRemote :: Assistant [(Remote, ScanInfo)] +getScanRemote = do + v <- getAssistant scanRemoteMap + liftIO $ atomically $ + reverse . sortBy (compare `on` scanPriority . snd) . M.toList + <$> takeTMVar v + +{- Adds new remotes that need scanning. -} +addScanRemotes :: Bool -> [Remote] -> Assistant () +addScanRemotes _ [] = noop +addScanRemotes full rs = do + v <- getAssistant scanRemoteMap + liftIO $ atomically $ do + m <- fromMaybe M.empty <$> tryTakeTMVar v + putTMVar v $ M.unionWith merge (M.fromList $ zip rs (map info rs)) m + where + info r = ScanInfo (-1 * Remote.cost r) full + merge x y = ScanInfo + { scanPriority = max (scanPriority x) (scanPriority y) + , fullScan = fullScan x || fullScan y + } diff --git a/Assistant/Ssh.hs b/Assistant/Ssh.hs new file mode 100644 index 0000000000..4c8d8afbf0 --- /dev/null +++ b/Assistant/Ssh.hs @@ -0,0 +1,225 @@ +{- git-annex assistant ssh utilities + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Ssh where + +import Common.Annex +import Utility.TempFile +import Utility.UserInfo +import Utility.Shell +import Git.Remote + +import Data.Text (Text) +import qualified Data.Text as T +import Data.Char + +data SshData = SshData + { sshHostName :: Text + , sshUserName :: Maybe Text + , sshDirectory :: Text + , sshRepoName :: String + , sshPort :: Int + , needsPubKey :: Bool + , rsyncOnly :: Bool + } + deriving (Read, Show, Eq) + +data SshKeyPair = SshKeyPair + { sshPubKey :: String + , sshPrivKey :: String + } + +instance Show SshKeyPair where + show = sshPubKey + +type SshPubKey = String + +{- ssh -ofoo=bar command-line option -} +sshOpt :: String -> String -> String +sshOpt k v = concat ["-o", k, "=", v] + +sshDir :: IO FilePath +sshDir = do + home <- myHomeDir + return $ home ".ssh" + +{- user@host or host -} +genSshHost :: Text -> Maybe Text -> String +genSshHost host user = maybe "" (\v -> T.unpack v ++ "@") user ++ T.unpack host + +{- Generates a git remote name, like host_dir or host -} +genSshRepoName :: String -> FilePath -> String +genSshRepoName host dir + | null dir = makeLegalName host + | otherwise = makeLegalName $ host ++ "_" ++ dir + +{- The output of ssh, including both stdout and stderr. -} +sshTranscript :: [String] -> (Maybe String) -> IO (String, Bool) +sshTranscript opts input = processTranscript "ssh" opts input + +{- Ensure that the ssh public key doesn't include any ssh options, like + - command=foo, or other weirdness -} +validateSshPubKey :: SshPubKey -> IO () +validateSshPubKey pubkey = either error return $ check $ words pubkey + where + check [prefix, _key, comment] = do + checkprefix prefix + checkcomment comment + check [prefix, _key] = + checkprefix prefix + check _ = err "wrong number of words in ssh public key" + + ok = Right () + err msg = Left $ unwords [msg, pubkey] + + checkprefix prefix + | ssh == "ssh" && all isAlphaNum keytype = ok + | otherwise = err "bad ssh public key prefix" + where + (ssh, keytype) = separate (== '-') prefix + + checkcomment comment + | all (\c -> isAlphaNum c || c == '@' || c == '-' || c == '_' || c == '.') comment = ok + | otherwise = err "bad comment in ssh public key" + +addAuthorizedKeys :: Bool -> FilePath -> SshPubKey -> IO Bool +addAuthorizedKeys rsynconly dir pubkey = boolSystem "sh" + [ Param "-c" , Param $ addAuthorizedKeysCommand rsynconly dir pubkey ] + +removeAuthorizedKeys :: Bool -> FilePath -> SshPubKey -> IO () +removeAuthorizedKeys rsynconly dir pubkey = do + let keyline = authorizedKeysLine rsynconly dir pubkey + sshdir <- sshDir + let keyfile = sshdir "authorized_keys" + ls <- lines <$> readFileStrict keyfile + writeFile keyfile $ unlines $ filter (/= keyline) ls + +{- Implemented as a shell command, so it can be run on remote servers over + - ssh. + - + - The ~/.ssh/git-annex-shell wrapper script is created if not already + - present. + -} +addAuthorizedKeysCommand :: Bool -> FilePath -> SshPubKey -> String +addAuthorizedKeysCommand rsynconly dir pubkey = join "&&" + [ "mkdir -p ~/.ssh" + , join "; " + [ "if [ ! -e " ++ wrapper ++ " ]" + , "then (" ++ join ";" (map echoval script) ++ ") > " ++ wrapper + , "fi" + ] + , "chmod 700 " ++ wrapper + , "touch ~/.ssh/authorized_keys" + , "chmod 600 ~/.ssh/authorized_keys" + , unwords + [ "echo" + , shellEscape $ authorizedKeysLine rsynconly dir pubkey + , ">>~/.ssh/authorized_keys" + ] + ] + where + echoval v = "echo " ++ shellEscape v + wrapper = "~/.ssh/git-annex-shell" + script = + [ shebang + , "set -e" + , "exec git-annex-shell -c \"$SSH_ORIGINAL_COMMAND\"" + ] + +authorizedKeysLine :: Bool -> FilePath -> SshPubKey -> String +authorizedKeysLine rsynconly dir pubkey + {- TODO: Locking down rsync is difficult, requiring a rather + - long perl script. -} + | rsynconly = pubkey + | otherwise = limitcommand ++ pubkey + where + limitcommand = "command=\"GIT_ANNEX_SHELL_DIRECTORY="++shellEscape dir++" ~/.ssh/git-annex-shell\",no-agent-forwarding,no-port-forwarding,no-X11-forwarding " + +{- Generates a ssh key pair. -} +genSshKeyPair :: IO SshKeyPair +genSshKeyPair = withTempDir "git-annex-keygen" $ \dir -> do + ok <- boolSystem "ssh-keygen" + [ Param "-P", Param "" -- no password + , Param "-f", File $ dir "key" + ] + unless ok $ + error "ssh-keygen failed" + SshKeyPair + <$> readFile (dir "key.pub") + <*> readFile (dir "key") + +{- Installs a ssh key pair, and sets up ssh config with a mangled hostname + - that will enable use of the key. This way we avoid changing the user's + - regular ssh experience at all. Returns a modified SshData containing the + - mangled hostname. -} +setupSshKeyPair :: SshKeyPair -> SshData -> IO SshData +setupSshKeyPair sshkeypair sshdata = do + sshdir <- sshDir + createDirectoryIfMissing True sshdir + + unlessM (doesFileExist $ sshdir sshprivkeyfile) $ do + h <- fdToHandle =<< + createFile (sshdir sshprivkeyfile) + (unionFileModes ownerWriteMode ownerReadMode) + hPutStr h (sshPrivKey sshkeypair) + hClose h + unlessM (doesFileExist $ sshdir sshpubkeyfile) $ + writeFile (sshdir sshpubkeyfile) (sshPubKey sshkeypair) + + setSshConfig sshdata + [ ("IdentityFile", "~/.ssh/" ++ sshprivkeyfile) ] + where + sshprivkeyfile = "key." ++ mangleSshHostName sshdata + sshpubkeyfile = sshprivkeyfile ++ ".pub" + +{- Setups up a ssh config with a mangled hostname. + - Returns a modified SshData containing the mangled hostname. -} +setSshConfig :: SshData -> [(String, String)] -> IO SshData +setSshConfig sshdata config = do + sshdir <- sshDir + createDirectoryIfMissing True sshdir + let configfile = sshdir "config" + unlessM (catchBoolIO $ isInfixOf mangledhost <$> readFile configfile) $ + appendFile configfile $ unlines $ + [ "" + , "# Added automatically by git-annex" + , "Host " ++ mangledhost + ] ++ map (\(k, v) -> "\t" ++ k ++ " " ++ v) + (settings ++ config) + return $ sshdata { sshHostName = T.pack mangledhost } + where + mangledhost = mangleSshHostName sshdata + settings = + [ ("Hostname", T.unpack $ sshHostName sshdata) + , ("Port", show $ sshPort sshdata) + ] + +mangleSshHostName :: SshData -> String +mangleSshHostName sshdata = "git-annex-" ++ host ++ (maybe "-" ('-':) user) + where + host = T.unpack $ sshHostName sshdata + user = T.unpack <$> sshUserName sshdata + +unMangleSshHostName :: String -> String +unMangleSshHostName h + | "git-annex-" `isPrefixOf` h = join "-" (beginning $ drop 2 dashbits) + | otherwise = h + where + dashbits = split "-" h + +{- Does ssh have known_hosts data for a hostname? -} +knownHost :: Text -> IO Bool +knownHost hostname = do + sshdir <- sshDir + ifM (doesFileExist $ sshdir "known_hosts") + ( not . null <$> checkhost + , return False + ) + where + {- ssh-keygen -F can crash on some old known_hosts file -} + checkhost = catchDefaultIO "" $ + readProcess "ssh-keygen" ["-F", T.unpack hostname] diff --git a/Assistant/Sync.hs b/Assistant/Sync.hs new file mode 100644 index 0000000000..8546aa318f --- /dev/null +++ b/Assistant/Sync.hs @@ -0,0 +1,178 @@ +{- git-annex assistant repo syncing + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Sync where + +import Assistant.Common +import Assistant.Pushes +import Assistant.NetMessager +import Assistant.Types.NetMessager +import Assistant.Alert +import Assistant.DaemonStatus +import Assistant.ScanRemotes +import qualified Command.Sync +import Utility.Parallel +import qualified Git +import qualified Git.Branch +import qualified Git.Ref +import qualified Git.Command +import qualified Remote +import qualified Types.Remote as Remote +import qualified Annex.Branch +import Annex.UUID + +import Data.Time.Clock +import qualified Data.Map as M +import Control.Concurrent + +{- Syncs with remotes that may have been disconnected for a while. + - + - First gets git in sync, and then prepares any necessary file transfers. + - + - An expensive full scan is queued when the git-annex branches of some of + - the remotes have diverged from the local git-annex branch. Otherwise, + - it's sufficient to requeue failed transfers. + -} +reconnectRemotes :: Bool -> [Remote] -> Assistant () +reconnectRemotes _ [] = noop +reconnectRemotes notifypushes rs = void $ do + alertWhile (syncAlert rs) $ do + (ok, diverged) <- sync + =<< liftAnnex (inRepo Git.Branch.current) + addScanRemotes diverged rs + return ok + where + gitremotes = filter (notspecialremote . Remote.repo) rs + notspecialremote r + | Git.repoIsUrl r = True + | Git.repoIsLocal r = True + | otherwise = False + sync (Just branch) = do + diverged <- snd <$> manualPull (Just branch) gitremotes + now <- liftIO getCurrentTime + ok <- pushToRemotes now notifypushes gitremotes + return (ok, diverged) + {- No local branch exists yet, but we can try pulling. -} + sync Nothing = do + diverged <- snd <$> manualPull Nothing gitremotes + return (True, diverged) + +{- Updates the local sync branch, then pushes it to all remotes, in + - parallel, along with the git-annex branch. This is the same + - as "git annex sync", except in parallel, and will co-exist with use of + - "git annex sync". + - + - After the pushes to normal git remotes, also signals XMPP clients that + - they can request an XMPP push. + - + - Avoids running possibly long-duration commands in the Annex monad, so + - as not to block other threads. + - + - This can fail, when the remote's sync branch (or git-annex branch) has + - been updated by some other remote pushing into it, or by the remote + - itself. To handle failure, a manual pull and merge is done, and the push + - is retried. + - + - When there's a lot of activity, we may fail more than once. + - On the other hand, we may fail because the remote is not available. + - Rather than retrying indefinitely, after the first retry we enter a + - fallback mode, where our push is guarenteed to succeed if the remote is + - reachable. If the fallback fails, the push is queued to be retried + - later. + -} +pushToRemotes :: UTCTime -> Bool -> [Remote] -> Assistant Bool +pushToRemotes now notifypushes remotes = do + (g, branch, u) <- liftAnnex $ do + Annex.Branch.commit "update" + (,,) + <$> gitRepo + <*> inRepo Git.Branch.current + <*> getUUID + let (xmppremotes, normalremotes) = partition isXMPPRemote remotes + ret <- go True branch g u normalremotes + forM_ xmppremotes $ \r -> + sendNetMessage $ Pushing (getXMPPClientID r) CanPush + return ret + where + go _ Nothing _ _ _ = return True -- no branch, so nothing to do + go _ _ _ _ [] = return True -- no remotes, so nothing to do + go shouldretry (Just branch) g u rs = do + debug ["pushing to", show rs] + liftIO $ Command.Sync.updateBranch (Command.Sync.syncBranch branch) g + (succeeded, failed) <- liftIO $ inParallel (push g branch) rs + updatemap succeeded [] + if null failed + then do + when notifypushes $ + sendNetMessage $ NotifyPush $ + map Remote.uuid succeeded + return True + else if shouldretry + then retry branch g u failed + else fallback branch g u failed + + updatemap succeeded failed = changeFailedPushMap $ \m -> + M.union (makemap failed) $ + M.difference m (makemap succeeded) + makemap l = M.fromList $ zip l (repeat now) + + retry branch g u rs = do + debug ["trying manual pull to resolve failed pushes"] + void $ manualPull (Just branch) rs + go False (Just branch) g u rs + + fallback branch g u rs = do + debug ["fallback pushing to", show rs] + (succeeded, failed) <- liftIO $ + inParallel (\r -> pushFallback u branch r g) rs + updatemap succeeded failed + when (notifypushes && (not $ null succeeded)) $ + sendNetMessage $ NotifyPush $ + map Remote.uuid succeeded + return $ null failed + + push g branch remote = Command.Sync.pushBranch remote branch g + +{- This fallback push mode pushes to branches on the remote that have our + - uuid in them. While ugly, those branches are reserved for pushing by us, + - and so our pushes will never conflict with other pushes. -} +pushFallback :: UUID -> Git.Ref -> Remote -> Git.Repo -> IO Bool +pushFallback u branch remote = Git.Command.runBool + [ Param "push" + , Param $ Remote.name remote + , Param $ refspec Annex.Branch.name + , Param $ refspec branch + ] + where + {- Push to refs/synced/uuid/branch; this + - avoids cluttering up the branch display. -} + refspec b = concat + [ s + , ":" + , "refs/synced/" ++ fromUUID u ++ "/" ++ s + ] + where s = show $ Git.Ref.base b + +{- Manually pull from remotes and merge their branches. -} +manualPull :: Maybe Git.Ref -> [Remote] -> Assistant ([Bool], Bool) +manualPull currentbranch remotes = do + g <- liftAnnex gitRepo + results <- liftIO $ forM remotes $ \r -> + Git.Command.runBool [Param "fetch", Param $ Remote.name r] g + haddiverged <- liftAnnex Annex.Branch.forceUpdate + forM_ remotes $ \r -> + liftAnnex $ Command.Sync.mergeRemote r currentbranch + return (results, haddiverged) + +{- Start syncing a newly added remote, using a background thread. -} +syncNewRemote :: Remote -> Assistant () +syncNewRemote remote = do + updateSyncRemotes + thread <- asIO $ do + reconnectRemotes False [remote] + addScanRemotes True [remote] + void $ liftIO $ forkIO $ thread diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs new file mode 100644 index 0000000000..5887f5e43c --- /dev/null +++ b/Assistant/Threads/Committer.hs @@ -0,0 +1,301 @@ +{- git-annex assistant commit thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP, BangPatterns #-} + +module Assistant.Threads.Committer where + +import Assistant.Common +import Assistant.Changes +import Assistant.Types.Changes +import Assistant.Commits +import Assistant.Alert +import Assistant.DaemonStatus +import Assistant.TransferQueue +import Logs.Transfer +import Logs.Location +import qualified Annex.Queue +import qualified Git.Command +import qualified Git.LsFiles +import qualified Git.Version +import qualified Command.Add +import Utility.ThreadScheduler +import qualified Utility.Lsof as Lsof +import qualified Utility.DirWatcher as DirWatcher +import Types.KeySource +import Config +import Annex.Exception +import Annex.Content +import Annex.Link +import qualified Annex + +import Data.Time.Clock +import Data.Tuple.Utils +import qualified Data.Set as S +import Data.Either + +{- This thread makes git commits at appropriate times. -} +commitThread :: NamedThread +commitThread = namedThread "Committer" $ do + delayadd <- liftAnnex $ + maybe delayaddDefault (return . Just . Seconds) + =<< annexDelayAdd <$> Annex.getGitConfig + runEvery (Seconds 1) <~> do + -- We already waited one second as a simple rate limiter. + -- Next, wait until at least one change is available for + -- processing. + changes <- getChanges + -- Now see if now's a good time to commit. + time <- liftIO getCurrentTime + if shouldCommit time changes + then do + readychanges <- handleAdds delayadd changes + if shouldCommit time readychanges + then do + debug + [ "committing" + , show (length readychanges) + , "changes" + ] + void $ alertWhile commitAlert $ + liftAnnex commitStaged + recordCommit + else refill readychanges + else refill changes + where + refill [] = noop + refill cs = do + debug ["delaying commit of", show (length cs), "changes"] + refillChanges cs + +commitStaged :: Annex Bool +commitStaged = do + {- This could fail if there's another commit being made by + - something else. -} + v <- tryAnnex Annex.Queue.flush + case v of + Left _ -> return False + Right _ -> do + direct <- isDirect + let params = nomessage $ catMaybes + [ Just $ Param "--quiet" + {- In indirect mode, avoid running the + - usual git-annex pre-commit hook; + - watch does the same symlink fixing, + - and we don't want to deal with unlocked + - files in these commits. -} + , if direct then Nothing else Just $ Param "--no-verify" + ] + {- Empty commits may be made if tree changes cancel + - each other out, etc. Git returns nonzero on those, + - so don't propigate out commit failures. -} + void $ inRepo $ catchMaybeIO . + Git.Command.runQuiet (Param "commit" : params) + return True + where + nomessage ps + | Git.Version.older "1.7.2" = Param "-m" + : Param "autocommit" : ps + | otherwise = Param "--allow-empty-message" + : Param "-m" : Param "" : ps + +{- Decide if now is a good time to make a commit. + - Note that the list of change times has an undefined order. + - + - Current strategy: If there have been 10 changes within the past second, + - a batch activity is taking place, so wait for later. + -} +shouldCommit :: UTCTime -> [Change] -> Bool +shouldCommit now changes + | len == 0 = False + | len > 10000 = True -- avoid bloating queue too much + | length (filter thisSecond changes) < 10 = True + | otherwise = False -- batch activity + where + len = length changes + thisSecond c = now `diffUTCTime` changeTime c <= 1 + +{- OSX needs a short delay after a file is added before locking it down, + - when using a non-direct mode repository, as pasting a file seems to + - try to set file permissions or otherwise access the file after closing + - it. -} +delayaddDefault :: Annex (Maybe Seconds) +#ifdef darwin_HOST_OS +delayaddDefault = ifM isDirect + ( return Nothing + , return $ Just $ Seconds 1 + ) +#else +delayaddDefault = return Nothing +#endif + +{- If there are PendingAddChanges, or InProcessAddChanges, the files + - have not yet actually been added to the annex, and that has to be done + - now, before committing. + - + - Deferring the adds to this point causes batches to be bundled together, + - which allows faster checking with lsof that the files are not still open + - for write by some other process, and faster checking with git-ls-files + - that the files are not already checked into git. + - + - When a file is added, Inotify will notice the new symlink. So this waits + - for additional Changes to arrive, so that the symlink has hopefully been + - staged before returning, and will be committed immediately. + - + - OTOH, for kqueue, eventsCoalesce, so instead the symlink is directly + - created and staged. + - + - Returns a list of all changes that are ready to be committed. + - Any pending adds that are not ready yet are put back into the ChangeChan, + - where they will be retried later. + -} +handleAdds :: Maybe Seconds -> [Change] -> Assistant [Change] +handleAdds delayadd cs = returnWhen (null incomplete) $ do + let (pending, inprocess) = partition isPendingAddChange incomplete + direct <- liftAnnex isDirect + pending' <- if direct + then return pending + else findnew pending + (postponed, toadd) <- partitionEithers <$> safeToAdd delayadd pending' inprocess + + unless (null postponed) $ + refillChanges postponed + + returnWhen (null toadd) $ do + added <- catMaybes <$> forM toadd add + if DirWatcher.eventsCoalesce || null added || direct + then return $ added ++ otherchanges + else do + r <- handleAdds delayadd =<< getChanges + return $ r ++ added ++ otherchanges + where + (incomplete, otherchanges) = partition (\c -> isPendingAddChange c || isInProcessAddChange c) cs + + findnew [] = return [] + findnew pending@(exemplar:_) = do + (!newfiles, cleanup) <- liftAnnex $ + inRepo (Git.LsFiles.notInRepo False $ map changeFile pending) + void $ liftIO cleanup + -- note: timestamp info is lost here + let ts = changeTime exemplar + return $ map (PendingAddChange ts) newfiles + + returnWhen c a + | c = return otherchanges + | otherwise = a + + add :: Change -> Assistant (Maybe Change) + add change@(InProcessAddChange { keySource = ks }) = do + alertWhile' (addFileAlert $ keyFilename ks) $ + liftM ret $ catchMaybeIO <~> do + sanitycheck ks $ do + key <- liftAnnex $ do + showStart "add" $ keyFilename ks + Command.Add.ingest $ Just ks + done (finishedChange change) (keyFilename ks) key + where + {- Add errors tend to be transient and will be automatically + - dealt with, so don't pass to the alert code. -} + ret (Just j@(Just _)) = (True, j) + ret _ = (True, Nothing) + add _ = return Nothing + + done _ _ Nothing = do + liftAnnex showEndFail + return Nothing + done change file (Just key) = do + liftAnnex $ do + logStatus key InfoPresent + link <- ifM isDirect + ( calcGitLink file key + , Command.Add.link file key True + ) + whenM (pure DirWatcher.eventsCoalesce <||> isDirect) $ do + stageSymlink file =<< hashSymlink link + showEndOk + queueTransfers "newly added file" Next key (Just file) Upload + return $ Just change + + {- Check that the keysource's keyFilename still exists, + - and is still a hard link to its contentLocation, + - before ingesting it. -} + sanitycheck keysource a = do + fs <- liftIO $ getSymbolicLinkStatus $ keyFilename keysource + ks <- liftIO $ getSymbolicLinkStatus $ contentLocation keysource + if deviceID ks == deviceID fs && fileID ks == fileID fs + then a + else do + -- remove the hard link + when (contentLocation keysource /= keyFilename keysource) $ + void $ liftIO $ tryIO $ removeFile $ contentLocation keysource + return Nothing + +{- Files can Either be Right to be added now, + - or are unsafe, and must be Left for later. + - + - Check by running lsof on the repository. + -} +safeToAdd :: Maybe Seconds -> [Change] -> [Change] -> Assistant [Either Change Change] +safeToAdd _ [] [] = return [] +safeToAdd delayadd pending inprocess = do + maybe noop (liftIO . threadDelaySeconds) delayadd + liftAnnex $ do + keysources <- mapM Command.Add.lockDown (map changeFile pending) + let inprocess' = inprocess ++ catMaybes (map mkinprocess $ zip pending keysources) + openfiles <- S.fromList . map fst3 . filter openwrite <$> + findopenfiles (map keySource inprocess') + let checked = map (check openfiles) inprocess' + + {- If new events are received when files are closed, + - there's no need to retry any changes that cannot + - be done now. -} + if DirWatcher.closingTracked + then do + mapM_ canceladd $ lefts checked + allRight $ rights checked + else return checked + where + check openfiles change@(InProcessAddChange { keySource = ks }) + | S.member (contentLocation ks) openfiles = Left change + check _ change = Right change + + mkinprocess (c, Just ks) = Just $ InProcessAddChange + { changeTime = changeTime c + , keySource = ks + } + mkinprocess (_, Nothing) = Nothing + + canceladd (InProcessAddChange { keySource = ks }) = do + warning $ keyFilename ks + ++ " still has writers, not adding" + -- remove the hard link + when (contentLocation ks /= keyFilename ks) $ + void $ liftIO $ tryIO $ removeFile $ contentLocation ks + canceladd _ = noop + + openwrite (_file, mode, _pid) + | mode == Lsof.OpenWriteOnly = True + | mode == Lsof.OpenReadWrite = True + | mode == Lsof.OpenUnknown = True + | otherwise = False + + allRight = return . map Right + + {- Normally the KeySources are locked down inside the temp directory, + - so can just lsof that, which is quite efficient. + - + - In crippled filesystem mode, there is no lock down, so must run lsof + - on each individual file. + -} + findopenfiles keysources = ifM crippledFileSystem + ( liftIO $ do + let segments = segmentXargs $ map keyFilename keysources + concat <$> forM segments (\fs -> Lsof.query $ "--" : fs) + , do + tmpdir <- fromRepo gitAnnexTmpDir + liftIO $ Lsof.queryDir tmpdir + ) diff --git a/Assistant/Threads/ConfigMonitor.hs b/Assistant/Threads/ConfigMonitor.hs new file mode 100644 index 0000000000..6a01ff35ee --- /dev/null +++ b/Assistant/Threads/ConfigMonitor.hs @@ -0,0 +1,86 @@ +{- git-annex assistant config monitor thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.ConfigMonitor where + +import Assistant.Common +import Assistant.BranchChange +import Assistant.DaemonStatus +import Assistant.Commits +import Utility.ThreadScheduler +import Logs.UUID +import Logs.Trust +import Logs.Remote +import Logs.PreferredContent +import Logs.Group +import Remote.List (remoteListRefresh) +import qualified Git.LsTree as LsTree +import qualified Annex.Branch + +import qualified Data.Set as S + +{- This thread detects when configuration changes have been made to the + - git-annex branch and reloads cached configuration. + - + - If the branch is frequently changing, it's checked for configuration + - changes no more often than once every 60 seconds. On the other hand, + - if the branch has not changed in a while, configuration changes will + - be detected immediately. + -} +configMonitorThread :: NamedThread +configMonitorThread = namedThread "ConfigMonitor" $ loop =<< getConfigs + where + loop old = do + waitBranchChange + new <- getConfigs + when (old /= new) $ do + let changedconfigs = new `S.difference` old + debug $ "reloading config" : + map fst (S.toList changedconfigs) + reloadConfigs new + {- Record a commit to get this config + - change pushed out to remotes. -} + recordCommit + liftIO $ threadDelaySeconds (Seconds 60) + loop new + +{- Config files, and their checksums. -} +type Configs = S.Set (FilePath, String) + +{- All git-annex's config files, and actions to run when they change. -} +configFilesActions :: [(FilePath, Annex ())] +configFilesActions = + [ (uuidLog, void $ uuidMapLoad) + , (remoteLog, void remoteListRefresh) + , (trustLog, void trustMapLoad) + , (groupLog, void groupMapLoad) + -- Preferred content settings depend on most of the other configs, + -- so will be reloaded whenever any configs change. + , (preferredContentLog, noop) + ] + +reloadConfigs :: Configs -> Assistant () +reloadConfigs changedconfigs = do + liftAnnex $ do + sequence_ as + void preferredContentMapLoad + {- Changes to the remote log, or the trust log, can affect the + - syncRemotes list. Changes to the uuid log may affect its + - display so are also included. -} + when (any (`elem` fs) [remoteLog, trustLog, uuidLog]) $ + updateSyncRemotes + where + (fs, as) = unzip $ filter (flip S.member changedfiles . fst) + configFilesActions + changedfiles = S.map fst changedconfigs + +getConfigs :: Assistant Configs +getConfigs = S.fromList . map extract + <$> liftAnnex (inRepo $ LsTree.lsTreeFiles Annex.Branch.fullname files) + where + files = map fst configFilesActions + extract treeitem = (LsTree.file treeitem, LsTree.sha treeitem) diff --git a/Assistant/Threads/DaemonStatus.hs b/Assistant/Threads/DaemonStatus.hs new file mode 100644 index 0000000000..fffc6ed37f --- /dev/null +++ b/Assistant/Threads/DaemonStatus.hs @@ -0,0 +1,29 @@ +{- git-annex assistant daemon status thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.DaemonStatus where + +import Assistant.Common +import Assistant.DaemonStatus +import Utility.ThreadScheduler +import Utility.NotificationBroadcaster + +{- This writes the daemon status to disk, when it changes, but no more + - frequently than once every ten minutes. + -} +daemonStatusThread :: NamedThread +daemonStatusThread = namedThread "DaemonStatus" $ do + notifier <- liftIO . newNotificationHandle + =<< changeNotifier <$> getDaemonStatus + checkpoint + runEvery (Seconds tenMinutes) <~> do + liftIO $ waitNotification notifier + checkpoint + where + checkpoint = do + file <- liftAnnex $ fromRepo gitAnnexDaemonStatusFile + liftIO . writeDaemonStatusFile file =<< getDaemonStatus diff --git a/Assistant/Threads/Glacier.hs b/Assistant/Threads/Glacier.hs new file mode 100644 index 0000000000..46f64cd561 --- /dev/null +++ b/Assistant/Threads/Glacier.hs @@ -0,0 +1,43 @@ +{- git-annex assistant Amazon Glacier retrieval + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Threads.Glacier where + +import Assistant.Common +import Utility.ThreadScheduler +import qualified Types.Remote as Remote +import qualified Remote.Glacier as Glacier +import Logs.Transfer +import Assistant.DaemonStatus +import Assistant.TransferQueue + +import qualified Data.Set as S + +{- Wakes up every half hour and checks if any glacier remotes have failed + - downloads. If so, runs glacier-cli to check if the files are now + - available, and queues the downloads. -} +glacierThread :: NamedThread +glacierThread = namedThread "Glacier" $ runEvery (Seconds 3600) <~> go + where + isglacier r = Remote.remotetype r == Glacier.remote + go = do + rs <- filter isglacier . syncDataRemotes <$> getDaemonStatus + forM_ rs $ \r -> + check r =<< (liftAnnex $ getFailedTransfers $ Remote.uuid r) + check _ [] = noop + check r l = do + let keys = map getkey l + (availkeys, failedkeys) <- liftAnnex $ Glacier.jobList r keys + let s = S.fromList (failedkeys ++ availkeys) + let l' = filter (\p -> S.member (getkey p) s) l + forM_ l' $ \(t, info) -> do + liftAnnex $ removeFailedTransfer t + queueTransferWhenSmall "object available from glacier" (associatedFile info) t r + getkey = transferKey . fst diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs new file mode 100644 index 0000000000..1488a2f0dc --- /dev/null +++ b/Assistant/Threads/Merger.hs @@ -0,0 +1,96 @@ +{- git-annex assistant git merge thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.Merger where + +import Assistant.Common +import Assistant.TransferQueue +import Assistant.BranchChange +import Utility.DirWatcher +import Utility.Types.DirWatcher +import qualified Annex.Branch +import qualified Git +import qualified Git.Branch +import qualified Command.Sync + +{- This thread watches for changes to .git/refs/, and handles incoming + - pushes. -} +mergeThread :: NamedThread +mergeThread = namedThread "Merger" $ do + g <- liftAnnex gitRepo + let dir = Git.localGitDir g "refs" + liftIO $ createDirectoryIfMissing True dir + let hook a = Just <$> asIO2 (runHandler a) + addhook <- hook onAdd + errhook <- hook onErr + let hooks = mkWatchHooks + { addHook = addhook + , errHook = errhook + } + void $ liftIO $ watchDir dir (const False) hooks id + debug ["watching", dir] + +type Handler = FilePath -> Assistant () + +{- Runs an action handler. + - + - Exceptions are ignored, otherwise a whole thread could be crashed. + -} +runHandler :: Handler -> FilePath -> Maybe FileStatus -> Assistant () +runHandler handler file _filestatus = + either (liftIO . print) (const noop) =<< tryIO <~> handler file + +{- Called when there's an error with inotify. -} +onErr :: Handler +onErr msg = error msg + +{- Called when a new branch ref is written. + - + - This relies on git's atomic method of updating branch ref files, + - which is to first write the new file to .lock, and then rename it + - over the old file. So, ignore .lock files, and the rename ensures + - the watcher sees a new file being added on each update. + - + - At startup, synthetic add events fire, causing this to run, but that's + - ok; it ensures that any changes pushed since the last time the assistant + - ran are merged in. + -} +onAdd :: Handler +onAdd file + | ".lock" `isSuffixOf` file = noop + | isAnnexBranch file = do + branchChanged + whenM (liftAnnex Annex.Branch.forceUpdate) $ + queueDeferredDownloads "retrying deferred download" Later + | "/synced/" `isInfixOf` file = do + mergecurrent =<< liftAnnex (inRepo Git.Branch.current) + | otherwise = noop + where + changedbranch = fileToBranch file + mergecurrent (Just current) + | equivBranches changedbranch current = do + debug + [ "merging", show changedbranch + , "into", show current + ] + void $ liftAnnex $ Command.Sync.mergeFrom changedbranch + mergecurrent _ = noop + +equivBranches :: Git.Ref -> Git.Ref -> Bool +equivBranches x y = base x == base y + where + base = takeFileName . show + +isAnnexBranch :: FilePath -> Bool +isAnnexBranch f = n `isSuffixOf` f + where + n = "/" ++ show Annex.Branch.name + +fileToBranch :: FilePath -> Git.Ref +fileToBranch f = Git.Ref $ "refs" base + where + base = Prelude.last $ split "/refs/" f diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs new file mode 100644 index 0000000000..143ae9cee4 --- /dev/null +++ b/Assistant/Threads/MountWatcher.hs @@ -0,0 +1,192 @@ +{- git-annex assistant mount watcher, using either dbus or mtab polling + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Threads.MountWatcher where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.Sync +import qualified Annex +import qualified Git +import Utility.ThreadScheduler +import Utility.Mounts +import Remote.List +import qualified Types.Remote as Remote + +import qualified Data.Set as S + +#if WITH_DBUS +import Utility.DBus +import DBus.Client +import DBus +import Data.Word (Word32) +import Control.Concurrent +import qualified Control.Exception as E +#else +#warning Building without dbus support; will use mtab polling +#endif + +mountWatcherThread :: NamedThread +mountWatcherThread = namedThread "MountWatcher" $ +#if WITH_DBUS + dbusThread +#else + pollingThread +#endif + +#if WITH_DBUS + +dbusThread :: Assistant () +dbusThread = do + runclient <- asIO1 go + r <- liftIO $ E.try $ runClient getSessionAddress runclient + either onerr (const noop) r + where + go client = ifM (checkMountMonitor client) + ( do + {- Store the current mount points in an MVar, to be + - compared later. We could in theory work out the + - mount point from the dbus message, but this is + - easier. -} + mvar <- liftIO $ newMVar =<< currentMountPoints + handleevent <- asIO1 $ \_event -> do + nowmounted <- liftIO $ currentMountPoints + wasmounted <- liftIO $ swapMVar mvar nowmounted + handleMounts wasmounted nowmounted + liftIO $ forM_ mountChanged $ \matcher -> + listen client matcher handleevent + , do + liftAnnex $ + warning "No known volume monitor available through dbus; falling back to mtab polling" + pollingThread + ) + onerr :: E.SomeException -> Assistant () + onerr e = do + {- If the session dbus fails, the user probably + - logged out of their desktop. Even if they log + - back in, we won't have access to the dbus + - session key, so polling is the best that can be + - done in this situation. -} + liftAnnex $ + warning $ "dbus failed; falling back to mtab polling (" ++ show e ++ ")" + pollingThread + +{- Examine the list of services connected to dbus, to see if there + - are any we can use to monitor mounts. If not, will attempt to start one. -} +checkMountMonitor :: Client -> Assistant Bool +checkMountMonitor client = do + running <- filter (`elem` usableservices) + <$> liftIO (listServiceNames client) + case running of + [] -> startOneService client startableservices + (service:_) -> do + debug [ "Using running DBUS service" + , service + , "to monitor mount events." + ] + return True + where + startableservices = [gvfs, gvfsgdu] + usableservices = startableservices ++ [kde] + gvfs = "org.gtk.Private.UDisks2VolumeMonitor" + gvfsgdu = "org.gtk.Private.GduVolumeMonitor" + kde = "org.kde.DeviceNotifications" + +startOneService :: Client -> [ServiceName] -> Assistant Bool +startOneService _ [] = return False +startOneService client (x:xs) = do + _ <- liftIO $ tryNonAsync $ callDBus client "StartServiceByName" + [toVariant x, toVariant (0 :: Word32)] + ifM (liftIO $ elem x <$> listServiceNames client) + ( do + debug + [ "Started DBUS service", x + , "to monitor mount events." + ] + return True + , startOneService client xs + ) + +{- Filter matching events recieved when drives are mounted and unmounted. -} +mountChanged :: [MatchRule] +mountChanged = [gvfs True, gvfs False, kde, kdefallback] + where + {- gvfs reliably generates this event whenever a + - drive is mounted/unmounted, whether automatically, or manually -} + gvfs mount = matchAny + { matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor" + , matchMember = Just $ if mount then "MountAdded" else "MountRemoved" + } + {- This event fires when KDE prompts the user what to do with a drive, + - but maybe not at other times. And it's not received -} + kde = matchAny + { matchInterface = Just "org.kde.Solid.Device" + , matchMember = Just "setupDone" + } + {- This event may not be closely related to mounting a drive, but it's + - observed reliably when a drive gets mounted or unmounted. -} + kdefallback = matchAny + { matchInterface = Just "org.kde.KDirNotify" + , matchMember = Just "enteredDirectory" + } + +#endif + +pollingThread :: Assistant () +pollingThread = go =<< liftIO currentMountPoints + where + go wasmounted = do + liftIO $ threadDelaySeconds (Seconds 10) + nowmounted <- liftIO currentMountPoints + handleMounts wasmounted nowmounted + go nowmounted + +handleMounts :: MountPoints -> MountPoints -> Assistant () +handleMounts wasmounted nowmounted = + mapM_ (handleMount . mnt_dir) $ + S.toList $ newMountPoints wasmounted nowmounted + +handleMount :: FilePath -> Assistant () +handleMount dir = do + debug ["detected mount of", dir] + rs <- filter (Git.repoIsLocal . Remote.repo) <$> remotesUnder dir + reconnectRemotes True rs + +{- Finds remotes located underneath the mount point. + - + - Updates state to include the remotes. + - + - The config of git remotes is re-read, as it may not have been available + - at startup time, or may have changed (it could even be a different + - repository at the same remote location..) + -} +remotesUnder :: FilePath -> Assistant [Remote] +remotesUnder dir = do + repotop <- liftAnnex $ fromRepo Git.repoPath + rs <- liftAnnex remoteList + pairs <- liftAnnex $ mapM (checkremote repotop) rs + let (waschanged, rs') = unzip pairs + when (any id waschanged) $ do + liftAnnex $ Annex.changeState $ \s -> s { Annex.remotes = rs' } + updateSyncRemotes + return $ map snd $ filter fst pairs + where + checkremote repotop r = case Remote.localpath r of + Just p | dirContains dir (absPathFrom repotop p) -> + (,) <$> pure True <*> updateRemote r + _ -> return (False, r) + +type MountPoints = S.Set Mntent + +currentMountPoints :: IO MountPoints +currentMountPoints = S.fromList <$> getMounts + +newMountPoints :: MountPoints -> MountPoints -> MountPoints +newMountPoints old new = S.difference new old diff --git a/Assistant/Threads/NetWatcher.hs b/Assistant/Threads/NetWatcher.hs new file mode 100644 index 0000000000..6ac7203b0f --- /dev/null +++ b/Assistant/Threads/NetWatcher.hs @@ -0,0 +1,131 @@ +{- git-annex assistant network connection watcher, using dbus + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Threads.NetWatcher where + +import Assistant.Common +import Assistant.Sync +import Utility.ThreadScheduler +import Remote.List +import qualified Types.Remote as Remote + +#if WITH_DBUS +import Utility.DBus +import DBus.Client +import DBus +import Data.Word (Word32) +import Assistant.NetMessager +#else +#warning Building without dbus support; will poll for network connection changes +#endif + +netWatcherThread :: NamedThread +#if WITH_DBUS +netWatcherThread = thread dbusThread +#else +netWatcherThread = thread noop +#endif + where + thread = namedThread "NetWatcher" + +{- This is a fallback for when dbus cannot be used to detect + - network connection changes, but it also ensures that + - any networked remotes that may have not been routable for a + - while (despite the local network staying up), are synced with + - periodically. -} +netWatcherFallbackThread :: NamedThread +netWatcherFallbackThread = namedThread "NetWatcherFallback" $ + runEvery (Seconds 3600) <~> handleConnection + +#if WITH_DBUS + +dbusThread :: Assistant () +dbusThread = do + handleerr <- asIO2 onerr + runclient <- asIO1 go + liftIO $ persistentClient getSystemAddress () handleerr runclient + where + go client = ifM (checkNetMonitor client) + ( do + listenNMConnections client <~> handleconn + listenWicdConnections client <~> handleconn + , do + liftAnnex $ + warning "No known network monitor available through dbus; falling back to polling" + ) + handleconn = do + debug ["detected network connection"] + notifyNetMessagerRestart + handleConnection + onerr e _ = do + liftAnnex $ + warning $ "lost dbus connection; falling back to polling (" ++ show e ++ ")" + {- Wait, in hope that dbus will come back -} + liftIO $ threadDelaySeconds (Seconds 60) + +{- Examine the list of services connected to dbus, to see if there + - are any we can use to monitor network connections. -} +checkNetMonitor :: Client -> Assistant Bool +checkNetMonitor client = do + running <- liftIO $ filter (`elem` [networkmanager, wicd]) + <$> listServiceNames client + case running of + [] -> return False + (service:_) -> do + debug [ "Using running DBUS service" + , service + , "to monitor network connection events." + ] + return True + where + networkmanager = "org.freedesktop.NetworkManager" + wicd = "org.wicd.daemon" + +{- Listens for new NetworkManager connections. -} +listenNMConnections :: Client -> IO () -> IO () +listenNMConnections client callback = + listen client matcher $ \event -> + when (Just True == anyM activeconnection (signalBody event)) $ + callback + where + matcher = matchAny + { matchInterface = Just "org.freedesktop.NetworkManager.Connection.Active" + , matchMember = Just "PropertiesChanged" + } + nm_connection_activated = toVariant (2 :: Word32) + nm_state_key = toVariant ("State" :: String) + activeconnection v = do + m <- fromVariant v + vstate <- lookup nm_state_key $ dictionaryItems m + state <- fromVariant vstate + return $ state == nm_connection_activated + +{- Listens for new Wicd connections. -} +listenWicdConnections :: Client -> IO () -> IO () +listenWicdConnections client callback = + listen client matcher $ \event -> + when (any (== wicd_success) (signalBody event)) $ + callback + where + matcher = matchAny + { matchInterface = Just "org.wicd.daemon" + , matchMember = Just "ConnectResultsSent" + } + wicd_success = toVariant ("success" :: String) + +#endif + +handleConnection :: Assistant () +handleConnection = reconnectRemotes True =<< networkRemotes + +{- Finds network remotes. -} +networkRemotes :: Assistant [Remote] +networkRemotes = liftAnnex $ + filter (isNothing . Remote.localpath) <$> remoteList diff --git a/Assistant/Threads/PairListener.hs b/Assistant/Threads/PairListener.hs new file mode 100644 index 0000000000..4459ee13cf --- /dev/null +++ b/Assistant/Threads/PairListener.hs @@ -0,0 +1,153 @@ +{- git-annex assistant thread to listen for incoming pairing traffic + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.PairListener where + +import Assistant.Common +import Assistant.Pairing +import Assistant.Pairing.Network +import Assistant.Pairing.MakeRemote +import Assistant.WebApp (UrlRenderer, renderUrl) +import Assistant.WebApp.Types +import Assistant.Alert +import Assistant.DaemonStatus +import Utility.ThreadScheduler +import Git + +import Network.Multicast +import Network.Socket +import qualified Data.Text as T +import Data.Char + +pairListenerThread :: UrlRenderer -> NamedThread +pairListenerThread urlrenderer = namedThread "PairListener" $ do + listener <- asIO1 $ go [] [] + liftIO $ withSocketsDo $ + runEvery (Seconds 1) $ void $ tryIO $ + listener =<< getsock + where + {- Note this can crash if there's no network interface, + - or only one like lo that doesn't support multicast. -} + getsock = multicastReceiver (multicastAddress $ IPv4Addr undefined) pairingPort + + go reqs cache sock = liftIO (getmsg sock []) >>= \msg -> case readish msg of + Nothing -> go reqs cache sock + Just m -> do + sane <- checkSane msg + (pip, verified) <- verificationCheck m + =<< (pairingInProgress <$> getDaemonStatus) + let wrongstage = maybe False (\p -> pairMsgStage m <= inProgressPairStage p) pip + case (wrongstage, sane, pairMsgStage m) of + -- ignore our own messages, and + -- out of order messages + (True, _, _) -> go reqs cache sock + (_, False, _) -> go reqs cache sock + (_, _, PairReq) -> if m `elem` reqs + then go reqs (invalidateCache m cache) sock + else do + pairReqReceived verified urlrenderer m + go (m:take 10 reqs) (invalidateCache m cache) sock + (_, _, PairAck) -> do + cache' <- pairAckReceived verified pip m cache + go reqs cache' sock + (_, _, PairDone) -> do + pairDoneReceived verified pip m + go reqs cache sock + + {- As well as verifying the message using the shared secret, + - check its UUID against the UUID we have stored. If + - they're the same, someone is sending bogus messages, + - which could be an attempt to brute force the shared secret. -} + verificationCheck _ Nothing = return (Nothing, False) + verificationCheck m (Just pip) + | not verified && sameuuid = do + liftAnnex $ warning + "detected possible pairing brute force attempt; disabled pairing" + stopSending pip + return (Nothing, False) + |otherwise = return (Just pip, verified && sameuuid) + where + verified = verifiedPairMsg m pip + sameuuid = pairUUID (inProgressPairData pip) == pairUUID (pairMsgData m) + + {- Various sanity checks on the content of the message. -} + checkSane msg + {- Control characters could be used in a + - console poisoning attack. -} + | any isControl msg || any (`elem` "\r\n") msg = do + liftAnnex $ warning + "illegal control characters in pairing message; ignoring" + return False + | otherwise = return True + + {- PairReqs invalidate the cache of recently finished pairings. + - This is so that, if a new pairing is started with the + - same secret used before, a bogus PairDone is not sent. -} + invalidateCache msg = filter (not . verifiedPairMsg msg) + + getmsg sock c = do + (msg, n, _) <- recvFrom sock chunksz + if n < chunksz + then return $ c ++ msg + else getmsg sock $ c ++ msg + where + chunksz = 1024 + +{- Show an alert when a PairReq is seen. -} +pairReqReceived :: Bool -> UrlRenderer -> PairMsg -> Assistant () +pairReqReceived True _ _ = noop -- ignore our own PairReq +pairReqReceived False urlrenderer msg = do + url <- liftIO $ renderUrl urlrenderer (FinishLocalPairR msg) [] + closealert <- asIO1 removeAlert + void $ addAlert $ pairRequestReceivedAlert repo + AlertButton + { buttonUrl = url + , buttonLabel = T.pack "Respond" + , buttonAction = Just closealert + } + where + repo = pairRepo msg + +{- When a verified PairAck is seen, a host is ready to pair with us, and has + - already configured our ssh key. Stop sending PairReqs, finish the pairing, + - and send a single PairDone. -} +pairAckReceived :: Bool -> Maybe PairingInProgress -> PairMsg -> [PairingInProgress] -> Assistant [PairingInProgress] +pairAckReceived True (Just pip) msg cache = do + stopSending pip + repodir <- repoPath <$> liftAnnex gitRepo + liftIO $ setupAuthorizedKeys msg repodir + finishedLocalPairing msg (inProgressSshKeyPair pip) + startSending pip PairDone $ multicastPairMsg + (Just 1) (inProgressSecret pip) (inProgressPairData pip) + return $ pip : take 10 cache +{- A stale PairAck might also be seen, after we've finished pairing. + - Perhaps our PairDone was not received. To handle this, we keep + - a cache of recently finished pairings, and re-send PairDone in + - response to stale PairAcks for them. -} +pairAckReceived _ _ msg cache = do + let pips = filter (verifiedPairMsg msg) cache + unless (null pips) $ + forM_ pips $ \pip -> + startSending pip PairDone $ multicastPairMsg + (Just 1) (inProgressSecret pip) (inProgressPairData pip) + return cache + +{- If we get a verified PairDone, the host has accepted our PairAck, and + - has paired with us. Stop sending PairAcks, and finish pairing with them. + - + - TODO: Should third-party hosts remove their pair request alert when they + - see a PairDone? + - Complication: The user could have already clicked on the alert and be + - entering the secret. Would be better to start a fresh pair request in this + - situation. + -} +pairDoneReceived :: Bool -> Maybe PairingInProgress -> PairMsg -> Assistant () +pairDoneReceived False _ _ = noop -- not verified +pairDoneReceived True Nothing _ = noop -- not in progress +pairDoneReceived True (Just pip) msg = do + stopSending pip + finishedLocalPairing msg (inProgressSshKeyPair pip) diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs new file mode 100644 index 0000000000..a444a8530b --- /dev/null +++ b/Assistant/Threads/Pusher.hs @@ -0,0 +1,64 @@ +{- git-annex assistant git pushing thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.Pusher where + +import Assistant.Common +import Assistant.Commits +import Assistant.Types.Commits +import Assistant.Pushes +import Assistant.Alert +import Assistant.DaemonStatus +import Assistant.Sync +import Utility.ThreadScheduler +import qualified Types.Remote as Remote + +import Data.Time.Clock + +{- This thread retries pushes that failed before. -} +pushRetryThread :: NamedThread +pushRetryThread = namedThread "PushRetrier" $ runEvery (Seconds halfhour) <~> do + -- We already waited half an hour, now wait until there are failed + -- pushes to retry. + topush <- getFailedPushesBefore (fromIntegral halfhour) + unless (null topush) $ do + debug ["retrying", show (length topush), "failed pushes"] + void $ alertWhile (pushRetryAlert topush) $ do + now <- liftIO $ getCurrentTime + pushToRemotes now True topush + where + halfhour = 1800 + +{- This thread pushes git commits out to remotes soon after they are made. -} +pushThread :: NamedThread +pushThread = namedThread "Pusher" $ runEvery (Seconds 2) <~> do + -- We already waited two seconds as a simple rate limiter. + -- Next, wait until at least one commit has been made + commits <- getCommits + -- Now see if now's a good time to push. + if shouldPush commits + then do + remotes <- filter (not . Remote.readonly) + . syncGitRemotes <$> getDaemonStatus + unless (null remotes) $ + void $ alertWhile (pushAlert remotes) $ do + now <- liftIO $ getCurrentTime + pushToRemotes now True remotes + else do + debug ["delaying push of", show (length commits), "commits"] + refillCommits commits + +{- Decide if now is a good time to push to remotes. + - + - Current strategy: Immediately push all commits. The commit machinery + - already determines batches of changes, so we can't easily determine + - batches better. + -} +shouldPush :: [Commit] -> Bool +shouldPush commits + | not (null commits) = True + | otherwise = False diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs new file mode 100644 index 0000000000..a6166214a9 --- /dev/null +++ b/Assistant/Threads/SanityChecker.hs @@ -0,0 +1,137 @@ +{- git-annex assistant sanity checker + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.SanityChecker ( + sanityCheckerDailyThread, + sanityCheckerHourlyThread +) where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.Alert +import qualified Git.LsFiles +import qualified Git.Command +import qualified Git.Config +import Utility.ThreadScheduler +import qualified Assistant.Threads.Watcher as Watcher +import Utility.LogFile +import Config + +import Data.Time.Clock.POSIX + +{- This thread wakes up hourly for inxepensive frequent sanity checks. -} +sanityCheckerHourlyThread :: NamedThread +sanityCheckerHourlyThread = namedThread "SanityCheckerHourly" $ forever $ do + liftIO $ threadDelaySeconds $ Seconds oneHour + hourlyCheck + +{- This thread wakes up daily to make sure the tree is in good shape. -} +sanityCheckerDailyThread :: NamedThread +sanityCheckerDailyThread = namedThread "SanityCheckerDaily" $ forever $ do + waitForNextCheck + + debug ["starting sanity check"] + void $ alertWhile sanityCheckAlert go + debug ["sanity check complete"] + where + go = do + modifyDaemonStatus_ $ \s -> s { sanityCheckRunning = True } + + now <- liftIO $ getPOSIXTime -- before check started + r <- either showerr return =<< tryIO <~> dailyCheck + + modifyDaemonStatus_ $ \s -> s + { sanityCheckRunning = False + , lastSanityCheck = Just now + } + + return r + + showerr e = do + liftAnnex $ warning $ show e + return False + +{- Only run one check per day, from the time of the last check. -} +waitForNextCheck :: Assistant () +waitForNextCheck = do + v <- lastSanityCheck <$> getDaemonStatus + now <- liftIO getPOSIXTime + liftIO $ threadDelaySeconds $ Seconds $ calcdelay now v + where + calcdelay _ Nothing = oneDay + calcdelay now (Just lastcheck) + | lastcheck < now = max oneDay $ + oneDay - truncate (now - lastcheck) + | otherwise = oneDay + +{- It's important to stay out of the Annex monad as much as possible while + - running potentially expensive parts of this check, since remaining in it + - will block the watcher. -} +dailyCheck :: Assistant Bool +dailyCheck = do + g <- liftAnnex gitRepo + + -- Find old unstaged symlinks, and add them to git. + (unstaged, cleanup) <- liftIO $ Git.LsFiles.notInRepo False ["."] g + now <- liftIO $ getPOSIXTime + forM_ unstaged $ \file -> do + ms <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file + case ms of + Just s | toonew (statusChangeTime s) now -> noop + | isSymbolicLink s -> addsymlink file ms + _ -> noop + liftIO $ void cleanup + + {- Allow git-gc to run once per day. More frequent gc is avoided + - by default to avoid slowing things down. Only run repacks when 100x + - the usual number of loose objects are present; we tend + - to have a lot of small objects and they should not be a + - significant size. -} + when (Git.Config.getMaybe "gc.auto" g == Just "0") $ + liftIO $ void $ Git.Command.runBool + [ Param "-c", Param "gc.auto=670000" + , Param "gc" + , Param "--auto" + ] g + + return True + where + toonew timestamp now = now < (realToFrac (timestamp + slop) :: POSIXTime) + slop = fromIntegral tenMinutes + insanity msg = do + liftAnnex $ warning msg + void $ addAlert $ sanityCheckFixAlert msg + addsymlink file s = do + isdirect <- liftAnnex isDirect + Watcher.runHandler (Watcher.onAddSymlink isdirect) file s + insanity $ "found unstaged symlink: " ++ file + +hourlyCheck :: Assistant () +hourlyCheck = checkLogSize 0 + +{- Rotate logs until log file size is < 1 mb. -} +checkLogSize :: Int -> Assistant () +checkLogSize n = do + f <- liftAnnex $ fromRepo gitAnnexLogFile + logs <- liftIO $ listLogs f + totalsize <- liftIO $ sum <$> mapM filesize logs + when (totalsize > oneMegabyte) $ do + notice ["Rotated logs due to size:", show totalsize] + liftIO $ openLog f >>= redirLog + when (n < maxLogs + 1) $ + checkLogSize $ n + 1 + where + filesize f = fromIntegral . fileSize <$> liftIO (getFileStatus f) + +oneMegabyte :: Int +oneMegabyte = 1000000 + +oneHour :: Int +oneHour = 60 * 60 + +oneDay :: Int +oneDay = 24 * oneHour diff --git a/Assistant/Threads/TransferPoller.hs b/Assistant/Threads/TransferPoller.hs new file mode 100644 index 0000000000..20b832652d --- /dev/null +++ b/Assistant/Threads/TransferPoller.hs @@ -0,0 +1,56 @@ +{- git-annex assistant transfer polling thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferPoller where + +import Assistant.Common +import Assistant.DaemonStatus +import Logs.Transfer +import Utility.NotificationBroadcaster +import qualified Assistant.Threads.TransferWatcher as TransferWatcher + +import Control.Concurrent +import qualified Data.Map as M + +{- This thread polls the status of ongoing transfers, determining how much + - of each transfer is complete. -} +transferPollerThread :: NamedThread +transferPollerThread = namedThread "TransferPoller" $ do + g <- liftAnnex gitRepo + tn <- liftIO . newNotificationHandle =<< + transferNotifier <$> getDaemonStatus + forever $ do + liftIO $ threadDelay 500000 -- 0.5 seconds + ts <- currentTransfers <$> getDaemonStatus + if M.null ts + -- block until transfers running + then liftIO $ waitNotification tn + else mapM_ (poll g) $ M.toList ts + where + poll g (t, info) + {- Downloads are polled by checking the size of the + - temp file being used for the transfer. -} + | transferDirection t == Download = do + let f = gitAnnexTmpLocation (transferKey t) g + sz <- liftIO $ catchMaybeIO $ + fromIntegral . fileSize <$> getFileStatus f + newsize t info sz + {- Uploads don't need to be polled for when the TransferWatcher + - thread can track file modifications. -} + | TransferWatcher.watchesTransferSize = noop + {- Otherwise, this code polls the upload progress + - by reading the transfer info file. -} + | otherwise = do + let f = transferFile t g + mi <- liftIO $ catchDefaultIO Nothing $ + readTransferInfoFile Nothing f + maybe noop (newsize t info . bytesComplete) mi + + newsize t info sz + | bytesComplete info /= sz && isJust sz = + alterTransferInfo t $ \i -> i { bytesComplete = sz } + | otherwise = noop diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs new file mode 100644 index 0000000000..c6bb8c586b --- /dev/null +++ b/Assistant/Threads/TransferScanner.hs @@ -0,0 +1,142 @@ +{- git-annex assistant thread to scan remotes to find needed transfers + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferScanner where + +import Assistant.Common +import Assistant.Types.ScanRemotes +import Assistant.ScanRemotes +import Assistant.TransferQueue +import Assistant.DaemonStatus +import Assistant.Alert +import Assistant.Drop +import Logs.Transfer +import Logs.Location +import Logs.Web (webUUID) +import qualified Remote +import qualified Types.Remote as Remote +import Utility.ThreadScheduler +import qualified Git.LsFiles as LsFiles +import qualified Backend +import Annex.Content +import Annex.Wanted + +import qualified Data.Set as S + +{- This thread waits until a remote needs to be scanned, to find transfers + - that need to be made, to keep data in sync. + -} +transferScannerThread :: NamedThread +transferScannerThread = namedThread "TransferScanner" $ do + startupScan + go S.empty + where + go scanned = do + liftIO $ threadDelaySeconds (Seconds 2) + (rs, infos) <- unzip <$> getScanRemote + if any fullScan infos || any (`S.notMember` scanned) rs + then do + expensiveScan rs + go $ scanned `S.union` S.fromList rs + else do + mapM_ failedTransferScan rs + go scanned + {- All available remotes are scanned in full on startup, + - for multiple reasons, including: + - + - * This may be the first run, and there may be remotes + - already in place, that need to be synced. + - * We may have run before, and scanned a remote, but + - only been in a subdirectory of the git remote, and so + - not synced it all. + - * We may have run before, and had transfers queued, + - and then the system (or us) crashed, and that info was + - lost. + -} + startupScan = addScanRemotes True =<< syncDataRemotes <$> getDaemonStatus + +{- This is a cheap scan for failed transfers involving a remote. -} +failedTransferScan :: Remote -> Assistant () +failedTransferScan r = do + failed <- liftAnnex $ getFailedTransfers (Remote.uuid r) + liftAnnex $ mapM_ removeFailedTransfer $ map fst failed + mapM_ retry failed + where + retry (t, info) + | transferDirection t == Download = do + {- Check if the remote still has the key. + - If not, relies on the expensiveScan to + - get it queued from some other remote. -} + whenM (liftAnnex $ remoteHas r $ transferKey t) $ + requeue t info + | otherwise = do + {- The Transferrer checks when uploading + - that the remote doesn't already have the + - key, so it's not redundantly checked here. -} + requeue t info + requeue t info = queueTransferWhenSmall "retrying failed transfer" (associatedFile info) t r + +{- This is a expensive scan through the full git work tree, finding + - files to transfer. The scan is blocked when the transfer queue gets + - too large. + - + - This also finds files that are present either here or on a remote + - but that are not preferred content, and drops them. Searching for files + - to drop is done concurrently with the scan for transfers. + - + - TODO: It would be better to first drop as much as we can, before + - transferring much, to minimise disk use. + -} +expensiveScan :: [Remote] -> Assistant () +expensiveScan rs = unless onlyweb $ do + debug ["starting scan of", show visiblers] + void $ alertWhile (scanAlert visiblers) $ do + g <- liftAnnex gitRepo + (files, cleanup) <- liftIO $ LsFiles.inRepo [] g + forM_ files $ \f -> do + ts <- maybe (return []) (findtransfers f) + =<< liftAnnex (Backend.lookupFile f) + mapM_ (enqueue f) ts + void $ liftIO cleanup + return True + debug ["finished scan of", show visiblers] + where + onlyweb = all (== webUUID) $ map Remote.uuid rs + visiblers = let rs' = filter (not . Remote.readonly) rs + in if null rs' then rs else rs' + enqueue f (r, t) = + queueTransferWhenSmall "expensive scan found missing object" + (Just f) t r + findtransfers f (key, _) = do + {- The syncable remotes may have changed since this + - scan began. -} + syncrs <- syncDataRemotes <$> getDaemonStatus + locs <- liftAnnex $ loggedLocations key + present <- liftAnnex $ inAnnex key + handleDropsFrom locs syncrs + "expensive scan found too many copies of object" + present key (Just f) Nothing + liftAnnex $ do + let slocs = S.fromList locs + let use a = return $ catMaybes $ map (a key slocs) syncrs + if present + then filterM (wantSend True (Just f) . Remote.uuid . fst) + =<< use (genTransfer Upload False) + else ifM (wantGet True $ Just f) + ( use (genTransfer Download True) , return [] ) + +genTransfer :: Direction -> Bool -> Key -> S.Set UUID -> Remote -> Maybe (Remote, Transfer) +genTransfer direction want key slocs r + | direction == Upload && Remote.readonly r = Nothing + | (S.member (Remote.uuid r) slocs) == want = Just + (r, Transfer direction (Remote.uuid r) key) + | otherwise = Nothing + +remoteHas :: Remote -> Key -> Annex Bool +remoteHas r key = elem + <$> pure (Remote.uuid r) + <*> loggedLocations key diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs new file mode 100644 index 0000000000..7a6e426b32 --- /dev/null +++ b/Assistant/Threads/TransferWatcher.hs @@ -0,0 +1,128 @@ +{- git-annex assistant transfer watching thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferWatcher where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.TransferQueue +import Assistant.Drop +import Annex.Content +import Logs.Transfer +import Utility.DirWatcher +import Utility.Types.DirWatcher +import qualified Remote + +import Control.Concurrent + +{- This thread watches for changes to the gitAnnexTransferDir, + - and updates the DaemonStatus's map of ongoing transfers. -} +transferWatcherThread :: NamedThread +transferWatcherThread = namedThread "TransferWatcher" $ do + dir <- liftAnnex $ gitAnnexTransferDir <$> gitRepo + liftIO $ createDirectoryIfMissing True dir + let hook a = Just <$> asIO2 (runHandler a) + addhook <- hook onAdd + delhook <- hook onDel + modifyhook <- hook onModify + errhook <- hook onErr + let hooks = mkWatchHooks + { addHook = addhook + , delHook = delhook + , modifyHook = modifyhook + , errHook = errhook + } + void $ liftIO $ watchDir dir (const False) hooks id + debug ["watching for transfers"] + +type Handler = FilePath -> Assistant () + +{- Runs an action handler. + - + - Exceptions are ignored, otherwise a whole thread could be crashed. + -} +runHandler :: Handler -> FilePath -> Maybe FileStatus -> Assistant () +runHandler handler file _filestatus = + either (liftIO . print) (const noop) =<< tryIO <~> handler file + +{- Called when there's an error with inotify. -} +onErr :: Handler +onErr msg = error msg + +{- Called when a new transfer information file is written. -} +onAdd :: Handler +onAdd file = case parseTransferFile file of + Nothing -> noop + Just t -> go t =<< liftAnnex (checkTransfer t) + where + go _ Nothing = noop -- transfer already finished + go t (Just info) = do + debug [ "transfer starting:", describeTransfer t info ] + r <- headMaybe . filter (sameuuid t) + <$> liftAnnex Remote.remoteList + updateTransferInfo t info { transferRemote = r } + sameuuid t r = Remote.uuid r == transferUUID t + +{- Called when a transfer information file is updated. + - + - The only thing that should change in the transfer info is the + - bytesComplete, so that's the only thing updated in the DaemonStatus. -} +onModify :: Handler +onModify file = do + case parseTransferFile file of + Nothing -> noop + Just t -> go t =<< liftIO (readTransferInfoFile Nothing file) + where + go _ Nothing = noop + go t (Just newinfo) = alterTransferInfo t $ + \i -> i { bytesComplete = bytesComplete newinfo } + +{- This thread can only watch transfer sizes when the DirWatcher supports + - tracking modificatons to files. -} +watchesTransferSize :: Bool +watchesTransferSize = modifyTracked + +{- Called when a transfer information file is removed. -} +onDel :: Handler +onDel file = case parseTransferFile file of + Nothing -> noop + Just t -> do + debug [ "transfer finishing:", show t] + minfo <- removeTransfer t + + finished <- asIO2 finishedTransfer + void $ liftIO $ forkIO $ do + {- XXX race workaround delay. The location + - log needs to be updated before finishedTransfer + - runs. -} + threadDelay 10000000 -- 10 seconds + finished t minfo + +{- Queue uploads of files downloaded to us, spreading them + - out to other reachable remotes. + - + - Downloading a file may have caused a remote to not want it; + - so check for drops from remotes. + - + - Uploading a file may cause the local repo, or some other remote to not + - want it; handle that too. + -} +finishedTransfer :: Transfer -> Maybe TransferInfo -> Assistant () +finishedTransfer t (Just info) + | transferDirection t == Download = + whenM (liftAnnex $ inAnnex $ transferKey t) $ do + dodrops False + queueTransfersMatching (/= transferUUID t) + "newly received object" + Later (transferKey t) (associatedFile info) Upload + | otherwise = dodrops True + where + dodrops fromhere = handleDrops + ("drop wanted after " ++ describeTransfer t info) + fromhere (transferKey t) (associatedFile info) Nothing +finishedTransfer _ _ = noop + diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs new file mode 100644 index 0000000000..d8212768a3 --- /dev/null +++ b/Assistant/Threads/Transferrer.hs @@ -0,0 +1,117 @@ +{- git-annex assistant data transferrer thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.Transferrer where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.TransferQueue +import Assistant.TransferSlots +import Assistant.Alert +import Assistant.Commits +import Assistant.Drop +import Logs.Transfer +import Logs.Location +import Annex.Content +import qualified Remote +import Types.Key +import Locations.UserConfig +import Assistant.Threads.TransferWatcher + +import System.Process (create_group) + +{- Dispatches transfers from the queue. -} +transfererThread :: NamedThread +transfererThread = namedThread "Transferrer" $ do + program <- liftIO readProgramFile + forever $ inTransferSlot $ + maybe (return Nothing) (uncurry $ startTransfer program) + =<< getNextTransfer notrunning + where + {- Skip transfers that are already running. -} + notrunning = isNothing . startedTime + +{- By the time this is called, the daemonstatus's transfer map should + - already have been updated to include the transfer. -} +startTransfer :: FilePath -> Transfer -> TransferInfo -> Assistant (Maybe (Transfer, TransferInfo, Assistant ())) +startTransfer program t info = case (transferRemote info, associatedFile info) of + (Just remote, Just file) -> ifM (liftAnnex $ shouldTransfer t info) + ( do + debug [ "Transferring:" , describeTransfer t info ] + notifyTransfer + return $ Just (t, info, transferprocess remote file) + , do + debug [ "Skipping unnecessary transfer:" , describeTransfer t info ] + void $ removeTransfer t + finishedTransfer t (Just info) + return Nothing + ) + _ -> return Nothing + where + direction = transferDirection t + isdownload = direction == Download + + transferprocess remote file = void $ do + (_, _, _, pid) + <- liftIO $ createProcess (proc program $ toCommand params) + { create_group = True } + {- Alerts are only shown for successful transfers. + - Transfers can temporarily fail for many reasons, + - so there's no point in bothering the user about + - those. The assistant should recover. + - + - After a successful upload, handle dropping it from + - here, if desired. In this case, the remote it was + - uploaded to is known to have it. + - + - Also, after a successful transfer, the location + - log has changed. Indicate that a commit has been + - made, in order to queue a push of the git-annex + - branch out to remotes that did not participate + - in the transfer. + -} + whenM (liftIO $ (==) ExitSuccess <$> waitForProcess pid) $ do + void $ addAlert $ makeAlertFiller True $ + transferFileAlert direction True file + unless isdownload $ + handleDrops + ("object uploaded to " ++ show remote) + True (transferKey t) + (associatedFile info) + (Just remote) + recordCommit + where + params = + [ Param "transferkey" + , Param "--quiet" + , Param $ key2file $ transferKey t + , Param $ if isdownload + then "--from" + else "--to" + , Param $ Remote.name remote + , Param "--file" + , File file + ] + +{- Checks if the file to download is already present, or the remote + - being uploaded to isn't known to have the file. -} +shouldTransfer :: Transfer -> TransferInfo -> Annex Bool +shouldTransfer t info + | transferDirection t == Download = + not <$> inAnnex key + | transferDirection t == Upload = + {- Trust the location log to check if the + - remote already has the key. This avoids + - a roundtrip to the remote. -} + case transferRemote info of + Nothing -> return False + Just remote -> + notElem (Remote.uuid remote) + <$> loggedLocations key + | otherwise = return False + where + key = transferKey t diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs new file mode 100644 index 0000000000..38c5c138f3 --- /dev/null +++ b/Assistant/Threads/Watcher.hs @@ -0,0 +1,290 @@ +{- git-annex assistant tree watcher + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE DeriveDataTypeable, CPP #-} + +module Assistant.Threads.Watcher ( + watchThread, + WatcherException(..), + checkCanWatch, + needLsof, + onAddSymlink, + runHandler, +) where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.Changes +import Assistant.Types.Changes +import Assistant.TransferQueue +import Assistant.Alert +import Assistant.Drop +import Logs.Transfer +import Utility.DirWatcher +import Utility.Types.DirWatcher +import Utility.Lsof +import qualified Annex +import qualified Annex.Queue +import qualified Git +import qualified Git.UpdateIndex +import qualified Git.LsFiles as LsFiles +import qualified Backend +import Annex.Content +import Annex.Direct +import Annex.Content.Direct +import Annex.CatFile +import Annex.Link +import Git.Types +import Config +import Utility.ThreadScheduler + +import Data.Bits.Utils +import Data.Typeable +import qualified Data.ByteString.Lazy as L +import qualified Control.Exception as E + +checkCanWatch :: Annex () +checkCanWatch + | canWatch = do + liftIO setupLsof + unlessM (liftIO (inPath "lsof") <||> Annex.getState Annex.force) + needLsof + | otherwise = error "watch mode is not available on this system" + +needLsof :: Annex () +needLsof = error $ unlines + [ "The lsof command is needed for watch mode to be safe, and is not in PATH." + , "To override lsof checks to ensure that files are not open for writing" + , "when added to the annex, you can use --force" + , "Be warned: This can corrupt data in the annex, and make fsck complain." + ] + +{- A special exception that can be thrown to pause or resume the watcher. -} +data WatcherException = PauseWatcher | ResumeWatcher + deriving (Show, Eq, Typeable) + +instance E.Exception WatcherException + +watchThread :: NamedThread +watchThread = namedThread "Watcher" $ + ifM (liftAnnex $ annexAutoCommit <$> Annex.getGitConfig) + ( runWatcher + , waitFor ResumeWatcher runWatcher + ) + +runWatcher :: Assistant () +runWatcher = do + startup <- asIO1 startupScan + direct <- liftAnnex isDirect + addhook <- hook $ if direct then onAddDirect else onAdd + delhook <- hook onDel + addsymlinkhook <- hook $ onAddSymlink direct + deldirhook <- hook onDelDir + errhook <- hook onErr + let hooks = mkWatchHooks + { addHook = addhook + , delHook = delhook + , addSymlinkHook = addsymlinkhook + , delDirHook = deldirhook + , errHook = errhook + } + handle <- liftIO $ watchDir "." ignored hooks startup + debug [ "watching", "."] + + {- Let the DirWatcher thread run until signalled to pause it, + - then wait for a resume signal, and restart. -} + waitFor PauseWatcher $ do + liftIO $ stopWatchDir handle + waitFor ResumeWatcher runWatcher + where + hook a = Just <$> asIO2 (runHandler a) + +waitFor :: WatcherException -> Assistant () -> Assistant () +waitFor sig next = do + r <- liftIO $ (E.try pause :: IO (Either E.SomeException ())) + case r of + Left e -> case E.fromException e of + Just s + | s == sig -> next + _ -> noop + _ -> noop + where + pause = runEvery (Seconds 86400) noop + +{- Initial scartup scan. The action should return once the scan is complete. -} +startupScan :: IO a -> Assistant a +startupScan scanner = do + liftAnnex $ showAction "scanning" + alertWhile' startupScanAlert $ do + r <- liftIO $ scanner + + -- Notice any files that were deleted before + -- watching was started. + top <- liftAnnex $ fromRepo Git.repoPath + (fs, cleanup) <- liftAnnex $ inRepo $ LsFiles.deleted [top] + forM_ fs $ \f -> do + liftAnnex $ Annex.Queue.addUpdateIndex =<< + inRepo (Git.UpdateIndex.unstageFile f) + maybe noop recordChange =<< madeChange f RmChange + void $ liftIO $ cleanup + + liftAnnex $ showAction "started" + liftIO $ putStrLn "" + + modifyDaemonStatus_ $ \s -> s { scanComplete = True } + + return (True, r) + +ignored :: FilePath -> Bool +ignored = ig . takeFileName + where + ig ".git" = True + ig ".gitignore" = True + ig ".gitattributes" = True +#ifdef darwin_HOST_OS + ig ".DS_Store" = True +#endif + ig _ = False + +type Handler = FilePath -> Maybe FileStatus -> Assistant (Maybe Change) + +{- Runs an action handler, and if there was a change, adds it to the ChangeChan. + - + - Exceptions are ignored, otherwise a whole watcher thread could be crashed. + -} +runHandler :: Handler -> FilePath -> Maybe FileStatus -> Assistant () +runHandler handler file filestatus = void $ do + r <- tryIO <~> handler file filestatus + case r of + Left e -> liftIO $ print e + Right Nothing -> noop + Right (Just change) -> do + -- Just in case the commit thread is not + -- flushing the queue fast enough. + liftAnnex $ Annex.Queue.flushWhenFull + recordChange change + +onAdd :: Handler +onAdd file filestatus + | maybe False isRegularFile filestatus = pendingAddChange file + | otherwise = noChange + +{- In direct mode, add events are received for both new files, and + - modified existing files. Or, in some cases, existing files that have not + - really been modified. -} +onAddDirect :: Handler +onAddDirect file fs = do + v <- liftAnnex $ catKeyFile file + case (v, fs) of + (Just key, Just filestatus) -> + ifM (liftAnnex $ sameFileStatus key filestatus) + ( noChange + , do + liftAnnex $ changedDirect key file + pendingAddChange file + ) + _ -> pendingAddChange file + +{- A symlink might be an arbitrary symlink, which is just added. + - Or, if it is a git-annex symlink, ensure it points to the content + - before adding it. + -} +onAddSymlink :: Bool -> Handler +onAddSymlink isdirect file filestatus = go =<< liftAnnex (Backend.lookupFile file) + where + go (Just (key, _)) = do + when isdirect $ + liftAnnex $ void $ addAssociatedFile key file + link <- liftAnnex $ calcGitLink file key + ifM ((==) (Just link) <$> liftIO (catchMaybeIO $ readSymbolicLink file)) + ( do + s <- getDaemonStatus + checkcontent key s + ensurestaged (Just link) s + , do + unless isdirect $ do + liftIO $ removeFile file + liftAnnex $ Backend.makeAnnexLink link file + checkcontent key =<< getDaemonStatus + addlink link + ) + go Nothing = do -- other symlink + mlink <- liftIO (catchMaybeIO $ readSymbolicLink file) + ensurestaged mlink =<< getDaemonStatus + + {- This is often called on symlinks that are already + - staged correctly. A symlink may have been deleted + - and being re-added, or added when the watcher was + - not running. So they're normally restaged to make sure. + - + - As an optimisation, during the startup scan, avoid + - restaging everything. Only links that were created since + - the last time the daemon was running are staged. + - (If the daemon has never ran before, avoid staging + - links too.) + -} + ensurestaged (Just link) daemonstatus + | scanComplete daemonstatus = addlink link + | otherwise = case filestatus of + Just s + | not (afterLastDaemonRun (statusChangeTime s) daemonstatus) -> noChange + _ -> addlink link + ensurestaged Nothing _ = noChange + + {- For speed, tries to reuse the existing blob for symlink target. -} + addlink link = do + debug ["add symlink", file] + liftAnnex $ do + v <- catObjectDetails $ Ref $ ':':file + case v of + Just (currlink, sha) + | s2w8 link == L.unpack currlink -> + stageSymlink file sha + _ -> stageSymlink file =<< hashSymlink link + madeChange file LinkChange + + {- When a new link appears, or a link is changed, after the startup + - scan, handle getting or dropping the key's content. + - Also, moving or copying a link may caused it be be transferred + - elsewhere, so check that too. -} + checkcontent key daemonstatus + | scanComplete daemonstatus = do + present <- liftAnnex $ inAnnex key + if present + then queueTransfers "new file created" Next key (Just file) Upload + else queueTransfers "new or renamed file wanted" Next key (Just file) Download + handleDrops "file renamed" present key (Just file) Nothing + | otherwise = noop + +onDel :: Handler +onDel file _ = do + debug ["file deleted", file] + liftAnnex $ + Annex.Queue.addUpdateIndex =<< + inRepo (Git.UpdateIndex.unstageFile file) + madeChange file RmChange + +{- A directory has been deleted, or moved, so tell git to remove anything + - that was inside it from its cache. Since it could reappear at any time, + - use --cached to only delete it from the index. + - + - Note: This could use unstageFile, but would need to run another git + - command to get the recursive list of files in the directory, so rm is + - just as good. -} +onDelDir :: Handler +onDelDir dir _ = do + debug ["directory deleted", dir] + liftAnnex $ Annex.Queue.addCommand "rm" + [Params "--quiet -r --cached --ignore-unmatch --"] [dir] + madeChange dir RmDirChange + +{- Called when there's an error with inotify or kqueue. -} +onErr :: Handler +onErr msg _ = do + liftAnnex $ warning msg + void $ addAlert $ warningAlert "watcher" msg + noChange diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs new file mode 100644 index 0000000000..39b9c95c32 --- /dev/null +++ b/Assistant/Threads/WebApp.hs @@ -0,0 +1,92 @@ +{- git-annex assistant webapp thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Assistant.Threads.WebApp where + +import Assistant.Common +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.WebApp.DashBoard +import Assistant.WebApp.SideBar +import Assistant.WebApp.Notifications +import Assistant.WebApp.Configurators +import Assistant.WebApp.Configurators.Edit +import Assistant.WebApp.Configurators.Local +import Assistant.WebApp.Configurators.Ssh +import Assistant.WebApp.Configurators.Pairing +import Assistant.WebApp.Configurators.AWS +import Assistant.WebApp.Configurators.WebDAV +import Assistant.WebApp.Configurators.XMPP +import Assistant.WebApp.Configurators.Preferences +import Assistant.WebApp.Documentation +import Assistant.WebApp.Control +import Assistant.WebApp.OtherRepos +import Assistant.Types.ThreadedMonad +import Utility.WebApp +import Utility.TempFile +import Utility.FileMode +import Git + +import Yesod +import Yesod.Static +import Network.Socket (SockAddr) +import Data.Text (pack, unpack) + +mkYesodDispatch "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") + +type Url = String + +webAppThread + :: AssistantData + -> UrlRenderer + -> Bool + -> Maybe (IO String) + -> Maybe (Url -> FilePath -> IO ()) + -> NamedThread +webAppThread assistantdata urlrenderer noannex postfirstrun onstartup = thread $ liftIO $ do + webapp <- WebApp + <$> pure assistantdata + <*> (pack <$> genRandomToken) + <*> getreldir + <*> pure $(embed "static") + <*> newWebAppState + <*> pure postfirstrun + <*> pure noannex + setUrlRenderer urlrenderer $ yesodRender webapp (pack "") + app <- toWaiAppPlain webapp + app' <- ifM debugEnabled + ( return $ httpDebugLogger app + , return app + ) + runWebApp app' $ \addr -> if noannex + then withTempFile "webapp.html" $ \tmpfile _ -> + go addr webapp tmpfile Nothing + else do + let st = threadState assistantdata + htmlshim <- runThreadState st $ fromRepo gitAnnexHtmlShim + urlfile <- runThreadState st $ fromRepo gitAnnexUrlFile + go addr webapp htmlshim (Just urlfile) + where + thread = namedThread "WebApp" + getreldir + | noannex = return Nothing + | otherwise = Just <$> + (relHome =<< absPath + =<< runThreadState (threadState assistantdata) (fromRepo repoPath)) + go addr webapp htmlshim urlfile = do + let url = myUrl webapp addr + maybe noop (`writeFileProtected` url) urlfile + writeHtmlShim "Starting webapp..." url htmlshim + maybe noop (\a -> a url htmlshim) onstartup + +myUrl :: WebApp -> SockAddr -> Url +myUrl webapp addr = unpack $ yesodRender webapp urlbase HomeR [] + where + urlbase = pack $ "http://" ++ show addr diff --git a/Assistant/Threads/XMPPClient.hs b/Assistant/Threads/XMPPClient.hs new file mode 100644 index 0000000000..ebface7964 --- /dev/null +++ b/Assistant/Threads/XMPPClient.hs @@ -0,0 +1,257 @@ +{- git-annex XMPP client + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.XMPPClient where + +import Assistant.Common +import Assistant.XMPP +import Assistant.XMPP.Client +import Assistant.NetMessager +import Assistant.Types.NetMessager +import Assistant.Types.Buddies +import Assistant.XMPP.Buddies +import Assistant.Sync +import Assistant.DaemonStatus +import qualified Remote +import Utility.ThreadScheduler +import Assistant.WebApp (UrlRenderer, renderUrl) +import Assistant.WebApp.Types +import Assistant.Alert +import Assistant.Pairing +import Assistant.XMPP.Git +import Annex.UUID + +import Network.Protocol.XMPP +import Control.Concurrent +import qualified Data.Text as T +import qualified Data.Set as S +import qualified Data.Map as M +import qualified Git.Branch +import Data.Time.Clock + +xmppClientThread :: UrlRenderer -> NamedThread +xmppClientThread urlrenderer = namedThread "XMPPClient" $ + restartableClient . xmppClient urlrenderer =<< getAssistant id + +{- Runs the client, handing restart events. -} +restartableClient :: IO () -> Assistant () +restartableClient a = forever $ do + tid <- liftIO $ forkIO a + waitNetMessagerRestart + liftIO $ killThread tid + +xmppClient :: UrlRenderer -> AssistantData -> IO () +xmppClient urlrenderer d = do + v <- liftAssistant $ liftAnnex getXMPPCreds + case v of + Nothing -> noop -- will be restarted once creds get configured + Just c -> retry (runclient c) =<< getCurrentTime + where + liftAssistant = runAssistant d + inAssistant = liftIO . liftAssistant + + {- When the client exits, it's restarted; + - if it keeps failing, back off to wait 5 minutes before + - trying it again. -} + retry client starttime = do + e <- client + now <- getCurrentTime + if diffUTCTime now starttime > 300 + then do + liftAssistant $ debug ["connection lost; reconnecting", show e] + retry client now + else do + liftAssistant $ debug ["connection failed; will retry", show e] + threadDelaySeconds (Seconds 300) + retry client =<< getCurrentTime + + runclient c = liftIO $ connectXMPP c $ \jid -> do + selfjid <- bindJID jid + putStanza gitAnnexSignature + + inAssistant $ debug ["connected", show selfjid] + {- The buddy list starts empty each time + - the client connects, so that stale info + - is not retained. -} + void $ inAssistant $ + updateBuddyList (const noBuddies) <<~ buddyList + + xmppThread $ receivenotifications selfjid + forever $ do + a <- inAssistant $ relayNetMessage selfjid + a + + receivenotifications selfjid = forever $ do + l <- decodeStanza selfjid <$> getStanza + -- inAssistant $ debug ["received:", show l] + mapM_ (handle selfjid) l + + handle _ (PresenceMessage p) = void $ inAssistant $ + updateBuddyList (updateBuddies p) <<~ buddyList + handle _ (GotNetMessage QueryPresence) = putStanza gitAnnexSignature + handle _ (GotNetMessage (NotifyPush us)) = void $ inAssistant $ pull us + handle selfjid (GotNetMessage (PairingNotification stage c u)) = + maybe noop (inAssistant . pairMsgReceived urlrenderer stage u selfjid) (parseJID c) + handle _ (GotNetMessage m@(Pushing _ pushstage)) + | isPushInitiation pushstage = inAssistant $ + unlessM (queueNetPushMessage m) $ + void $ forkIO <~> handlePushInitiation m + | otherwise = void $ inAssistant $ queueNetPushMessage m + handle _ (Ignorable _) = noop + handle _ (Unknown _) = noop + handle _ (ProtocolError _) = noop + + +data XMPPEvent + = GotNetMessage NetMessage + | PresenceMessage Presence + | Ignorable ReceivedStanza + | Unknown ReceivedStanza + | ProtocolError ReceivedStanza + deriving Show + +{- Decodes an XMPP stanza into one or more events. -} +decodeStanza :: JID -> ReceivedStanza -> [XMPPEvent] +decodeStanza selfjid s@(ReceivedPresence p) + | presenceType p == PresenceError = [ProtocolError s] + | presenceFrom p == Nothing = [Ignorable s] + | presenceFrom p == Just selfjid = [Ignorable s] + | otherwise = maybe [PresenceMessage p] decode (gitAnnexTagInfo p) + where + decode i + | tagAttr i == pushAttr = impliedp $ GotNetMessage $ NotifyPush $ + decodePushNotification (tagValue i) + | tagAttr i == queryAttr = impliedp $ GotNetMessage QueryPresence + | otherwise = [Unknown s] + {- Things sent via presence imply a presence message, + - along with their real meaning. -} + impliedp v = [PresenceMessage p, v] +decodeStanza selfjid s@(ReceivedMessage m) + | messageFrom m == Nothing = [Ignorable s] + | messageFrom m == Just selfjid = [Ignorable s] + | messageType m == MessageError = [ProtocolError s] + | otherwise = [fromMaybe (Unknown s) (GotNetMessage <$> decodeMessage m)] +decodeStanza _ s = [Unknown s] + +{- Waits for a NetMessager message to be sent, and relays it to XMPP. + - + - Chat messages must be directed to specific clients, not a base + - account JID, due to git-annex clients using a negative presence priority. + - PairingNotification messages are always directed at specific + - clients, but Pushing messages are sometimes not, and need to be exploded. + -} +relayNetMessage :: JID -> Assistant (XMPP ()) +relayNetMessage selfjid = convert =<< waitNetMessage + where + convert (NotifyPush us) = return $ putStanza $ pushNotification us + convert QueryPresence = return $ putStanza presenceQuery + convert (PairingNotification stage c u) = withclient c $ \tojid -> do + changeBuddyPairing tojid True + return $ putStanza $ pairingNotification stage u tojid selfjid + convert (Pushing c pushstage) = withclient c $ \tojid -> do + if tojid == baseJID tojid + then do + bud <- getBuddy (genBuddyKey tojid) <<~ buddyList + return $ forM_ (maybe [] (S.toList . buddyAssistants) bud) $ \(Client jid) -> + putStanza $ pushMessage pushstage jid selfjid + else return $ putStanza $ pushMessage pushstage tojid selfjid + + withclient c a = case parseJID c of + Nothing -> return noop + Just tojid + | tojid == selfjid -> return noop + | otherwise -> a tojid + +{- Runs a XMPP action in a separate thread, using a session to allow it + - to access the same XMPP client. -} +xmppThread :: XMPP () -> XMPP () +xmppThread a = do + s <- getSession + void $ liftIO $ forkIO $ + void $ runXMPP s a + +{- We only pull from one remote out of the set listed in the push + - notification, as an optimisation. + - + - Note that it might be possible (though very unlikely) for the push + - notification to take a while to be sent, and multiple pushes happen + - before it is sent, so it includes multiple remotes that were pushed + - to at different times. + - + - It could then be the case that the remote we choose had the earlier + - push sent to it, but then failed to get the later push, and so is not + - fully up-to-date. If that happens, the pushRetryThread will come along + - and retry the push, and we'll get another notification once it succeeds, + - and pull again. -} +pull :: [UUID] -> Assistant () +pull [] = noop +pull us = do + rs <- filter matching . syncGitRemotes <$> getDaemonStatus + debug $ "push notification for" : map (fromUUID . Remote.uuid ) rs + pullone rs =<< liftAnnex (inRepo Git.Branch.current) + where + matching r = Remote.uuid r `S.member` s + s = S.fromList us + + pullone [] _ = noop + pullone (r:rs) branch = + unlessM (all id . fst <$> manualPull branch [r]) $ + pullone rs branch + +pairMsgReceived :: UrlRenderer -> PairStage -> UUID -> JID -> JID -> Assistant () +pairMsgReceived urlrenderer PairReq theiruuid selfjid theirjid + | baseJID selfjid == baseJID theirjid = autoaccept + | otherwise = do + knownjids <- catMaybes . map (parseJID . getXMPPClientID) + . filter isXMPPRemote . syncRemotes <$> getDaemonStatus + if any (== baseJID theirjid) knownjids + then autoaccept + else showalert + + where + -- PairReq from another client using our JID, or the JID of + -- any repo we're already paired with is automatically accepted. + autoaccept = do + selfuuid <- liftAnnex getUUID + sendNetMessage $ + PairingNotification PairAck (formatJID theirjid) selfuuid + finishXMPPPairing theirjid theiruuid + -- Show an alert to let the user decide if they want to pair. + showalert = do + let route = ConfirmXMPPPairR (PairKey theiruuid $ formatJID theirjid) + url <- liftIO $ renderUrl urlrenderer route [] + close <- asIO1 removeAlert + void $ addAlert $ pairRequestReceivedAlert (T.unpack $ buddyName theirjid) + AlertButton + { buttonUrl = url + , buttonLabel = T.pack "Respond" + , buttonAction = Just close + } + +pairMsgReceived _ PairAck theiruuid _selfjid theirjid = + {- PairAck must come from one of the buddies we are pairing with; + - don't pair with just anyone. -} + whenM (isBuddyPairing theirjid) $ do + changeBuddyPairing theirjid False + selfuuid <- liftAnnex getUUID + sendNetMessage $ + PairingNotification PairDone (formatJID theirjid) selfuuid + finishXMPPPairing theirjid theiruuid + +pairMsgReceived _ PairDone _theiruuid _selfjid theirjid = + changeBuddyPairing theirjid False + +isBuddyPairing :: JID -> Assistant Bool +isBuddyPairing jid = maybe False buddyPairing <$> + getBuddy (genBuddyKey jid) <<~ buddyList + +changeBuddyPairing :: JID -> Bool -> Assistant () +changeBuddyPairing jid ispairing = + updateBuddyList (M.adjust set key) <<~ buddyList + where + key = genBuddyKey jid + set b = b { buddyPairing = ispairing } diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs new file mode 100644 index 0000000000..7c41f0399d --- /dev/null +++ b/Assistant/TransferQueue.hs @@ -0,0 +1,207 @@ +{- git-annex assistant pending transfer queue + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.TransferQueue ( + TransferQueue, + Schedule(..), + newTransferQueue, + getTransferQueue, + queueTransfers, + queueTransfersMatching, + queueDeferredDownloads, + queueTransfer, + queueTransferAt, + queueTransferWhenSmall, + getNextTransfer, + getMatchingTransfers, + dequeueTransfers, +) where + +import Assistant.Common +import Assistant.DaemonStatus +import Assistant.Types.TransferQueue +import Logs.Transfer +import Types.Remote +import qualified Remote +import qualified Types.Remote as Remote +import Annex.Wanted + +import Control.Concurrent.STM +import qualified Data.Map as M + +type Reason = String + +{- Reads the queue's content without blocking or changing it. -} +getTransferQueue :: Assistant [(Transfer, TransferInfo)] +getTransferQueue = (atomically . readTVar . queuelist) <<~ transferQueue + +stubInfo :: AssociatedFile -> Remote -> TransferInfo +stubInfo f r = stubTransferInfo + { transferRemote = Just r + , associatedFile = f + } + +{- Adds transfers to queue for some of the known remotes. + - Honors preferred content settings, only transferring wanted files. -} +queueTransfers :: Reason -> Schedule -> Key -> AssociatedFile -> Direction -> Assistant () +queueTransfers = queueTransfersMatching (const True) + +{- Adds transfers to queue for some of the known remotes, that match a + - condition. Honors preferred content settings. -} +queueTransfersMatching :: (UUID -> Bool) -> Reason -> Schedule -> Key -> AssociatedFile -> Direction -> Assistant () +queueTransfersMatching matching reason schedule k f direction + | direction == Download = whenM (liftAnnex $ wantGet True f) go + | otherwise = go + where + go = do + rs <- liftAnnex . sufficientremotes + =<< syncDataRemotes <$> getDaemonStatus + let matchingrs = filter (matching . Remote.uuid) rs + if null matchingrs + then defer + else forM_ matchingrs $ \r -> + enqueue reason schedule (gentransfer r) (stubInfo f r) + sufficientremotes rs + {- Queue downloads from all remotes that + - have the key, with the cheapest ones first. + - More expensive ones will only be tried if + - downloading from a cheap one fails. -} + | direction == Download = do + uuids <- Remote.keyLocations k + return $ filter (\r -> uuid r `elem` uuids) rs + {- Upload to all remotes that want the content. -} + | otherwise = filterM (wantSend True f . Remote.uuid) $ + filter (not . Remote.readonly) rs + gentransfer r = Transfer + { transferDirection = direction + , transferKey = k + , transferUUID = Remote.uuid r + } + defer + {- Defer this download, as no known remote has the key. -} + | direction == Download = do + q <- getAssistant transferQueue + void $ liftIO $ atomically $ + modifyTVar' (deferreddownloads q) $ + \l -> (k, f):l + | otherwise = noop + +{- Queues any deferred downloads that can now be accomplished, leaving + - any others in the list to try again later. -} +queueDeferredDownloads :: Reason -> Schedule -> Assistant () +queueDeferredDownloads reason schedule = do + q <- getAssistant transferQueue + l <- liftIO $ atomically $ swapTVar (deferreddownloads q) [] + rs <- syncDataRemotes <$> getDaemonStatus + left <- filterM (queue rs) l + unless (null left) $ + liftIO $ atomically $ modifyTVar' (deferreddownloads q) $ + \new -> new ++ left + where + queue rs (k, f) = do + uuids <- liftAnnex $ Remote.keyLocations k + let sources = filter (\r -> uuid r `elem` uuids) rs + unless (null sources) $ + forM_ sources $ \r -> + enqueue reason schedule (gentransfer r) (stubInfo f r) + return $ null sources + where + gentransfer r = Transfer + { transferDirection = Download + , transferKey = k + , transferUUID = Remote.uuid r + } + +enqueue :: Reason -> Schedule -> Transfer -> TransferInfo -> Assistant () +enqueue reason schedule t info + | schedule == Next = go (new:) + | otherwise = go (\l -> l++[new]) + where + new = (t, info) + go modlist = do + q <- getAssistant transferQueue + liftIO $ atomically $ do + void $ modifyTVar' (queuesize q) succ + void $ modifyTVar' (queuelist q) modlist + debug [ "queued", describeTransfer t info, ": " ++ reason ] + notifyTransfer + +{- Adds a transfer to the queue. -} +queueTransfer :: Reason -> Schedule -> AssociatedFile -> Transfer -> Remote -> Assistant () +queueTransfer reason schedule f t remote = + enqueue reason schedule t (stubInfo f remote) + +{- Blocks until the queue is no larger than a given size, and then adds a + - transfer to the queue. -} +queueTransferAt :: Int -> Reason -> Schedule -> AssociatedFile -> Transfer -> Remote -> Assistant () +queueTransferAt wantsz reason schedule f t remote = do + q <- getAssistant transferQueue + liftIO $ atomically $ do + sz <- readTVar (queuesize q) + unless (sz <= wantsz) $ + retry -- blocks until queuesize changes + enqueue reason schedule t (stubInfo f remote) + +queueTransferWhenSmall :: Reason -> AssociatedFile -> Transfer -> Remote -> Assistant () +queueTransferWhenSmall reason = queueTransferAt 10 reason Later + +{- Blocks until a pending transfer is available in the queue, + - and removes it. + - + - Checks that it's acceptable, before adding it to the + - currentTransfers map. If it's not acceptable, it's discarded. + - + - This is done in a single STM transaction, so there is no window + - where an observer sees an inconsistent status. -} +getNextTransfer :: (TransferInfo -> Bool) -> Assistant (Maybe (Transfer, TransferInfo)) +getNextTransfer acceptable = do + q <- getAssistant transferQueue + dstatus <- getAssistant daemonStatusHandle + liftIO $ atomically $ do + sz <- readTVar (queuesize q) + if sz < 1 + then retry -- blocks until queuesize changes + else do + (r@(t,info):rest) <- readTVar (queuelist q) + writeTVar (queuelist q) rest + void $ modifyTVar' (queuesize q) pred + if acceptable info + then do + adjustTransfersSTM dstatus $ + M.insertWith' const t info + return $ Just r + else return Nothing + +{- Moves transfers matching a condition from the queue, to the + - currentTransfers map. -} +getMatchingTransfers :: (Transfer -> Bool) -> Assistant [(Transfer, TransferInfo)] +getMatchingTransfers c = do + q <- getAssistant transferQueue + dstatus <- getAssistant daemonStatusHandle + liftIO $ atomically $ do + ts <- dequeueTransfersSTM q c + unless (null ts) $ + adjustTransfersSTM dstatus $ \m -> M.union m $ M.fromList ts + return ts + +{- Removes transfers matching a condition from the queue, and returns the + - removed transfers. -} +dequeueTransfers :: (Transfer -> Bool) -> Assistant [(Transfer, TransferInfo)] +dequeueTransfers c = do + q <- getAssistant transferQueue + removed <- liftIO $ atomically $ dequeueTransfersSTM q c + unless (null removed) $ + notifyTransfer + return removed + +dequeueTransfersSTM :: TransferQueue -> (Transfer -> Bool) -> STM [(Transfer, TransferInfo)] +dequeueTransfersSTM q c = do + (removed, ts) <- partition (c . fst) + <$> readTVar (queuelist q) + void $ writeTVar (queuesize q) (length ts) + void $ writeTVar (queuelist q) ts + return removed diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs new file mode 100644 index 0000000000..7c9f747023 --- /dev/null +++ b/Assistant/TransferSlots.hs @@ -0,0 +1,73 @@ +{- git-annex assistant transfer slots + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.TransferSlots where + +import Assistant.Common +import Utility.ThreadScheduler +import Assistant.Types.TransferSlots +import Assistant.DaemonStatus +import Logs.Transfer + +import qualified Control.Exception as E +import Control.Concurrent +import qualified Control.Concurrent.MSemN as MSemN + +type TransferGenerator = Assistant (Maybe (Transfer, TransferInfo, Assistant ())) + +{- Waits until a transfer slot becomes available, then runs a + - TransferGenerator, and then runs the transfer action in its own thread. + -} +inTransferSlot :: TransferGenerator -> Assistant () +inTransferSlot gen = do + flip MSemN.wait 1 <<~ transferSlots + runTransferThread =<< gen + +{- Runs a TransferGenerator, and its transfer action, + - without waiting for a slot to become available. -} +inImmediateTransferSlot :: TransferGenerator -> Assistant () +inImmediateTransferSlot gen = do + flip MSemN.signal (-1) <<~ transferSlots + runTransferThread =<< gen + +{- Runs a transfer action, in an already allocated transfer slot. + - Once it finishes, frees the transfer slot. + - + - Note that the action is subject to being killed when the transfer + - is canceled or paused. + - + - A PauseTransfer exception is handled by letting the action be killed, + - then pausing the thread until a ResumeTransfer exception is raised, + - then rerunning the action. + -} +runTransferThread :: Maybe (Transfer, TransferInfo, Assistant ()) -> Assistant () +runTransferThread Nothing = flip MSemN.signal 1 <<~ transferSlots +runTransferThread (Just (t, info, a)) = do + d <- getAssistant id + aio <- asIO a + tid <- liftIO $ forkIO $ runTransferThread' d aio + updateTransferInfo t $ info { transferTid = Just tid } + +runTransferThread' :: AssistantData -> IO () -> IO () +runTransferThread' d a = go + where + go = catchPauseResume a + pause = catchPauseResume $ runEvery (Seconds 86400) noop + {- Note: This must use E.try, rather than E.catch. + - When E.catch is used, and has called go in its exception + - handler, Control.Concurrent.throwTo will block sometimes + - when signaling. Using E.try avoids the problem. -} + catchPauseResume a' = do + r <- E.try a' :: IO (Either E.SomeException ()) + case r of + Left e -> case E.fromException e of + Just PauseTransfer -> pause + Just ResumeTransfer -> go + _ -> done + _ -> done + done = runAssistant d $ + flip MSemN.signal 1 <<~ transferSlots diff --git a/Assistant/Types/BranchChange.hs b/Assistant/Types/BranchChange.hs new file mode 100644 index 0000000000..399abee54d --- /dev/null +++ b/Assistant/Types/BranchChange.hs @@ -0,0 +1,19 @@ +{- git-annex assistant git-annex branch change tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.BranchChange where + +import Control.Concurrent.MSampleVar +import Common.Annex + +newtype BranchChangeHandle = BranchChangeHandle (MSampleVar ()) + +newBranchChangeHandle :: IO BranchChangeHandle +newBranchChangeHandle = BranchChangeHandle <$> newEmptySV + +fromBranchChangeHandle :: BranchChangeHandle -> MSampleVar () +fromBranchChangeHandle (BranchChangeHandle v) = v diff --git a/Assistant/Types/Buddies.hs b/Assistant/Types/Buddies.hs new file mode 100644 index 0000000000..36d8a4fedc --- /dev/null +++ b/Assistant/Types/Buddies.hs @@ -0,0 +1,80 @@ +{- git-annex assistant buddies + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Assistant.Types.Buddies where + +import Common.Annex + +import qualified Data.Map as M +import Control.Concurrent.STM +import Utility.NotificationBroadcaster +import Data.Text as T + +{- For simplicity, dummy types are defined even when XMPP is disabled. -} +#ifdef WITH_XMPP +import Network.Protocol.XMPP +import Data.Set as S +import Data.Ord + +newtype Client = Client JID + deriving (Eq, Show) + +instance Ord Client where + compare = comparing show + +data Buddy = Buddy + { buddyPresent :: S.Set Client + , buddyAway :: S.Set Client + , buddyAssistants :: S.Set Client + , buddyPairing :: Bool + } +#else +data Buddy = Buddy +#endif + deriving (Eq, Show) + +data BuddyKey = BuddyKey T.Text + deriving (Eq, Ord, Show, Read) + +data PairKey = PairKey UUID T.Text + deriving (Eq, Ord, Show, Read) + +type Buddies = M.Map BuddyKey Buddy + +{- A list of buddies, and a way to notify when it changes. -} +type BuddyList = (TMVar Buddies, NotificationBroadcaster) + +noBuddies :: Buddies +noBuddies = M.empty + +newBuddyList :: IO BuddyList +newBuddyList = (,) + <$> atomically (newTMVar noBuddies) + <*> newNotificationBroadcaster + +getBuddyList :: BuddyList -> IO [Buddy] +getBuddyList (v, _) = M.elems <$> atomically (readTMVar v) + +getBuddy :: BuddyKey -> BuddyList -> IO (Maybe Buddy) +getBuddy k (v, _) = M.lookup k <$> atomically (readTMVar v) + +getBuddyBroadcaster :: BuddyList -> NotificationBroadcaster +getBuddyBroadcaster (_, h) = h + +{- Applies a function to modify the buddy list, and if it's changed, + - sends notifications to any listeners. -} +updateBuddyList :: (Buddies -> Buddies) -> BuddyList -> IO () +updateBuddyList a (v, caster) = do + changed <- atomically $ do + buds <- takeTMVar v + let buds' = a buds + putTMVar v buds' + return $ buds /= buds' + when changed $ + sendNotification caster diff --git a/Assistant/Types/Changes.hs b/Assistant/Types/Changes.hs new file mode 100644 index 0000000000..887aa819eb --- /dev/null +++ b/Assistant/Types/Changes.hs @@ -0,0 +1,54 @@ +{- git-annex assistant change tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.Changes where + +import Types.KeySource +import Utility.TSet + +import Data.Time.Clock + +data ChangeType = AddChange | LinkChange | RmChange | RmDirChange + deriving (Show, Eq) + +type ChangeChan = TSet Change + +data Change + = Change + { changeTime :: UTCTime + , changeFile :: FilePath + , changeType :: ChangeType + } + | PendingAddChange + { changeTime ::UTCTime + , changeFile :: FilePath + } + | InProcessAddChange + { changeTime ::UTCTime + , keySource :: KeySource + } + deriving (Show) + +newChangeChan :: IO ChangeChan +newChangeChan = newTSet + +isPendingAddChange :: Change -> Bool +isPendingAddChange (PendingAddChange {}) = True +isPendingAddChange _ = False + +isInProcessAddChange :: Change -> Bool +isInProcessAddChange (InProcessAddChange {}) = True +isInProcessAddChange _ = False + +finishedChange :: Change -> Change +finishedChange c@(InProcessAddChange { keySource = ks }) = Change + { changeTime = changeTime c + , changeFile = keyFilename ks + , changeType = AddChange + } +finishedChange c = c + diff --git a/Assistant/Types/Commits.hs b/Assistant/Types/Commits.hs new file mode 100644 index 0000000000..bb17c578b6 --- /dev/null +++ b/Assistant/Types/Commits.hs @@ -0,0 +1,17 @@ +{- git-annex assistant commit tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.Commits where + +import Utility.TSet + +type CommitChan = TSet Commit + +data Commit = Commit + +newCommitChan :: IO CommitChan +newCommitChan = newTSet diff --git a/Assistant/Types/DaemonStatus.hs b/Assistant/Types/DaemonStatus.hs new file mode 100644 index 0000000000..b60d49edf2 --- /dev/null +++ b/Assistant/Types/DaemonStatus.hs @@ -0,0 +1,81 @@ +{- git-annex assistant daemon status + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE RankNTypes, ImpredicativeTypes #-} + +module Assistant.Types.DaemonStatus where + +import Common.Annex +import Assistant.Alert +import Assistant.Pairing +import Utility.NotificationBroadcaster +import Logs.Transfer +import Assistant.Types.ThreadName + +import Control.Concurrent.STM +import Control.Concurrent.Async +import Data.Time.Clock.POSIX +import qualified Data.Map as M + +data DaemonStatus = DaemonStatus + -- All the named threads that comprise the daemon, + -- and actions to run to restart them. + { startedThreads :: M.Map ThreadName (Async (), IO ()) + -- False when the daemon is performing its startup scan + , scanComplete :: Bool + -- Time when a previous process of the daemon was running ok + , lastRunning :: Maybe POSIXTime + -- True when the sanity checker is running + , sanityCheckRunning :: Bool + -- Last time the sanity checker ran + , lastSanityCheck :: Maybe POSIXTime + -- Currently running file content transfers + , currentTransfers :: TransferMap + -- Messages to display to the user. + , alertMap :: AlertMap + , lastAlertId :: AlertId + -- Ordered list of all remotes that can be synced with + , syncRemotes :: [Remote] + -- Ordered list of remotes to sync git with + , syncGitRemotes :: [Remote] + -- Ordered list of remotes to sync data with + , syncDataRemotes :: [Remote] + -- Pairing request that is in progress. + , pairingInProgress :: Maybe PairingInProgress + -- Broadcasts notifications about all changes to the DaemonStatus + , changeNotifier :: NotificationBroadcaster + -- Broadcasts notifications when queued or current transfers change. + , transferNotifier :: NotificationBroadcaster + -- Broadcasts notifications when there's a change to the alerts + , alertNotifier :: NotificationBroadcaster + -- Broadcasts notifications when the syncRemotes change + , syncRemotesNotifier :: NotificationBroadcaster + } + +type TransferMap = M.Map Transfer TransferInfo + +{- This TMVar is never left empty, so accessing it will never block. -} +type DaemonStatusHandle = TMVar DaemonStatus + +newDaemonStatus :: IO DaemonStatus +newDaemonStatus = DaemonStatus + <$> pure M.empty + <*> pure False + <*> pure Nothing + <*> pure False + <*> pure Nothing + <*> pure M.empty + <*> pure M.empty + <*> pure firstAlertId + <*> pure [] + <*> pure [] + <*> pure [] + <*> pure Nothing + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster + <*> newNotificationBroadcaster diff --git a/Assistant/Types/NamedThread.hs b/Assistant/Types/NamedThread.hs new file mode 100644 index 0000000000..0e884637a3 --- /dev/null +++ b/Assistant/Types/NamedThread.hs @@ -0,0 +1,17 @@ +{- named threads + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.NamedThread where + +import Assistant.Monad +import Assistant.Types.ThreadName + +{- Information about a named thread that can be run. -} +data NamedThread = NamedThread ThreadName (Assistant ()) + +namedThread :: String -> Assistant () -> NamedThread +namedThread name a = NamedThread (ThreadName name) a diff --git a/Assistant/Types/NetMessager.hs b/Assistant/Types/NetMessager.hs new file mode 100644 index 0000000000..c036d624ae --- /dev/null +++ b/Assistant/Types/NetMessager.hs @@ -0,0 +1,101 @@ +{- git-annex assistant out of band network messager types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.NetMessager where + +import Common.Annex +import Assistant.Pairing + +import Data.Text (Text) +import Control.Concurrent.STM +import Control.Concurrent.MSampleVar +import Data.ByteString (ByteString) +import qualified Data.Set as S + +{- Messages that can be sent out of band by a network messager. -} +data NetMessage + -- indicate that pushes have been made to the repos with these uuids + = NotifyPush [UUID] + -- requests other clients to inform us of their presence + | QueryPresence + -- notification about a stage in the pairing process, + -- involving a client, and a UUID. + | PairingNotification PairStage ClientID UUID + -- used for git push over the network messager + | Pushing ClientID PushStage + deriving (Show, Eq, Ord) + +{- Something used to identify the client, or clients to send the message to. -} +type ClientID = Text + +data PushStage + -- indicates that we have data to push over the out of band network + = CanPush + -- request that a git push be sent over the out of band network + | PushRequest + -- indicates that a push is starting + | StartingPush + -- a chunk of output of git receive-pack + | ReceivePackOutput ByteString + -- a chuck of output of git send-pack + | SendPackOutput ByteString + -- sent when git receive-pack exits, with its exit code + | ReceivePackDone ExitCode + deriving (Show, Eq, Ord) + +{- Things that initiate either side of a push, but do not actually send data. -} +isPushInitiation :: PushStage -> Bool +isPushInitiation CanPush = True +isPushInitiation PushRequest = True +isPushInitiation StartingPush = True +isPushInitiation _ = False + +data PushSide = SendPack | ReceivePack + deriving (Eq, Ord) + +pushDestinationSide :: PushStage -> PushSide +pushDestinationSide CanPush = ReceivePack +pushDestinationSide PushRequest = SendPack +pushDestinationSide StartingPush = ReceivePack +pushDestinationSide (ReceivePackOutput _) = SendPack +pushDestinationSide (SendPackOutput _) = ReceivePack +pushDestinationSide (ReceivePackDone _) = SendPack + +type SideMap a = PushSide -> a + +mkSideMap :: STM a -> IO (SideMap a) +mkSideMap gen = do + (sp, rp) <- atomically $ (,) <$> gen <*> gen + return $ lookupside sp rp + where + lookupside sp _ SendPack = sp + lookupside _ rp ReceivePack = rp + +getSide :: PushSide -> SideMap a -> a +getSide side m = m side + +data NetMessager = NetMessager + -- outgoing messages + { netMessages :: TChan (NetMessage) + -- write to this to restart the net messager + , netMessagerRestart :: MSampleVar () + -- only one side of a push can be running at a time + , netMessagerPushRunning :: SideMap (TMVar (Maybe ClientID)) + -- incoming messages related to a running push + , netMessagesPush :: SideMap (TChan NetMessage) + -- incoming push messages, deferred to be processed later + , netMessagesPushDeferred :: SideMap (TMVar (S.Set NetMessage)) + } + +newNetMessager :: IO NetMessager +newNetMessager = NetMessager + <$> atomically newTChan + <*> newEmptySV + <*> mkSideMap (newTMVar Nothing) + <*> mkSideMap newTChan + <*> mkSideMap (newTMVar S.empty) + where diff --git a/Assistant/Types/Pushes.hs b/Assistant/Types/Pushes.hs new file mode 100644 index 0000000000..99e0ee1628 --- /dev/null +++ b/Assistant/Types/Pushes.hs @@ -0,0 +1,24 @@ +{- git-annex assistant push tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.Pushes where + +import Common.Annex + +import Control.Concurrent.STM +import Data.Time.Clock +import qualified Data.Map as M + +{- Track the most recent push failure for each remote. -} +type PushMap = M.Map Remote UTCTime +type FailedPushMap = TMVar PushMap + +{- The TMVar starts empty, and is left empty when there are no + - failed pushes. This way we can block until there are some failed pushes. + -} +newFailedPushMap :: IO FailedPushMap +newFailedPushMap = atomically newEmptyTMVar diff --git a/Assistant/Types/ScanRemotes.hs b/Assistant/Types/ScanRemotes.hs new file mode 100644 index 0000000000..d2f0c588fb --- /dev/null +++ b/Assistant/Types/ScanRemotes.hs @@ -0,0 +1,25 @@ +{- git-annex assistant remotes needing scanning + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.ScanRemotes where + +import Common.Annex + +import Control.Concurrent.STM +import qualified Data.Map as M + +data ScanInfo = ScanInfo + { scanPriority :: Int + , fullScan :: Bool + } + +type ScanRemoteMap = TMVar (M.Map Remote ScanInfo) + +{- The TMVar starts empty, and is left empty when there are no remotes + - to scan. -} +newScanRemoteMap :: IO ScanRemoteMap +newScanRemoteMap = atomically newEmptyTMVar diff --git a/Assistant/Types/ThreadName.hs b/Assistant/Types/ThreadName.hs new file mode 100644 index 0000000000..c8d264a381 --- /dev/null +++ b/Assistant/Types/ThreadName.hs @@ -0,0 +1,14 @@ +{- name of a thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.ThreadName where + +newtype ThreadName = ThreadName String + deriving (Eq, Read, Show, Ord) + +fromThreadName :: ThreadName -> String +fromThreadName (ThreadName n) = n diff --git a/Assistant/Types/ThreadedMonad.hs b/Assistant/Types/ThreadedMonad.hs new file mode 100644 index 0000000000..1a2aa7eb7f --- /dev/null +++ b/Assistant/Types/ThreadedMonad.hs @@ -0,0 +1,38 @@ +{- making the Annex monad available across threads + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.ThreadedMonad where + +import Common.Annex +import qualified Annex + +import Control.Concurrent +import Data.Tuple + +{- The Annex state is stored in a MVar, so that threaded actions can access + - it. -} +type ThreadState = MVar Annex.AnnexState + +{- Stores the Annex state in a MVar. + - + - Once the action is finished, retrieves the state from the MVar. + -} +withThreadState :: (ThreadState -> Annex a) -> Annex a +withThreadState a = do + state <- Annex.getState id + mvar <- liftIO $ newMVar state + r <- a mvar + newstate <- liftIO $ takeMVar mvar + Annex.changeState (const newstate) + return r + +{- Runs an Annex action, using the state from the MVar. + - + - This serializes calls by threads; only one thread can run in Annex at a + - time. -} +runThreadState :: ThreadState -> Annex a -> IO a +runThreadState mvar a = modifyMVar mvar $ \state -> swap <$> Annex.run state a diff --git a/Assistant/Types/TransferQueue.hs b/Assistant/Types/TransferQueue.hs new file mode 100644 index 0000000000..6620ebdf6d --- /dev/null +++ b/Assistant/Types/TransferQueue.hs @@ -0,0 +1,29 @@ +{- git-annex assistant pending transfer queue + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Types.TransferQueue where + +import Common.Annex +import Logs.Transfer +import Types.Remote + +import Control.Concurrent.STM + +data TransferQueue = TransferQueue + { queuesize :: TVar Int + , queuelist :: TVar [(Transfer, TransferInfo)] + , deferreddownloads :: TVar [(Key, AssociatedFile)] + } + +data Schedule = Next | Later + deriving (Eq) + +newTransferQueue :: IO TransferQueue +newTransferQueue = atomically $ TransferQueue + <$> newTVar 0 + <*> newTVar [] + <*> newTVar [] diff --git a/Assistant/Types/TransferSlots.hs b/Assistant/Types/TransferSlots.hs new file mode 100644 index 0000000000..5140995a37 --- /dev/null +++ b/Assistant/Types/TransferSlots.hs @@ -0,0 +1,34 @@ +{- git-annex assistant transfer slots + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE DeriveDataTypeable #-} + +module Assistant.Types.TransferSlots where + +import qualified Control.Exception as E +import qualified Control.Concurrent.MSemN as MSemN +import Data.Typeable + +type TransferSlots = MSemN.MSemN Int + +{- A special exception that can be thrown to pause or resume a transfer, while + - keeping its slot in use. -} +data TransferException = PauseTransfer | ResumeTransfer + deriving (Show, Eq, Typeable) + +instance E.Exception TransferException + +{- Number of concurrent transfers allowed to be run from the assistant. + - + - Transfers launched by other means, including by remote assistants, + - do not currently take up slots. + -} +numSlots :: Int +numSlots = 1 + +newTransferSlots :: IO TransferSlots +newTransferSlots = MSemN.new numSlots diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs new file mode 100644 index 0000000000..0e5dc6d535 --- /dev/null +++ b/Assistant/WebApp.hs @@ -0,0 +1,104 @@ +{- git-annex assistant webapp core + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp where + +import Assistant.WebApp.Types +import Assistant.Common hiding (liftAnnex) +import qualified Assistant.Monad as Assistant +import Utility.NotificationBroadcaster +import Utility.Yesod + +import Yesod +import Data.Text (Text) +import Control.Concurrent.STM +import Control.Concurrent + +inFirstRun :: Handler Bool +inFirstRun = isNothing . relDir <$> getYesod + +newWebAppState :: IO (TMVar WebAppState) +newWebAppState = atomically $ newTMVar $ WebAppState { showIntro = True } + +getWebAppState :: forall sub. GHandler sub WebApp WebAppState +getWebAppState = liftIO . atomically . readTMVar =<< webAppState <$> getYesod + +modifyWebAppState :: forall sub. (WebAppState -> WebAppState) -> GHandler sub WebApp () +modifyWebAppState a = go =<< webAppState <$> getYesod + where + go s = liftIO $ atomically $ do + v <- takeTMVar s + putTMVar s $ a v + +{- Runs an Annex action from the webapp. + - + - When the webapp is run outside a git-annex repository, the fallback + - value is returned. + -} +liftAnnexOr :: forall sub a. a -> Annex a -> GHandler sub WebApp a +liftAnnexOr fallback a = ifM (noAnnex <$> getYesod) + ( return fallback + , liftAssistant $ Assistant.liftAnnex a + ) + +liftAnnex :: forall sub a. Annex a -> GHandler sub WebApp a +liftAnnex = liftAnnexOr $ error "internal runAnnex" + +liftAssistant :: forall sub a. (Assistant a) -> GHandler sub WebApp a +liftAssistant a = liftIO . flip runAssistant a =<< assistantData <$> getYesod + +waitNotifier :: forall sub. (Assistant NotificationBroadcaster) -> NotificationId -> GHandler sub WebApp () +waitNotifier getbroadcaster nid = liftAssistant $ do + b <- getbroadcaster + liftIO $ waitNotification $ notificationHandleFromId b nid + +newNotifier :: forall sub. (Assistant NotificationBroadcaster) -> GHandler sub WebApp NotificationId +newNotifier getbroadcaster = liftAssistant $ do + b <- getbroadcaster + liftIO $ notificationHandleToId <$> newNotificationHandle b + +{- Adds the auth parameter as a hidden field on a form. Must be put into + - every form. -} +webAppFormAuthToken :: Widget +webAppFormAuthToken = do + webapp <- lift getYesod + [whamlet||] + +{- A button with an icon, and maybe label or tooltip, that can be + - clicked to perform some action. + - With javascript, clicking it POSTs the Route, and remains on the same + - page. + - With noscript, clicking it GETs the Route. -} +actionButton :: Route WebApp -> (Maybe String) -> (Maybe String) -> String -> String -> Widget +actionButton route label tooltip buttonclass iconclass = $(widgetFile "actionbutton") + +type UrlRenderFunc = Route WebApp -> [(Text, Text)] -> Text +type UrlRenderer = MVar (UrlRenderFunc) + +newUrlRenderer :: IO UrlRenderer +newUrlRenderer = newEmptyMVar + +setUrlRenderer :: UrlRenderer -> (UrlRenderFunc) -> IO () +setUrlRenderer = putMVar + +{- Blocks until the webapp is running and has called setUrlRenderer. -} +renderUrl :: UrlRenderer -> Route WebApp -> [(Text, Text)] -> IO Text +renderUrl urlrenderer route params = do + r <- readMVar urlrenderer + return $ r route params + +{- Redirects back to the referring page, or if there's none, HomeR -} +redirectBack :: Handler () +redirectBack = do + clearUltDest + setUltDestReferer + redirectUltDest HomeR + +controlMenu :: Widget +controlMenu = $(widgetFile "controlmenu") diff --git a/Assistant/WebApp/Common.hs b/Assistant/WebApp/Common.hs new file mode 100644 index 0000000000..0c6bcdd11b --- /dev/null +++ b/Assistant/WebApp/Common.hs @@ -0,0 +1,18 @@ +{- git-annex assistant webapp, common imports + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Assistant.WebApp.Common (module X) where + +import Assistant.Common as X hiding (liftAnnex) +import Assistant.WebApp as X +import Assistant.WebApp.Page as X +import Assistant.WebApp.Form as X +import Assistant.WebApp.Types as X +import Utility.Yesod as X + +import Data.Text as X (Text) +import Yesod as X hiding (textField, passwordField, insertBy, replace, joinPath, deleteBy, delete, insert, Key, Option) diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs new file mode 100644 index 0000000000..21289206cc --- /dev/null +++ b/Assistant/WebApp/Configurators.hs @@ -0,0 +1,202 @@ +{- git-annex assistant webapp configurators + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes, CPP #-} + +module Assistant.WebApp.Configurators where + +import Assistant.WebApp.Common +import Assistant.DaemonStatus +import Assistant.WebApp.Notifications +import Assistant.WebApp.Utility +import Assistant.WebApp.Configurators.Local +import qualified Annex +import qualified Remote +import qualified Types.Remote as Remote +import Annex.UUID (getUUID) +import Logs.Remote +import Logs.Trust +import qualified Git +#ifdef WITH_XMPP +import Assistant.XMPP.Client +#endif + +import qualified Data.Map as M + +{- The main configuration screen. -} +getConfigurationR :: Handler RepHtml +getConfigurationR = ifM (inFirstRun) + ( getFirstRepositoryR + , page "Configuration" (Just Configuration) $ do +#ifdef WITH_XMPP + xmppconfigured <- lift $ liftAnnex $ isJust <$> getXMPPCreds +#else + let xmppconfigured = False +#endif + $(widgetFile "configurators/main") + ) + +{- An intro message, list of repositories, and nudge to make more. -} +introDisplay :: Text -> Widget +introDisplay ident = do + webapp <- lift getYesod + repolist <- lift $ repoList $ RepoSelector + { onlyCloud = False + , onlyConfigured = True + , includeHere = False + } + let n = length repolist + let numrepos = show n + $(widgetFile "configurators/intro") + lift $ modifyWebAppState $ \s -> s { showIntro = False } + +makeMiscRepositories :: Widget +makeMiscRepositories = $(widgetFile "configurators/repositories/misc") + +makeCloudRepositories :: Widget +makeCloudRepositories = $(widgetFile "configurators/repositories/cloud") + +{- Lists known repositories, followed by options to add more. -} +getRepositoriesR :: Handler RepHtml +getRepositoriesR = page "Repositories" (Just Configuration) $ do + let repolist = repoListDisplay $ RepoSelector + { onlyCloud = False + , onlyConfigured = False + , includeHere = True + } + $(widgetFile "configurators/repositories") + +data Actions + = DisabledRepoActions + { setupRepoLink :: Route WebApp } + | SyncingRepoActions + { setupRepoLink :: Route WebApp + , syncToggleLink :: Route WebApp + } + | NotSyncingRepoActions + { setupRepoLink :: Route WebApp + , syncToggleLink :: Route WebApp + } + +mkSyncingRepoActions :: UUID -> Actions +mkSyncingRepoActions u = SyncingRepoActions + { setupRepoLink = EditRepositoryR u + , syncToggleLink = DisableSyncR u + } + +mkNotSyncingRepoActions :: UUID -> Actions +mkNotSyncingRepoActions u = NotSyncingRepoActions + { setupRepoLink = EditRepositoryR u + , syncToggleLink = EnableSyncR u + } + +needsEnabled :: Actions -> Bool +needsEnabled (DisabledRepoActions _) = True +needsEnabled _ = False + +notSyncing :: Actions -> Bool +notSyncing (SyncingRepoActions _ _) = False +notSyncing _ = True + +{- Called by client to get a list of repos, that refreshes + - when new repos as added. + - + - Returns a div, which will be inserted into the calling page. + -} +getRepoListR :: RepoListNotificationId -> Handler RepHtml +getRepoListR (RepoListNotificationId nid reposelector) = do + waitNotifier getRepoListBroadcaster nid + p <- widgetToPageContent $ repoListDisplay reposelector + hamletToRepHtml $ [hamlet|^{pageBody p}|] + +repoListDisplay :: RepoSelector -> Widget +repoListDisplay reposelector = do + autoUpdate ident (NotifierRepoListR reposelector) (10 :: Int) (10 :: Int) + + repolist <- lift $ repoList reposelector + + $(widgetFile "configurators/repositories/list") + + where + ident = "repolist" + +type RepoList = [(String, String, Actions)] + +{- A numbered list of known repositories, + - with actions that can be taken on them. -} +repoList :: RepoSelector -> Handler RepoList +repoList reposelector + | onlyConfigured reposelector = list =<< configured + | otherwise = list =<< (++) <$> configured <*> rest + where + configured = do + rs <- filter wantedrepo . syncRemotes + <$> liftAssistant getDaemonStatus + liftAnnex $ do + let us = map Remote.uuid rs + let l = zip us $ map mkSyncingRepoActions us + if includeHere reposelector + then do + u <- getUUID + autocommit <- annexAutoCommit <$> Annex.getGitConfig + let hereactions = if autocommit + then mkSyncingRepoActions u + else mkNotSyncingRepoActions u + let here = (u, hereactions) + return $ here : l + else return l + rest = liftAnnex $ do + m <- readRemoteLog + unconfigured <- map snd . catMaybes . filter wantedremote + . map (findinfo m) + <$> (trustExclude DeadTrusted $ M.keys m) + unsyncable <- map Remote.uuid . filter wantedrepo . + filter (not . remoteAnnexSync . Remote.gitconfig) + <$> Remote.enabledRemoteList + return $ zip unsyncable (map mkNotSyncingRepoActions unsyncable) ++ unconfigured + wantedrepo r + | Remote.readonly r = False + | onlyCloud reposelector = Git.repoIsUrl (Remote.repo r) && not (isXMPPRemote r) + | otherwise = True + wantedremote Nothing = False + wantedremote (Just (iscloud, _)) + | onlyCloud reposelector = iscloud + | otherwise = True + findinfo m u = case M.lookup u m of + Nothing -> Nothing + Just c -> case M.lookup "type" c of + Just "rsync" -> val True EnableRsyncR + Just "directory" -> val False EnableDirectoryR +#ifdef WITH_S3 + Just "S3" -> val True EnableS3R +#endif + Just "glacier" -> val True EnableGlacierR +#ifdef WITH_WEBDAV + Just "webdav" -> val True EnableWebDAVR +#endif + _ -> Nothing + where + val iscloud r = Just (iscloud, (u, DisabledRepoActions $ r u)) + list l = liftAnnex $ do + let l' = nubBy (\x y -> fst x == fst y) l + zip3 + <$> pure counter + <*> Remote.prettyListUUIDs (map fst l') + <*> pure (map snd l') + counter = map show ([1..] :: [Int]) + +getEnableSyncR :: UUID -> Handler () +getEnableSyncR = flipSync True + +getDisableSyncR :: UUID -> Handler () +getDisableSyncR = flipSync False + +flipSync :: Bool -> UUID -> Handler () +flipSync enable uuid = do + mremote <- liftAnnex $ Remote.remoteFromUUID uuid + changeSyncable mremote enable + redirect RepositoriesR diff --git a/Assistant/WebApp/Configurators/AWS.hs b/Assistant/WebApp/Configurators/AWS.hs new file mode 100644 index 0000000000..b70e70c940 --- /dev/null +++ b/Assistant/WebApp/Configurators/AWS.hs @@ -0,0 +1,189 @@ +{- git-annex assistant webapp configurators for Amazon AWS services + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, FlexibleContexts, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Configurators.AWS where + +import Assistant.WebApp.Common +import Assistant.MakeRemote +import Assistant.Sync +#ifdef WITH_S3 +import qualified Remote.S3 as S3 +#endif +import qualified Remote.Glacier as Glacier +import qualified Remote.Helper.AWS as AWS +import Logs.Remote +import qualified Remote +import Types.Remote (RemoteConfig) +import Types.StandardGroups +import Logs.PreferredContent + +import qualified Data.Text as T +import qualified Data.Map as M +import Data.Char + +awsConfigurator :: Widget -> Handler RepHtml +awsConfigurator = page "Add an Amazon repository" (Just Configuration) + +glacierConfigurator :: Widget -> Handler RepHtml +glacierConfigurator a = do + ifM (liftIO $ inPath "glacier") + ( awsConfigurator a + , awsConfigurator needglaciercli + ) + where + needglaciercli = $(widgetFile "configurators/needglaciercli") + +data StorageClass = StandardRedundancy | ReducedRedundancy + deriving (Eq, Enum, Bounded) + +instance Show StorageClass where + show StandardRedundancy = "STANDARD" + show ReducedRedundancy = "REDUCED_REDUNDANCY" + +data AWSInput = AWSInput + { accessKeyID :: Text + , secretAccessKey :: Text + , datacenter :: Text + -- Only used for S3, not Glacier. + , storageClass :: StorageClass + , repoName :: Text + , enableEncryption :: EnableEncryption + } + +data AWSCreds = AWSCreds Text Text + +extractCreds :: AWSInput -> AWSCreds +extractCreds i = AWSCreds (accessKeyID i) (secretAccessKey i) + +s3InputAForm :: AForm WebApp WebApp AWSInput +s3InputAForm = AWSInput + <$> accessKeyIDField + <*> secretAccessKeyField + <*> datacenterField AWS.S3 + <*> areq (selectFieldList storageclasses) "Storage class" (Just StandardRedundancy) + <*> areq textField "Repository name" (Just "S3") + <*> enableEncryptionField + where + storageclasses :: [(Text, StorageClass)] + storageclasses = + [ ("Standard redundancy", StandardRedundancy) + , ("Reduced redundancy (costs less)", ReducedRedundancy) + ] + +glacierInputAForm :: AForm WebApp WebApp AWSInput +glacierInputAForm = AWSInput + <$> accessKeyIDField + <*> secretAccessKeyField + <*> datacenterField AWS.Glacier + <*> pure StandardRedundancy + <*> areq textField "Repository name" (Just "glacier") + <*> enableEncryptionField + +awsCredsAForm :: AForm WebApp WebApp AWSCreds +awsCredsAForm = AWSCreds + <$> accessKeyIDField + <*> secretAccessKeyField + +accessKeyIDField :: AForm WebApp WebApp Text +accessKeyIDField = areq (textField `withNote` help) "Access Key ID" Nothing + where + help = [whamlet| + + Get Amazon access keys +|] + +secretAccessKeyField :: AForm WebApp WebApp Text +secretAccessKeyField = areq passwordField "Secret Access Key" Nothing + +datacenterField :: AWS.Service -> AForm WebApp WebApp Text +datacenterField service = areq (selectFieldList list) "Datacenter" defregion + where + list = M.toList $ AWS.regionMap service + defregion = Just $ AWS.defaultRegion service + +getAddS3R :: Handler RepHtml +#ifdef WITH_S3 +getAddS3R = awsConfigurator $ do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap s3InputAForm + case result of + FormSuccess input -> lift $ do + let name = T.unpack $ repoName input + makeAWSRemote S3.remote (extractCreds input) name setgroup $ M.fromList + [ configureEncryption $ enableEncryption input + , ("type", "S3") + , ("datacenter", T.unpack $ datacenter input) + , ("storageclass", show $ storageClass input) + ] + _ -> $(widgetFile "configurators/adds3") + where + setgroup r = liftAnnex $ + setStandardGroup (Remote.uuid r) TransferGroup +#else +getAddS3R = error "S3 not supported by this build" +#endif + +getAddGlacierR :: Handler RepHtml +getAddGlacierR = glacierConfigurator $ do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap glacierInputAForm + case result of + FormSuccess input -> lift $ do + let name = T.unpack $ repoName input + makeAWSRemote Glacier.remote (extractCreds input) name setgroup $ M.fromList + [ configureEncryption $ enableEncryption input + , ("type", "glacier") + , ("datacenter", T.unpack $ datacenter input) + ] + _ -> $(widgetFile "configurators/addglacier") + where + setgroup r = liftAnnex $ + setStandardGroup (Remote.uuid r) SmallArchiveGroup + +getEnableS3R :: UUID -> Handler RepHtml +#ifdef WITH_S3 +getEnableS3R = awsConfigurator . enableAWSRemote S3.remote +#else +getEnableS3R _ = error "S3 not supported by this build" +#endif + +getEnableGlacierR :: UUID -> Handler RepHtml +getEnableGlacierR = glacierConfigurator . enableAWSRemote Glacier.remote + +enableAWSRemote :: RemoteType -> UUID -> Widget +enableAWSRemote remotetype uuid = do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap awsCredsAForm + case result of + FormSuccess creds -> lift $ do + m <- liftAnnex readRemoteLog + let name = fromJust $ M.lookup "name" $ + fromJust $ M.lookup uuid m + makeAWSRemote remotetype creds name (const noop) M.empty + _ -> do + description <- lift $ liftAnnex $ + T.pack . concat <$> Remote.prettyListUUIDs [uuid] + $(widgetFile "configurators/enableaws") + +makeAWSRemote :: RemoteType -> AWSCreds -> String -> (Remote -> Handler ()) -> RemoteConfig -> Handler () +makeAWSRemote remotetype (AWSCreds ak sk) name setup config = do + remotename <- liftAnnex $ fromRepo $ uniqueRemoteName name 0 + liftIO $ AWS.setCredsEnv (T.unpack ak, T.unpack sk) + r <- liftAnnex $ addRemote $ do + makeSpecialRemote hostname remotetype config + return remotename + setup r + liftAssistant $ syncNewRemote r + redirect $ EditNewCloudRepositoryR $ Remote.uuid r + where + {- AWS services use the remote name as the basis for a host + - name, so filter it to contain valid characters. -} + hostname = case filter isAlphaNum name of + [] -> "aws" + n -> n diff --git a/Assistant/WebApp/Configurators/Edit.hs b/Assistant/WebApp/Configurators/Edit.hs new file mode 100644 index 0000000000..32e5158ed2 --- /dev/null +++ b/Assistant/WebApp/Configurators/Edit.hs @@ -0,0 +1,150 @@ +{- git-annex assistant webapp configurator for editing existing repos + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Configurators.Edit where + +import Assistant.WebApp.Common +import Assistant.WebApp.Utility +import Assistant.DaemonStatus +import Assistant.MakeRemote (uniqueRemoteName) +import Assistant.WebApp.Configurators.XMPP (xmppNeeded) +import qualified Remote +import qualified Types.Remote as Remote +import qualified Remote.List as Remote +import Logs.UUID +import Logs.Group +import Logs.PreferredContent +import Types.StandardGroups +import qualified Git +import qualified Git.Command +import qualified Git.Config +import qualified Annex +import Git.Remote + +import qualified Data.Text as T +import qualified Data.Map as M +import qualified Data.Set as S + +data RepoGroup = RepoGroupCustom String | RepoGroupStandard StandardGroup + deriving (Show, Eq) + +data RepoConfig = RepoConfig + { repoName :: Text + , repoDescription :: Maybe Text + , repoGroup :: RepoGroup + , repoSyncable :: Bool + } + deriving (Show) + +getRepoConfig :: UUID -> Maybe Remote -> Annex RepoConfig +getRepoConfig uuid mremote = RepoConfig + <$> pure (T.pack $ maybe "here" Remote.name mremote) + <*> (maybe Nothing (Just . T.pack) . M.lookup uuid <$> uuidMap) + <*> getrepogroup + <*> getsyncing + where + getrepogroup = do + groups <- lookupGroups uuid + return $ + maybe (RepoGroupCustom $ unwords $ S.toList groups) RepoGroupStandard + (getStandardGroup groups) + getsyncing = case mremote of + Just r -> return $ remoteAnnexSync $ Remote.gitconfig r + Nothing -> annexAutoCommit <$> Annex.getGitConfig + +setRepoConfig :: UUID -> Maybe Remote -> RepoConfig -> RepoConfig -> Handler () +setRepoConfig uuid mremote oldc newc = do + when (repoDescription oldc /= repoDescription newc) $ liftAnnex $ do + maybe noop (describeUUID uuid . T.unpack) (repoDescription newc) + void uuidMapLoad + when (repoGroup oldc /= repoGroup newc) $ liftAnnex $ + case repoGroup newc of + RepoGroupStandard g -> setStandardGroup uuid g + RepoGroupCustom s -> groupSet uuid $ S.fromList $ words s + when (repoSyncable oldc /= repoSyncable newc) $ + changeSyncable mremote (repoSyncable newc) + when (isJust mremote && makeLegalName (T.unpack $ repoName oldc) /= makeLegalName (T.unpack $ repoName newc)) $ do + liftAnnex $ do + name <- fromRepo $ uniqueRemoteName (T.unpack $ repoName newc) 0 + {- git remote rename expects there to be a + - remote..fetch, and exits nonzero if + - there's not. Special remotes don't normally + - have that, and don't use it. Temporarily add + - it if it's missing. -} + let remotefetch = "remote." ++ T.unpack (repoName oldc) ++ ".fetch" + needfetch <- isNothing <$> fromRepo (Git.Config.getMaybe remotefetch) + when needfetch $ + inRepo $ Git.Command.run + [Param "config", Param remotefetch, Param ""] + inRepo $ Git.Command.run + [ Param "remote" + , Param "rename" + , Param $ T.unpack $ repoName oldc + , Param name + ] + void $ Remote.remoteListRefresh + liftAssistant updateSyncRemotes + +editRepositoryAForm :: RepoConfig -> AForm WebApp WebApp RepoConfig +editRepositoryAForm def = RepoConfig + <$> areq textField "Name" (Just $ repoName def) + <*> aopt textField "Description" (Just $ repoDescription def) + <*> areq (selectFieldList groups `withNote` help) "Repository group" (Just $ repoGroup def) + <*> areq checkBoxField "Syncing enabled" (Just $ repoSyncable def) + where + groups = customgroups ++ standardgroups + standardgroups :: [(Text, RepoGroup)] + standardgroups = map (\g -> (T.pack $ descStandardGroup g , RepoGroupStandard g)) + [minBound :: StandardGroup .. maxBound :: StandardGroup] + customgroups :: [(Text, RepoGroup)] + customgroups = case repoGroup def of + RepoGroupCustom s -> [(T.pack s, RepoGroupCustom s)] + _ -> [] + help = [whamlet|What's this?|] + +getEditRepositoryR :: UUID -> Handler RepHtml +getEditRepositoryR = editForm False + +getEditNewRepositoryR :: UUID -> Handler RepHtml +getEditNewRepositoryR = editForm True + +getEditNewCloudRepositoryR :: UUID -> Handler RepHtml +getEditNewCloudRepositoryR uuid = xmppNeeded >> editForm True uuid + +editForm :: Bool -> UUID -> Handler RepHtml +editForm new uuid = page "Configure repository" (Just Configuration) $ do + mremote <- lift $ liftAnnex $ Remote.remoteFromUUID uuid + curr <- lift $ liftAnnex $ getRepoConfig uuid mremote + lift $ checkarchivedirectory curr + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap $ editRepositoryAForm curr + case result of + FormSuccess input -> lift $ do + checkarchivedirectory input + setRepoConfig uuid mremote curr input + redirect RepositoriesR + _ -> showform form enctype curr + where + showform form enctype curr = do + let istransfer = repoGroup curr == RepoGroupStandard TransferGroup + $(widgetFile "configurators/editrepository") + + {- Makes a toplevel archive directory, so the user can get on with + - using it. This is done both when displaying the form, as well + - as after it's posted, because the user may not post the form, + - but may see that the repo is set up to use the archive + - directory. -} + checkarchivedirectory cfg + | repoGroup cfg == RepoGroupStandard SmallArchiveGroup = go + | repoGroup cfg == RepoGroupStandard FullArchiveGroup = go + | otherwise = noop + where + go = liftAnnex $ inRepo $ \g -> + createDirectoryIfMissing True $ + Git.repoPath g "archive" diff --git a/Assistant/WebApp/Configurators/Local.hs b/Assistant/WebApp/Configurators/Local.hs new file mode 100644 index 0000000000..d222331055 --- /dev/null +++ b/Assistant/WebApp/Configurators/Local.hs @@ -0,0 +1,334 @@ +{- git-annex assistant webapp configurators for making local repositories + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +#if defined VERSION_yesod_form +#if ! MIN_VERSION_yesod_form(1,2,0) +#define WITH_OLD_YESOD +#endif +#endif + +module Assistant.WebApp.Configurators.Local where + +import Assistant.WebApp.Common +import Assistant.WebApp.Utility +import Assistant.WebApp.OtherRepos +import Assistant.MakeRemote +import Init +import qualified Git +import qualified Git.Construct +import qualified Git.Config +import qualified Git.Command +import qualified Annex +import Locations.UserConfig +import Utility.FreeDesktop +import Utility.Mounts +import Utility.DiskFree +import Utility.DataUnits +import Utility.Network +import Remote (prettyListUUIDs) +import Annex.UUID +import Types.StandardGroups +import Logs.PreferredContent +import Utility.UserInfo +import Config + +import qualified Data.Text as T +import Data.Char +import System.Posix.Directory +import qualified Control.Exception as E + +data RepositoryPath = RepositoryPath Text + deriving Show + +{- Custom field display for a RepositoryPath, with an icon etc. + - + - Validates that the path entered is not empty, and is a safe value + - to use as a repository. -} +repositoryPathField :: forall sub. Bool -> Field sub WebApp Text +repositoryPathField autofocus = Field +#ifdef WITH_OLD_YESOD + { fieldParse = parse +#else + { fieldParse = \l _ -> parse l +#endif + , fieldView = view +#ifndef WITH_OLD_YESOD + , fieldEnctype = UrlEncoded +#endif + } + where + view idAttr nameAttr attrs val isReq = + [whamlet||] + + parse [path] + | T.null path = nopath + | otherwise = liftIO $ checkRepositoryPath path + parse [] = return $ Right Nothing + parse _ = nopath + + nopath = return $ Left "Enter a location for the repository" + +{- As well as checking the path for a lot of silly things, tilde is + - expanded in the returned path. -} +checkRepositoryPath :: Text -> IO (Either (SomeMessage WebApp) (Maybe Text)) +checkRepositoryPath p = do + home <- myHomeDir + let basepath = expandTilde home $ T.unpack p + path <- absPath basepath + let parent = parentDir path + problems <- catMaybes <$> mapM runcheck + [ (return $ path == "/", "Enter the full path to use for the repository.") + , (return $ all isSpace basepath, "A blank path? Seems unlikely.") + , (doesFileExist path, "A file already exists with that name.") + , (return $ path == home, "Sorry, using git-annex for your whole home directory is not currently supported.") + , (not <$> doesDirectoryExist parent, "Parent directory does not exist.") + , (not <$> canWrite path, "Cannot write a repository there.") + ] + return $ + case headMaybe problems of + Nothing -> Right $ Just $ T.pack basepath + Just prob -> Left prob + where + runcheck (chk, msg) = ifM (chk) ( return $ Just msg, return Nothing ) + expandTilde home ('~':'/':path) = home path + expandTilde _ path = path + +{- On first run, if run in the home directory, default to putting it in + - ~/Desktop/annex, when a Desktop directory exists, and ~/annex otherwise. + - + - If run in another directory, that the user can write to, + - the user probably wants to put it there. -} +defaultRepositoryPath :: Bool -> IO FilePath +defaultRepositoryPath firstrun = do + cwd <- liftIO $ getCurrentDirectory + home <- myHomeDir + if home == cwd && firstrun + then inhome + else ifM (canWrite cwd) ( return cwd, inhome ) + where + inhome = do + desktop <- userDesktopDir + ifM (doesDirectoryExist desktop) + ( relHome $ desktop gitAnnexAssistantDefaultDir + , return $ "~" gitAnnexAssistantDefaultDir + ) + +newRepositoryForm :: FilePath -> Form RepositoryPath +newRepositoryForm defpath msg = do + (pathRes, pathView) <- mreq (repositoryPathField True) "" + (Just $ T.pack $ addTrailingPathSeparator defpath) + let (err, errmsg) = case pathRes of + FormMissing -> (False, "") + FormFailure l -> (True, concat $ map T.unpack l) + FormSuccess _ -> (False, "") + let form = do + webAppFormAuthToken + $(widgetFile "configurators/newrepository/form") + return (RepositoryPath <$> pathRes, form) + +{- Making the first repository, when starting the webapp for the first time. -} +getFirstRepositoryR :: Handler RepHtml +getFirstRepositoryR = page "Getting started" (Just Configuration) $ do + path <- liftIO . defaultRepositoryPath =<< lift inFirstRun + ((res, form), enctype) <- lift $ runFormGet $ newRepositoryForm path + case res of + FormSuccess (RepositoryPath p) -> lift $ + startFullAssistant $ T.unpack p + _ -> $(widgetFile "configurators/newrepository/first") + +{- Adding a new local repository, which may be entirely separate, or may + - be connected to the current repository. -} +getNewRepositoryR :: Handler RepHtml +getNewRepositoryR = page "Add another repository" (Just Configuration) $ do + home <- liftIO myHomeDir + ((res, form), enctype) <- lift $ runFormGet $ newRepositoryForm home + case res of + FormSuccess (RepositoryPath p) -> do + let path = T.unpack p + liftIO $ makeRepo path False + u <- liftIO $ initRepo True path Nothing + lift $ liftAnnexOr () $ setStandardGroup u ClientGroup + liftIO $ addAutoStartFile path + liftIO $ startAssistant path + askcombine u path + _ -> $(widgetFile "configurators/newrepository") + where + askcombine newrepouuid newrepopath = do + newrepo <- liftIO $ relHome newrepopath + mainrepo <- fromJust . relDir <$> lift getYesod + $(widgetFile "configurators/newrepository/combine") + +getCombineRepositoryR :: FilePathAndUUID -> Handler RepHtml +getCombineRepositoryR (FilePathAndUUID newrepopath newrepouuid) = do + r <- combineRepos newrepopath remotename + syncRemote r + redirect $ EditRepositoryR newrepouuid + where + remotename = takeFileName newrepopath + +data RemovableDrive = RemovableDrive + { diskFree :: Maybe Integer + , mountPoint :: Text + } + deriving (Show, Eq, Ord) + +selectDriveForm :: [RemovableDrive] -> Maybe RemovableDrive -> Form RemovableDrive +selectDriveForm drives def = renderBootstrap $ RemovableDrive + <$> pure Nothing + <*> areq (selectFieldList pairs) "Select drive:" (mountPoint <$> def) + where + pairs = zip (map describe drives) (map mountPoint drives) + describe drive = case diskFree drive of + Nothing -> mountPoint drive + Just free -> + let sz = roughSize storageUnits True free + in T.unwords + [ mountPoint drive + , T.concat ["(", T.pack sz] + , "free)" + ] + +{- Adding a removable drive. -} +getAddDriveR :: Handler RepHtml +getAddDriveR = page "Add a removable drive" (Just Configuration) $ do + removabledrives <- liftIO $ driveList + writabledrives <- liftIO $ + filterM (canWrite . T.unpack . mountPoint) removabledrives + ((res, form), enctype) <- lift $ runFormGet $ + selectDriveForm (sort writabledrives) Nothing + case res of + FormSuccess (RemovableDrive { mountPoint = d }) -> lift $ + make (T.unpack d) >>= redirect . EditNewRepositoryR + _ -> $(widgetFile "configurators/adddrive") + where + make mountpoint = do + liftIO $ makerepo dir + u <- liftIO $ initRepo False dir $ Just remotename + r <- combineRepos dir remotename + liftAnnex $ setStandardGroup u TransferGroup + syncRemote r + return u + where + dir = mountpoint gitAnnexAssistantDefaultDir + remotename = takeFileName mountpoint + {- The repo may already exist, when adding removable media + - that has already been used elsewhere. -} + makerepo dir = liftIO $ do + r <- E.try (inDir dir $ getUUID) :: IO (Either E.SomeException UUID) + case r of + Right u | u /= NoUUID -> noop + _ -> do + createDirectoryIfMissing True dir + makeRepo dir True + +{- Each repository is made a remote of the other. + - Next call syncRemote to get them in sync. -} +combineRepos :: FilePath -> String -> Handler Remote +combineRepos dir name = liftAnnex $ do + hostname <- maybe "host" id <$> liftIO getHostname + hostlocation <- fromRepo Git.repoLocation + liftIO $ inDir dir $ void $ makeGitRemote hostname hostlocation + addRemote $ makeGitRemote name dir + +getEnableDirectoryR :: UUID -> Handler RepHtml +getEnableDirectoryR uuid = page "Enable a repository" (Just Configuration) $ do + description <- lift $ liftAnnex $ + T.pack . concat <$> prettyListUUIDs [uuid] + $(widgetFile "configurators/enabledirectory") + +{- List of removable drives. -} +driveList :: IO [RemovableDrive] +driveList = mapM (gen . mnt_dir) =<< filter sane <$> getMounts + where + gen dir = RemovableDrive + <$> getDiskFree dir + <*> pure (T.pack dir) + -- filter out some things that are surely not removable drives + sane Mntent { mnt_dir = dir, mnt_fsname = dev } + {- We want real disks like /dev/foo, not + - dummy mount points like proc or tmpfs or + - gvfs-fuse-daemon. -} + | not ('/' `elem` dev) = False + {- Just in case: These mount points are surely not + - removable disks. -} + | dir == "/" = False + | dir == "/tmp" = False + | dir == "/run/shm" = False + | dir == "/run/lock" = False + | otherwise = True + +{- Bootstraps from first run mode to a fully running assistant in a + - repository, by running the postFirstRun callback, which returns the + - url to the new webapp. -} +startFullAssistant :: FilePath -> Handler () +startFullAssistant path = do + webapp <- getYesod + url <- liftIO $ do + makeRepo path False + u <- initRepo True path Nothing + inDir path $ + setStandardGroup u ClientGroup + addAutoStartFile path + changeWorkingDirectory path + fromJust $ postFirstRun webapp + redirect $ T.pack url + +{- Makes a new git repository. -} +makeRepo :: FilePath -> Bool -> IO () +makeRepo path bare = do + (transcript, ok) <- processTranscript "git" (toCommand params) Nothing + unless ok $ + error $ "git init failed!\nOutput:\n" ++ transcript + where + baseparams = [Param "init", Param "--quiet"] + params + | bare = baseparams ++ [Param "--bare", File path] + | otherwise = baseparams ++ [File path] + +{- Runs an action in the git-annex repository in the specified directory. -} +inDir :: FilePath -> Annex a -> IO a +inDir dir a = do + state <- Annex.new =<< Git.Config.read =<< Git.Construct.fromPath dir + Annex.eval state a + +initRepo :: Bool -> FilePath -> Maybe String -> IO UUID +initRepo primary_assistant_repo dir desc = inDir dir $ do + {- Initialize a git-annex repository in a directory with a description. -} + unlessM isInitialized $ + initialize desc + {- Initialize the master branch, so things that expect + - to have it will work, before any files are added. -} + unlessM (Git.Config.isBare <$> gitRepo) $ + void $ inRepo $ Git.Command.runBool + [ Param "commit" + , Param "--quiet" + , Param "--allow-empty" + , Param "-m" + , Param "created repository" + ] + {- Repositories directly managed by the assistant use direct mode. + - + - Automatic gc is disabled, as it can be slow. Insted, gc is done + - once a day. + -} + when primary_assistant_repo $ do + setDirect True + inRepo $ Git.Command.run + [Param "config", Param "gc.auto", Param "0"] + getUUID + +{- Checks if the user can write to a directory. + - + - The directory may be in the process of being created; if so + - the parent directory is checked instead. -} +canWrite :: FilePath -> IO Bool +canWrite dir = do + tocheck <- ifM (doesDirectoryExist dir) + (return dir, return $ parentDir dir) + catchBoolIO $ fileAccess tocheck False True False diff --git a/Assistant/WebApp/Configurators/Pairing.hs b/Assistant/WebApp/Configurators/Pairing.hs new file mode 100644 index 0000000000..db2bb429af --- /dev/null +++ b/Assistant/WebApp/Configurators/Pairing.hs @@ -0,0 +1,292 @@ +{- git-annex assistant webapp configurator for pairing + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} +{-# LANGUAGE CPP #-} + +module Assistant.WebApp.Configurators.Pairing where + +import Assistant.Pairing +import Assistant.WebApp.Common +import Assistant.Types.Buddies +#ifdef WITH_PAIRING +import Assistant.Pairing.Network +import Assistant.Pairing.MakeRemote +import Assistant.Ssh +import Assistant.Alert +import Assistant.DaemonStatus +import Utility.Verifiable +import Utility.Network +import Annex.UUID +#endif +#ifdef WITH_XMPP +import Assistant.XMPP +import Assistant.XMPP.Client +import Assistant.XMPP.Buddies +import Assistant.XMPP.Git +import Network.Protocol.XMPP +import Assistant.Types.NetMessager +import Assistant.NetMessager +import Assistant.WebApp.Configurators +import Assistant.WebApp.Configurators.XMPP +#endif +import Utility.UserInfo +import Git + +#ifdef WITH_PAIRING +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Data.ByteString.Lazy as B +import Data.Char +import qualified Control.Exception as E +import Control.Concurrent +#endif +#ifdef WITH_XMPP +import qualified Data.Set as S +#endif + +getStartXMPPPairR :: Handler RepHtml +#ifdef WITH_XMPP +getStartXMPPPairR = ifM (isJust <$> liftAnnex getXMPPCreds) + ( do + {- Ask buddies to send presence info, to get + - the buddy list populated. -} + liftAssistant $ sendNetMessage QueryPresence + pairPage $ + $(widgetFile "configurators/pairing/xmpp/prompt") + , redirect XMPPR -- go get XMPP configured, then come back + ) +#else +getStartXMPPPairR = noXMPPPairing + +noXMPPPairing :: Handler RepHtml +noXMPPPairing = noPairing "XMPP" +#endif + +{- Does pairing with an XMPP buddy, or with other clients sharing an + - XMPP account. -} +getRunningXMPPPairR :: BuddyKey -> Handler RepHtml +#ifdef WITH_XMPP +getRunningXMPPPairR bid = do + buddy <- liftAssistant $ getBuddy bid <<~ buddyList + go $ S.toList . buddyAssistants <$> buddy + where + go (Just (clients@((Client exemplar):_))) = do + creds <- liftAnnex getXMPPCreds + let ourjid = fromJust $ parseJID =<< xmppJID <$> creds + let samejid = baseJID ourjid == baseJID exemplar + u <- liftAnnex getUUID + liftAssistant $ forM_ clients $ \(Client c) -> sendNetMessage $ + PairingNotification PairReq (formatJID c) u + xmppPairEnd True $ if samejid then Nothing else Just exemplar + -- A buddy could have logged out, or the XMPP client restarted, + -- and there be no clients to message; handle unforseen by going back. + go _ = redirect StartXMPPPairR +#else +getRunningXMPPPairR _ = noXMPPPairing +#endif + +{- Starts local pairing. -} +getStartLocalPairR :: Handler RepHtml +#ifdef WITH_PAIRING +getStartLocalPairR = promptSecret Nothing $ + startLocalPairing PairReq noop pairingAlert Nothing +#else +getStartLocalPairR = noLocalPairing + +noLocalPairing :: Handler RepHtml +noLocalPairing = noPairing "local" +#endif + +{- Runs on the system that responds to a local pair request; sets up the ssh + - authorized key first so that the originating host can immediately sync + - with us. -} +getFinishLocalPairR :: PairMsg -> Handler RepHtml +#ifdef WITH_PAIRING +getFinishLocalPairR msg = promptSecret (Just msg) $ \_ secret -> do + repodir <- lift $ repoPath <$> liftAnnex gitRepo + liftIO $ setup repodir + startLocalPairing PairAck (cleanup repodir) alert uuid "" secret + where + alert = pairRequestAcknowledgedAlert (pairRepo msg) . Just + setup repodir = setupAuthorizedKeys msg repodir + cleanup repodir = removeAuthorizedKeys False repodir $ + remoteSshPubKey $ pairMsgData msg + uuid = Just $ pairUUID $ pairMsgData msg +#else +getFinishLocalPairR _ = noLocalPairing +#endif + +getConfirmXMPPPairR :: PairKey -> Handler RepHtml +#ifdef WITH_XMPP +getConfirmXMPPPairR pairkey@(PairKey _ t) = case parseJID t of + Nothing -> error "bad JID" + Just theirjid -> pairPage $ do + let name = buddyName theirjid + $(widgetFile "configurators/pairing/xmpp/confirm") +#else +getConfirmXMPPPairR _ = noXMPPPairing +#endif + +getFinishXMPPPairR :: PairKey -> Handler RepHtml +#ifdef WITH_XMPP +getFinishXMPPPairR (PairKey theiruuid t) = case parseJID t of + Nothing -> error "bad JID" + Just theirjid -> do + selfuuid <- liftAnnex getUUID + liftAssistant $ do + sendNetMessage $ + PairingNotification PairAck (formatJID theirjid) selfuuid + finishXMPPPairing theirjid theiruuid + xmppPairEnd False $ Just theirjid +#else +getFinishXMPPPairR _ = noXMPPPairing +#endif + +#ifdef WITH_XMPP +xmppPairEnd :: Bool -> Maybe JID -> Handler RepHtml +xmppPairEnd inprogress theirjid = pairPage $ do + let friend = buddyName <$> theirjid + let cloudrepolist = repoListDisplay $ RepoSelector + { onlyCloud = True + , onlyConfigured = False + , includeHere = False + } + $(widgetFile "configurators/pairing/xmpp/end") +#endif + +getRunningLocalPairR :: SecretReminder -> Handler RepHtml +#ifdef WITH_PAIRING +getRunningLocalPairR s = pairPage $ do + let secret = fromSecretReminder s + $(widgetFile "configurators/pairing/local/inprogress") +#else +getRunningLocalPairR _ = noLocalPairing +#endif + +#ifdef WITH_PAIRING + +{- Starts local pairing, at either the PairReq (initiating host) or + - PairAck (responding host) stage. + - + - Displays an alert, and starts a thread sending the pairing message, + - which will continue running until the other host responds, or until + - canceled by the user. If canceled by the user, runs the oncancel action. + - + - Redirects to the pairing in progress page. + -} +startLocalPairing :: PairStage -> IO () -> (AlertButton -> Alert) -> Maybe UUID -> Text -> Secret -> Widget +startLocalPairing stage oncancel alert muuid displaysecret secret = do + urlrender <- lift getUrlRender + reldir <- fromJust . relDir <$> lift getYesod + + sendrequests <- lift $ liftAssistant $ asIO2 $ mksendrequests urlrender + {- Generating a ssh key pair can take a while, so do it in the + - background. -} + thread <- lift $ liftAssistant $ asIO $ do + keypair <- liftIO $ genSshKeyPair + pairdata <- liftIO $ PairData + <$> getHostname + <*> myUserName + <*> pure reldir + <*> pure (sshPubKey keypair) + <*> (maybe genUUID return muuid) + let sender = multicastPairMsg Nothing secret pairdata + let pip = PairingInProgress secret Nothing keypair pairdata stage + startSending pip stage $ sendrequests sender + void $ liftIO $ forkIO thread + + lift $ redirect $ RunningLocalPairR $ toSecretReminder displaysecret + where + {- Sends pairing messages until the thread is killed, + - and shows an activity alert while doing it. + - + - The cancel button returns the user to the HomeR. This is + - not ideal, but they have to be sent somewhere, and could + - have been on a page specific to the in-process pairing + - that just stopped, so can't go back there. + -} + mksendrequests urlrender sender _stage = do + tid <- liftIO myThreadId + let selfdestruct = AlertButton + { buttonLabel = "Cancel" + , buttonUrl = urlrender HomeR + , buttonAction = Just $ const $ do + oncancel + killThread tid + } + alertDuring (alert selfdestruct) $ liftIO $ do + _ <- E.try (sender stage) :: IO (Either E.SomeException ()) + return () + +data InputSecret = InputSecret { secretText :: Maybe Text } + +{- If a PairMsg is passed in, ensures that the user enters a secret + - that can validate it. -} +promptSecret :: Maybe PairMsg -> (Text -> Secret -> Widget) -> Handler RepHtml +promptSecret msg cont = pairPage $ do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap $ + InputSecret <$> aopt textField "Secret phrase" Nothing + case result of + FormSuccess v -> do + let rawsecret = fromMaybe "" $ secretText v + let secret = toSecret rawsecret + case msg of + Nothing -> case secretProblem secret of + Nothing -> cont rawsecret secret + Just problem -> + showform form enctype $ Just problem + Just m -> + if verify (fromPairMsg m) secret + then cont rawsecret secret + else showform form enctype $ Just + "That's not the right secret phrase." + _ -> showform form enctype Nothing + where + showform form enctype mproblem = do + let start = isNothing msg + let badphrase = isJust mproblem + let problem = fromMaybe "" mproblem + let (username, hostname) = maybe ("", "") + (\(_, v, a) -> (T.pack $ remoteUserName v, T.pack $ fromMaybe (showAddr a) (remoteHostName v))) + (verifiableVal . fromPairMsg <$> msg) + u <- T.pack <$> liftIO myUserName + let sameusername = username == u + $(widgetFile "configurators/pairing/local/prompt") + +{- This counts unicode characters as more than one character, + - but that's ok; they *do* provide additional entropy. -} +secretProblem :: Secret -> Maybe Text +secretProblem s + | B.null s = Just "The secret phrase cannot be left empty. (Remember that punctuation and white space is ignored.)" + | B.length s < 7 = Just "Enter a longer secret phrase, at least 6 characters, but really, a phrase is best! This is not a password you'll need to enter every day." + | s == toSecret sampleQuote = Just "Speaking of foolishness, don't paste in the example I gave. Enter a different phrase, please!" + | otherwise = Nothing + +toSecret :: Text -> Secret +toSecret s = B.fromChunks [T.encodeUtf8 $ T.toLower $ T.filter isAlphaNum s] + +{- From Dickens -} +sampleQuote :: Text +sampleQuote = T.unwords + [ "It was the best of times," + , "it was the worst of times," + , "it was the age of wisdom," + , "it was the age of foolishness." + ] + +#else + +#endif + +pairPage :: Widget -> Handler RepHtml +pairPage = page "Pairing" (Just Configuration) + +noPairing :: Text -> Handler RepHtml +noPairing pairingtype = pairPage $ + $(widgetFile "configurators/pairing/disabled") diff --git a/Assistant/WebApp/Configurators/Preferences.hs b/Assistant/WebApp/Configurators/Preferences.hs new file mode 100644 index 0000000000..997d9d47bd --- /dev/null +++ b/Assistant/WebApp/Configurators/Preferences.hs @@ -0,0 +1,98 @@ +{- git-annex assistant general preferences + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE QuasiQuotes, TemplateHaskell, OverloadedStrings #-} + +module Assistant.WebApp.Configurators.Preferences ( + getPreferencesR +) where + +import Assistant.WebApp.Common +import qualified Annex +import qualified Git +import Config +import Locations.UserConfig +import Utility.DataUnits + +import qualified Data.Text as T +import System.Log.Logger + +data PrefsForm = PrefsForm + { diskReserve :: Text + , numCopies :: Int + , autoStart :: Bool + , debugEnabled :: Bool + } + +prefsAForm :: PrefsForm -> AForm WebApp WebApp PrefsForm +prefsAForm def = PrefsForm + <$> areq (storageField `withNote` diskreservenote) + "Disk reserve" (Just $ diskReserve def) + <*> areq (positiveIntField `withNote` numcopiesnote) + "Number of copies" (Just $ numCopies def) + <*> areq (checkBoxField `withNote` autostartnote) + "Auto start" (Just $ autoStart def) + <*> areq (checkBoxField `withNote` debugnote) + "Enable debug logging" (Just $ debugEnabled def) + where + diskreservenote = [whamlet|
Avoid downloading files from other repositories when there is too little free disk space.|] + numcopiesnote = [whamlet|
Only drop a file after verifying that other repositories contain this many copies.|] + debugnote = [whamlet|View Log|] + autostartnote = [whamlet|Start the git-annex assistant at boot or on login.|] + + positiveIntField = check isPositive intField + where + isPositive i + | i > 0 = Right i + | otherwise = Left notPositive + notPositive :: Text + notPositive = "This should be 1 or more!" + + storageField = check validStorage textField + where + validStorage t + | T.null t = Right t + | otherwise = case readSize dataUnits $ T.unpack t of + Nothing -> Left badParse + Just _ -> Right t + badParse :: Text + badParse = "Parse error. Expected something like \"100 megabytes\" or \"2 gb\"" + +getPrefs :: Annex PrefsForm +getPrefs = PrefsForm + <$> (T.pack . roughSize storageUnits False . annexDiskReserve <$> Annex.getGitConfig) + <*> (annexNumCopies <$> Annex.getGitConfig) + <*> inAutoStartFile + <*> ((==) <$> (pure $ Just DEBUG) <*> (liftIO $ getLevel <$> getRootLogger)) + +storePrefs :: PrefsForm -> Annex () +storePrefs p = do + setConfig (annexConfig "diskreserve") (T.unpack $ diskReserve p) + setConfig (annexConfig "numcopies") (show $ numCopies p) + unlessM ((==) <$> pure (autoStart p) <*> inAutoStartFile) $ do + here <- fromRepo Git.repoPath + liftIO $ if autoStart p + then addAutoStartFile here + else removeAutoStartFile here + liftIO $ updateGlobalLogger rootLoggerName $ setLevel $ + if debugEnabled p then DEBUG else WARNING + +getPreferencesR :: Handler RepHtml +getPreferencesR = page "Preferences" (Just Configuration) $ do + ((result, form), enctype) <- lift $ do + current <- liftAnnex getPrefs + runFormGet $ renderBootstrap $ prefsAForm current + case result of + FormSuccess new -> lift $ do + liftAnnex $ storePrefs new + redirect ConfigurationR + _ -> $(widgetFile "configurators/preferences") + +inAutoStartFile :: Annex Bool +inAutoStartFile = do + here <- fromRepo Git.repoPath + any (`equalFilePath` here) <$> liftIO readAutoStartFile diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs new file mode 100644 index 0000000000..7df02b3bd3 --- /dev/null +++ b/Assistant/WebApp/Configurators/Ssh.hs @@ -0,0 +1,353 @@ +{- git-annex assistant webapp configurator for ssh-based remotes + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Configurators.Ssh where + +import Assistant.WebApp.Common +import Assistant.Ssh +import Assistant.MakeRemote +import Utility.Rsync (rsyncUrlIsShell) +import Logs.Remote +import Remote +import Logs.PreferredContent +import Types.StandardGroups +import Utility.UserInfo + +import qualified Data.Text as T +import qualified Data.Map as M +import Network.Socket + +sshConfigurator :: Widget -> Handler RepHtml +sshConfigurator = page "Add a remote server" (Just Configuration) + +data SshInput = SshInput + { inputHostname :: Maybe Text + , inputUsername :: Maybe Text + , inputDirectory :: Maybe Text + , inputPort :: Int + } + deriving (Show) + +{- SshInput is only used for applicative form prompting, this converts + - the result of such a form into a SshData. -} +mkSshData :: SshInput -> SshData +mkSshData s = SshData + { sshHostName = fromMaybe "" $ inputHostname s + , sshUserName = inputUsername s + , sshDirectory = fromMaybe "" $ inputDirectory s + , sshRepoName = genSshRepoName + (T.unpack $ fromJust $ inputHostname s) + (maybe "" T.unpack $ inputDirectory s) + , sshPort = inputPort s + , needsPubKey = False + , rsyncOnly = False + } + +sshInputAForm :: (Field WebApp WebApp Text) -> SshInput -> AForm WebApp WebApp SshInput +sshInputAForm hostnamefield def = SshInput + <$> aopt check_hostname "Host name" (Just $ inputHostname def) + <*> aopt check_username "User name" (Just $ inputUsername def) + <*> aopt textField "Directory" (Just $ Just $ fromMaybe (T.pack gitAnnexAssistantDefaultDir) $ inputDirectory def) + <*> areq intField "Port" (Just $ inputPort def) + where + check_hostname = checkM (liftIO . checkdns) hostnamefield + checkdns t = do + let h = T.unpack t + r <- catchMaybeIO $ getAddrInfo canonname (Just h) Nothing + return $ case catMaybes . map addrCanonName <$> r of + -- canonicalize input hostname if it had no dot + Just (fullname:_) + | '.' `elem` h -> Right t + | otherwise -> Right $ T.pack fullname + Just [] -> Right t + Nothing -> Left bad_hostname + canonname = Just $ defaultHints { addrFlags = [AI_CANONNAME] } + + check_username = checkBool (all (`notElem` "/:@ \t") . T.unpack) + bad_username textField + + bad_hostname = "cannot resolve host name" :: Text + bad_username = "bad user name" :: Text + +data ServerStatus + = UntestedServer + | UnusableServer Text -- reason why it's not usable + | UsableRsyncServer + | UsableSshInput + deriving (Eq) + +usable :: ServerStatus -> Bool +usable UntestedServer = False +usable (UnusableServer _) = False +usable UsableRsyncServer = True +usable UsableSshInput = True + +getAddSshR :: Handler RepHtml +getAddSshR = sshConfigurator $ do + u <- liftIO $ T.pack <$> myUserName + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap $ sshInputAForm textField $ + SshInput Nothing (Just u) Nothing 22 + case result of + FormSuccess sshinput -> do + s <- liftIO $ testServer sshinput + case s of + Left status -> showform form enctype status + Right sshdata -> lift $ redirect $ ConfirmSshR sshdata + _ -> showform form enctype UntestedServer + where + showform form enctype status = $(widgetFile "configurators/ssh/add") + +{- To enable an existing rsync special remote, parse the SshInput from + - its rsyncurl, and display a form whose only real purpose is to check + - if ssh public keys need to be set up. From there, we can proceed with + - the usual repo setup; all that code is idempotent. + - + - Note that there's no EnableSshR because ssh remotes are not special + - remotes, and so their configuration is not shared between repositories. + -} +getEnableRsyncR :: UUID -> Handler RepHtml +getEnableRsyncR u = do + m <- fromMaybe M.empty . M.lookup u <$> liftAnnex readRemoteLog + case (parseSshRsyncUrl =<< M.lookup "rsyncurl" m, M.lookup "name" m) of + (Just sshinput, Just reponame) -> sshConfigurator $ do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap $ sshInputAForm textField sshinput + case result of + FormSuccess sshinput' + | isRsyncNet (inputHostname sshinput') -> + void $ lift $ makeRsyncNet sshinput' reponame (const noop) + | otherwise -> do + s <- liftIO $ testServer sshinput' + case s of + Left status -> showform form enctype status + Right sshdata -> enable sshdata + { sshRepoName = reponame } + _ -> showform form enctype UntestedServer + _ -> redirect AddSshR + where + showform form enctype status = do + description <- lift $ liftAnnex $ + T.pack . concat <$> prettyListUUIDs [u] + $(widgetFile "configurators/ssh/enable") + enable sshdata = lift $ redirect $ ConfirmSshR $ + sshdata { rsyncOnly = True } + +{- Converts a rsyncurl value to a SshInput. But only if it's a ssh rsync + - url; rsync:// urls or bare path names are not supported. + - + - The hostname is stored mangled in the remote log for rsync special + - remotes configured by this webapp. So that mangling has to reversed + - here to get back the original hostname. + -} +parseSshRsyncUrl :: String -> Maybe SshInput +parseSshRsyncUrl u + | not (rsyncUrlIsShell u) = Nothing + | otherwise = Just $ SshInput + { inputHostname = val $ unMangleSshHostName host + , inputUsername = if null user then Nothing else val user + , inputDirectory = val dir + , inputPort = 22 + } + where + val = Just . T.pack + (userhost, dir) = separate (== ':') u + (user, host) = if '@' `elem` userhost + then separate (== '@') userhost + else (userhost, "") + +{- Test if we can ssh into the server. + - + - Two probe attempts are made. First, try sshing in using the existing + - configuration, but don't let ssh prompt for any password. If + - passwordless login is already enabled, use it. Otherwise, + - a special ssh key will need to be generated just for this server. + - + - Once logged into the server, probe to see if git-annex-shell is + - available, or rsync. Note that, ~/.ssh/git-annex-shell may be + - present, while git-annex-shell is not in PATH. + -} +testServer :: SshInput -> IO (Either ServerStatus SshData) +testServer (SshInput { inputHostname = Nothing }) = return $ + Left $ UnusableServer "Please enter a host name." +testServer sshinput@(SshInput { inputHostname = Just hn }) = do + status <- probe [sshOpt "NumberOfPasswordPrompts" "0"] + if usable status + then ret status False + else do + status' <- probe [] + if usable status' + then ret status' True + else return $ Left status' + where + ret status needspubkey = return $ Right $ (mkSshData sshinput) + { needsPubKey = needspubkey + , rsyncOnly = status == UsableRsyncServer + } + probe extraopts = do + let remotecommand = join ";" + [ report "loggedin" + , checkcommand "git-annex-shell" + , checkcommand "rsync" + , checkcommand shim + ] + knownhost <- knownHost hn + let sshopts = filter (not . null) $ extraopts ++ + {- If this is an already known host, let + - ssh check it as usual. + - Otherwise, trust the host key. -} + [ if knownhost then "" else sshOpt "StrictHostKeyChecking" "no" + , "-n" -- don't read from stdin + , "-p", show (inputPort sshinput) + , genSshHost + (fromJust $ inputHostname sshinput) + (inputUsername sshinput) + , remotecommand + ] + parsetranscript . fst <$> sshTranscript sshopts Nothing + parsetranscript s + | reported "git-annex-shell" = UsableSshInput + | reported shim = UsableSshInput + | reported "rsync" = UsableRsyncServer + | reported "loggedin" = UnusableServer + "Neither rsync nor git-annex are installed on the server. Perhaps you should go install them?" + | otherwise = UnusableServer $ T.pack $ + "Failed to ssh to the server. Transcript: " ++ s + where + reported r = token r `isInfixOf` s + checkcommand c = "if which " ++ c ++ "; then " ++ report c ++ "; fi" + token r = "git-annex-probe " ++ r + report r = "echo " ++ token r + shim = "~/.ssh/git-annex-shell" + +{- Runs a ssh command; if it fails shows the user the transcript, + - and if it succeeds, runs an action. -} +sshSetup :: [String] -> String -> Handler RepHtml -> Handler RepHtml +sshSetup opts input a = do + (transcript, ok) <- liftIO $ sshTranscript opts (Just input) + if ok + then a + else showSshErr transcript + +showSshErr :: String -> Handler RepHtml +showSshErr msg = sshConfigurator $ + $(widgetFile "configurators/ssh/error") + +getConfirmSshR :: SshData -> Handler RepHtml +getConfirmSshR sshdata = sshConfigurator $ + $(widgetFile "configurators/ssh/confirm") + +getMakeSshGitR :: SshData -> Handler RepHtml +getMakeSshGitR = makeSsh False setupGroup + +getMakeSshRsyncR :: SshData -> Handler RepHtml +getMakeSshRsyncR = makeSsh True setupGroup + +makeSsh :: Bool -> (Remote -> Handler ()) -> SshData -> Handler RepHtml +makeSsh rsync setup sshdata + | needsPubKey sshdata = do + keypair <- liftIO genSshKeyPair + sshdata' <- liftIO $ setupSshKeyPair keypair sshdata + makeSsh' rsync setup sshdata' (Just keypair) + | sshPort sshdata /= 22 = do + sshdata' <- liftIO $ setSshConfig sshdata [] + makeSsh' rsync setup sshdata' Nothing + | otherwise = makeSsh' rsync setup sshdata Nothing + +makeSsh' :: Bool -> (Remote -> Handler ()) -> SshData -> Maybe SshKeyPair -> Handler RepHtml +makeSsh' rsync setup sshdata keypair = + sshSetup [sshhost, remoteCommand] "" $ + makeSshRepo rsync setup sshdata + where + sshhost = genSshHost (sshHostName sshdata) (sshUserName sshdata) + remotedir = T.unpack $ sshDirectory sshdata + remoteCommand = join "&&" $ catMaybes + [ Just $ "mkdir -p " ++ shellEscape remotedir + , Just $ "cd " ++ shellEscape remotedir + , if rsync then Nothing else Just "git init --bare --shared" + , if rsync then Nothing else Just "git annex init" + , if needsPubKey sshdata + then addAuthorizedKeysCommand (rsyncOnly sshdata) remotedir . sshPubKey <$> keypair + else Nothing + ] + +makeSshRepo :: Bool -> (Remote -> Handler ()) -> SshData -> Handler RepHtml +makeSshRepo forcersync setup sshdata = do + r <- liftAssistant $ makeSshRemote forcersync sshdata + setup r + redirect $ EditNewCloudRepositoryR $ Remote.uuid r + +getAddRsyncNetR :: Handler RepHtml +getAddRsyncNetR = do + ((result, form), enctype) <- runFormGet $ + renderBootstrap $ sshInputAForm hostnamefield $ + SshInput Nothing Nothing Nothing 22 + let showform status = page "Add a Rsync.net repository" (Just Configuration) $ + $(widgetFile "configurators/addrsync.net") + case result of + FormSuccess sshinput + | isRsyncNet (inputHostname sshinput) -> do + let reponame = genSshRepoName "rsync.net" + (maybe "" T.unpack $ inputDirectory sshinput) + makeRsyncNet sshinput reponame setupGroup + | otherwise -> + showform $ UnusableServer + "That is not a rsync.net host name." + _ -> showform UntestedServer + where + hostnamefield = textField `withNote` help + help = [whamlet| + + Help +
+
+ When you sign up for a Rsync.net account, you should receive an # + email from them with the host name and user name to put here. +
+ The host name will be something like "usw-s001.rsync.net", and the # + user name something like "7491" +|] + +makeRsyncNet :: SshInput -> String -> (Remote -> Handler ()) -> Handler RepHtml +makeRsyncNet sshinput reponame setup = do + knownhost <- liftIO $ maybe (return False) knownHost (inputHostname sshinput) + keypair <- liftIO $ genSshKeyPair + sshdata <- liftIO $ setupSshKeyPair keypair $ + (mkSshData sshinput) + { sshRepoName = reponame + , needsPubKey = True + , rsyncOnly = True + } + {- I'd prefer to separate commands with && , but + - rsync.net's shell does not support that. + - + - The dd method of appending to the authorized_keys file is the + - one recommended by rsync.net documentation. I touch the file first + - to not need to use a different method to create it. + -} + let remotecommand = join ";" + [ "mkdir -p .ssh" + , "touch .ssh/authorized_keys" + , "dd of=.ssh/authorized_keys oflag=append conv=notrunc" + , "mkdir -p " ++ T.unpack (sshDirectory sshdata) + ] + let sshopts = filter (not . null) + [ if knownhost then "" else sshOpt "StrictHostKeyChecking" "no" + , genSshHost (sshHostName sshdata) (sshUserName sshdata) + , remotecommand + ] + sshSetup sshopts (sshPubKey keypair) $ + makeSshRepo True setup sshdata + +isRsyncNet :: Maybe Text -> Bool +isRsyncNet Nothing = False +isRsyncNet (Just host) = ".rsync.net" `T.isSuffixOf` T.toLower host + +setupGroup :: Remote -> Handler () +setupGroup r = liftAnnex $ setStandardGroup (Remote.uuid r) TransferGroup diff --git a/Assistant/WebApp/Configurators/WebDAV.hs b/Assistant/WebApp/Configurators/WebDAV.hs new file mode 100644 index 0000000000..1afa1f5212 --- /dev/null +++ b/Assistant/WebApp/Configurators/WebDAV.hs @@ -0,0 +1,129 @@ +{- git-annex assistant webapp configurators for WebDAV remotes + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Configurators.WebDAV where + +import Assistant.WebApp.Common +import Assistant.MakeRemote +import Assistant.Sync +#ifdef WITH_WEBDAV +import qualified Remote.WebDAV as WebDAV +#endif +import qualified Remote +import Types.Remote (RemoteConfig) +import Types.StandardGroups +import Logs.PreferredContent +import Logs.Remote +import Creds + +import qualified Data.Text as T +import qualified Data.Map as M + +webDAVConfigurator :: Widget -> Handler RepHtml +webDAVConfigurator = page "Add a WebDAV repository" (Just Configuration) + +boxConfigurator :: Widget -> Handler RepHtml +boxConfigurator = page "Add a Box.com repository" (Just Configuration) + +data WebDAVInput = WebDAVInput + { user :: Text + , password :: Text + , embedCreds :: Bool + , directory :: Text + , enableEncryption :: EnableEncryption + } + +toCredPair :: WebDAVInput -> CredPair +toCredPair input = (T.unpack $ user input, T.unpack $ password input) + +boxComAForm :: AForm WebApp WebApp WebDAVInput +boxComAForm = WebDAVInput + <$> areq textField "Username or Email" Nothing + <*> areq passwordField "Box.com Password" Nothing + <*> areq checkBoxField "Share this account with friends?" (Just True) + <*> areq textField "Directory" (Just "annex") + <*> enableEncryptionField + +webDAVCredsAForm :: AForm WebApp WebApp WebDAVInput +webDAVCredsAForm = WebDAVInput + <$> areq textField "Username or Email" Nothing + <*> areq passwordField "Password" Nothing + <*> pure False + <*> pure T.empty + <*> pure NoEncryption -- not used! + +getAddBoxComR :: Handler RepHtml +#ifdef WITH_WEBDAV +getAddBoxComR = boxConfigurator $ do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap boxComAForm + case result of + FormSuccess input -> lift $ + makeWebDavRemote "box.com" (toCredPair input) setgroup $ M.fromList + [ configureEncryption $ enableEncryption input + , ("embedcreds", if embedCreds input then "yes" else "no") + , ("type", "webdav") + , ("url", "https://www.box.com/dav/" ++ T.unpack (directory input)) + -- Box.com has a max file size of 100 mb, but + -- using smaller chunks has better memory + -- performance. + , ("chunksize", "10mb") + ] + _ -> $(widgetFile "configurators/addbox.com") + where + setgroup r = liftAnnex $ + setStandardGroup (Remote.uuid r) TransferGroup +#else +getAddBoxComR = error "WebDAV not supported by this build" +#endif + +getEnableWebDAVR :: UUID -> Handler RepHtml +#ifdef WITH_WEBDAV +getEnableWebDAVR uuid = do + m <- liftAnnex readRemoteLog + let c = fromJust $ M.lookup uuid m + let name = fromJust $ M.lookup "name" c + let url = fromJust $ M.lookup "url" c + mcreds <- liftAnnex $ + getRemoteCredPairFor "webdav" c (WebDAV.davCreds uuid) + case mcreds of + Just creds -> webDAVConfigurator $ lift $ + makeWebDavRemote name creds (const noop) M.empty + Nothing + | "box.com/" `isInfixOf` url -> + boxConfigurator $ showform name url + | otherwise -> + webDAVConfigurator $ showform name url + where + showform name url = do + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap webDAVCredsAForm + case result of + FormSuccess input -> lift $ + makeWebDavRemote name (toCredPair input) (const noop) M.empty + _ -> do + description <- lift $ liftAnnex $ + T.pack . concat <$> Remote.prettyListUUIDs [uuid] + $(widgetFile "configurators/enablewebdav") +#else +getEnableWebDAVR _ = error "WebDAV not supported by this build" +#endif + +#ifdef WITH_WEBDAV +makeWebDavRemote :: String -> CredPair -> (Remote -> Handler ()) -> RemoteConfig -> Handler () +makeWebDavRemote name creds setup config = do + remotename <- liftAnnex $ fromRepo $ uniqueRemoteName name 0 + liftIO $ WebDAV.setCredsEnv creds + r <- liftAnnex $ addRemote $ do + makeSpecialRemote name WebDAV.remote config + return remotename + setup r + liftAssistant $ syncNewRemote r + redirect $ EditNewCloudRepositoryR $ Remote.uuid r +#endif diff --git a/Assistant/WebApp/Configurators/XMPP.hs b/Assistant/WebApp/Configurators/XMPP.hs new file mode 100644 index 0000000000..b81a31c5a3 --- /dev/null +++ b/Assistant/WebApp/Configurators/XMPP.hs @@ -0,0 +1,145 @@ +{- git-annex assistant XMPP configuration + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings #-} +{-# LANGUAGE CPP #-} + +module Assistant.WebApp.Configurators.XMPP where + +import Assistant.WebApp.Common +import Assistant.WebApp.Notifications +import Utility.NotificationBroadcaster +#ifdef WITH_XMPP +import Assistant.XMPP.Client +import Assistant.XMPP.Buddies +import Assistant.Types.Buddies +import Assistant.NetMessager +import Assistant.Alert +import Assistant.DaemonStatus +import Utility.SRV +#endif + +#ifdef WITH_XMPP +import Network +import Network.Protocol.XMPP +import qualified Data.Text as T +import Control.Exception (SomeException) +#endif + +{- Displays an alert suggesting to configure XMPP, with a button. -} +xmppNeeded :: Handler () +#ifdef WITH_XMPP +xmppNeeded = whenM (isNothing <$> liftAnnex getXMPPCreds) $ do + urlrender <- getUrlRender + void $ liftAssistant $ do + close <- asIO1 removeAlert + addAlert $ xmppNeededAlert $ AlertButton + { buttonLabel = "Configure a Jabber account" + , buttonUrl = urlrender XMPPR + , buttonAction = Just close + } +#else +xmppNeeded = return () +#endif + +#ifdef WITH_XMPP +getXMPPR :: Handler RepHtml +getXMPPR = xmppPage $ do + ((result, form), enctype) <- lift $ do + oldcreds <- liftAnnex getXMPPCreds + runFormGet $ renderBootstrap $ xmppAForm $ + creds2Form <$> oldcreds + let showform problem = $(widgetFile "configurators/xmpp") + case result of + FormSuccess f -> either (showform . Just . show) (lift . storecreds) + =<< liftIO (validateForm f) + _ -> showform Nothing + where + storecreds creds = do + void $ liftAnnex $ setXMPPCreds creds + liftAssistant notifyNetMessagerRestart + redirect StartXMPPPairR +#else +getXMPPR = xmppPage $ + $(widgetFile "configurators/xmpp/disabled") +#endif + +{- Called by client to get a list of buddies. + - + - Returns a div, which will be inserted into the calling page. + -} +getBuddyListR :: NotificationId -> Handler RepHtml +getBuddyListR nid = do + waitNotifier getBuddyListBroadcaster nid + + p <- widgetToPageContent buddyListDisplay + hamletToRepHtml $ [hamlet|^{pageBody p}|] + +buddyListDisplay :: Widget +buddyListDisplay = do + autoUpdate ident NotifierBuddyListR (10 :: Int) (10 :: Int) +#ifdef WITH_XMPP + buddies <- lift $ liftAssistant $ do + rs <- filter isXMPPRemote . syncGitRemotes <$> getDaemonStatus + let pairedwith = catMaybes $ map (parseJID . getXMPPClientID) rs + catMaybes . map (buddySummary pairedwith) + <$> (getBuddyList <<~ buddyList) + $(widgetFile "configurators/xmpp/buddylist") +#endif + where + ident = "buddylist" + +#ifdef WITH_XMPP + +data XMPPForm = XMPPForm + { formJID :: Text + , formPassword :: Text } + +creds2Form :: XMPPCreds -> XMPPForm +creds2Form c = XMPPForm (xmppJID c) (xmppPassword c) + +xmppAForm :: (Maybe XMPPForm) -> AForm WebApp WebApp XMPPForm +xmppAForm def = XMPPForm + <$> areq jidField "Jabber address" (formJID <$> def) + <*> areq passwordField "Password" Nothing + +jidField :: Field WebApp WebApp Text +jidField = checkBool (isJust . parseJID) bad textField + where + bad :: Text + bad = "This should look like an email address.." + +validateForm :: XMPPForm -> IO (Either SomeException XMPPCreds) +validateForm f = do + let jid = fromMaybe (error "bad JID") $ parseJID (formJID f) + let domain = T.unpack $ strDomain $ jidDomain jid + hostports <- lookupSRV $ mkSRVTcp "xmpp-client" domain + let username = fromMaybe "" (strNode <$> jidNode jid) + case hostports of + ((h, PortNumber p):_) -> testXMPP $ XMPPCreds + { xmppUsername = username + , xmppPassword = formPassword f + , xmppHostname = h + , xmppPort = fromIntegral p + , xmppJID = formJID f + } + _ -> testXMPP $ XMPPCreds + { xmppUsername = username + , xmppPassword = formPassword f + , xmppHostname = T.unpack $ strDomain $ jidDomain jid + , xmppPort = 5222 + , xmppJID = formJID f + } + +testXMPP :: XMPPCreds -> IO (Either SomeException XMPPCreds) +testXMPP creds = either Left (const $ Right creds) + <$> connectXMPP creds (const noop) + +#endif + +xmppPage :: Widget -> Handler RepHtml +xmppPage = page "Jabber" (Just Configuration) diff --git a/Assistant/WebApp/Control.hs b/Assistant/WebApp/Control.hs new file mode 100644 index 0000000000..e18c9890ef --- /dev/null +++ b/Assistant/WebApp/Control.hs @@ -0,0 +1,58 @@ +{- git-annex assistant webapp control + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Control where + +import Assistant.WebApp.Common +import Locations.UserConfig +import Utility.LogFile +import Assistant.DaemonStatus + +import Control.Concurrent +import System.Posix (getProcessID, signalProcess, sigTERM) +import qualified Data.Map as M + +getShutdownR :: Handler RepHtml +getShutdownR = page "Shutdown" Nothing $ + $(widgetFile "control/shutdown") + +getShutdownConfirmedR :: Handler RepHtml +getShutdownConfirmedR = page "Shutdown" Nothing $ do + {- Wait 2 seconds before shutting down, to give the web page time + - to display. -} + void $ liftIO $ forkIO $ do + threadDelay 2000000 + signalProcess sigTERM =<< getProcessID + $(widgetFile "control/shutdownconfirmed") + +{- Quite a hack, and doesn't redirect the browser window. -} +getRestartR :: Handler RepHtml +getRestartR = page "Restarting" Nothing $ do + void $ liftIO $ forkIO $ do + threadDelay 2000000 + program <- readProgramFile + unlessM (boolSystem "sh" [Param "-c", Param $ restartcommand program]) $ + error "restart failed" + $(widgetFile "control/restarting") + where + restartcommand program = program ++ " assistant --stop; " ++ + program ++ " webapp" + +getRestartThreadR :: ThreadName -> Handler () +getRestartThreadR name = do + m <- liftAssistant $ startedThreads <$> getDaemonStatus + liftIO $ maybe noop snd $ M.lookup name m + redirectBack + +getLogR :: Handler RepHtml +getLogR = page "Logs" Nothing $ do + logfile <- lift $ liftAnnex $ fromRepo gitAnnexLogFile + logs <- liftIO $ listLogs logfile + logcontent <- liftIO $ concat <$> mapM readFile logs + $(widgetFile "control/log") diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs new file mode 100644 index 0000000000..d71f240cce --- /dev/null +++ b/Assistant/WebApp/DashBoard.hs @@ -0,0 +1,150 @@ +{- git-annex assistant webapp dashboard + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.DashBoard where + +import Assistant.WebApp.Common +import Assistant.WebApp.Utility +import Assistant.WebApp.Notifications +import Assistant.WebApp.Configurators +import Assistant.TransferQueue +import Utility.NotificationBroadcaster +import Logs.Transfer +import Utility.Percentage +import Utility.DataUnits +import Types.Key +import qualified Remote +import qualified Git + +import Text.Hamlet +import qualified Data.Map as M +import Control.Concurrent + +{- A display of currently running and queued transfers. + - + - Or, if there have never been any this run, an intro display. -} +transfersDisplay :: Bool -> Widget +transfersDisplay warnNoScript = do + webapp <- lift getYesod + current <- lift $ M.toList <$> getCurrentTransfers + queued <- lift $ take 10 <$> liftAssistant getTransferQueue + autoUpdate ident NotifierTransfersR (10 :: Int) (10 :: Int) + let transfers = simplifyTransfers $ current ++ queued + if null transfers + then ifM (lift $ showIntro <$> getWebAppState) + ( introDisplay ident + , $(widgetFile "dashboard/transfers") + ) + else $(widgetFile "dashboard/transfers") + where + ident = "transfers" + isrunning info = not $ + transferPaused info || isNothing (startedTime info) + +{- Simplifies a list of transfers, avoiding display of redundant + - equivilant transfers. -} +simplifyTransfers :: [(Transfer, TransferInfo)] -> [(Transfer, TransferInfo)] +simplifyTransfers [] = [] +simplifyTransfers (x:[]) = [x] +simplifyTransfers (v@(t1, _):r@((t2, _):l)) + | equivilantTransfer t1 t2 = simplifyTransfers (v:l) + | otherwise = v : (simplifyTransfers r) + +{- Called by client to get a display of currently in process transfers. + - + - Returns a div, which will be inserted into the calling page. + - + - Note that the head of the widget is not included, only its + - body is. To get the widget head content, the widget is also + - inserted onto the getHomeR page. + -} +getTransfersR :: NotificationId -> Handler RepHtml +getTransfersR nid = do + waitNotifier getTransferBroadcaster nid + + p <- widgetToPageContent $ transfersDisplay False + hamletToRepHtml $ [hamlet|^{pageBody p}|] + +{- The main dashboard. -} +dashboard :: Bool -> Widget +dashboard warnNoScript = do + let content = transfersDisplay warnNoScript + $(widgetFile "dashboard/main") + +getHomeR :: Handler RepHtml +getHomeR = ifM (inFirstRun) + ( redirect ConfigurationR + , page "" (Just DashBoard) $ dashboard True + ) + +{- Used to test if the webapp is running. -} +headHomeR :: Handler () +headHomeR = noop + +{- Same as HomeR, except no autorefresh at all (and no noscript warning). -} +getNoScriptR :: Handler RepHtml +getNoScriptR = page "" (Just DashBoard) $ dashboard False + +{- Same as HomeR, except with autorefreshing via meta refresh. -} +getNoScriptAutoR :: Handler RepHtml +getNoScriptAutoR = page "" (Just DashBoard) $ do + let ident = NoScriptR + let delayseconds = 3 :: Int + let this = NoScriptAutoR + toWidgetHead $(hamletFile $ hamletTemplate "dashboard/metarefresh") + dashboard False + +{- The javascript code does a post. -} +postFileBrowserR :: Handler () +postFileBrowserR = void openFileBrowser + +{- Used by non-javascript browsers, where clicking on the link actually + - opens this page, so we redirect back to the referrer. -} +getFileBrowserR :: Handler () +getFileBrowserR = whenM openFileBrowser $ redirectBack + +{- Opens the system file browser on the repo, or, as a fallback, + - goes to a file:// url. Returns True if it's ok to redirect away + - from the page (ie, the system file browser was opened). + - + - Note that the command is opened using a different thread, to avoid + - blocking the response to the browser on it. -} +openFileBrowser :: Handler Bool +openFileBrowser = do + path <- liftAnnex $ fromRepo Git.repoPath + ifM (liftIO $ inPath cmd <&&> inPath cmd) + ( do + void $ liftIO $ forkIO $ void $ + boolSystem cmd [Param path] + return True + , do + void $ redirect $ "file://" ++ path + return False + ) + where +#ifdef darwin_HOST_OS + cmd = "open" +#else + cmd = "xdg-open" +#endif + +{- Transfer controls. The GET is done in noscript mode and redirects back + - to the referring page. The POST is called by javascript. -} +getPauseTransferR :: Transfer -> Handler () +getPauseTransferR t = pauseTransfer t >> redirectBack +postPauseTransferR :: Transfer -> Handler () +postPauseTransferR t = pauseTransfer t +getStartTransferR :: Transfer -> Handler () +getStartTransferR t = startTransfer t >> redirectBack +postStartTransferR :: Transfer -> Handler () +postStartTransferR t = startTransfer t +getCancelTransferR :: Transfer -> Handler () +getCancelTransferR t = cancelTransfer False t >> redirectBack +postCancelTransferR :: Transfer -> Handler () +postCancelTransferR t = cancelTransfer False t diff --git a/Assistant/WebApp/Documentation.hs b/Assistant/WebApp/Documentation.hs new file mode 100644 index 0000000000..bee472dce1 --- /dev/null +++ b/Assistant/WebApp/Documentation.hs @@ -0,0 +1,41 @@ +{- git-annex assistant webapp documentation + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Documentation where + +import Assistant.WebApp.Common +import Assistant.Install (standaloneAppBase) +import Build.SysConfig (packageversion) + +{- The full license info may be included in a file on disk that can + - be read in and displayed. -} +licenseFile :: IO (Maybe FilePath) +licenseFile = do + base <- standaloneAppBase + return $ ( "LICENSE") <$> base + +getAboutR :: Handler RepHtml +getAboutR = page "About git-annex" (Just About) $ do + builtinlicense <- isJust <$> liftIO licenseFile + $(widgetFile "documentation/about") + +getLicenseR :: Handler RepHtml +getLicenseR = do + v <- liftIO licenseFile + case v of + Nothing -> redirect AboutR + Just f -> customPage (Just About) $ do + -- no sidebar, just pages of legalese.. + setTitle "License" + license <- liftIO $ readFile f + $(widgetFile "documentation/license") + +getRepoGroupR :: Handler RepHtml +getRepoGroupR = page "About repository groups" (Just About) $ do + $(widgetFile "documentation/repogroup") diff --git a/Assistant/WebApp/Form.hs b/Assistant/WebApp/Form.hs new file mode 100644 index 0000000000..20d2952850 --- /dev/null +++ b/Assistant/WebApp/Form.hs @@ -0,0 +1,64 @@ +{- git-annex assistant webapp form utilities + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE FlexibleContexts, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Form where + +import Types.Remote (RemoteConfigKey) + +import Yesod hiding (textField, passwordField) +import Yesod.Form.Fields as F +import Data.Text (Text) + +{- Yesod's textField sets the required attribute for required fields. + - We don't want this, because many of the forms used in this webapp + - display a modal dialog when submitted, which interacts badly with + - required field handling by the browser. + - + - Required fields are still checked by Yesod. + -} +textField :: RenderMessage master FormMessage => Field sub master Text +textField = F.textField + { fieldView = \theId name attrs val _isReq -> [whamlet| + +|] + } + +{- Also without required attribute. -} +passwordField :: RenderMessage master FormMessage => Field sub master Text +passwordField = F.passwordField + { fieldView = \theId name attrs val _isReq -> toWidget [hamlet| + +|] + } + +{- Makes a note widget be displayed after a field. -} +withNote :: Field sub master v -> GWidget sub master () -> Field sub master v +withNote field note = field { fieldView = newview } + where + newview theId name attrs val isReq = + let fieldwidget = (fieldView field) theId name attrs val isReq + in [whamlet|^{fieldwidget}  ^{note}|] + +data EnableEncryption = SharedEncryption | NoEncryption + deriving (Eq) + +{- Adds a check box to an AForm to control encryption. -} +enableEncryptionField :: RenderMessage master FormMessage => AForm sub master EnableEncryption +enableEncryptionField = areq (selectFieldList choices) "Encryption" (Just SharedEncryption) + where + choices :: [(Text, EnableEncryption)] + choices = + [ ("Encrypt all data", SharedEncryption) + , ("Disable encryption", NoEncryption) + ] + +{- Generates Remote configuration for encryption. -} +configureEncryption :: EnableEncryption -> (RemoteConfigKey, String) +configureEncryption SharedEncryption = ("encryption", "shared") +configureEncryption NoEncryption = ("encryption", "none") diff --git a/Assistant/WebApp/Notifications.hs b/Assistant/WebApp/Notifications.hs new file mode 100644 index 0000000000..06f1033603 --- /dev/null +++ b/Assistant/WebApp/Notifications.hs @@ -0,0 +1,96 @@ +{- git-annex assistant webapp notifications + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +#if defined VERSION_yesod_default +#if ! MIN_VERSION_yesod_default(1,1,0) +#define WITH_OLD_YESOD +#endif +#endif + +module Assistant.WebApp.Notifications where + +import Assistant.Common +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.DaemonStatus +import Assistant.Types.Buddies +import Utility.NotificationBroadcaster +import Utility.Yesod + +import Yesod +import Data.Text (Text) +import qualified Data.Text as T +#ifndef WITH_OLD_YESOD +import qualified Data.Aeson.Types as Aeson +#endif + +{- Add to any widget to make it auto-update using long polling. + - + - The widget should have a html element with an id=ident, which will be + - replaced when it's updated. + - + - The geturl route should return the notifier url to use for polling. + - + - ms_delay is how long to delay between AJAX updates + - ms_startdelay is how long to delay before updating with AJAX at the start + -} +autoUpdate :: Text -> Route WebApp -> Int -> Int -> Widget +autoUpdate tident geturl ms_delay ms_startdelay = do +#ifdef WITH_OLD_YESOD + let delay = show ms_delay + let startdelay = show ms_startdelay + let ident = "'" ++ T.unpack tident ++ "'" +#else + let delay = Aeson.String (T.pack (show ms_delay)) + let startdelay = Aeson.String (T.pack (show ms_startdelay)) + let ident = Aeson.String tident +#endif + addScript $ StaticR longpolling_js + $(widgetFile "notifications/longpolling") + +{- Notifier urls are requested by the javascript, to avoid allocation + - of NotificationIds when noscript pages are loaded. This constructs a + - notifier url for a given Route and NotificationBroadcaster. + -} +notifierUrl :: (NotificationId -> Route WebApp) -> Assistant NotificationBroadcaster -> Handler RepPlain +notifierUrl route broadcaster = do + (urlbits, _params) <- renderRoute . route <$> newNotifier broadcaster + webapp <- getYesod + return $ RepPlain $ toContent $ T.concat + [ "/" + , T.intercalate "/" urlbits + , "?auth=" + , secretToken webapp + ] + +getNotifierTransfersR :: Handler RepPlain +getNotifierTransfersR = notifierUrl TransfersR getTransferBroadcaster + +getNotifierSideBarR :: Handler RepPlain +getNotifierSideBarR = notifierUrl SideBarR getAlertBroadcaster + +getNotifierBuddyListR :: Handler RepPlain +getNotifierBuddyListR = notifierUrl BuddyListR getBuddyListBroadcaster + +getNotifierRepoListR :: RepoSelector -> Handler RepPlain +getNotifierRepoListR reposelector = notifierUrl route getRepoListBroadcaster + where + route nid = RepoListR $ RepoListNotificationId nid reposelector + +getTransferBroadcaster :: Assistant NotificationBroadcaster +getTransferBroadcaster = transferNotifier <$> getDaemonStatus + +getAlertBroadcaster :: Assistant NotificationBroadcaster +getAlertBroadcaster = alertNotifier <$> getDaemonStatus + +getBuddyListBroadcaster :: Assistant NotificationBroadcaster +getBuddyListBroadcaster = getBuddyBroadcaster <$> getAssistant buddyList + +getRepoListBroadcaster :: Assistant NotificationBroadcaster +getRepoListBroadcaster = syncRemotesNotifier <$> getDaemonStatus diff --git a/Assistant/WebApp/OtherRepos.hs b/Assistant/WebApp/OtherRepos.hs new file mode 100644 index 0000000000..b408dd8a5b --- /dev/null +++ b/Assistant/WebApp/OtherRepos.hs @@ -0,0 +1,68 @@ +{- git-annex assistant webapp switching to other repos + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.OtherRepos where + +import Assistant.Common +import Assistant.WebApp.Types +import Assistant.WebApp.Page +import qualified Git.Construct +import qualified Git.Config +import Locations.UserConfig +import qualified Utility.Url as Url +import Utility.Yesod + +import Yesod +import Control.Concurrent +import System.Process (cwd) + +getRepositorySwitcherR :: Handler RepHtml +getRepositorySwitcherR = page "Switch repository" Nothing $ do + repolist <- liftIO listOtherRepos + $(widgetFile "control/repositoryswitcher") + +listOtherRepos :: IO [(String, String)] +listOtherRepos = do + dirs <- readAutoStartFile + pwd <- getCurrentDirectory + gooddirs <- filterM doesDirectoryExist $ + filter (\d -> not $ d `dirContains` pwd) dirs + names <- mapM relHome gooddirs + return $ sort $ zip names gooddirs + +{- Starts up the assistant in the repository, and waits for it to create + - a gitAnnexUrlFile. Waits for the assistant to be up and listening for + - connections by testing the url. Once it's running, redirect to it. + -} +getSwitchToRepositoryR :: FilePath -> Handler RepHtml +getSwitchToRepositoryR repo = do + liftIO $ startAssistant repo + redirect =<< liftIO geturl + where + geturl = do + r <- Git.Config.read =<< Git.Construct.fromPath repo + waiturl $ gitAnnexUrlFile r + waiturl urlfile = do + v <- tryIO $ readFile urlfile + case v of + Left _ -> delayed $ waiturl urlfile + Right url -> ifM (listening url) + ( return url + , delayed $ waiturl urlfile + ) + listening url = catchBoolIO $ fst <$> Url.exists url [] + delayed a = do + threadDelay 100000 -- 1/10th of a second + a + +startAssistant :: FilePath -> IO () +startAssistant repo = do + program <- readProgramFile + void $ forkIO $ void $ createProcess $ + (proc program ["assistant"]) { cwd = Just repo } diff --git a/Assistant/WebApp/Page.hs b/Assistant/WebApp/Page.hs new file mode 100644 index 0000000000..8e6dd1b0af --- /dev/null +++ b/Assistant/WebApp/Page.hs @@ -0,0 +1,67 @@ +{- git-annex assistant webapp page display + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.Page where + +import Assistant.Common +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.WebApp.SideBar +import Utility.Yesod + +import Yesod +import Text.Hamlet +import Data.Text (Text) + +data NavBarItem = DashBoard | Configuration | About + deriving (Eq) + +navBarName :: NavBarItem -> Text +navBarName DashBoard = "Dashboard" +navBarName Configuration = "Configuration" +navBarName About = "About" + +navBarRoute :: NavBarItem -> Route WebApp +navBarRoute DashBoard = HomeR +navBarRoute Configuration = ConfigurationR +navBarRoute About = AboutR + +defaultNavBar :: [NavBarItem] +defaultNavBar = [DashBoard, Configuration, About] + +firstRunNavBar :: [NavBarItem] +firstRunNavBar = [Configuration, About] + +selectNavBar :: Handler [NavBarItem] +selectNavBar = ifM (inFirstRun) (return firstRunNavBar, return defaultNavBar) + +{- A standard page of the webapp, with a title, a sidebar, and that may + - be highlighted on the navbar. -} +page :: Html -> Maybe NavBarItem -> Widget -> Handler RepHtml +page title navbaritem content = customPage navbaritem $ do + setTitle title + sideBarDisplay + content + +{- A custom page, with no title or sidebar set. -} +customPage :: Maybe NavBarItem -> Widget -> Handler RepHtml +customPage navbaritem content = do + webapp <- getYesod + navbar <- map navdetails <$> selectNavBar + pageinfo <- widgetToPageContent $ do + addStylesheet $ StaticR css_bootstrap_css + addStylesheet $ StaticR css_bootstrap_responsive_css + addScript $ StaticR jquery_full_js + addScript $ StaticR js_bootstrap_dropdown_js + addScript $ StaticR js_bootstrap_modal_js + addScript $ StaticR js_bootstrap_collapse_js + $(widgetFile "page") + hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap") + where + navdetails i = (navBarName i, navBarRoute i, Just i == navbaritem) diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs new file mode 100644 index 0000000000..a87065fb56 --- /dev/null +++ b/Assistant/WebApp/SideBar.hs @@ -0,0 +1,100 @@ +{- git-annex assistant webapp sidebar + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} + +module Assistant.WebApp.SideBar where + +import Assistant.Common +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.WebApp.Notifications +import Assistant.Alert +import Assistant.DaemonStatus +import Utility.NotificationBroadcaster +import Utility.Yesod + +import Yesod +import Data.Text (Text) +import qualified Data.Map as M +import Control.Concurrent + +sideBarDisplay :: Widget +sideBarDisplay = do + let content = do + {- Add newest alerts to the sidebar. -} + alertpairs <- lift $ M.toList . alertMap + <$> liftAssistant getDaemonStatus + mapM_ renderalert $ + take displayAlerts $ reverse $ sortAlertPairs alertpairs + let ident = "sidebar" + $(widgetFile "sidebar/main") + autoUpdate ident NotifierSideBarR (10 :: Int) (10 :: Int) + where + bootstrapclass :: AlertClass -> Text + bootstrapclass Activity = "alert-info" + bootstrapclass Warning = "alert" + bootstrapclass Error = "alert-error" + bootstrapclass Success = "alert-success" + bootstrapclass Message = "alert-info" + + renderalert (aid, alert) = do + let alertid = show aid + let closable = alertClosable alert + let block = alertBlockDisplay alert + let divclass = bootstrapclass $ alertClass alert + $(widgetFile "sidebar/alert") + +{- Called by client to get a sidebar display. + - + - Returns a div, which will be inserted into the calling page. + - + - Note that the head of the widget is not included, only its + - body is. To get the widget head content, the widget is also + - inserted onto all pages. + -} +getSideBarR :: NotificationId -> Handler RepHtml +getSideBarR nid = do + waitNotifier getAlertBroadcaster nid + + {- This 0.1 second delay avoids very transient notifications from + - being displayed and churning the sidebar unnecesarily. + - + - This needs to be below the level perceptable by the user, + - to avoid slowing down user actions like closing alerts. -} + liftIO $ threadDelay 100000 + + page <- widgetToPageContent sideBarDisplay + hamletToRepHtml $ [hamlet|^{pageBody page}|] + +{- Called by the client to close an alert. -} +getCloseAlert :: AlertId -> Handler () +getCloseAlert = liftAssistant . removeAlert + +{- When an alert with a button is clicked on, the button takes us here. -} +getClickAlert :: AlertId -> Handler () +getClickAlert i = do + m <- alertMap <$> liftAssistant getDaemonStatus + case M.lookup i m of + Just (Alert { alertButton = Just b }) -> do + {- Spawn a thread to run the action while redirecting. -} + case buttonAction b of + Nothing -> noop + Just a -> liftIO $ void $ forkIO $ a i + redirect $ buttonUrl b + _ -> redirectBack + +htmlIcon :: AlertIcon -> GWidget sub master () +htmlIcon ActivityIcon = bootstrapIcon "refresh" +htmlIcon InfoIcon = bootstrapIcon "info-sign" +htmlIcon SuccessIcon = bootstrapIcon "ok" +htmlIcon ErrorIcon = bootstrapIcon "exclamation-sign" +-- utf-8 umbrella (utf-8 cloud looks too stormy) +htmlIcon TheCloud = [whamlet|☂|] + +bootstrapIcon :: Text -> GWidget sub master () +bootstrapIcon name = [whamlet||] diff --git a/Assistant/WebApp/Types.hs b/Assistant/WebApp/Types.hs new file mode 100644 index 0000000000..c9a00797e0 --- /dev/null +++ b/Assistant/WebApp/Types.hs @@ -0,0 +1,146 @@ +{- git-annex assistant webapp types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Assistant.WebApp.Types where + +import Assistant.Common +import Assistant.Ssh +import Assistant.Alert +import Assistant.Pairing +import Assistant.Types.Buddies +import Utility.NotificationBroadcaster +import Utility.WebApp +import Utility.Yesod +import Logs.Transfer +import Build.SysConfig (packageversion) + +import Yesod +import Yesod.Static +import Text.Hamlet +import Data.Text (Text, pack, unpack) +import Control.Concurrent.STM + +publicFiles "static" + +mkYesodData "WebApp" $(parseRoutesFile "Assistant/WebApp/routes") + +data WebApp = WebApp + { assistantData :: AssistantData + , secretToken :: Text + , relDir :: Maybe FilePath + , getStatic :: Static + , webAppState :: TMVar WebAppState + , postFirstRun :: Maybe (IO String) + , noAnnex :: Bool + } + +instance Yesod WebApp where + {- Require an auth token be set when accessing any (non-static) route -} + isAuthorized _ _ = checkAuthToken secretToken + + {- Add the auth token to every url generated, except static subsite + - urls (which can show up in Permission Denied pages). -} + joinPath = insertAuthToken secretToken excludeStatic + where + excludeStatic [] = True + excludeStatic (p:_) = p /= "static" + + makeSessionBackend = webAppSessionBackend + jsLoader _ = BottomOfHeadBlocking + + {- The webapp does not use defaultLayout, so this is only used + - for error pages or any other built-in yesod page. + - + - This can use static routes, but should use no other routes, + - as that would expose the auth token. + -} + defaultLayout content = do + webapp <- getYesod + pageinfo <- widgetToPageContent $ do + addStylesheet $ StaticR css_bootstrap_css + addStylesheet $ StaticR css_bootstrap_responsive_css + $(widgetFile "error") + hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap") + +instance RenderMessage WebApp FormMessage where + renderMessage _ _ = defaultFormMessage + +type Form x = Html -> MForm WebApp WebApp (FormResult x, Widget) + +data WebAppState = WebAppState + { showIntro :: Bool -- should the into message be displayed? + } + +data RepoSelector = RepoSelector + { onlyCloud :: Bool + , onlyConfigured :: Bool + , includeHere :: Bool + } + deriving (Read, Show, Eq) + +data RepoListNotificationId = RepoListNotificationId NotificationId RepoSelector + deriving (Read, Show, Eq) + +{- Only needed to work around old-yesod bug that emits a warning message + - when a route has two parameters. -} +data FilePathAndUUID = FilePathAndUUID FilePath UUID + deriving (Read, Show, Eq) + +instance PathPiece FilePathAndUUID where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece SshData where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece NotificationId where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece AlertId where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece Transfer where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece PairMsg where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece SecretReminder where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece UUID where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece BuddyKey where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece PairKey where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece RepoListNotificationId where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece RepoSelector where + toPathPiece = pack . show + fromPathPiece = readish . unpack + +instance PathPiece ThreadName where + toPathPiece = pack . show + fromPathPiece = readish . unpack diff --git a/Assistant/WebApp/Utility.hs b/Assistant/WebApp/Utility.hs new file mode 100644 index 0000000000..311bbfddca --- /dev/null +++ b/Assistant/WebApp/Utility.hs @@ -0,0 +1,125 @@ +{- git-annex assistant webapp utilities + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Assistant.WebApp.Utility where + +import Assistant.Common hiding (liftAnnex) +import Assistant.WebApp +import Assistant.WebApp.Types +import Assistant.DaemonStatus +import Assistant.TransferQueue +import Assistant.Types.TransferSlots +import Assistant.TransferSlots +import Assistant.Sync +import qualified Remote +import qualified Types.Remote as Remote +import qualified Remote.List as Remote +import qualified Assistant.Threads.Transferrer as Transferrer +import Logs.Transfer +import Locations.UserConfig +import qualified Config +import Git.Config +import Assistant.Threads.Watcher +import Assistant.NamedThread + +import qualified Data.Map as M +import Control.Concurrent +import System.Posix.Signals (signalProcessGroup, sigTERM, sigKILL) +import System.Posix.Process (getProcessGroupIDOf) + +{- Use Nothing to change autocommit setting; or a remote to change + - its sync setting. -} +changeSyncable :: (Maybe Remote) -> Bool -> Handler () +changeSyncable Nothing enable = do + liftAnnex $ Config.setConfig key (boolConfig enable) + liftIO . maybe noop (`throwTo` signal) + =<< liftAssistant (namedThreadId watchThread) + where + key = Config.annexConfig "autocommit" + signal + | enable = ResumeWatcher + | otherwise = PauseWatcher +changeSyncable (Just r) True = do + changeSyncFlag r True + syncRemote r +changeSyncable (Just r) False = do + changeSyncFlag r False + liftAssistant $ updateSyncRemotes + {- Stop all transfers to or from this remote. + - XXX Can't stop any ongoing scan, or git syncs. -} + void $ liftAssistant $ dequeueTransfers tofrom + mapM_ (cancelTransfer False) =<< + filter tofrom . M.keys <$> + liftAssistant (currentTransfers <$> getDaemonStatus) + where + tofrom t = transferUUID t == Remote.uuid r + +changeSyncFlag :: Remote -> Bool -> Handler () +changeSyncFlag r enabled = liftAnnex $ do + Config.setConfig key (boolConfig enabled) + void $ Remote.remoteListRefresh + where + key = Config.remoteConfig (Remote.repo r) "sync" + +{- Start syncing remote, using a background thread. -} +syncRemote :: Remote -> Handler () +syncRemote = liftAssistant . syncNewRemote + +pauseTransfer :: Transfer -> Handler () +pauseTransfer = cancelTransfer True + +cancelTransfer :: Bool -> Transfer -> Handler () +cancelTransfer pause t = do + m <- getCurrentTransfers + unless pause $ + {- remove queued transfer -} + void $ liftAssistant $ dequeueTransfers $ equivilantTransfer t + {- stop running transfer -} + maybe noop stop (M.lookup t m) + where + stop info = liftAssistant $ do + {- When there's a thread associated with the + - transfer, it's signaled first, to avoid it + - displaying any alert about the transfer having + - failed when the transfer process is killed. -} + liftIO $ maybe noop signalthread $ transferTid info + liftIO $ maybe noop killproc $ transferPid info + if pause + then void $ alterTransferInfo t $ + \i -> i { transferPaused = True } + else void $ removeTransfer t + signalthread tid + | pause = throwTo tid PauseTransfer + | otherwise = killThread tid + {- In order to stop helper processes like rsync, + - kill the whole process group of the process running the transfer. -} + killproc pid = void $ tryIO $ do + g <- getProcessGroupIDOf pid + void $ tryIO $ signalProcessGroup sigTERM g + threadDelay 50000 -- 0.05 second grace period + void $ tryIO $ signalProcessGroup sigKILL g + +startTransfer :: Transfer -> Handler () +startTransfer t = do + m <- getCurrentTransfers + maybe startqueued go (M.lookup t m) + where + go info = maybe (start info) resume $ transferTid info + startqueued = do + is <- liftAssistant $ map snd <$> getMatchingTransfers (== t) + maybe noop start $ headMaybe is + resume tid = do + liftAssistant $ alterTransferInfo t $ + \i -> i { transferPaused = False } + liftIO $ throwTo tid ResumeTransfer + start info = liftAssistant $ do + program <- liftIO readProgramFile + inImmediateTransferSlot $ + Transferrer.startTransfer program t info + +getCurrentTransfers :: Handler TransferMap +getCurrentTransfers = currentTransfers <$> liftAssistant getDaemonStatus diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes new file mode 100644 index 0000000000..e258966971 --- /dev/null +++ b/Assistant/WebApp/routes @@ -0,0 +1,74 @@ +/ HomeR GET HEAD +/noscript NoScriptR GET +/noscript/auto NoScriptAutoR GET +/about AboutR GET +/about/license LicenseR GET +/about/repogroups RepoGroupR GET + +/shutdown ShutdownR GET +/shutdown/confirm ShutdownConfirmedR GET +/restart RestartR GET +/restart/thread/#ThreadName RestartThreadR GET +/log LogR GET + +/config ConfigurationR GET +/config/repository RepositoriesR GET +/config/preferences PreferencesR GET +/config/xmpp XMPPR GET + +/config/repository/new/first FirstRepositoryR GET +/config/repository/new NewRepositoryR GET +/config/repository/switcher RepositorySwitcherR GET +/config/repository/switchto/#FilePath SwitchToRepositoryR GET +/config/repository/combine/#FilePathAndUUID CombineRepositoryR GET +/config/repository/edit/#UUID EditRepositoryR GET +/config/repository/edit/new/#UUID EditNewRepositoryR GET +/config/repository/edit/new/cloud/#UUID EditNewCloudRepositoryR GET +/config/repository/sync/disable/#UUID DisableSyncR GET +/config/repository/sync/enable/#UUID EnableSyncR GET + +/config/repository/add/drive AddDriveR GET +/config/repository/add/ssh AddSshR GET +/config/repository/add/ssh/confirm/#SshData ConfirmSshR GET +/config/repository/add/ssh/make/git/#SshData MakeSshGitR GET +/config/repository/add/ssh/make/rsync/#SshData MakeSshRsyncR GET +/config/repository/add/cloud/rsync.net AddRsyncNetR GET +/config/repository/add/cloud/S3 AddS3R GET +/config/repository/add/cloud/glacier AddGlacierR GET +/config/repository/add/cloud/box.com AddBoxComR GET + +/config/repository/pair/local/start StartLocalPairR GET +/config/repository/pair/local/running/#SecretReminder RunningLocalPairR GET +/config/repository/pair/local/finish/#PairMsg FinishLocalPairR GET +/config/repository/pair/xmpp/start StartXMPPPairR GET +/config/repository/pair/xmpp/running/#BuddyKey RunningXMPPPairR GET +/config/repository/pair/xmpp/accept/#PairKey ConfirmXMPPPairR GET +/config/repository/pair/xmpp/finish/#PairKey FinishXMPPPairR GET + +/config/repository/enable/rsync/#UUID EnableRsyncR GET +/config/repository/enable/directory/#UUID EnableDirectoryR GET +/config/repository/enable/S3/#UUID EnableS3R GET +/config/repository/enable/glacier/#UUID EnableGlacierR GET +/config/repository/enable/webdav/#UUID EnableWebDAVR GET + +/transfers/#NotificationId TransfersR GET +/notifier/transfers NotifierTransfersR GET + +/sidebar/#NotificationId SideBarR GET +/notifier/sidebar NotifierSideBarR GET + +/buddylist/#NotificationId BuddyListR GET +/notifier/buddylist NotifierBuddyListR GET + +/repolist/#RepoListNotificationId RepoListR GET +/notifier/repolist/#RepoSelector NotifierRepoListR GET + +/alert/close/#AlertId CloseAlert GET +/alert/click/#AlertId ClickAlert GET +/filebrowser FileBrowserR GET POST + +/transfer/pause/#Transfer PauseTransferR GET POST +/transfer/start/#Transfer StartTransferR GET POST +/transfer/cancel/#Transfer CancelTransferR GET POST + +/static StaticR Static getStatic diff --git a/Assistant/XMPP.hs b/Assistant/XMPP.hs new file mode 100644 index 0000000000..2c00044036 --- /dev/null +++ b/Assistant/XMPP.hs @@ -0,0 +1,241 @@ +{- core xmpp support + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.XMPP where + +import Assistant.Common +import Assistant.Types.NetMessager +import Assistant.Pairing + +import Network.Protocol.XMPP hiding (Node) +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Map as M +import Data.ByteString (ByteString) +import qualified Data.ByteString as B +import Data.XML.Types +import qualified Codec.Binary.Base64 as B64 + +{- Name of the git-annex tag, in our own XML namespace. + - (Not using a namespace URL to avoid unnecessary bloat.) -} +gitAnnexTagName :: Name +gitAnnexTagName = "{git-annex}git-annex" + +{- Creates a git-annex tag containing a particular attribute and value. -} +gitAnnexTag :: Name -> Text -> Element +gitAnnexTag attr val = gitAnnexTagContent attr val [] + +{- Also with some content. -} +gitAnnexTagContent :: Name -> Text -> [Node] -> Element +gitAnnexTagContent attr val = Element gitAnnexTagName [(attr, [ContentText val])] + +isGitAnnexTag :: Element -> Bool +isGitAnnexTag t = elementName t == gitAnnexTagName + +{- Things that a git-annex tag can inserted into. -} +class GitAnnexTaggable a where + insertGitAnnexTag :: a -> Element -> a + + extractGitAnnexTag :: a -> Maybe Element + + hasGitAnnexTag :: a -> Bool + hasGitAnnexTag = isJust . extractGitAnnexTag + +instance GitAnnexTaggable Message where + insertGitAnnexTag m elt = m { messagePayloads = elt : messagePayloads m } + extractGitAnnexTag = headMaybe . filter isGitAnnexTag . messagePayloads + +instance GitAnnexTaggable Presence where + -- always mark extended away and set presence priority to negative + insertGitAnnexTag p elt = p + { presencePayloads = extendedAway : negativePriority : elt : presencePayloads p } + extractGitAnnexTag = headMaybe . filter isGitAnnexTag . presencePayloads + +data GitAnnexTagInfo = GitAnnexTagInfo + { tagAttr :: Name + , tagValue :: Text + , tagElement :: Element + } + +type Decoder = Message -> GitAnnexTagInfo -> Maybe NetMessage + +gitAnnexTagInfo :: GitAnnexTaggable a => a -> Maybe GitAnnexTagInfo +gitAnnexTagInfo v = case extractGitAnnexTag v of + {- Each git-annex tag has a single attribute. -} + Just (tag@(Element _ [(attr, _)] _)) -> GitAnnexTagInfo + <$> pure attr + <*> attributeText attr tag + <*> pure tag + _ -> Nothing + +{- A presence with a git-annex tag in it. -} +gitAnnexPresence :: Element -> Presence +gitAnnexPresence = insertGitAnnexTag $ emptyPresence PresenceAvailable + +{- A presence with an empty git-annex tag in it, used for letting other + - clients know we're around and are a git-annex client. -} +gitAnnexSignature :: Presence +gitAnnexSignature = gitAnnexPresence $ Element gitAnnexTagName [] [] + +{- A message with a git-annex tag in it. -} +gitAnnexMessage :: Element -> JID -> JID -> Message +gitAnnexMessage elt tojid fromjid = (insertGitAnnexTag silentMessage elt) + { messageTo = Just tojid + , messageFrom = Just fromjid + } + +{- A notification that we've pushed to some repositories, listing their + - UUIDs. -} +pushNotification :: [UUID] -> Presence +pushNotification = gitAnnexPresence . gitAnnexTag pushAttr . encodePushNotification + +encodePushNotification :: [UUID] -> Text +encodePushNotification = T.intercalate uuidSep . map (T.pack . fromUUID) + +decodePushNotification :: Text -> [UUID] +decodePushNotification = map (toUUID . T.unpack) . T.splitOn uuidSep + +uuidSep :: Text +uuidSep = "," + +{- A request for other git-annex clients to send presence. -} +presenceQuery :: Presence +presenceQuery = gitAnnexPresence $ gitAnnexTag queryAttr T.empty + +{- A notification about a stage of pairing. -} +pairingNotification :: PairStage -> UUID -> JID -> JID -> Message +pairingNotification pairstage u = gitAnnexMessage $ + gitAnnexTag pairAttr $ encodePairingNotification pairstage u + +encodePairingNotification :: PairStage -> UUID -> Text +encodePairingNotification pairstage u = T.unwords $ map T.pack + [ show pairstage + , fromUUID u + ] + +decodePairingNotification :: Decoder +decodePairingNotification m = parse . words . T.unpack . tagValue + where + parse [stage, u] = PairingNotification + <$> readish stage + <*> (formatJID <$> messageFrom m) + <*> pure (toUUID u) + parse _ = Nothing + +pushMessage :: PushStage -> JID -> JID -> Message +pushMessage = gitAnnexMessage . encode + where + encode CanPush = gitAnnexTag canPushAttr T.empty + encode PushRequest = gitAnnexTag pushRequestAttr T.empty + encode StartingPush = gitAnnexTag startingPushAttr T.empty + encode (ReceivePackOutput b) = + gitAnnexTagContent receivePackAttr T.empty $ encodeTagContent b + encode (SendPackOutput b) = + gitAnnexTagContent sendPackAttr T.empty $ encodeTagContent b + encode (ReceivePackDone code) = + gitAnnexTag receivePackDoneAttr $ + T.pack $ show $ encodeExitCode code + +decodeMessage :: Message -> Maybe NetMessage +decodeMessage m = decode =<< gitAnnexTagInfo m + where + decode i = M.lookup (tagAttr i) decoders >>= rundecoder i + rundecoder i d = d m i + decoders = M.fromList $ zip + [ pairAttr + , canPushAttr + , pushRequestAttr + , startingPushAttr + , receivePackAttr + , sendPackAttr + , receivePackDoneAttr + ] + [ decodePairingNotification + , pushdecoder $ const $ Just CanPush + , pushdecoder $ const $ Just PushRequest + , pushdecoder $ const $ Just StartingPush + , pushdecoder $ + fmap ReceivePackOutput . decodeTagContent . tagElement + , pushdecoder $ + fmap SendPackOutput . decodeTagContent . tagElement + , pushdecoder $ + fmap (ReceivePackDone . decodeExitCode) . readish . + T.unpack . tagValue + ] + pushdecoder a m' i = Pushing + <$> (formatJID <$> messageFrom m') + <*> a i + +decodeExitCode :: Int -> ExitCode +decodeExitCode 0 = ExitSuccess +decodeExitCode n = ExitFailure n + +encodeExitCode :: ExitCode -> Int +encodeExitCode ExitSuccess = 0 +encodeExitCode (ExitFailure n) = n + +{- Base 64 encoding a ByteString to use as the content of a tag. -} +encodeTagContent :: ByteString -> [Node] +encodeTagContent b = [NodeContent $ ContentText $ T.pack $ B64.encode $ B.unpack b] + +decodeTagContent :: Element -> Maybe ByteString +decodeTagContent elt = B.pack <$> B64.decode s + where + s = T.unpack $ T.concat $ elementText elt + +{- The JID without the client part. -} +baseJID :: JID -> JID +baseJID j = JID (jidNode j) (jidDomain j) Nothing + +{- An XMPP chat message with an empty body. This should not be displayed + - by clients, but can be used for communications. -} +silentMessage :: Message +silentMessage = (emptyMessage MessageChat) + { messagePayloads = [ emptybody ] } + where + emptybody = Element + { elementName = "body" + , elementAttributes = [] + , elementNodes = [] + } + +{- Add to a presence to mark its client as extended away. -} +extendedAway :: Element +extendedAway = Element "show" [] [NodeContent $ ContentText "xa"] + +{- Add to a presence to give it a negative priority. -} +negativePriority :: Element +negativePriority = Element "priority" [] [NodeContent $ ContentText "-1"] + +pushAttr :: Name +pushAttr = "push" + +queryAttr :: Name +queryAttr = "query" + +pairAttr :: Name +pairAttr = "pair" + +canPushAttr :: Name +canPushAttr = "canpush" + +pushRequestAttr :: Name +pushRequestAttr = "pushrequest" + +startingPushAttr :: Name +startingPushAttr = "startingpush" + +receivePackAttr :: Name +receivePackAttr = "rp" + +sendPackAttr :: Name +sendPackAttr = "sp" + +receivePackDoneAttr :: Name +receivePackDoneAttr = "rpdone" diff --git a/Assistant/XMPP/Buddies.hs b/Assistant/XMPP/Buddies.hs new file mode 100644 index 0000000000..0c466e51c9 --- /dev/null +++ b/Assistant/XMPP/Buddies.hs @@ -0,0 +1,87 @@ +{- xmpp buddies + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.XMPP.Buddies where + +import Assistant.XMPP +import Common.Annex +import Assistant.Types.Buddies + +import Network.Protocol.XMPP +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Text (Text) +import qualified Data.Text as T + +genBuddyKey :: JID -> BuddyKey +genBuddyKey j = BuddyKey $ formatJID $ baseJID j + +buddyName :: JID -> Text +buddyName j = maybe (T.pack "") strNode (jidNode j) + +ucFirst :: Text -> Text +ucFirst s = let (first, rest) = T.splitAt 1 s + in T.concat [T.toUpper first, rest] + +{- Summary of info about a buddy. + - + - If the buddy has no clients at all anymore, returns Nothing. -} +buddySummary :: [JID] -> Buddy -> Maybe (Text, Bool, Bool, Bool, BuddyKey) +buddySummary pairedwith b = case clients of + ((Client j):_) -> Just (buddyName j, away, canpair, alreadypaired j, genBuddyKey j) + [] -> Nothing + where + away = S.null (buddyPresent b) && S.null (buddyAssistants b) + canpair = not $ S.null (buddyAssistants b) + clients = S.toList $ buddyPresent b `S.union` buddyAway b `S.union` buddyAssistants b + alreadypaired j = baseJID j `elem` pairedwith + +{- Updates the buddies with XMPP presence info. -} +updateBuddies :: Presence -> Buddies -> Buddies +updateBuddies p@(Presence { presenceFrom = Just jid }) = M.alter update key + where + key = genBuddyKey jid + update (Just b) = Just $ applyPresence p b + update Nothing = newBuddy p +updateBuddies _ = id + +{- Creates a new buddy based on XMPP presence info. -} +newBuddy :: Presence -> Maybe Buddy +newBuddy p + | presenceType p == PresenceAvailable = go + | presenceType p == PresenceUnavailable = go + | otherwise = Nothing + where + go = make <$> presenceFrom p + make _jid = applyPresence p $ Buddy + { buddyPresent = S.empty + , buddyAway = S.empty + , buddyAssistants = S.empty + , buddyPairing = False + } + +applyPresence :: Presence -> Buddy -> Buddy +applyPresence p b = fromMaybe b $! go <$> presenceFrom p + where + go jid + | presenceType p == PresenceUnavailable = b + { buddyAway = addto $ buddyAway b + , buddyPresent = removefrom $ buddyPresent b + , buddyAssistants = removefrom $ buddyAssistants b + } + | hasGitAnnexTag p = b + { buddyAssistants = addto $ buddyAssistants b + , buddyAway = removefrom $ buddyAway b } + | presenceType p == PresenceAvailable = b + { buddyPresent = addto $ buddyPresent b + , buddyAway = removefrom $ buddyAway b + } + | otherwise = b + where + client = Client jid + removefrom = S.filter (/= client) + addto = S.insert client diff --git a/Assistant/XMPP/Client.hs b/Assistant/XMPP/Client.hs new file mode 100644 index 0000000000..c2a86cb411 --- /dev/null +++ b/Assistant/XMPP/Client.hs @@ -0,0 +1,74 @@ +{- xmpp client support + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.XMPP.Client where + +import Assistant.Common +import Utility.SRV +import Creds + +import Network.Protocol.XMPP +import Network +import Control.Concurrent +import qualified Data.Text as T +import Control.Exception (SomeException) + +{- Everything we need to know to connect to an XMPP server. -} +data XMPPCreds = XMPPCreds + { xmppUsername :: T.Text + , xmppPassword :: T.Text + , xmppHostname :: HostName + , xmppPort :: Int + , xmppJID :: T.Text + } + deriving (Read, Show) + +connectXMPP :: XMPPCreds -> (JID -> XMPP a) -> IO (Either SomeException ()) +connectXMPP c a = case parseJID (xmppJID c) of + Nothing -> error "bad JID" + Just jid -> connectXMPP' jid c a + +{- Do a SRV lookup, but if it fails, fall back to the cached xmppHostname. -} +connectXMPP' :: JID -> XMPPCreds -> (JID -> XMPP a) -> IO (Either SomeException ()) +connectXMPP' jid c a = go =<< lookupSRV srvrecord + where + srvrecord = mkSRVTcp "xmpp-client" $ + T.unpack $ strDomain $ jidDomain jid + serverjid = JID Nothing (jidDomain jid) Nothing + + go [] = run (xmppHostname c) + (PortNumber $ fromIntegral $ xmppPort c) + (a jid) + go ((h,p):rest) = do + {- Try each SRV record in turn, until one connects, + - at which point the MVar will be full. -} + mv <- newEmptyMVar + r <- run h p $ do + liftIO $ putMVar mv () + a jid + ifM (isEmptyMVar mv) (go rest, return r) + + {- Async exceptions are let through so the XMPP thread can + - be killed. -} + run h p a' = tryNonAsync $ + runClientError (Server serverjid h p) jid + (xmppUsername c) (xmppPassword c) (void a') + +{- XMPP runClient, that throws errors rather than returning an Either -} +runClientError :: Server -> JID -> T.Text -> T.Text -> XMPP a -> IO a +runClientError s j u p x = either (error . show) return =<< runClient s j u p x + +getXMPPCreds :: Annex (Maybe XMPPCreds) +getXMPPCreds = parse <$> readCacheCreds xmppCredsFile + where + parse s = readish =<< s + +setXMPPCreds :: XMPPCreds -> Annex () +setXMPPCreds creds = writeCacheCreds (show creds) xmppCredsFile + +xmppCredsFile :: FilePath +xmppCredsFile = "xmpp" diff --git a/Assistant/XMPP/Git.hs b/Assistant/XMPP/Git.hs new file mode 100644 index 0000000000..8dba309a88 --- /dev/null +++ b/Assistant/XMPP/Git.hs @@ -0,0 +1,296 @@ +{- git over XMPP + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.XMPP.Git where + +import Assistant.Common +import Assistant.NetMessager +import Assistant.Types.NetMessager +import Assistant.XMPP +import Assistant.XMPP.Buddies +import Assistant.DaemonStatus +import Assistant.Alert +import Assistant.MakeRemote +import Assistant.Sync +import qualified Command.Sync +import qualified Annex.Branch +import Annex.UUID +import Config +import Git +import qualified Git.Branch +import Locations.UserConfig +import qualified Types.Remote as Remote +import Utility.FileMode +import Utility.Shell + +import Network.Protocol.XMPP +import qualified Data.Text as T +import System.Posix.Env +import System.Posix.Types +import System.Process (std_in, std_out, std_err) +import Control.Concurrent +import System.Timeout +import qualified Data.ByteString as B +import qualified Data.Map as M + +finishXMPPPairing :: JID -> UUID -> Assistant () +finishXMPPPairing jid u = void $ alertWhile alert $ + makeXMPPGitRemote buddy (baseJID jid) u + where + buddy = T.unpack $ buddyName jid + alert = pairRequestAcknowledgedAlert buddy Nothing + +gitXMPPLocation :: JID -> String +gitXMPPLocation jid = "xmpp::" ++ T.unpack (formatJID $ baseJID jid) + +makeXMPPGitRemote :: String -> JID -> UUID -> Assistant Bool +makeXMPPGitRemote buddyname jid u = do + remote <- liftAnnex $ addRemote $ + makeGitRemote buddyname $ gitXMPPLocation jid + liftAnnex $ storeUUID (remoteConfig (Remote.repo remote) "uuid") u + syncNewRemote remote + return True + +{- Pushes over XMPP, communicating with a specific client. + - Runs an arbitrary IO action to push, which should run git-push with + - an xmpp:: url. + - + - To handle xmpp:: urls, git push will run git-remote-xmpp, which is + - injected into its PATH, and in turn runs git-annex xmppgit. The + - dataflow them becomes: + - + - git push <--> git-annex xmppgit <--> xmppPush <-------> xmpp + - | + - git receive-pack <--> xmppReceivePack <---------------> xmpp + - + - The pipe between git-annex xmppgit and us is set up and communicated + - using two environment variables, relayIn and relayOut, that are set + - to the file descriptors to use. Another, relayControl, is used to + - propigate the exit status of git receive-pack. + - + - We listen at the other end of the pipe and relay to and from XMPP. + -} +xmppPush :: ClientID -> (Git.Repo -> IO Bool) -> Assistant Bool +xmppPush cid gitpush = runPush SendPack cid handleDeferred $ do + sendNetMessage $ Pushing cid StartingPush + + (Fd inf, writepush) <- liftIO createPipe + (readpush, Fd outf) <- liftIO createPipe + (Fd controlf, writecontrol) <- liftIO createPipe + + tmp <- liftAnnex $ fromRepo gitAnnexTmpDir + let tmpdir = tmp "xmppgit" + installwrapper tmpdir + + env <- liftIO getEnvironment + path <- liftIO getSearchPath + let myenv = M.fromList + [ ("PATH", join [searchPathSeparator] $ tmpdir:path) + , (relayIn, show inf) + , (relayOut, show outf) + , (relayControl, show controlf) + ] + `M.union` M.fromList env + + inh <- liftIO $ fdToHandle readpush + outh <- liftIO $ fdToHandle writepush + controlh <- liftIO $ fdToHandle writecontrol + + t1 <- forkIO <~> toxmpp inh + t2 <- forkIO <~> fromxmpp outh controlh + + {- This can take a long time to run, so avoid running it in the + - Annex monad. Also, override environment. -} + g <- liftAnnex gitRepo + r <- liftIO $ gitpush $ g { gitEnv = Just $ M.toList myenv } + + liftIO $ do + mapM_ killThread [t1, t2] + mapM_ hClose [inh, outh, controlh] + + return r + where + toxmpp inh = forever $ do + b <- liftIO $ B.hGetSome inh chunkSize + if B.null b + then liftIO $ killThread =<< myThreadId + else sendNetMessage $ Pushing cid $ SendPackOutput b + fromxmpp outh controlh = forever $ do + m <- timeout xmppTimeout <~> waitNetPushMessage SendPack + case m of + (Just (Pushing _ (ReceivePackOutput b))) -> + liftIO $ writeChunk outh b + (Just (Pushing _ (ReceivePackDone exitcode))) -> + liftIO $ do + hPrint controlh exitcode + hFlush controlh + (Just _) -> noop + Nothing -> do + debug ["timeout waiting for git receive-pack output via XMPP"] + -- Send a synthetic exit code to git-annex + -- xmppgit, which will exit and cause git push + -- to die. + liftIO $ do + hPrint controlh (ExitFailure 1) + hFlush controlh + installwrapper tmpdir = liftIO $ do + createDirectoryIfMissing True tmpdir + let wrapper = tmpdir "git-remote-xmpp" + program <- readProgramFile + writeFile wrapper $ unlines + [ shebang + , "exec " ++ program ++ " xmppgit" + ] + modifyFileMode wrapper $ addModes executeModes + +type EnvVar = String + +envVar :: String -> EnvVar +envVar s = "GIT_ANNEX_XMPPGIT_" ++ s + +relayIn :: EnvVar +relayIn = envVar "IN" + +relayOut :: EnvVar +relayOut = envVar "OUT" + +relayControl :: EnvVar +relayControl = envVar "CONTROL" + +relayHandle :: EnvVar -> IO Handle +relayHandle var = do + v <- getEnv var + case readish =<< v of + Nothing -> error $ var ++ " not set" + Just n -> fdToHandle $ Fd n + +{- Called by git-annex xmppgit. + - + - git-push is talking to us on stdin + - we're talking to git-push on stdout + - git-receive-pack is talking to us on relayIn (via XMPP) + - we're talking to git-receive-pack on relayOut (via XMPP) + - git-receive-pack's exit code will be passed to us on relayControl + -} +xmppGitRelay :: IO () +xmppGitRelay = do + flip relay stdout =<< relayHandle relayIn + relay stdin =<< relayHandle relayOut + code <- hGetLine =<< relayHandle relayControl + exitWith $ fromMaybe (ExitFailure 1) $ readish code + where + {- Is it possible to set up pipes and not need to copy the data + - ourselves? See splice(2) -} + relay fromh toh = void $ forkIO $ forever $ do + b <- B.hGetSome fromh chunkSize + when (B.null b) $ do + hClose fromh + hClose toh + killThread =<< myThreadId + writeChunk toh b + +{- Relays git receive-pack stdin and stdout via XMPP, as well as propigating + - its exit status to XMPP. -} +xmppReceivePack :: ClientID -> Assistant Bool +xmppReceivePack cid = runPush ReceivePack cid handleDeferred $ do + repodir <- liftAnnex $ fromRepo repoPath + let p = (proc "git" ["receive-pack", repodir]) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = Inherit + } + (Just inh, Just outh, _, pid) <- liftIO $ createProcess p + readertid <- forkIO <~> relayfromxmpp inh + relaytoxmpp outh + code <- liftIO $ waitForProcess pid + void $ sendNetMessage $ Pushing cid $ ReceivePackDone code + liftIO $ do + killThread readertid + hClose inh + hClose outh + return $ code == ExitSuccess + where + relaytoxmpp outh = do + b <- liftIO $ B.hGetSome outh chunkSize + -- empty is EOF, so exit + unless (B.null b) $ do + sendNetMessage $ Pushing cid $ ReceivePackOutput b + relaytoxmpp outh + relayfromxmpp inh = forever $ do + m <- timeout xmppTimeout <~> waitNetPushMessage ReceivePack + case m of + (Just (Pushing _ (SendPackOutput b))) -> + liftIO $ writeChunk inh b + (Just _) -> noop + Nothing -> do + debug ["timeout waiting for git send-pack output via XMPP"] + -- closing the handle will make + -- git receive-pack exit + liftIO $ do + hClose inh + killThread =<< myThreadId + +xmppRemotes :: ClientID -> Assistant [Remote] +xmppRemotes cid = case baseJID <$> parseJID cid of + Nothing -> return [] + Just jid -> do + let loc = gitXMPPLocation jid + filter (matching loc . Remote.repo) . syncGitRemotes + <$> getDaemonStatus + where + matching loc r = repoIsUrl r && repoLocation r == loc + +whenXMPPRemote :: ClientID -> Assistant () -> Assistant () +whenXMPPRemote cid = unlessM (null <$> xmppRemotes cid) + +handlePushInitiation :: NetMessage -> Assistant () +handlePushInitiation (Pushing cid CanPush) = + whenXMPPRemote cid $ + sendNetMessage $ Pushing cid PushRequest + +handlePushInitiation (Pushing cid PushRequest) = + go =<< liftAnnex (inRepo Git.Branch.current) + where + go Nothing = noop + go (Just branch) = do + rs <- xmppRemotes cid + liftAnnex $ Annex.Branch.commit "update" + (g, u) <- liftAnnex $ (,) + <$> gitRepo + <*> getUUID + liftIO $ Command.Sync.updateBranch (Command.Sync.syncBranch branch) g + debug ["pushing to", show rs] + forM_ rs $ \r -> xmppPush cid $ pushFallback u branch r + +handlePushInitiation (Pushing cid StartingPush) = + whenXMPPRemote cid $ + void $ xmppReceivePack cid +handlePushInitiation _ = noop + +handleDeferred :: NetMessage -> Assistant () +handleDeferred = handlePushInitiation + +writeChunk :: Handle -> B.ByteString -> IO () +writeChunk h b = do + B.hPut h b + hFlush h + +{- Largest chunk of data to send in a single XMPP message. -} +chunkSize :: Int +chunkSize = 4096 + +{- How long to wait for an expected message before assuming the other side + - has gone away and canceling a push. + - + - This needs to be long enough to allow a message of up to 2+ times + - chunkSize to propigate up to a XMPP server, perhaps across to another + - server, and back down to us. On the other hand, other XMPP pushes can be + - delayed for running until the timeout is reached, so it should not be + - excessive. + -} +xmppTimeout :: Int +xmppTimeout = 120000000 -- 120 seconds diff --git a/Backend.hs b/Backend.hs new file mode 100644 index 0000000000..6bbf3f75e1 --- /dev/null +++ b/Backend.hs @@ -0,0 +1,122 @@ +{- git-annex key/value backends + - + - Copyright 2010,2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend ( + list, + orderedList, + genKey, + lookupFile, + isAnnexLink, + makeAnnexLink, + chooseBackend, + lookupBackendName, + maybeLookupBackendName +) where + +import Common.Annex +import qualified Annex +import Annex.CheckAttr +import Annex.CatFile +import Annex.Link +import Types.Key +import Types.KeySource +import qualified Types.Backend as B +import Config + +-- When adding a new backend, import it here and add it to the list. +import qualified Backend.SHA +import qualified Backend.WORM +import qualified Backend.URL + +list :: [Backend] +list = Backend.SHA.backends ++ Backend.WORM.backends ++ Backend.URL.backends + +{- List of backends in the order to try them when storing a new key. -} +orderedList :: Annex [Backend] +orderedList = do + l <- Annex.getState Annex.backends -- list is cached here + if not $ null l + then return l + else do + f <- Annex.getState Annex.forcebackend + case f of + Just name | not (null name) -> + return [lookupBackendName name] + _ -> do + l' <- gen . annexBackends <$> Annex.getGitConfig + Annex.changeState $ \s -> s { Annex.backends = l' } + return l' + where + gen [] = list + gen l = map lookupBackendName l + +{- Generates a key for a file, trying each backend in turn until one + - accepts it. -} +genKey :: KeySource -> Maybe Backend -> Annex (Maybe (Key, Backend)) +genKey source trybackend = do + bs <- orderedList + let bs' = maybe bs (: bs) trybackend + genKey' bs' source +genKey' :: [Backend] -> KeySource -> Annex (Maybe (Key, Backend)) +genKey' [] _ = return Nothing +genKey' (b:bs) source = do + r <- B.getKey b source + case r of + Nothing -> genKey' bs source + Just k -> return $ Just (makesane k, b) + where + -- keyNames should not contain newline characters. + makesane k = k { keyName = map fixbadchar (keyName k) } + fixbadchar c + | c == '\n' = '_' + | otherwise = c + +{- Looks up the key and backend corresponding to an annexed file, + - by examining what the file links to. + - + - In direct mode, there is often no link on disk, in which case + - the symlink is looked up in git instead. However, a real link + - on disk still takes precedence over what was committed to git in direct + - mode. + -} +lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) +lookupFile file = do + mkey <- isAnnexLink file + case mkey of + Just key -> makeret key + Nothing -> ifM isDirect + ( maybe (return Nothing) makeret =<< catKeyFile file + , return Nothing + ) + where + makeret k = let bname = keyBackendName k in + case maybeLookupBackendName bname of + Just backend -> do + return $ Just (k, backend) + Nothing -> do + warning $ + "skipping " ++ file ++ + " (unknown backend " ++ bname ++ ")" + return Nothing + +{- Looks up the backend that should be used for a file. + - That can be configured on a per-file basis in the gitattributes file. -} +chooseBackend :: FilePath -> Annex (Maybe Backend) +chooseBackend f = Annex.getState Annex.forcebackend >>= go + where + go Nothing = maybeLookupBackendName <$> checkAttr "annex.backend" f + go (Just _) = Just . Prelude.head <$> orderedList + +{- Looks up a backend by name. May fail if unknown. -} +lookupBackendName :: String -> Backend +lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s + where + unknown = error $ "unknown backend " ++ s +maybeLookupBackendName :: String -> Maybe Backend +maybeLookupBackendName s = headMaybe matches + where + matches = filter (\b -> s == B.name b) list diff --git a/Backend/SHA.hs b/Backend/SHA.hs new file mode 100644 index 0000000000..34faa4922e --- /dev/null +++ b/Backend/SHA.hs @@ -0,0 +1,164 @@ +{- git-annex SHA backends + - + - Copyright 2011,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA (backends) where + +import Common.Annex +import qualified Annex +import Types.Backend +import Types.Key +import Types.KeySource + +import qualified Build.SysConfig as SysConfig +import Data.Digest.Pure.SHA +import qualified Data.ByteString.Lazy as L +import System.Process +import Data.Char + +type SHASize = Int + +{- Order is slightly significant; want SHA256 first, and more general + - sizes earlier. -} +sizes :: [Int] +sizes = [256, 1, 512, 224, 384] + +{- The SHA256E backend is the default. -} +backends :: [Backend] +backends = catMaybes $ map genBackendE sizes ++ map genBackend sizes + +genBackend :: SHASize -> Maybe Backend +genBackend size = Just $ Backend + { name = shaName size + , getKey = keyValue size + , fsckKey = Just $ checkKeyChecksum size + , canUpgradeKey = Just $ needsUpgrade + } + +genBackendE :: SHASize -> Maybe Backend +genBackendE size = do + b <- genBackend size + return $ b + { name = shaNameE size + , getKey = keyValueE size + } + +shaName :: SHASize -> String +shaName size = "SHA" ++ show size + +shaNameE :: SHASize -> String +shaNameE size = shaName size ++ "E" + +shaN :: SHASize -> FilePath -> Integer -> Annex String +shaN shasize file filesize = do + showAction "checksum" + case shaCommand shasize filesize of + Left sha -> liftIO $ sha <$> L.readFile file + Right command -> liftIO $ parse command . lines <$> + readsha command (toCommand [File file]) + where + parse command [] = bad command + parse command (l:_) + | null sha = bad command + -- sha is prefixed with \ when filename contains certian chars + | "\\" `isPrefixOf` sha = drop 1 sha + | otherwise = sha + where + sha = fst $ separate (== ' ') l + bad command = error $ command ++ " parse error" + {- sha commands output the filename, so need to set fileEncoding -} + readsha command args = + withHandle StdoutHandle createProcessSuccess p $ \h -> do + fileEncoding h + output <- hGetContentsStrict h + hClose h + return output + where + p = (proc command args) { std_out = CreatePipe } + +shaCommand :: SHASize -> Integer -> Either (L.ByteString -> String) String +shaCommand shasize filesize + | shasize == 1 = use SysConfig.sha1 sha1 + | shasize == 256 = use SysConfig.sha256 sha256 + | shasize == 224 = use SysConfig.sha224 sha224 + | shasize == 384 = use SysConfig.sha384 sha384 + | shasize == 512 = use SysConfig.sha512 sha512 + | otherwise = error $ "bad sha size " ++ show shasize + where + use Nothing sha = Left $ showDigest . sha + use (Just c) sha + {- use builtin, but slower sha for small files + - benchmarking indicates it's faster up to + - and slightly beyond 50 kb files -} + | filesize < 51200 = use Nothing sha + | otherwise = Right c + +{- A key is a checksum of its contents. -} +keyValue :: SHASize -> KeySource -> Annex (Maybe Key) +keyValue shasize source = do + let file = contentLocation source + stat <- liftIO $ getFileStatus file + let filesize = fromIntegral $ fileSize stat + s <- shaN shasize file filesize + return $ Just $ stubKey + { keyName = s + , keyBackendName = shaName shasize + , keySize = Just filesize + } + +{- Extension preserving keys. -} +keyValueE :: SHASize -> KeySource -> Annex (Maybe Key) +keyValueE size source = keyValue size source >>= maybe (return Nothing) addE + where + addE k = return $ Just $ k + { keyName = keyName k ++ selectExtension (keyFilename source) + , keyBackendName = shaNameE size + } + +selectExtension :: FilePath -> String +selectExtension f + | null es = "" + | otherwise = join "." ("":es) + where + es = filter (not . null) $ reverse $ + take 2 $ takeWhile shortenough $ + reverse $ split "." $ filter validExtension $ takeExtensions f + shortenough e = length e <= 4 -- long enough for "jpeg" + +{- A key's checksum is checked during fsck. -} +checkKeyChecksum :: SHASize -> Key -> FilePath -> Annex Bool +checkKeyChecksum size key file = do + fast <- Annex.getState Annex.fast + mstat <- liftIO $ catchMaybeIO $ getFileStatus file + case (mstat, fast) of + (Just stat, False) -> do + let filesize = fromIntegral $ fileSize stat + check <$> shaN size file filesize + _ -> return True + where + sha = keySha key + check s + | s == sha = True + {- A bug caused checksums to be prefixed with \ in some + - cases; still accept these as legal now that the bug has been + - fixed. -} + | '\\' : s == sha = True + | otherwise = False + +keySha :: Key -> String +keySha key = dropExtensions (keyName key) + +validExtension :: Char -> Bool +validExtension c + | isAlphaNum c = True + | c == '.' = True + | otherwise = False + +{- Upgrade keys that have the \ prefix on their sha due to a bug, or + - that contain non-alphanumeric characters in their extension. -} +needsUpgrade :: Key -> Bool +needsUpgrade key = "\\" `isPrefixOf` keySha key || + any (not . validExtension) (takeExtensions $ keyName key) diff --git a/Backend/URL.hs b/Backend/URL.hs new file mode 100644 index 0000000000..9e1652970a --- /dev/null +++ b/Backend/URL.hs @@ -0,0 +1,42 @@ +{- git-annex "URL" backend -- keys whose content is available from urls. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.URL ( + backends, + fromUrl +) where + +import Data.Hash.MD5 + +import Common.Annex +import Types.Backend +import Types.Key + +backends :: [Backend] +backends = [backend] + +backend :: Backend +backend = Backend + { name = "URL" + , getKey = const $ return Nothing + , fsckKey = Nothing + , canUpgradeKey = Nothing + } + +fromUrl :: String -> Maybe Integer -> Key +fromUrl url size = stubKey + { keyName = key + , keyBackendName = "URL" + , keySize = size + } + where + {- when it's not too long, use the url as the key name + - 256 is the absolute filename max, but use a shorter + - length because this is not the entire key filename. -} + key + | length url < 128 = url + | otherwise = take 128 url ++ "-" ++ md5s (Str url) diff --git a/Backend/WORM.hs b/Backend/WORM.hs new file mode 100644 index 0000000000..3471eedc14 --- /dev/null +++ b/Backend/WORM.hs @@ -0,0 +1,41 @@ +{- git-annex "WORM" backend -- Write Once, Read Many + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.WORM (backends) where + +import Common.Annex +import Types.Backend +import Types.Key +import Types.KeySource + +backends :: [Backend] +backends = [backend] + +backend :: Backend +backend = Backend + { name = "WORM" + , getKey = keyValue + , fsckKey = Nothing + , canUpgradeKey = Nothing + } + +{- The key includes the file size, modification time, and the + - basename of the filename. + - + - That allows multiple files with the same names to have different keys, + - while also allowing a file to be moved around while retaining the + - same key. + -} +keyValue :: KeySource -> Annex (Maybe Key) +keyValue source = do + stat <- liftIO $ getFileStatus $ contentLocation source + return $ Just Key { + keyName = takeFileName $ keyFilename source, + keyBackendName = name backend, + keySize = Just $ fromIntegral $ fileSize stat, + keyMtime = Just $ modificationTime stat + } diff --git a/Build/Configure.hs b/Build/Configure.hs new file mode 100644 index 0000000000..b6f2b773d2 --- /dev/null +++ b/Build/Configure.hs @@ -0,0 +1,146 @@ +{- Checks system configuration and generates SysConfig.hs. -} + +module Build.Configure where + +import System.Directory +import Data.List +import System.Process +import Control.Applicative +import System.FilePath +import System.Environment + +import Build.TestConfig +import Utility.SafeCommand + +tests :: [TestCase] +tests = + [ TestCase "version" getVersion + , TestCase "git" $ requireCmd "git" "git --version >/dev/null" + , TestCase "git version" getGitVersion + , testCp "cp_a" "-a" + , testCp "cp_p" "-p" + , testCp "cp_reflink_auto" "--reflink=auto" + , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" + , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" + , TestCase "wget" $ testCmd "wget" "wget --version >/dev/null" + , TestCase "bup" $ testCmd "bup" "bup --version >/dev/null" + , TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null" + , TestCase "lsof" $ findCmdPath "lsof" "lsof" + , TestCase "ssh connection caching" getSshConnectionCaching + ] ++ shaTestCases + [ (1, "da39a3ee5e6b4b0d3255bfef95601890afd80709") + , (256, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + , (512, "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e") + , (224, "d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f") + , (384, "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b") + ] + +{- shaNsum are the program names used by coreutils. Some systems like OSX + - sometimes install these with 'g' prefixes. + - + - On some systems, shaN is used instead, but on other + - systems, it might be "hashalot", which does not produce + - usable checksums. Only accept programs that produce + - known-good hashes. -} +shaTestCases :: [(Int, String)] -> [TestCase] +shaTestCases l = map make l + where + make (n, knowngood) = TestCase key $ maybeSelectCmd key $ + zip (shacmds n) (repeat check) + where + key = "sha" ++ show n + check = "/dev/null | grep -q '" ++ knowngood ++ "'" + shacmds n = concatMap (\x -> [x, 'g':x, osxpath x]) $ + map (\x -> "sha" ++ show n ++ x) ["sum", ""] + {- Max OSX sometimes puts GNU tools outside PATH, so look in + - the location it uses, and remember where to run them + - from. -} + osxpath = "/opt/local/libexec/gnubin" + +tmpDir :: String +tmpDir = "tmp" + +testFile :: String +testFile = tmpDir ++ "/testfile" + +testCp :: ConfigKey -> String -> TestCase +testCp k option = TestCase cmd $ testCmd k cmdline + where + cmd = "cp " ++ option + cmdline = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new" + +{- Pulls package version out of the changelog. -} +getVersion :: Test +getVersion = do + version <- getVersionString + return $ Config "packageversion" (StringConfig version) + +getVersionString :: IO String +getVersionString = do + changelog <- readFile "CHANGELOG" + let verline = head $ lines changelog + return $ middle (words verline !! 1) + where + middle = drop 1 . init + +getGitVersion :: Test +getGitVersion = do + s <- readProcess "git" ["--version"] "" + let version = unwords $ drop 2 $ words $ head $ lines s + return $ Config "gitversion" (StringConfig version) + +getSshConnectionCaching :: Test +getSshConnectionCaching = Config "sshconnectioncaching" . BoolConfig <$> + boolSystem "sh" [Param "-c", Param "ssh -o ControlPersist=yes -V >/dev/null 2>/dev/null"] + +{- Set up cabal file with version. -} +cabalSetup :: IO () +cabalSetup = do + version <- getVersionString + cabal <- readFile cabalfile + writeFile tmpcabalfile $ unlines $ + map (setfield "Version" version) $ + lines cabal + renameFile tmpcabalfile cabalfile + where + cabalfile = "git-annex.cabal" + tmpcabalfile = cabalfile++".tmp" + setfield field value s + | fullfield `isPrefixOf` s = fullfield ++ value + | otherwise = s + where + fullfield = field ++ ": " + +setup :: IO () +setup = do + createDirectoryIfMissing True tmpDir + writeFile testFile "test file contents" + +cleanup :: IO () +cleanup = removeDirectoryRecursive tmpDir + +run :: [TestCase] -> IO () +run ts = do + args <- getArgs + setup + config <- runTests ts + if args == ["Android"] + then writeSysConfig $ androidConfig config + else writeSysConfig config + cleanup + cabalSetup + +{- Hard codes some settings to cross-compile for Android. -} +androidConfig :: [Config] -> [Config] +androidConfig c = overrides ++ filter (not . overridden) c + where + overrides = + [ Config "cp_reflink_auto" $ BoolConfig False + , Config "curl" $ BoolConfig False + , Config "sha224" $ MaybeStringConfig Nothing + , Config "sha384" $ MaybeStringConfig Nothing + ] + overridden (Config k _) = k `elem` overridekeys + overridekeys = map (\(Config k _) -> k) overrides + diff --git a/Build/InstallDesktopFile.hs b/Build/InstallDesktopFile.hs new file mode 100644 index 0000000000..6339791554 --- /dev/null +++ b/Build/InstallDesktopFile.hs @@ -0,0 +1,96 @@ +{- Generating and installing a desktop menu entry file + - and a desktop autostart file. (And OSX equivilants.) + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Build.InstallDesktopFile where + +import Utility.Exception +import Utility.FreeDesktop +import Utility.Path +import Utility.Monad +import Locations.UserConfig +import Utility.OSX +import Assistant.Install.AutoStart + +import Control.Applicative +import System.Directory +import System.Environment +import System.Posix.User +import System.Posix.Files +import System.FilePath +import Data.Maybe + +{- The command can be either just "git-annex", or the full path to use + - to run it. -} +desktop :: FilePath -> DesktopEntry +desktop command = genDesktopEntry + "Git Annex" + "Track and sync the files in your Git Annex" + False + (command ++ " webapp") + ["Network", "FileTransfer"] + +autostart :: FilePath -> DesktopEntry +autostart command = genDesktopEntry + "Git Annex Assistant" + "Autostart" + False + (command ++ " assistant --autostart") + [] + +systemwideInstall :: IO Bool +systemwideInstall = isroot <||> destdirset + where + isroot = do + uid <- fromIntegral <$> getRealUserID + return $ uid == (0 :: Int) + destdirset = isJust <$> catchMaybeIO (getEnv "DESTDIR") + +inDestDir :: FilePath -> IO FilePath +inDestDir f = do + destdir <- catchDefaultIO "" (getEnv "DESTDIR") + return $ destdir ++ "/" ++ f + +writeFDODesktop :: FilePath -> IO () +writeFDODesktop command = do + datadir <- ifM systemwideInstall ( return systemDataDir, userDataDir ) + writeDesktopMenuFile (desktop command) + =<< inDestDir (desktopMenuFilePath "git-annex" datadir) + + configdir <- ifM systemwideInstall ( return systemConfigDir, userConfigDir ) + installAutoStart command + =<< inDestDir (autoStartPath "git-annex" configdir) + +writeOSXDesktop :: FilePath -> IO () +writeOSXDesktop command = do + installAutoStart command =<< inDestDir =<< ifM systemwideInstall + ( return $ systemAutoStart osxAutoStartLabel + , userAutoStart osxAutoStartLabel + ) + +install :: FilePath -> IO () +install command = do +#ifdef darwin_HOST_OS + writeOSXDesktop command +#else + writeFDODesktop command +#endif + ifM systemwideInstall + ( return () + , do + programfile <- inDestDir =<< programFile + createDirectoryIfMissing True (parentDir programfile) + writeFile programfile command + ) + +main :: IO () +main = getArgs >>= go + where + go [] = error "specify git-annex command" + go (command:_) = install command diff --git a/Build/OSXMkLibs.hs b/Build/OSXMkLibs.hs new file mode 100644 index 0000000000..b0f87d153b --- /dev/null +++ b/Build/OSXMkLibs.hs @@ -0,0 +1,126 @@ +{- OSX library copier + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Build.OSXMkLibs where + +import Control.Applicative +import System.Environment +import Data.Maybe +import System.FilePath +import System.Directory +import System.IO +import Control.Monad +import Data.List + +import Utility.PartialPrelude +import Utility.Directory +import Utility.Process +import Utility.Monad +import Utility.SafeCommand +import Utility.Path + +import qualified Data.Map as M +import qualified Data.Set as S + +type LibMap = M.Map FilePath String + +{- Recursively find and install libs, until nothing new to install is found. -} +mklibs :: FilePath -> [FilePath] -> LibMap -> IO () +mklibs appbase libdirs libmap = do + (new, libmap') <- installLibs appbase libmap + unless (null new) $ + mklibs appbase (libdirs++new) libmap' + +{- Returns directories into which new libs were installed. -} +installLibs :: FilePath -> LibMap -> IO ([FilePath], LibMap) +installLibs appbase libmap = do + (needlibs, libmap') <- otool appbase libmap + libs <- forM needlibs $ \lib -> do + let shortlib = fromMaybe (error "internal") (M.lookup lib libmap') + let fulllib = dropWhile (== '/') lib + let dest = appbase fulllib + let symdest = appbase shortlib + ifM (doesFileExist dest) + ( return Nothing + , do + createDirectoryIfMissing True (parentDir dest) + putStrLn $ "installing " ++ lib ++ " as " ++ shortlib + _ <- boolSystem "cp" [File lib, File dest] + _ <- boolSystem "chmod" [Param "644", File dest] + _ <- boolSystem "ln" [Param "-s", File fulllib, File symdest] + return $ Just appbase + ) + return (catMaybes libs, libmap') + +{- Returns libraries to install. -} +otool :: FilePath -> LibMap -> IO ([FilePath], LibMap) +otool appbase libmap = do + files <- filterM doesFileExist =<< dirContentsRecursive appbase + process [] files libmap + where + want s = not ("@executable_path" `isInfixOf` s) + && not (".framework" `isInfixOf` s) + && not ("libSystem.B" `isInfixOf` s) + process c [] m = return (nub $ concat c, m) + process c (file:rest) m = do + _ <- boolSystem "chmod" [Param "755", File file] + libs <- filter want . parseOtool + <$> readProcess "otool" ["-L", file] + m' <- install_name_tool file libs m + process (libs:c) rest m' + +parseOtool :: String -> [FilePath] +parseOtool = catMaybes . map parse . lines + where + parse l + | "\t" `isPrefixOf` l = headMaybe $ words l + | otherwise = Nothing + +{- Adjusts binaries to use libraries bundled with it, rather than the + - system libraries. -} +install_name_tool :: FilePath -> [FilePath] -> LibMap -> IO LibMap +install_name_tool _ [] libmap = return libmap +install_name_tool binary libs libmap = do + let (libnames, libmap') = getLibNames libs libmap + let params = concatMap change $ zip libs libnames + ok <- boolSystem "install_name_tool" $ params ++ [File binary] + unless ok $ + error $ "install_name_tool failed for " ++ binary + return libmap' + where + change (lib, libname) = + [ Param "-change" + , File lib + , Param $ "@executable_path/" ++ libname + ] + +getLibNames :: [FilePath] -> LibMap -> ([FilePath], LibMap) +getLibNames libs libmap = go [] libs libmap + where + go c [] m = (reverse c, m) + go c (l:rest) m = + let (f, m') = getLibName l m + in go (f:c) rest m' + +{- Uses really short names for the library files it installs, because + - binaries have arbitrarily short RPATH field limits. -} +getLibName :: FilePath -> LibMap -> (FilePath, LibMap) +getLibName lib libmap = case M.lookup lib libmap of + Just n -> (n, libmap) + Nothing -> (nextfreename, M.insert lib nextfreename libmap) + where + names = map (\c -> [c]) ['A' .. 'Z'] ++ + [[n, l] | n <- ['0' .. '9'], l <- ['A' .. 'Z']] + used = S.fromList $ M.elems libmap + nextfreename = fromMaybe (error "ran out of short library names!") $ + headMaybe $ dropWhile (`S.member` used) names + +main :: IO () +main = getArgs >>= go + where + go [] = error "specify OSXAPP_BASE" + go (appbase:_) = mklibs appbase [] M.empty diff --git a/Build/Standalone.hs b/Build/Standalone.hs new file mode 100644 index 0000000000..76ff7b25c3 --- /dev/null +++ b/Build/Standalone.hs @@ -0,0 +1,77 @@ +{- Makes standalone bundle. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Build.Standalone where + +import Control.Applicative +import Control.Monad.IfElse +import System.Environment +import Data.Maybe +import System.FilePath +import System.Directory +import System.IO +import Control.Monad +import Data.List +import Build.SysConfig as SysConfig + +import Utility.PartialPrelude +import Utility.Directory +import Utility.Process +import Utility.Monad +import Utility.SafeCommand +import Utility.Path + +{- Programs that git-annex uses, to include in the bundle. + - + - These may be just the command name, or the full path to it. -} +thirdpartyProgs :: [FilePath] +thirdpartyProgs = catMaybes + [ Just "git" + , Just "cp" + , Just "xargs" + , Just "gpg" + , Just "rsync" + , Just "ssh" + , Just "sh" + , ifset SysConfig.curl "curl" + , ifset SysConfig.wget "wget" + , ifset SysConfig.bup "bup" + , SysConfig.lsof + , SysConfig.sha1 + , SysConfig.sha256 + , SysConfig.sha512 + , SysConfig.sha224 + , SysConfig.sha384 + ] + where + ifset True s = Just s + ifset False _ = Nothing + +progDir :: FilePath -> FilePath +#ifdef darwin_HOST_OS +progDir topdir = topdir +#else +progDir topdir = topdir "bin" +#endif + +installProg :: FilePath -> FilePath -> IO () +installProg dir prog = searchPath prog >>= go + where + go Nothing = error $ "cannot find " ++ prog ++ " in PATH" + go (Just f) = unlessM (boolSystem "install" [File f, File dir]) $ + error $ "install failed for " ++ prog + +main = getArgs >>= go + where + go [] = error "specify topdir" + go (topdir:_) = do + let dir = progDir topdir + createDirectoryIfMissing True dir + forM_ thirdpartyProgs $ installProg dir + diff --git a/Build/TestConfig.hs b/Build/TestConfig.hs new file mode 100644 index 0000000000..9937f799f1 --- /dev/null +++ b/Build/TestConfig.hs @@ -0,0 +1,142 @@ +{- Tests the system and generates Build.SysConfig.hs. -} + +module Build.TestConfig where + +import Utility.Path +import Utility.Monad + +import System.IO +import System.Cmd +import System.Exit +import System.FilePath +import System.Directory + +type ConfigKey = String +data ConfigValue = + BoolConfig Bool | + StringConfig String | + MaybeStringConfig (Maybe String) | + MaybeBoolConfig (Maybe Bool) +data Config = Config ConfigKey ConfigValue + +type Test = IO Config +type TestName = String +data TestCase = TestCase TestName Test + +instance Show ConfigValue where + show (BoolConfig b) = show b + show (StringConfig s) = show s + show (MaybeStringConfig s) = show s + show (MaybeBoolConfig s) = show s + +instance Show Config where + show (Config key value) = unlines + [ key ++ " :: " ++ valuetype value + , key ++ " = " ++ show value + ] + where + valuetype (BoolConfig _) = "Bool" + valuetype (StringConfig _) = "String" + valuetype (MaybeStringConfig _) = "Maybe String" + valuetype (MaybeBoolConfig _) = "Maybe Bool" + +writeSysConfig :: [Config] -> IO () +writeSysConfig config = writeFile "Build/SysConfig.hs" body + where + body = unlines $ header ++ map show config ++ footer + header = [ + "{- Automatically generated. -}" + , "module Build.SysConfig where" + , "" + ] + footer = [] + +runTests :: [TestCase] -> IO [Config] +runTests [] = return [] +runTests (TestCase tname t : ts) = do + testStart tname + c <- t + testEnd c + rest <- runTests ts + return $ c:rest + +{- Tests that a command is available, aborting if not. -} +requireCmd :: ConfigKey -> String -> Test +requireCmd k cmdline = do + ret <- testCmd k cmdline + handle ret + where + handle r@(Config _ (BoolConfig True)) = return r + handle r = do + testEnd r + error $ "** the " ++ c ++ " command is required" + c = head $ words cmdline + +{- Checks if a command is available by running a command line. -} +testCmd :: ConfigKey -> String -> Test +testCmd k cmdline = do + ret <- system $ quiet cmdline + return $ Config k (BoolConfig $ ret == ExitSuccess) + +{- Ensures that one of a set of commands is available by running each in + - turn. The Config is set to the first one found. -} +selectCmd :: ConfigKey -> [(String, String)] -> Test +selectCmd k = searchCmd + (return . Config k . StringConfig) + (\cmds -> do + testEnd $ Config k $ BoolConfig False + error $ "* need one of these commands, but none are available: " ++ show cmds + ) + +maybeSelectCmd :: ConfigKey -> [(String, String)] -> Test +maybeSelectCmd k = searchCmd + (return . Config k . MaybeStringConfig . Just) + (\_ -> return $ Config k $ MaybeStringConfig Nothing) + +searchCmd :: (String -> Test) -> ([String] -> Test) -> [(String, String)] -> Test +searchCmd success failure cmdsparams = search cmdsparams + where + search [] = failure $ fst $ unzip cmdsparams + search ((c, params):cs) = do + ret <- system $ quiet $ c ++ " " ++ params + if ret == ExitSuccess + then success c + else search cs + +{- Finds a command, either in PATH or perhaps in a sbin directory not in + - PATH. If it's in PATH the config is set to just the command name, + - but if it's found outside PATH, the config is set to the full path to + - the command. -} +findCmdPath :: ConfigKey -> String -> Test +findCmdPath k command = do + ifM (inPath command) + ( return $ Config k $ MaybeStringConfig $ Just command + , do + r <- getM find ["/usr/sbin", "/sbin", "/usr/local/sbin"] + return $ Config k $ MaybeStringConfig r + ) + where + find d = + let f = d command + in ifM (doesFileExist f) ( return (Just f), return Nothing ) + +quiet :: String -> String +quiet s = s ++ " >/dev/null 2>&1" + +testStart :: TestName -> IO () +testStart s = do + putStr $ " checking " ++ s ++ "..." + hFlush stdout + +testEnd :: Config -> IO () +testEnd (Config _ (BoolConfig True)) = status "yes" +testEnd (Config _ (BoolConfig False)) = status "no" +testEnd (Config _ (StringConfig s)) = status s +testEnd (Config _ (MaybeStringConfig (Just s))) = status s +testEnd (Config _ (MaybeStringConfig Nothing)) = status "not available" +testEnd (Config _ (MaybeBoolConfig (Just True))) = status "yes" +testEnd (Config _ (MaybeBoolConfig (Just False))) = status "no" +testEnd (Config _ (MaybeBoolConfig Nothing)) = status "unknown" + +status :: String -> IO () +status s = putStrLn $ ' ':s diff --git a/Build/make-sdist.sh b/Build/make-sdist.sh new file mode 100755 index 0000000000..9503345327 --- /dev/null +++ b/Build/make-sdist.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# +# Workaround for `cabal sdist` requiring all included files to be listed +# in .cabal. + +# Create target directory +sdist_dir=git-annex-$(grep '^Version:' git-annex.cabal | sed -re 's/Version: *//') +mkdir --parents dist/$sdist_dir + +find . \( -name .git -or -name dist -or -name cabal-dev \) -prune \ + -or -not -name \\*.orig -not -type d -print \ +| perl -ne "print unless length >= 100 - length q{$sdist_dir}" \ +| xargs cp --parents --target-directory dist/$sdist_dir + +cd dist +tar -caf $sdist_dir.tar.gz $sdist_dir + +# Check that tarball can be unpacked by cabal. +# It's picky about tar longlinks etc. +rm -rf $sdist_dir +cabal unpack $sdist_dir.tar.gz diff --git a/Build/mdwn2man b/Build/mdwn2man new file mode 100755 index 0000000000..ba5919b385 --- /dev/null +++ b/Build/mdwn2man @@ -0,0 +1,43 @@ +#!/usr/bin/env perl +# Warning: hack + +my $prog=shift; +my $section=shift; + +print ".TH $prog $section\n"; + +while (<>) { + s{(\\?)\[\[([^\s\|\]]+)(\|[^\s\]]+)?\]\]}{$1 ? "[[$2]]" : $2}eg; + s/\`//g; + s/^\s*\./\\&./g; + if (/^#\s/) { + s/^#\s/.SH /; + <>; # blank; + } + s/^[ \n]+//; + s/^\t/ /; + s/-/\\-/g; + s/^Warning:.*//g; + s/^$/.PP\n/; + s/^\*\s+(.*)/.IP "$1"/; + next if $_ eq ".PP\n" && $skippara; + if (/^.IP /) { + $inlist=1; + $spippara=0; + } + elsif (/.SH/) { + $skippara=0; + $inlist=0; + } + elsif (/^\./) { + $skippara=1; + } + else { + $skippara=0; + } + if ($inlist && $_ eq ".PP\n") { + $_=".IP\n"; + } + + print $_; +} diff --git a/CHANGELOG b/CHANGELOG new file mode 120000 index 0000000000..d526672ce2 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1 @@ +debian/changelog \ No newline at end of file diff --git a/COPYRIGHT b/COPYRIGHT new file mode 120000 index 0000000000..9060ce8208 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1 @@ +debian/copyright \ No newline at end of file diff --git a/Checks.hs b/Checks.hs new file mode 100644 index 0000000000..92e9f7e38e --- /dev/null +++ b/Checks.hs @@ -0,0 +1,42 @@ +{- git-annex command checks + - + - Common sanity checks for commands, and an interface to selectively + - remove them, or add others. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Checks where + +import Common.Annex +import Types.Command +import Init +import Config +import qualified Git + +commonChecks :: [CommandCheck] +commonChecks = [repoExists] + +repoExists :: CommandCheck +repoExists = CommandCheck 0 ensureInitialized + +notDirect :: Command -> Command +notDirect = addCheck $ whenM isDirect $ + error "You cannot run this subcommand in a direct mode repository." + +notBareRepo :: Command -> Command +notBareRepo = addCheck $ whenM (fromRepo Git.repoIsLocalBare) $ + error "You cannot run this subcommand in a bare repository." + +dontCheck :: CommandCheck -> Command -> Command +dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c + +addCheck :: Annex () -> Command -> Command +addCheck check cmd = mutateCheck cmd $ \c -> + CommandCheck (length c + 100) check : c + +mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command +mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c } + diff --git a/CmdLine.hs b/CmdLine.hs new file mode 100644 index 0000000000..0b155215d7 --- /dev/null +++ b/CmdLine.hs @@ -0,0 +1,122 @@ +{- git-annex command line parsing and dispatch + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine ( + dispatch, + usage, + shutdown +) where + +import qualified Control.Exception as E +import qualified Data.Map as M +import Control.Exception (throw) +import System.Console.GetOpt +import System.Posix.Signals + +import Common.Annex +import qualified Annex +import qualified Annex.Queue +import qualified Git +import qualified Git.AutoCorrect +import Annex.Content +import Annex.Ssh +import Command + +type Params = [String] +type Flags = [Annex ()] + +{- Runs the passed command line. -} +dispatch :: Bool -> Params -> [Command] -> [Option] -> [(String, String)] -> String -> IO Git.Repo -> IO () +dispatch fuzzyok allargs allcmds commonoptions fields header getgitrepo = do + setupConsole + r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo) + case r of + Left e -> fromMaybe (throw e) (cmdnorepo cmd) + Right g -> do + state <- Annex.new g + (actions, state') <- Annex.run state $ do + checkfuzzy + forM_ fields $ \(f, v) -> Annex.setField f v + sequence_ flags + prepCommand cmd params + tryRun state' cmd $ [startup] ++ actions ++ [shutdown $ cmdnocommit cmd] + where + err msg = msg ++ "\n\n" ++ usage header allcmds commonoptions + cmd = Prelude.head cmds + (fuzzy, cmds, name, args) = findCmd fuzzyok allargs allcmds err + (flags, params) = getOptCmd args cmd commonoptions err + checkfuzzy = when fuzzy $ + inRepo $ Git.AutoCorrect.prepare name cmdname cmds + +{- Parses command line params far enough to find the Command to run, and + - returns the remaining params. + - Does fuzzy matching if necessary, which may result in multiple Commands. -} +findCmd :: Bool -> Params -> [Command] -> (String -> String) -> (Bool, [Command], String, Params) +findCmd fuzzyok argv cmds err + | isNothing name = error $ err "missing command" + | not (null exactcmds) = (False, exactcmds, fromJust name, args) + | fuzzyok && not (null inexactcmds) = (True, inexactcmds, fromJust name, args) + | otherwise = error $ err $ "unknown command " ++ fromJust name + where + (name, args) = findname argv [] + findname [] c = (Nothing, reverse c) + findname (a:as) c + | "-" `isPrefixOf` a = findname as (a:c) + | otherwise = (Just a, reverse c ++ as) + exactcmds = filter (\c -> name == Just (cmdname c)) cmds + inexactcmds = case name of + Nothing -> [] + Just n -> Git.AutoCorrect.fuzzymatches n cmdname cmds + +{- Parses command line options, and returns actions to run to configure flags + - and the remaining parameters for the command. -} +getOptCmd :: Params -> Command -> [Option] -> (String -> String) -> (Flags, Params) +getOptCmd argv cmd commonoptions err = check $ + getOpt Permute (commonoptions ++ cmdoptions cmd) argv + where + check (flags, rest, []) = (flags, rest) + check (_, _, errs) = error $ err $ concat errs + +{- Runs a list of Annex actions. Catches IO errors and continues + - (but explicitly thrown errors terminate the whole command). + -} +tryRun :: Annex.AnnexState -> Command -> [CommandCleanup] -> IO () +tryRun = tryRun' 0 +tryRun' :: Integer -> Annex.AnnexState -> Command -> [CommandCleanup] -> IO () +tryRun' errnum _ cmd [] + | errnum > 0 = error $ cmdname cmd ++ ": " ++ show errnum ++ " failed" + | otherwise = noop +tryRun' errnum state cmd (a:as) = do + r <- run + handle $! r + where + run = tryIO $ Annex.run state $ do + Annex.Queue.flushWhenFull + a + handle (Left err) = showerr err >> cont False state + handle (Right (success, state')) = cont success state' + cont success s = do + let errnum' = if success then errnum else errnum + 1 + (tryRun' $! errnum') s cmd as + showerr err = Annex.eval state $ do + showErr err + showEndFail + +{- Actions to perform each time ran. -} +startup :: Annex Bool +startup = liftIO $ do + void $ installHandler sigINT Default Nothing + return True + +{- Cleanup actions. -} +shutdown :: Bool -> Annex Bool +shutdown nocommit = do + saveState nocommit + sequence_ =<< M.elems <$> Annex.getState Annex.cleanup + liftIO reapZombies -- zombies from long-running git processes + sshCleanup -- ssh connection caching + return True diff --git a/Command.hs b/Command.hs new file mode 100644 index 0000000000..8225f7b1bb --- /dev/null +++ b/Command.hs @@ -0,0 +1,134 @@ +{- git-annex command infrastructure + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command ( + command, + noRepo, + noCommit, + withOptions, + next, + stop, + stopUnless, + prepCommand, + doCommand, + whenAnnexed, + ifAnnexed, + isBareRepo, + numCopies, + numCopiesCheck, + autoCopiesWith, + checkAuto, + module ReExported +) where + +import Common.Annex +import qualified Backend +import qualified Annex +import qualified Git +import qualified Remote +import Types.Command as ReExported +import Types.Option as ReExported +import Seek as ReExported +import Checks as ReExported +import Usage as ReExported +import Logs.Trust +import Config +import Annex.CheckAttr + +{- Generates a normal command -} +command :: String -> String -> [CommandSeek] -> String -> Command +command = Command [] Nothing commonChecks False + +{- Indicates that a command doesn't need to commit any changes to + - the git-annex branch. -} +noCommit :: Command -> Command +noCommit c = c { cmdnocommit = True } + +{- Adds a fallback action to a command, that will be run if it's used + - outside a git repository. -} +noRepo :: IO () -> Command -> Command +noRepo a c = c { cmdnorepo = Just a } + +{- Adds options to a command. -} +withOptions :: [Option] -> Command -> Command +withOptions o c = c { cmdoptions = o } + +{- For start and perform stages to indicate what step to run next. -} +next :: a -> Annex (Maybe a) +next a = return $ Just a + +{- Or to indicate nothing needs to be done. -} +stop :: Annex (Maybe a) +stop = return Nothing + +{- Stops unless a condition is met. -} +stopUnless :: Annex Bool -> Annex (Maybe a) -> Annex (Maybe a) +stopUnless c a = ifM c ( a , stop ) + +{- Prepares to run a command via the check and seek stages, returning a + - list of actions to perform to run the command. -} +prepCommand :: Command -> [String] -> Annex [CommandCleanup] +prepCommand Command { cmdseek = seek, cmdcheck = c } params = do + mapM_ runCheck c + map doCommand . concat <$> mapM (\s -> s params) seek + +{- Runs a command through the start, perform and cleanup stages -} +doCommand :: CommandStart -> CommandCleanup +doCommand = start + where + start = stage $ maybe skip perform + perform = stage $ maybe failure cleanup + cleanup = stage $ status + stage = (=<<) + skip = return True + failure = showEndFail >> return False + status r = showEndResult r >> return r + +{- Modifies an action to only act on files that are already annexed, + - and passes the key and backend on to it. -} +whenAnnexed :: (FilePath -> (Key, Backend) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a) +whenAnnexed a file = ifAnnexed file (a file) (return Nothing) + +ifAnnexed :: FilePath -> ((Key, Backend) -> Annex a) -> Annex a -> Annex a +ifAnnexed file yes no = maybe no yes =<< Backend.lookupFile file + +isBareRepo :: Annex Bool +isBareRepo = fromRepo Git.repoIsLocalBare + +numCopies :: FilePath -> Annex (Maybe Int) +numCopies file = readish <$> checkAttr "annex.numcopies" file + +numCopiesCheck :: FilePath -> Key -> (Int -> Int -> Bool) -> Annex Bool +numCopiesCheck file key vs = do + numcopiesattr <- numCopies file + needed <- getNumCopies numcopiesattr + have <- trustExclude UnTrusted =<< Remote.keyLocations key + return $ length have `vs` needed + +{- Used for commands that have an auto mode that checks the number of known + - copies of a key. + - + - In auto mode, first checks that the number of known + - copies of the key is > or < than the numcopies setting, before running + - the action. + -} +autoCopiesWith :: FilePath -> Key -> (Int -> Int -> Bool) -> (Maybe Int -> CommandStart) -> CommandStart +autoCopiesWith file key vs a = do + numcopiesattr <- numCopies file + Annex.getState Annex.auto >>= auto numcopiesattr + where + auto numcopiesattr False = a numcopiesattr + auto numcopiesattr True = do + needed <- getNumCopies numcopiesattr + have <- trustExclude UnTrusted =<< Remote.keyLocations key + if length have `vs` needed + then a numcopiesattr + else stop + +checkAuto :: Annex Bool -> Annex Bool +checkAuto checker = ifM (Annex.getState Annex.auto) + ( checker , return True ) diff --git a/Command/Add.hs b/Command/Add.hs new file mode 100644 index 0000000000..d41a0baead --- /dev/null +++ b/Command/Add.hs @@ -0,0 +1,212 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Command.Add where + +import Common.Annex +import Annex.Exception +import Command +import Types.KeySource +import Backend +import Logs.Location +import Annex.Content +import Annex.Content.Direct +import Annex.Perms +import Annex.Link +import qualified Annex +import qualified Annex.Queue +#ifndef __ANDROID__ +import Utility.Touch +#endif +import Utility.FileMode +import Config +import Utility.InodeCache + +def :: [Command] +def = [notBareRepo $ command "add" paramPaths seek "add files to annex"] + +{- Add acts on both files not checked into git yet, and unlocked files. + - + - In direct mode, it acts on any files that have changed. -} +seek :: [CommandSeek] +seek = + [ withFilesNotInGit start + , whenNotDirect $ withFilesUnlocked start + , whenDirect $ withFilesMaybeModified start + ] + +{- The add subcommand annexes a file, generating a key for it using a + - backend, and then moving it into the annex directory and setting up + - the symlink pointing to its content. -} +start :: FilePath -> CommandStart +start file = ifAnnexed file addpresent add + where + add = do + s <- liftIO $ getSymbolicLinkStatus file + if isSymbolicLink s || not (isRegularFile s) + then stop + else do + showStart "add" file + next $ perform file + addpresent (key, _) = ifM isDirect + ( ifM (goodContent key file) ( stop , add ) + , fixup key + ) + fixup key = do + -- fixup from an interrupted add; the symlink + -- is present but not yet added to git + showStart "add" file + liftIO $ removeFile file + next $ next $ cleanup file key =<< inAnnex key + +{- The file that's being added is locked down before a key is generated, + - to prevent it from being modified in between. It's hard linked into a + - temporary location, and its writable bits are removed. It could still be + - written to by a process that already has it open for writing. + - + - Lockdown can fail if a file gets deleted, and Nothing will be returned. + -} +lockDown :: FilePath -> Annex (Maybe KeySource) +lockDown file = ifM (crippledFileSystem) + ( liftIO $ catchMaybeIO $ do + cache <- genInodeCache file + return $ KeySource + { keyFilename = file + , contentLocation = file + , inodeCache = cache + } + , do + tmp <- fromRepo gitAnnexTmpDir + createAnnexDirectory tmp + liftIO $ catchMaybeIO $ do + preventWrite file + (tmpfile, h) <- openTempFile tmp (takeFileName file) + hClose h + nukeFile tmpfile + createLink file tmpfile + cache <- genInodeCache tmpfile + return $ KeySource + { keyFilename = file + , contentLocation = tmpfile + , inodeCache = cache + } + ) + +{- Ingests a locked down file into the annex. + - + - In direct mode, leaves the file alone, and just updates bookkeeping + - information. + -} +ingest :: (Maybe KeySource) -> Annex (Maybe Key) +ingest Nothing = return Nothing +ingest (Just source) = do + backend <- chooseBackend $ keyFilename source + k <- genKey source backend + cache <- liftIO $ genInodeCache $ contentLocation source + case inodeCache source of + Nothing -> go k cache + Just c + | (Just c == cache) -> go k cache + | otherwise -> failure + where + go k cache = ifM isDirect ( godirect k cache , goindirect k cache ) + + goindirect (Just (key, _)) _ = do + handle (undo (keyFilename source) key) $ + moveAnnex key $ contentLocation source + liftIO $ nukeFile $ keyFilename source + return $ Just key + goindirect Nothing _ = failure + + godirect (Just (key, _)) (Just cache) = do + writeInodeCache key cache + void $ addAssociatedFile key $ keyFilename source + unlessM crippledFileSystem $ + liftIO $ allowWrite $ keyFilename source + when (contentLocation source /= keyFilename source) $ + liftIO $ nukeFile $ contentLocation source + return $ Just key + godirect _ _ = failure + + failure = do + when (contentLocation source /= keyFilename source) $ + liftIO $ nukeFile $ contentLocation source + return Nothing + +perform :: FilePath -> CommandPerform +perform file = + maybe stop (\key -> next $ cleanup file key True) + =<< ingest =<< lockDown file + +{- On error, put the file back so it doesn't seem to have vanished. + - This can be called before or after the symlink is in place. -} +undo :: FilePath -> Key -> IOException -> Annex a +undo file key e = do + whenM (inAnnex key) $ do + liftIO $ nukeFile file + handle tryharder $ fromAnnex key file + logStatus key InfoMissing + throw e + where + -- fromAnnex could fail if the file ownership is weird + tryharder :: IOException -> Annex () + tryharder _ = do + src <- inRepo $ gitAnnexLocation key + liftIO $ moveFile src file + +{- Creates the symlink to the annexed content, returns the link target. -} +link :: FilePath -> Key -> Bool -> Annex String +link file key hascontent = handle (undo file key) $ do + l <- calcGitLink file key + makeAnnexLink l file + +#ifndef __ANDROID__ + when hascontent $ do + -- touch the symlink to have the same mtime as the + -- file it points to + liftIO $ do + mtime <- modificationTime <$> getFileStatus file + touch file (TimeSpec mtime) False +#endif + + return l + +{- Note: Several other commands call this, and expect it to + - create the link and add it. + - + - In direct mode, when we have the content of the file, it's left as-is, + - and we just stage a symlink to git. + - + - Otherwise, as long as the filesystem supports symlinks, we use + - git add, rather than directly staging the symlink to git. + - Using git add is best because it allows the queuing to work + - and is faster (staging the symlink runs hash-object commands each time). + - Also, using git add allows it to skip gitignored files, unless forced + - to include them. + -} +cleanup :: FilePath -> Key -> Bool -> CommandCleanup +cleanup file key hascontent = do + when hascontent $ + logStatus key InfoPresent + ifM (isDirect <&&> pure hascontent) + ( stageSymlink file =<< hashSymlink =<< calcGitLink file key + , ifM (coreSymlinks <$> Annex.getGitConfig) + ( do + _ <- link file key hascontent + params <- ifM (Annex.getState Annex.force) + ( return [Param "-f"] + , return [] + ) + Annex.Queue.addCommand "add" (params++[Param "--"]) [file] + , do + l <- link file key hascontent + addAnnexLink l file + ) + ) + return True diff --git a/Command/AddUnused.hs b/Command/AddUnused.hs new file mode 100644 index 0000000000..23dbdfcca1 --- /dev/null +++ b/Command/AddUnused.hs @@ -0,0 +1,37 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.AddUnused where + +import Common.Annex +import Logs.Unused +import Command +import qualified Command.Add +import Types.Key + +def :: [Command] +def = [notDirect $ command "addunused" (paramRepeating paramNumRange) + seek "add back unused files"] + +seek :: [CommandSeek] +seek = [withUnusedMaps start] + +start :: UnusedMaps -> Int -> CommandStart +start = startUnused "addunused" perform + (performOther "bad") + (performOther "tmp") + +perform :: Key -> CommandPerform +perform key = next $ Command.Add.cleanup file key True + where + file = "unused." ++ key2file key + +{- The content is not in the annex, but in another directory, and + - it seems better to error out, rather than moving bad/tmp content into + - the annex. -} +performOther :: String -> Key -> CommandPerform +performOther other _ = error $ "cannot addunused " ++ other ++ "content" diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs new file mode 100644 index 0000000000..41a947db8d --- /dev/null +++ b/Command/AddUrl.hs @@ -0,0 +1,121 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.AddUrl where + +import Network.URI + +import Common.Annex +import Command +import Backend +import qualified Command.Add +import qualified Annex +import qualified Backend.URL +import qualified Utility.Url as Url +import Annex.Content +import Logs.Web +import qualified Option +import Types.Key +import Types.KeySource +import Config +import Annex.Content.Direct + +def :: [Command] +def = [notBareRepo $ withOptions [fileOption, pathdepthOption] $ + command "addurl" (paramRepeating paramUrl) seek "add urls to annex"] + +fileOption :: Option +fileOption = Option.field [] "file" paramFile "specify what file the url is added to" + +pathdepthOption :: Option +pathdepthOption = Option.field [] "pathdepth" paramNumber "path components to use in filename" + +seek :: [CommandSeek] +seek = [withField fileOption return $ \f -> + withField pathdepthOption (return . maybe Nothing readish) $ \d -> + withStrings $ start f d] + +start :: Maybe FilePath -> Maybe Int -> String -> CommandStart +start optfile pathdepth s = go $ fromMaybe bad $ parseURI s + where + bad = fromMaybe (error $ "bad url " ++ s) $ + parseURI $ escapeURIString isUnescapedInURI s + go url = do + let file = fromMaybe (url2file url pathdepth) optfile + showStart "addurl" file + next $ perform s file + +perform :: String -> FilePath -> CommandPerform +perform url file = ifAnnexed file addurl geturl + where + geturl = do + liftIO $ createDirectoryIfMissing True (parentDir file) + ifM (Annex.getState Annex.fast) + ( nodownload url file , download url file ) + addurl (key, _backend) = do + headers <- getHttpHeaders + ifM (liftIO $ Url.check url headers $ keySize key) + ( do + setUrlPresent key url + next $ return True + , do + warning $ "failed to verify url: " ++ url + stop + ) + +download :: String -> FilePath -> CommandPerform +download url file = do + showAction $ "downloading " ++ url ++ " " + let dummykey = Backend.URL.fromUrl url Nothing + tmp <- fromRepo $ gitAnnexTmpLocation dummykey + liftIO $ createDirectoryIfMissing True (parentDir tmp) + stopUnless (downloadUrl [url] tmp) $ do + backend <- chooseBackend file + let source = KeySource + { keyFilename = file + , contentLocation = tmp + , inodeCache = Nothing + } + k <- genKey source backend + case k of + Nothing -> stop + Just (key, _) -> do + whenM isDirect $ + void $ addAssociatedFile key file + moveAnnex key tmp + setUrlPresent key url + next $ Command.Add.cleanup file key True + +nodownload :: String -> FilePath -> CommandPerform +nodownload url file = do + headers <- getHttpHeaders + (exists, size) <- liftIO $ Url.exists url headers + if exists + then do + let key = Backend.URL.fromUrl url size + whenM isDirect $ + void $ addAssociatedFile key file + setUrlPresent key url + next $ Command.Add.cleanup file key False + else do + warning $ "unable to access url: " ++ url + stop + +url2file :: URI -> Maybe Int -> FilePath +url2file url pathdepth = case pathdepth of + Nothing -> filesize $ escape fullurl + Just depth + | depth > 0 -> frombits $ drop depth + | depth < 0 -> frombits $ reverse . take (negate depth) . reverse + | otherwise -> error "bad --pathdepth" + where + fullurl = uriRegName auth ++ uriPath url ++ uriQuery url + frombits a = join "/" $ a urlbits + urlbits = map (filesize . escape) $ filter (not . null) $ split "/" fullurl + auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url + filesize = take 255 + escape = replace "/" "_" . replace "?" "_" diff --git a/Command/Assistant.hs b/Command/Assistant.hs new file mode 100644 index 0000000000..69a127b505 --- /dev/null +++ b/Command/Assistant.hs @@ -0,0 +1,67 @@ +{- git-annex assistant + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Assistant where + +import Common.Annex +import Command +import qualified Option +import qualified Command.Watch +import Init +import Locations.UserConfig + +import System.Environment +import System.Posix.Directory + +def :: [Command] +def = [noRepo checkAutoStart $ dontCheck repoExists $ + withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption, autoStartOption] $ + command "assistant" paramNothing seek "automatically handle changes"] + +autoStartOption :: Option +autoStartOption = Option.flag [] "autostart" "start in known repositories" + +seek :: [CommandSeek] +seek = [withFlag Command.Watch.stopOption $ \stopdaemon -> + withFlag Command.Watch.foregroundOption $ \foreground -> + withFlag autoStartOption $ \autostart -> + withNothing $ start foreground stopdaemon autostart] + +start :: Bool -> Bool -> Bool -> CommandStart +start foreground stopdaemon autostart + | autostart = do + liftIO autoStart + stop + | otherwise = do + ensureInitialized + Command.Watch.start True foreground stopdaemon + +{- Run outside a git repository. Check to see if any parameter is + - --autostart and enter autostart mode. -} +checkAutoStart :: IO () +checkAutoStart = ifM (elem "--autostart" <$> getArgs) + ( autoStart + , error "Not in a git repository." + ) + +autoStart :: IO () +autoStart = do + dirs <- liftIO readAutoStartFile + when (null dirs) $ do + f <- autoStartFile + error $ "Nothing listed in " ++ f + program <- readProgramFile + forM_ dirs $ \d -> do + putStrLn $ "git-annex autostart in " ++ d + ifM (catchBoolIO $ go program d) + ( putStrLn "ok" + , putStrLn "failed" + ) + where + go program dir = do + changeWorkingDirectory dir + boolSystem program [Param "assistant"] diff --git a/Command/Commit.hs b/Command/Commit.hs new file mode 100644 index 0000000000..1659061394 --- /dev/null +++ b/Command/Commit.hs @@ -0,0 +1,29 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Commit where + +import Common.Annex +import Command +import qualified Annex.Branch +import qualified Git + +def :: [Command] +def = [command "commit" paramNothing seek + "commits any staged changes to the git-annex branch"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = next $ next $ do + Annex.Branch.commit "update" + _ <- runhook <=< inRepo $ Git.hookPath "annex-content" + return True + where + runhook (Just hook) = liftIO $ boolSystem hook [] + runhook Nothing = return True diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs new file mode 100644 index 0000000000..505ad99e1b --- /dev/null +++ b/Command/ConfigList.hs @@ -0,0 +1,25 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ConfigList where + +import Common.Annex +import Command +import Annex.UUID + +def :: [Command] +def = [noCommit $ command "configlist" paramNothing seek + "outputs relevant git configuration"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = do + u <- getUUID + liftIO $ putStrLn $ "annex.uuid=" ++ fromUUID u + stop diff --git a/Command/Copy.hs b/Command/Copy.hs new file mode 100644 index 0000000000..6967c2f930 --- /dev/null +++ b/Command/Copy.hs @@ -0,0 +1,36 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Copy where + +import Common.Annex +import Command +import qualified Command.Move +import qualified Remote +import Annex.Wanted + +def :: [Command] +def = [withOptions Command.Move.options $ command "copy" paramPaths seek + "copy content of files to/from another repository"] + +seek :: [CommandSeek] +seek = [withField Command.Move.toOption Remote.byName $ \to -> + withField Command.Move.fromOption Remote.byName $ \from -> + withFilesInGit $ whenAnnexed $ start to from] + +{- A copy is just a move that does not delete the source file. + - However, --auto mode avoids unnecessary copies, and avoids getting or + - sending non-preferred content. -} +start :: Maybe Remote -> Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart +start to from file (key, backend) = stopUnless shouldCopy $ + Command.Move.start to from False file (key, backend) + where + shouldCopy = checkAuto (check <||> numCopiesCheck file key (<)) + check = case to of + Nothing -> wantGet False (Just file) + Just r -> wantSend False (Just file) (Remote.uuid r) + diff --git a/Command/Dead.hs b/Command/Dead.hs new file mode 100644 index 0000000000..34595769ff --- /dev/null +++ b/Command/Dead.hs @@ -0,0 +1,36 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Dead where + +import Common.Annex +import Command +import qualified Remote +import Logs.Trust +import Logs.Group + +import qualified Data.Set as S + +def :: [Command] +def = [command "dead" (paramRepeating paramRemote) seek + "hide a lost repository"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start ws = do + let name = unwords ws + showStart "dead " name + u <- Remote.nameToUUID name + next $ perform u + +perform :: UUID -> CommandPerform +perform uuid = do + trustSet uuid DeadTrusted + groupSet uuid S.empty + next $ return True diff --git a/Command/Describe.hs b/Command/Describe.hs new file mode 100644 index 0000000000..61297e77c7 --- /dev/null +++ b/Command/Describe.hs @@ -0,0 +1,32 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Describe where + +import Common.Annex +import Command +import qualified Remote +import Logs.UUID + +def :: [Command] +def = [command "describe" (paramPair paramRemote paramDesc) seek + "change description of a repository"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start (name:description) = do + showStart "describe" name + u <- Remote.nameToUUID name + next $ perform u $ unwords description +start _ = error "Specify a repository and a description." + +perform :: UUID -> String -> CommandPerform +perform u description = do + describeUUID u description + next $ return True diff --git a/Command/Direct.hs b/Command/Direct.hs new file mode 100644 index 0000000000..1617bd9c20 --- /dev/null +++ b/Command/Direct.hs @@ -0,0 +1,62 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Direct where + +import Common.Annex +import Command +import qualified Git +import qualified Git.Command +import qualified Git.LsFiles +import Config +import Annex.Direct +import Annex.Version + +def :: [Command] +def = [notBareRepo $ + command "direct" paramNothing seek "switch repository to direct mode"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = ifM isDirect ( stop , next perform ) + +perform :: CommandPerform +perform = do + showStart "commit" "" + showOutput + _ <- inRepo $ Git.Command.runBool + [ Param "commit" + , Param "-a" + , Param "-m" + , Param "commit before switching to direct mode" + ] + showEndOk + + top <- fromRepo Git.repoPath + (l, clean) <- inRepo $ Git.LsFiles.inRepo [top] + forM_ l go + void $ liftIO clean + next cleanup + where + go = whenAnnexed $ \f (k, _) -> do + r <- toDirectGen k f + case r of + Nothing -> noop + Just a -> do + showStart "direct" f + a + showEndOk + return Nothing + +cleanup :: CommandCleanup +cleanup = do + showStart "direct" "" + setDirect True + setVersion directModeVersion + return True diff --git a/Command/Drop.hs b/Command/Drop.hs new file mode 100644 index 0000000000..1683f3b570 --- /dev/null +++ b/Command/Drop.hs @@ -0,0 +1,142 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Drop where + +import Common.Annex +import Command +import qualified Remote +import qualified Annex +import Annex.UUID +import Logs.Location +import Logs.Trust +import Annex.Content +import Config +import qualified Option +import Annex.Wanted + +def :: [Command] +def = [withOptions [fromOption] $ command "drop" paramPaths seek + "indicate content of files not currently wanted"] + +fromOption :: Option +fromOption = Option.field ['f'] "from" paramRemote "drop content from a remote" + +seek :: [CommandSeek] +seek = [withField fromOption Remote.byName $ \from -> + withFilesInGit $ whenAnnexed $ start from] + +start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart +start from file (key, _) = autoCopiesWith file key (>) $ \numcopies -> + stopUnless (checkAuto $ wantDrop False (Remote.uuid <$> from) (Just file)) $ + case from of + Nothing -> startLocal file numcopies key Nothing + Just remote -> do + u <- getUUID + if Remote.uuid remote == u + then startLocal file numcopies key Nothing + else startRemote file numcopies key remote + +startLocal :: FilePath -> Maybe Int -> Key -> Maybe Remote -> CommandStart +startLocal file numcopies key knownpresentremote = stopUnless (inAnnex key) $ do + showStart "drop" file + next $ performLocal key numcopies knownpresentremote + +startRemote :: FilePath -> Maybe Int -> Key -> Remote -> CommandStart +startRemote file numcopies key remote = do + showStart ("drop " ++ Remote.name remote) file + next $ performRemote key numcopies remote + +performLocal :: Key -> Maybe Int -> Maybe Remote -> CommandPerform +performLocal key numcopies knownpresentremote = lockContent key $ do + (remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key + let trusteduuids' = case knownpresentremote of + Nothing -> trusteduuids + Just r -> nub (Remote.uuid r:trusteduuids) + untrusteduuids <- trustGet UnTrusted + let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids'++untrusteduuids) + stopUnless (canDropKey key numcopies trusteduuids' tocheck []) $ do + removeAnnex key + next $ cleanupLocal key + +performRemote :: Key -> Maybe Int -> Remote -> CommandPerform +performRemote key numcopies remote = lockContent key $ do + -- Filter the remote it's being dropped from out of the lists of + -- places assumed to have the key, and places to check. + -- When the local repo has the key, that's one additional copy. + (remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key + present <- inAnnex key + u <- getUUID + let have = filter (/= uuid) $ + if present then u:trusteduuids else trusteduuids + untrusteduuids <- trustGet UnTrusted + let tocheck = filter (/= remote) $ + Remote.remotesWithoutUUID remotes (have++untrusteduuids) + stopUnless (canDropKey key numcopies have tocheck [uuid]) $ do + ok <- Remote.removeKey remote key + next $ cleanupRemote key remote ok + where + uuid = Remote.uuid remote + +cleanupLocal :: Key -> CommandCleanup +cleanupLocal key = do + logStatus key InfoMissing + return True + +cleanupRemote :: Key -> Remote -> Bool -> CommandCleanup +cleanupRemote key remote ok = do + -- better safe than sorry: assume the remote dropped the key + -- even if it seemed to fail; the failure could have occurred + -- after it really dropped it + Remote.logStatus remote key InfoMissing + return ok + +{- Checks specified remotes to verify that enough copies of a key exist to + - allow it to be safely removed (with no data loss). Can be provided with + - some locations where the key is known/assumed to be present. -} +canDropKey :: Key -> Maybe Int -> [UUID] -> [Remote] -> [UUID] -> Annex Bool +canDropKey key numcopiesM have check skip = do + force <- Annex.getState Annex.force + if force || numcopiesM == Just 0 + then return True + else do + need <- getNumCopies numcopiesM + findCopies key need skip have check + +findCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote] -> Annex Bool +findCopies key need skip = helper [] [] + where + helper bad missing have [] + | length have >= need = return True + | otherwise = notEnoughCopies key need have (skip++missing) bad + helper bad missing have (r:rs) + | length have >= need = return True + | otherwise = do + let u = Remote.uuid r + let duplicate = u `elem` have + haskey <- Remote.hasKey r key + case (duplicate, haskey) of + (False, Right True) -> helper bad missing (u:have) rs + (False, Left _) -> helper (r:bad) missing have rs + (False, Right False) -> helper bad (u:missing) have rs + _ -> helper bad missing have rs + +notEnoughCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote] -> Annex Bool +notEnoughCopies key need have skip bad = do + unsafe + showLongNote $ + "Could only verify the existence of " ++ + show (length have) ++ " out of " ++ show need ++ + " necessary copies" + Remote.showTriedRemotes bad + Remote.showLocations key (have++skip) + "Rather than dropping this file, try using: git annex move" + hint + return False + where + unsafe = showNote "unsafe" + hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)" diff --git a/Command/DropKey.hs b/Command/DropKey.hs new file mode 100644 index 0000000000..c0d4f85cfe --- /dev/null +++ b/Command/DropKey.hs @@ -0,0 +1,39 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DropKey where + +import Common.Annex +import Command +import qualified Annex +import Logs.Location +import Annex.Content +import Types.Key + +def :: [Command] +def = [noCommit $ command "dropkey" (paramRepeating paramKey) seek + "drops annexed content for specified keys"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: Key -> CommandStart +start key = stopUnless (inAnnex key) $ do + unlessM (Annex.getState Annex.force) $ + error "dropkey can cause data loss; use --force if you're sure you want to do this" + showStart "dropkey" (key2file key) + next $ perform key + +perform :: Key -> CommandPerform +perform key = lockContent key $ do + removeAnnex key + next $ cleanup key + +cleanup :: Key -> CommandCleanup +cleanup key = do + logStatus key InfoMissing + return True diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs new file mode 100644 index 0000000000..95af062f5e --- /dev/null +++ b/Command/DropUnused.hs @@ -0,0 +1,44 @@ +{- git-annex command + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DropUnused where + +import Logs.Unused +import Common.Annex +import Command +import qualified Annex +import qualified Command.Drop +import qualified Remote +import qualified Git +import qualified Option + +def :: [Command] +def = [withOptions [Command.Drop.fromOption] $ + command "dropunused" (paramRepeating paramNumRange) + seek "drop unused file content"] + +seek :: [CommandSeek] +seek = [withUnusedMaps start] + +start :: UnusedMaps -> Int -> CommandStart +start = startUnused "dropunused" perform (performOther gitAnnexBadLocation) (performOther gitAnnexTmpLocation) + +perform :: Key -> CommandPerform +perform key = maybe droplocal dropremote =<< Remote.byName =<< from + where + dropremote r = do + showAction $ "from " ++ Remote.name r + ok <- Remote.removeKey r key + next $ Command.Drop.cleanupRemote key r ok + droplocal = Command.Drop.performLocal key (Just 0) Nothing -- force drop + from = Annex.getField $ Option.name Command.Drop.fromOption + +performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform +performOther filespec key = do + f <- fromRepo $ filespec key + liftIO $ nukeFile f + next $ return True diff --git a/Command/Find.hs b/Command/Find.hs new file mode 100644 index 0000000000..96f47ec877 --- /dev/null +++ b/Command/Find.hs @@ -0,0 +1,61 @@ +{- git-annex command + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Find where + +import qualified Data.Map as M + +import Common.Annex +import Command +import Annex.Content +import Limit +import qualified Annex +import qualified Utility.Format +import Utility.DataUnits +import Types.Key +import qualified Option + +def :: [Command] +def = [noCommit $ withOptions [formatOption, print0Option] $ + command "find" paramPaths seek "lists available files"] + +formatOption :: Option +formatOption = Option.field [] "format" paramFormat "control format of output" + +print0Option :: Option +print0Option = Option.Option [] ["print0"] (Option.NoArg set) + "terminate output with null" + where + set = Annex.setField (Option.name formatOption) "${file}\0" + +seek :: [CommandSeek] +seek = [withField formatOption formatconverter $ \f -> + withFilesInGit $ whenAnnexed $ start f] + where + formatconverter = return . fmap Utility.Format.gen + +start :: Maybe Utility.Format.Format -> FilePath -> (Key, Backend) -> CommandStart +start format file (key, _) = do + -- only files inAnnex are shown, unless the user has requested + -- others via a limit + whenM (limited <||> inAnnex key) $ + unlessM (showFullJSON vars) $ + case format of + Nothing -> liftIO $ putStrLn file + Just formatter -> liftIO $ putStr $ + Utility.Format.format formatter $ + M.fromList vars + stop + where + vars = + [ ("file", file) + , ("key", key2file key) + , ("backend", keyBackendName key) + , ("bytesize", size show) + , ("humansize", size $ roughSize storageUnits True) + ] + size c = maybe "unknown" c $ keySize key diff --git a/Command/Fix.hs b/Command/Fix.hs new file mode 100644 index 0000000000..e15951c213 --- /dev/null +++ b/Command/Fix.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Fix where + +import Common.Annex +import Command +import qualified Annex.Queue +import Annex.Content + +def :: [Command] +def = [notDirect $ noCommit $ command "fix" paramPaths seek + "fix up symlinks to point to annexed content"] + +seek :: [CommandSeek] +seek = [withFilesInGit $ whenAnnexed start] + +{- Fixes the symlink to an annexed file. -} +start :: FilePath -> (Key, Backend) -> CommandStart +start file (key, _) = do + link <- calcGitLink file key + stopUnless ((/=) (Just link) <$> liftIO (catchMaybeIO $ readSymbolicLink file)) $ do + showStart "fix" file + next $ perform file link + +perform :: FilePath -> FilePath -> CommandPerform +perform file link = do + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + next $ cleanup file + +cleanup :: FilePath -> CommandCleanup +cleanup file = do + Annex.Queue.addCommand "add" [Param "--force", Param "--"] [file] + return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs new file mode 100644 index 0000000000..d023be686c --- /dev/null +++ b/Command/FromKey.hs @@ -0,0 +1,44 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.FromKey where + +import Common.Annex +import Command +import qualified Annex.Queue +import Annex.Content +import Types.Key + +def :: [Command] +def = [notDirect $ notBareRepo $ + command "fromkey" (paramPair paramKey paramPath) seek + "adds a file using a specific key"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start (keyname:file:[]) = do + let key = fromMaybe (error "bad key") $ file2key keyname + inbackend <- inAnnex key + unless inbackend $ error $ + "key ("++ keyname ++") is not present in backend" + showStart "fromkey" file + next $ perform key file +start _ = error "specify a key and a dest file" + +perform :: Key -> FilePath -> CommandPerform +perform key file = do + link <- calcGitLink file key + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ createSymbolicLink link file + next $ cleanup file + +cleanup :: FilePath -> CommandCleanup +cleanup file = do + Annex.Queue.addCommand "add" [Param "--"] [file] + return True diff --git a/Command/Fsck.hs b/Command/Fsck.hs new file mode 100644 index 0000000000..b662ee5787 --- /dev/null +++ b/Command/Fsck.hs @@ -0,0 +1,497 @@ +{- git-annex command + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Fsck where + +import Common.Annex +import Command +import qualified Annex +import qualified Remote +import qualified Types.Backend +import qualified Types.Key +import qualified Backend +import Annex.Content +import Annex.Content.Direct +import Annex.Perms +import Annex.Link +import Logs.Location +import Logs.Trust +import Annex.UUID +import Utility.DataUnits +import Utility.FileMode +import Config +import qualified Option +import Types.Key +import Utility.HumanTime + +import System.Posix.Process (getProcessID) +import Data.Time.Clock.POSIX +import Data.Time +import System.Posix.Types (EpochTime) +import System.Locale + +def :: [Command] +def = [withOptions options $ command "fsck" paramPaths seek + "check for problems"] + +fromOption :: Option +fromOption = Option.field ['f'] "from" paramRemote "check remote" + +startIncrementalOption :: Option +startIncrementalOption = Option.flag ['S'] "incremental" "start an incremental fsck" + +moreIncrementalOption :: Option +moreIncrementalOption = Option.flag ['m'] "more" "continue an incremental fsck" + +incrementalScheduleOption :: Option +incrementalScheduleOption = Option.field [] "incremental-schedule" paramTime + "schedule incremental fscking" + +options :: [Option] +options = + [ fromOption + , startIncrementalOption + , moreIncrementalOption + , incrementalScheduleOption + ] + +seek :: [CommandSeek] +seek = + [ withField fromOption Remote.byName $ \from -> + withIncremental $ \i -> withFilesInGit $ whenAnnexed $ start from i + , withIncremental $ \i -> withBarePresentKeys $ startBare i + ] + +withIncremental :: (Incremental -> CommandSeek) -> CommandSeek +withIncremental = withValue $ do + i <- maybe (return False) (checkschedule . parseDuration) + =<< Annex.getField (Option.name incrementalScheduleOption) + starti <- Annex.getFlag (Option.name startIncrementalOption) + morei <- Annex.getFlag (Option.name moreIncrementalOption) + case (i, starti, morei) of + (False, False, False) -> return NonIncremental + (False, True, _) -> startIncremental + (False ,False, True) -> ContIncremental <$> getStartTime + (True, _, _) -> + maybe startIncremental (return . ContIncremental . Just) + =<< getStartTime + where + startIncremental = do + recordStartTime + return StartIncremental + + checkschedule Nothing = error "bad --incremental-schedule value" + checkschedule (Just delta) = do + Annex.addCleanup "" $ do + v <- getStartTime + case v of + Nothing -> noop + Just started -> do + now <- liftIO getPOSIXTime + when (now - realToFrac started >= delta) $ + resetStartTime + return True + +start :: Maybe Remote -> Incremental -> FilePath -> (Key, Backend) -> CommandStart +start from inc file (key, backend) = do + numcopies <- numCopies file + case from of + Nothing -> go $ perform key file backend numcopies + Just r -> go $ performRemote key file backend numcopies r + where + go = runFsck inc file key + +perform :: Key -> FilePath -> Backend -> Maybe Int -> Annex Bool +perform key file backend numcopies = check + -- order matters + [ fixLink key file + , verifyLocationLog key file + , verifyDirectMapping key file + , checkKeySize key + , checkBackend backend key + , checkKeyNumCopies key file numcopies + ] + +{- To fsck a remote, the content is retrieved to a tmp file, + - and checked locally. -} +performRemote :: Key -> FilePath -> Backend -> Maybe Int -> Remote -> Annex Bool +performRemote key file backend numcopies remote = + dispatch =<< Remote.hasKey remote key + where + dispatch (Left err) = do + showNote err + return False + dispatch (Right True) = withtmp $ \tmpfile -> + ifM (getfile tmpfile) + ( go True (Just tmpfile) + , go True Nothing + ) + dispatch (Right False) = go False Nothing + go present localcopy = check + [ verifyLocationLogRemote key file remote present + , checkKeySizeRemote key remote localcopy + , checkBackendRemote backend key remote localcopy + , checkKeyNumCopies key file numcopies + ] + withtmp a = do + pid <- liftIO getProcessID + t <- fromRepo gitAnnexTmpDir + createAnnexDirectory t + let tmp = t "fsck" ++ show pid ++ "." ++ keyFile key + let cleanup = liftIO $ catchIO (removeFile tmp) (const noop) + cleanup + cleanup `after` a tmp + getfile tmp = + ifM (Remote.retrieveKeyFileCheap remote key tmp) + ( return True + , ifM (Annex.getState Annex.fast) + ( return False + , Remote.retrieveKeyFile remote key Nothing tmp + ) + ) + +{- To fsck a bare repository, fsck each key in the location log. -} +withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek +withBarePresentKeys a params = isBareRepo >>= go + where + go False = return [] + go True = do + unless (null params) $ + error "fsck should be run without parameters in a bare repository" + map a <$> loggedKeys + +startBare :: Incremental -> Key -> CommandStart +startBare inc key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of + Nothing -> stop + Just backend -> runFsck inc (key2file key) key $ performBare key backend + +{- Note that numcopies cannot be checked in a bare repository, because + - getting the numcopies value requires a working copy with .gitattributes + - files. -} +performBare :: Key -> Backend -> Annex Bool +performBare key backend = check + [ verifyLocationLog key (key2file key) + , checkKeySize key + , checkBackend backend key + ] + +check :: [Annex Bool] -> Annex Bool +check cs = all id <$> sequence cs + +{- Checks that the file's link points correctly to the content. + - + - In direct mode, there is only a link when the content is not present. + -} +fixLink :: Key -> FilePath -> Annex Bool +fixLink key file = do + want <- calcGitLink file key + have <- getAnnexLinkTarget file + maybe noop (go want) have + return True + where + go want have = when (want /= have) $ do + {- Version 3.20120227 had a bug that could cause content + - to be stored in the wrong hash directory. Clean up + - after the bug by moving the content. + -} + whenM (liftIO $ doesFileExist file) $ + unlessM (inAnnex key) $ do + showNote "fixing content location" + dir <- liftIO $ parentDir <$> absPath file + let content = absPathFrom dir have + unlessM crippledFileSystem $ + liftIO $ allowWrite (parentDir content) + moveAnnex key content + + showNote "fixing link" + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + addAnnexLink want file + +{- Checks that the location log reflects the current status of the key, + - in this repository only. -} +verifyLocationLog :: Key -> String -> Annex Bool +verifyLocationLog key desc = do + present <- inAnnex key + direct <- isDirect + u <- getUUID + + {- Since we're checking that a key's file is present, throw + - in a permission fixup here too. -} + when (present && not direct) $ do + file <- inRepo $ gitAnnexLocation key + freezeContent file + freezeContentDir file + + {- In direct mode, modified files will show up as not present, + - but that is expected and not something to do anything about. -} + if (direct && not present) + then return True + else verifyLocationLog' key desc present u (logChange key u) + +verifyLocationLogRemote :: Key -> String -> Remote -> Bool -> Annex Bool +verifyLocationLogRemote key desc remote present = + verifyLocationLog' key desc present (Remote.uuid remote) + (Remote.logStatus remote key) + +verifyLocationLog' :: Key -> String -> Bool -> UUID -> (LogStatus -> Annex ()) -> Annex Bool +verifyLocationLog' key desc present u bad = do + uuids <- Remote.keyLocations key + case (present, u `elem` uuids) of + (True, False) -> do + fix InfoPresent + -- There is no data loss, so do not fail. + return True + (False, True) -> do + fix InfoMissing + warning $ + "** Based on the location log, " ++ desc + ++ "\n** was expected to be present, " ++ + "but its content is missing." + return False + _ -> return True + where + fix s = do + showNote "fixing location log" + bad s + +{- Ensures the direct mode mapping file is consistent. Each file + - it lists for the key should exist, and the specified file should be + - included in it. + -} +verifyDirectMapping :: Key -> FilePath -> Annex Bool +verifyDirectMapping key file = do + whenM isDirect $ do + fs <- addAssociatedFile key file + forM_ fs $ \f -> + unlessM (liftIO $ doesFileExist f) $ + void $ removeAssociatedFile key f + return True + +{- The size of the data for a key is checked against the size encoded in + - the key's metadata, if available. + - + - Not checked in direct mode, because files can be changed directly. + -} +checkKeySize :: Key -> Annex Bool +checkKeySize key = ifM isDirect + ( return True + , do + file <- inRepo $ gitAnnexLocation key + ifM (liftIO $ doesFileExist file) + ( checkKeySizeOr badContent key file + , return True + ) + ) + +checkKeySizeRemote :: Key -> Remote -> Maybe FilePath -> Annex Bool +checkKeySizeRemote _ _ Nothing = return True +checkKeySizeRemote key remote (Just file) = + checkKeySizeOr (badContentRemote remote) key file + +checkKeySizeOr :: (Key -> Annex String) -> Key -> FilePath -> Annex Bool +checkKeySizeOr bad key file = case Types.Key.keySize key of + Nothing -> return True + Just size -> do + size' <- fromIntegral . fileSize + <$> liftIO (getFileStatus file) + comparesizes size size' + where + comparesizes a b = do + let same = a == b + unless same $ badsize a b + return same + badsize a b = do + msg <- bad key + warning $ concat + [ "Bad file size (" + , compareSizes storageUnits True a b + , "); " + , msg + ] + +{- Runs the backend specific check on a key's content. + - + - In direct mode this is not done if the file has clearly been modified, + - because modification of direct mode files is allowed. It's still done + - if the file does not appear modified, to catch disk corruption, etc. + -} +checkBackend :: Backend -> Key -> Annex Bool +checkBackend backend key = do + file <- inRepo $ gitAnnexLocation key + ifM isDirect + ( ifM (goodContent key file) + ( checkBackendOr' (badContentDirect file) backend key file + (goodContent key file) + , return True + ) + , checkBackendOr badContent backend key file + ) + +checkBackendRemote :: Backend -> Key -> Remote -> Maybe FilePath -> Annex Bool +checkBackendRemote backend key remote = maybe (return True) go + where + go file = checkBackendOr (badContentRemote remote) backend key file + +checkBackendOr :: (Key -> Annex String) -> Backend -> Key -> FilePath -> Annex Bool +checkBackendOr bad backend key file = + checkBackendOr' bad backend key file (return True) + +checkBackendOr' :: (Key -> Annex String) -> Backend -> Key -> FilePath -> Annex Bool -> Annex Bool +checkBackendOr' bad backend key file postcheck = + case Types.Backend.fsckKey backend of + Nothing -> return True + Just a -> do + ok <- a key file + ifM postcheck + ( do + unless ok $ do + msg <- bad key + warning $ "Bad file content; " ++ msg + return ok + , return True + ) + +checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool +checkKeyNumCopies key file numcopies = do + needed <- getNumCopies numcopies + (untrustedlocations, safelocations) <- trustPartition UnTrusted =<< Remote.keyLocations key + let present = length safelocations + if present < needed + then do + ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations + warning $ missingNote file present needed ppuuids + return False + else return True + +missingNote :: String -> Int -> Int -> String -> String +missingNote file 0 _ [] = + "** No known copies exist of " ++ file +missingNote file 0 _ untrusted = + "Only these untrusted locations may have copies of " ++ file ++ + "\n" ++ untrusted ++ + "Back it up to trusted locations with git-annex copy." +missingNote file present needed [] = + "Only " ++ show present ++ " of " ++ show needed ++ + " trustworthy copies exist of " ++ file ++ + "\nBack it up with git-annex copy." +missingNote file present needed untrusted = + missingNote file present needed [] ++ + "\nThe following untrusted locations may also have copies: " ++ + "\n" ++ untrusted + +{- Bad content is moved aside. -} +badContent :: Key -> Annex String +badContent key = do + dest <- moveBad key + return $ "moved to " ++ dest + +{- Bad content is left where it is, but we touch the file, so it'll be + - committed to a new key. -} +badContentDirect :: FilePath -> Key -> Annex String +badContentDirect file key = do + void $ liftIO $ catchMaybeIO $ touchFile file + logStatus key InfoMissing + return $ "left in place for you to examine" + +badContentRemote :: Remote -> Key -> Annex String +badContentRemote remote key = do + ok <- Remote.removeKey remote key + -- better safe than sorry: assume the remote dropped the key + -- even if it seemed to fail; the failure could have occurred + -- after it really dropped it + Remote.logStatus remote key InfoMissing + return $ (if ok then "dropped from " else "failed to drop from ") + ++ Remote.name remote + +data Incremental = StartIncremental | ContIncremental (Maybe EpochTime) | NonIncremental + deriving (Eq) + +runFsck :: Incremental -> FilePath -> Key -> Annex Bool -> CommandStart +runFsck inc file key a = ifM (needFsck inc key) + ( do + showStart "fsck" file + next $ do + ok <- a + when ok $ + recordFsckTime key + next $ return ok + , stop + ) + +{- Check if a key needs to be fscked, with support for incremental fscks. -} +needFsck :: Incremental -> Key -> Annex Bool +needFsck (ContIncremental Nothing) _ = return True +needFsck (ContIncremental starttime) key = do + fscktime <- getFsckTime key + return $ fscktime < starttime +needFsck _ _ = return True + +{- To record the time that a key was last fscked, without + - modifying its mtime, we set the timestamp of its parent directory. + - Each annexed file is the only thing in its directory, so this is fine. + - + - To record that the file was fscked, the directory's sticky bit is set. + - (None of the normal unix behaviors of the sticky bit should matter, so + - we can reuse this permission bit.) + - + - Note that this relies on the parent directory being deleted when a file + - is dropped. That way, if it's later added back, the fsck record + - won't still be present. + -} +recordFsckTime :: Key -> Annex () +recordFsckTime key = do + parent <- parentDir <$> inRepo (gitAnnexLocation key) + liftIO $ void $ tryIO $ do + touchFile parent + setSticky parent + +getFsckTime :: Key -> Annex (Maybe EpochTime) +getFsckTime key = do + parent <- parentDir <$> inRepo (gitAnnexLocation key) + liftIO $ catchDefaultIO Nothing $ do + s <- getFileStatus parent + return $ if isSticky $ fileMode s + then Just $ modificationTime s + else Nothing + +{- Records the start time of an interactive fsck. + - + - To guard against time stamp damange (for example, if an annex directory + - is copied without -a), the fsckstate file contains a time that should + - be identical to its modification time. -} +recordStartTime :: Annex () +recordStartTime = do + f <- fromRepo gitAnnexFsckState + createAnnexDirectory $ parentDir f + liftIO $ do + nukeFile f + h <- openFile f WriteMode + t <- modificationTime <$> getFileStatus f + hPutStr h $ showTime $ realToFrac t + hClose h + where + showTime :: POSIXTime -> String + showTime = show + +resetStartTime :: Annex () +resetStartTime = liftIO . nukeFile =<< fromRepo gitAnnexFsckState + +{- Gets the incremental fsck start time. -} +getStartTime :: Annex (Maybe EpochTime) +getStartTime = do + f <- fromRepo gitAnnexFsckState + liftIO $ catchDefaultIO Nothing $ do + timestamp <- modificationTime <$> getFileStatus f + t <- readishTime <$> readFile f + return $ if Just (realToFrac timestamp) == t + then Just timestamp + else Nothing + where + readishTime :: String -> Maybe POSIXTime + readishTime s = utcTimeToPOSIXSeconds <$> + parseTime defaultTimeLocale "%s%Qs" s diff --git a/Command/Get.hs b/Command/Get.hs new file mode 100644 index 0000000000..9c58bde644 --- /dev/null +++ b/Command/Get.hs @@ -0,0 +1,75 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Get where + +import Common.Annex +import Command +import qualified Remote +import Annex.Content +import qualified Command.Move +import Logs.Transfer +import Annex.Wanted + +def :: [Command] +def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek + "make content of annexed files available"] + +seek :: [CommandSeek] +seek = [withField Command.Move.fromOption Remote.byName $ \from -> + withFilesInGit $ whenAnnexed $ start from] + +start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart +start from file (key, _) = stopUnless (not <$> inAnnex key) $ + stopUnless (checkAuto (numCopiesCheck file key (<) <||> wantGet False (Just file))) $ do + case from of + Nothing -> go $ perform key file + Just src -> + -- get --from = copy --from + stopUnless (Command.Move.fromOk src key) $ + go $ Command.Move.fromPerform src False key file + where + go a = do + showStart "get" file + next a + +perform :: Key -> FilePath -> CommandPerform +perform key file = stopUnless (getViaTmp key $ getKeyFile key file) $ + next $ return True -- no cleanup needed + +{- Try to find a copy of the file in one of the remotes, + - and copy it to here. -} +getKeyFile :: Key -> FilePath -> FilePath -> Annex Bool +getKeyFile key file dest = dispatch =<< Remote.keyPossibilities key + where + dispatch [] = do + showNote "not available" + showlocs + return False + dispatch remotes = trycopy remotes remotes + trycopy full [] = do + Remote.showTriedRemotes full + showlocs + return False + trycopy full (r:rs) = + ifM (probablyPresent r) + ( docopy r (trycopy full rs) + , trycopy full rs + ) + showlocs = Remote.showLocations key [] $ + "No other repository is known to contain the file." + -- This check is to avoid an ugly message if a remote is a + -- drive that is not mounted. + probablyPresent r + | Remote.hasKeyCheap r = + either (const False) id <$> Remote.hasKey r key + | otherwise = return True + docopy r continue = do + ok <- download (Remote.uuid r) key (Just file) noRetry $ do + showAction $ "from " ++ Remote.name r + Remote.retrieveKeyFile r key (Just file) dest + if ok then return ok else continue diff --git a/Command/Group.hs b/Command/Group.hs new file mode 100644 index 0000000000..5513ca3f72 --- /dev/null +++ b/Command/Group.hs @@ -0,0 +1,34 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Group where + +import Common.Annex +import Command +import qualified Remote +import Logs.Group +import Types.Group + +import qualified Data.Set as S + +def :: [Command] +def = [command "group" (paramPair paramRemote paramDesc) seek "add a repository to a group"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start (name:g:[]) = do + showStart "group" name + u <- Remote.nameToUUID name + next $ perform u g +start _ = error "Specify a repository and a group." + +perform :: UUID -> Group -> CommandPerform +perform uuid g = do + groupChange uuid (S.insert g) + next $ return True diff --git a/Command/Help.hs b/Command/Help.hs new file mode 100644 index 0000000000..95033eb7f4 --- /dev/null +++ b/Command/Help.hs @@ -0,0 +1,51 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Help where + +import Common.Annex +import Command +import qualified Command.Init +import qualified Command.Add +import qualified Command.Drop +import qualified Command.Get +import qualified Command.Move +import qualified Command.Copy +import qualified Command.Sync +import qualified Command.Whereis +import qualified Command.Fsck + +def :: [Command] +def = [noCommit $ noRepo showHelp $ dontCheck repoExists $ + command "help" paramNothing seek "display help"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start _ = do + liftIO showHelp + stop + +showHelp :: IO () +showHelp = liftIO $ putStrLn $ unlines + [ "The most commonly used git-annex commands are:" + , unlines $ map cmdline $ concat + [ Command.Init.def + , Command.Add.def + , Command.Drop.def + , Command.Get.def + , Command.Move.def + , Command.Copy.def + , Command.Sync.def + , Command.Whereis.def + , Command.Fsck.def + ] + , "Run git-annex without any options for a complete command and option list." + ] + where + cmdline c = "\t" ++ cmdname c ++ "\t" ++ cmddesc c diff --git a/Command/Import.hs b/Command/Import.hs new file mode 100644 index 0000000000..e8e839e4fa --- /dev/null +++ b/Command/Import.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Import where + +import Common.Annex +import Command +import qualified Annex +import qualified Command.Add + +def :: [Command] +def = [notDirect $ notBareRepo $ command "import" paramPaths seek + "move and add files from outside git working copy"] + +seek :: [CommandSeek] +seek = [withPathContents start] + +start :: (FilePath, FilePath) -> CommandStart +start (srcfile, destfile) = + ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile) + ( do + showStart "import" destfile + next $ perform srcfile destfile + , stop + ) + +perform :: FilePath -> FilePath -> CommandPerform +perform srcfile destfile = do + whenM (liftIO $ doesFileExist destfile) $ + unlessM (Annex.getState Annex.force) $ + error $ "not overwriting existing " ++ destfile ++ + " (use --force to override)" + + liftIO $ createDirectoryIfMissing True (parentDir destfile) + liftIO $ moveFile srcfile destfile + Command.Add.perform destfile diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs new file mode 100644 index 0000000000..cd4bff2c69 --- /dev/null +++ b/Command/InAnnex.hs @@ -0,0 +1,27 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.InAnnex where + +import Common.Annex +import Command +import Annex.Content + +def :: [Command] +def = [noCommit $ command "inannex" (paramRepeating paramKey) seek + "checks if keys are present in the annex"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: Key -> CommandStart +start key = inAnnexSafe key >>= dispatch + where + dispatch (Just True) = stop + dispatch (Just False) = exit 1 + dispatch Nothing = exit 100 + exit n = liftIO $ exitWith $ ExitFailure n diff --git a/Command/Indirect.hs b/Command/Indirect.hs new file mode 100644 index 0000000000..6290e6756e --- /dev/null +++ b/Command/Indirect.hs @@ -0,0 +1,98 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Indirect where + +import Common.Annex +import Command +import qualified Git +import qualified Git.Command +import qualified Git.LsFiles +import Config +import qualified Annex +import Annex.Direct +import Annex.Content +import Annex.CatFile +import Annex.Version +import Init + +def :: [Command] +def = [notBareRepo $ command "indirect" paramNothing seek + "switch repository to indirect mode"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = ifM isDirect + ( do + unlessM (coreSymlinks <$> Annex.getGitConfig) $ + error "Git is configured to not use symlinks, so you must use direct mode." + whenM probeCrippledFileSystem $ + error "This repository seems to be on a crippled filesystem, you must use direct mode." + next perform + , stop + ) + +perform :: CommandPerform +perform = do + showStart "commit" "" + whenM (stageDirect) $ do + showOutput + void $ inRepo $ Git.Command.runBool + [ Param "commit" + , Param "-m" + , Param "commit before switching to indirect mode" + ] + showEndOk + + -- Note that we set indirect mode early, so that we can use + -- moveAnnex in indirect mode. + setDirect False + + top <- fromRepo Git.repoPath + (l, clean) <- inRepo $ Git.LsFiles.stagedDetails [top] + forM_ l go + void $ liftIO clean + next cleanup + where + {- Walk tree from top and move all present direct mode files into + - the annex, replacing with symlinks. Also delete direct mode + - caches and mappings. -} + go (_, Nothing) = noop + go (f, Just sha) = do + r <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus f + case r of + Just s + | isSymbolicLink s -> void $ flip whenAnnexed f $ + \_ (k, _) -> do + cleandirect k + return Nothing + | otherwise -> + maybe noop (fromdirect f) + =<< catKey sha + _ -> noop + + fromdirect f k = do + showStart "indirect" f + cleandirect k -- clean before content directory gets frozen + whenM (liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f) $ do + moveAnnex k f + l <- calcGitLink f k + liftIO $ createSymbolicLink l f + showEndOk + + cleandirect k = do + liftIO . nukeFile =<< inRepo (gitAnnexInodeCache k) + liftIO . nukeFile =<< inRepo (gitAnnexMapping k) + +cleanup :: CommandCleanup +cleanup = do + setVersion defaultVersion + showStart "indirect" "" + showEndOk + return True diff --git a/Command/Init.hs b/Command/Init.hs new file mode 100644 index 0000000000..342ef84e15 --- /dev/null +++ b/Command/Init.hs @@ -0,0 +1,31 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Init where + +import Common.Annex +import Command +import Init + +def :: [Command] +def = [dontCheck repoExists $ + command "init" paramDesc seek "initialize git-annex"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start ws = do + showStart "init" description + next $ perform description + where + description = unwords ws + +perform :: String -> CommandPerform +perform description = do + initialize $ if null description then Nothing else Just description + next $ return True diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs new file mode 100644 index 0000000000..684f868efe --- /dev/null +++ b/Command/InitRemote.hs @@ -0,0 +1,97 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.InitRemote where + +import qualified Data.Map as M + +import Common.Annex +import Command +import qualified Remote +import qualified Logs.Remote +import qualified Types.Remote as R +import Annex.UUID +import Logs.UUID + +def :: [Command] +def = [command "initremote" + (paramPair paramName $ paramOptional $ paramRepeating paramKeyValue) + seek "sets up a special (non-git) remote"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start [] = do + names <- remoteNames + error $ "Specify a name for the remote. " ++ + if null names + then "" + else "Either a new name, or one of these existing special remotes: " ++ join " " names +start (name:ws) = do + (u, c) <- findByName name + let fullconfig = config `M.union` c + t <- findType fullconfig + + showStart "initremote" name + next $ perform t u name $ M.union config c + + where + config = Logs.Remote.keyValToConfig ws + +perform :: RemoteType -> UUID -> String -> R.RemoteConfig -> CommandPerform +perform t u name c = do + c' <- R.setup t u c + next $ cleanup u name c' + +cleanup :: UUID -> String -> R.RemoteConfig -> CommandCleanup +cleanup u name c = do + describeUUID u name + Logs.Remote.configSet u c + return True + +{- Look up existing remote's UUID and config by name, or generate a new one -} +findByName :: String -> Annex (UUID, R.RemoteConfig) +findByName name = do + m <- Logs.Remote.readRemoteLog + maybe generate return $ findByName' name m + where + generate = do + uuid <- liftIO genUUID + return (uuid, M.insert nameKey name M.empty) + +findByName' :: String -> M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig) +findByName' n = headMaybe . filter (matching . snd) . M.toList + where + matching c = case M.lookup nameKey c of + Nothing -> False + Just n' + | n' == n -> True + | otherwise -> False + +remoteNames :: Annex [String] +remoteNames = do + m <- Logs.Remote.readRemoteLog + return $ mapMaybe (M.lookup nameKey . snd) $ M.toList m + +{- find the specified remote type -} +findType :: R.RemoteConfig -> Annex RemoteType +findType config = maybe unspecified specified $ M.lookup typeKey config + where + unspecified = error "Specify the type of remote with type=" + specified s = case filter (findtype s) Remote.remoteTypes of + [] -> error $ "Unknown remote type " ++ s + (t:_) -> return t + findtype s i = R.typename i == s + +{- The name of a configured remote is stored in its config using this key. -} +nameKey :: String +nameKey = "name" + +{- The type of a remote is stored in its config using this key. -} +typeKey :: String +typeKey = "type" diff --git a/Command/Lock.hs b/Command/Lock.hs new file mode 100644 index 0000000000..c34e6a16b8 --- /dev/null +++ b/Command/Lock.hs @@ -0,0 +1,28 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Lock where + +import Common.Annex +import Command +import qualified Annex.Queue + +def :: [Command] +def = [notDirect $ command "lock" paramPaths seek "undo unlock command"] + +seek :: [CommandSeek] +seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start] + +start :: FilePath -> CommandStart +start file = do + showStart "lock" file + next $ perform file + +perform :: FilePath -> CommandPerform +perform file = do + Annex.Queue.addCommand "checkout" [Param "--"] [file] + next $ return True -- no cleanup needed diff --git a/Command/Log.hs b/Command/Log.hs new file mode 100644 index 0000000000..6608a99065 --- /dev/null +++ b/Command/Log.hs @@ -0,0 +1,171 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Log where + +import qualified Data.Set as S +import qualified Data.Map as M +import qualified Data.ByteString.Lazy.Char8 as L +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale +import Data.Char + +import Common.Annex +import Command +import qualified Logs.Location +import qualified Logs.Presence +import Annex.CatFile +import qualified Annex.Branch +import qualified Git +import Git.Command +import qualified Remote +import qualified Option +import qualified Annex + +data RefChange = RefChange + { changetime :: POSIXTime + , oldref :: Git.Ref + , newref :: Git.Ref + } + +type Outputter = Bool -> POSIXTime -> [UUID] -> Annex () + +def :: [Command] +def = [withOptions options $ + command "log" paramPaths seek "shows location log"] + +options :: [Option] +options = passthruOptions ++ [gourceOption] + +passthruOptions :: [Option] +passthruOptions = map odate ["since", "after", "until", "before"] ++ + [ Option.field ['n'] "max-count" paramNumber + "limit number of logs displayed" + ] + where + odate n = Option.field [] n paramDate $ "show log " ++ n ++ " date" + +gourceOption :: Option +gourceOption = Option.flag [] "gource" "format output for gource" + +seek :: [CommandSeek] +seek = [withValue Remote.uuidDescriptions $ \m -> + withValue (liftIO getCurrentTimeZone) $ \zone -> + withValue (concat <$> mapM getoption passthruOptions) $ \os -> + withFlag gourceOption $ \gource -> + withFilesInGit $ whenAnnexed $ start m zone os gource] + where + getoption o = maybe [] (use o) <$> + Annex.getField (Option.name o) + use o v = [Param ("--" ++ Option.name o), Param v] + +start :: M.Map UUID String -> TimeZone -> [CommandParam] -> Bool -> + FilePath -> (Key, Backend) -> CommandStart +start m zone os gource file (key, _) = do + showLog output =<< readLog <$> getLog key os + -- getLog produces a zombie; reap it + liftIO reapZombies + stop + where + output + | gource = gourceOutput lookupdescription file + | otherwise = normalOutput lookupdescription file zone + lookupdescription u = fromMaybe (fromUUID u) $ M.lookup u m + +showLog :: Outputter -> [RefChange] -> Annex () +showLog outputter ps = do + sets <- mapM (getset newref) ps + previous <- maybe (return genesis) (getset oldref) (lastMaybe ps) + sequence_ $ compareChanges outputter $ sets ++ [previous] + where + genesis = (0, S.empty) + getset select change = do + s <- S.fromList <$> get (select change) + return (changetime change, s) + get ref = map toUUID . Logs.Presence.getLog . L.unpack <$> + catObject ref + +normalOutput :: (UUID -> String) -> FilePath -> TimeZone -> Outputter +normalOutput lookupdescription file zone present ts us = + liftIO $ mapM_ (putStrLn . format) us + where + time = showTimeStamp zone ts + addel = if present then "+" else "-" + format u = unwords [ addel, time, file, "|", + fromUUID u ++ " -- " ++ lookupdescription u ] + +gourceOutput :: (UUID -> String) -> FilePath -> Outputter +gourceOutput lookupdescription file present ts us = + liftIO $ mapM_ (putStrLn . intercalate "|" . format) us + where + time = takeWhile isDigit $ show ts + addel = if present then "A" else "M" + format u = [ time, lookupdescription u, addel, file ] + +{- Generates a display of the changes (which are ordered with newest first), + - by comparing each change with the previous change. + - Uses a formatter to generate a display of items that are added and + - removed. -} +compareChanges :: Ord a => (Bool -> POSIXTime -> [a] -> b) -> [(POSIXTime, S.Set a)] -> [b] +compareChanges format changes = concatMap diff $ zip changes (drop 1 changes) + where + diff ((ts, new), (_, old)) = + [format True ts added, format False ts removed] + where + added = S.toList $ S.difference new old + removed = S.toList $ S.difference old new + +{- Gets the git log for a given location log file. + - + - This is complicated by git log using paths relative to the current + - directory, even when looking at files in a different branch. A wacky + - relative path to the log file has to be used. + - + - The --remove-empty is a significant optimisation. It relies on location + - log files never being deleted in normal operation. Letting git stop + - once the location log file is gone avoids it checking all the way back + - to commit 0 to see if it used to exist, so generally speeds things up a + - *lot* for newish files. -} +getLog :: Key -> [CommandParam] -> Annex [String] +getLog key os = do + top <- fromRepo Git.repoPath + p <- liftIO $ relPathCwdToFile top + let logfile = p Logs.Location.logFile key + inRepo $ pipeNullSplitZombie $ + [ Params "log -z --pretty=format:%ct --raw --abbrev=40" + , Param "--remove-empty" + ] ++ os ++ + [ Param $ show Annex.Branch.fullname + , Param "--" + , Param logfile + ] + +readLog :: [String] -> [RefChange] +readLog = mapMaybe (parse . lines) + where + parse (ts:raw:[]) = let (old, new) = parseRaw raw in + Just RefChange + { changetime = parseTimeStamp ts + , oldref = old + , newref = new + } + parse _ = Nothing + +-- Parses something like ":100644 100644 oldsha newsha M" +parseRaw :: String -> (Git.Ref, Git.Ref) +parseRaw l = go $ words l + where + go (_:_:oldsha:newsha:_) = (Git.Ref oldsha, Git.Ref newsha) + go _ = error $ "unable to parse git log output: " ++ l + +parseTimeStamp :: String -> POSIXTime +parseTimeStamp = utcTimeToPOSIXSeconds . fromMaybe (error "bad timestamp") . + parseTime defaultTimeLocale "%s" + +showTimeStamp :: TimeZone -> POSIXTime -> String +showTimeStamp zone = show . utcToLocalTime zone . posixSecondsToUTCTime diff --git a/Command/Map.hs b/Command/Map.hs new file mode 100644 index 0000000000..94b1289dc0 --- /dev/null +++ b/Command/Map.hs @@ -0,0 +1,238 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Map where + +import Control.Exception.Extensible +import qualified Data.Map as M + +import Common.Annex +import Command +import qualified Git +import qualified Git.Url +import qualified Git.Config +import qualified Git.Construct +import qualified Annex +import Annex.UUID +import Logs.UUID +import Logs.Trust +import Remote.Helper.Ssh +import qualified Utility.Dot as Dot + +-- a link from the first repository to the second (its remote) +data Link = Link Git.Repo Git.Repo + +def :: [Command] +def = [dontCheck repoExists $ + command "map" paramNothing seek "generate map of repositories"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = do + rs <- spider =<< gitRepo + + umap <- uuidMap + trusted <- trustGet Trusted + + file <- () <$> fromRepo gitAnnexDir <*> pure "map.dot" + + liftIO $ writeFile file (drawMap rs umap trusted) + next $ next $ + ifM (Annex.getState Annex.fast) + ( return True + , do + showLongNote $ "running: dot -Tx11 " ++ file + showOutput + liftIO $ boolSystem "dot" [Param "-Tx11", File file] + ) + +{- Generates a graph for dot(1). Each repository, and any other uuids, are + - displayed as a node, and each of its remotes is represented as an edge + - pointing at the node for the remote. + - + - The order nodes are added to the graph matters, since dot will draw + - the first ones near to the top and left. So it looks better to put + - the repositories first, followed by uuids that were not matched + - to a repository. + -} +drawMap :: [Git.Repo] -> M.Map UUID String -> [UUID] -> String +drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others + where + repos = map (node umap rs) rs + ruuids = ts ++ map getUncachedUUID rs + others = map (unreachable . uuidnode) $ + filter (`notElem` ruuids) (M.keys umap) + trusted = map (trustworthy . uuidnode) ts + uuidnode u = Dot.graphNode (fromUUID u) $ M.findWithDefault "" u umap + +hostname :: Git.Repo -> String +hostname r + | Git.repoIsUrl r = Git.Url.host r + | otherwise = "localhost" + +basehostname :: Git.Repo -> String +basehostname r = Prelude.head $ split "." $ hostname r + +{- A name to display for a repo. Uses the name from uuid.log if available, + - or the remote name if not. -} +repoName :: M.Map UUID String -> Git.Repo -> String +repoName umap r + | repouuid == NoUUID = fallback + | otherwise = M.findWithDefault fallback repouuid umap + where + repouuid = getUncachedUUID r + fallback = fromMaybe "unknown" $ Git.remoteName r + +{- A unique id for the node for a repo. Uses the annex.uuid if available. -} +nodeId :: Git.Repo -> String +nodeId r = + case getUncachedUUID r of + NoUUID -> Git.repoLocation r + UUID u -> u + +{- A node representing a repo. -} +node :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> String +node umap fullinfo r = unlines $ n:edges + where + n = Dot.subGraph (hostname r) (basehostname r) "lightblue" $ + decorate $ Dot.graphNode (nodeId r) (repoName umap r) + edges = map (edge umap fullinfo r) (Git.remotes r) + decorate + | Git.config r == M.empty = unreachable + | otherwise = reachable + +{- An edge between two repos. The second repo is a remote of the first. -} +edge :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> Git.Repo -> String +edge umap fullinfo from to = + Dot.graphEdge (nodeId from) (nodeId fullto) edgename + where + -- get the full info for the remote, to get its UUID + fullto = findfullinfo to + findfullinfo n = + case filter (same n) fullinfo of + [] -> n + (n':_) -> n' + {- Only name an edge if the name is different than the name + - that will be used for the destination node, and is + - different from its hostname. (This reduces visual clutter.) -} + edgename = maybe Nothing calcname $ Git.remoteName to + calcname n + | n `elem` [repoName umap fullto, hostname fullto] = Nothing + | otherwise = Just n + +unreachable :: String -> String +unreachable = Dot.fillColor "red" +reachable :: String -> String +reachable = Dot.fillColor "white" +trustworthy :: String -> String +trustworthy = Dot.fillColor "green" + +{- Recursively searches out remotes starting with the specified repo. -} +spider :: Git.Repo -> Annex [Git.Repo] +spider r = spider' [r] [] +spider' :: [Git.Repo] -> [Git.Repo] -> Annex [Git.Repo] +spider' [] known = return known +spider' (r:rs) known + | any (same r) known = spider' rs known + | otherwise = do + r' <- scan r + + -- The remotes will be relative to r', and need to be + -- made absolute for later use. + remotes <- mapM (absRepo r') (Git.remotes r') + let r'' = r' { Git.remotes = remotes } + + spider' (rs ++ remotes) (r'':known) + +{- Converts repos to a common absolute form. -} +absRepo :: Git.Repo -> Git.Repo -> Annex Git.Repo +absRepo reference r + | Git.repoIsUrl reference = return $ Git.Construct.localToUrl reference r + | Git.repoIsUrl r = return r + | otherwise = liftIO $ Git.Construct.fromAbsPath =<< absPath (Git.repoPath r) + +{- Checks if two repos are the same. -} +same :: Git.Repo -> Git.Repo -> Bool +same a b + | both Git.repoIsSsh = matching Git.Url.authority && matching Git.repoPath + | both Git.repoIsUrl && neither Git.repoIsSsh = matching show + | neither Git.repoIsSsh = matching Git.repoPath + | otherwise = False + where + matching t = t a == t b + both t = t a && t b + neither t = not (t a) && not (t b) + +{- reads the config of a remote, with progress display -} +scan :: Git.Repo -> Annex Git.Repo +scan r = do + showStart "map" $ Git.repoDescribe r + v <- tryScan r + case v of + Just r' -> do + showEndOk + return r' + Nothing -> do + showOutput + showEndFail + return r + +{- tries to read the config of a remote, returning it only if it can + - be accessed -} +tryScan :: Git.Repo -> Annex (Maybe Git.Repo) +tryScan r + | Git.repoIsSsh r = sshscan + | Git.repoIsUrl r = return Nothing + | otherwise = safely $ Git.Config.read r + where + safely a = do + result <- liftIO (try a :: IO (Either SomeException Git.Repo)) + case result of + Left _ -> return Nothing + Right r' -> return $ Just r' + pipedconfig cmd params = safely $ + withHandle StdoutHandle createProcessSuccess p $ + Git.Config.hRead r + where + p = proc cmd $ toCommand params + + configlist = onRemote r (pipedconfig, Nothing) "configlist" [] [] + manualconfiglist = do + sshparams <- sshToRepo r [Param sshcmd] + liftIO $ pipedconfig "ssh" sshparams + where + sshcmd = cddir ++ " && " ++ + "git config --null --list" + dir = Git.repoPath r + cddir + | "/~" `isPrefixOf` dir = + let (userhome, reldir) = span (/= '/') (drop 1 dir) + in "cd " ++ userhome ++ " && cd " ++ shellEscape (drop 1 reldir) + | otherwise = "cd " ++ shellEscape dir + + -- First, try sshing and running git config manually, + -- only fall back to git-annex-shell configlist if that + -- fails. + -- + -- This is done for two reasons, first I'd like this + -- subcommand to be usable on non-git-annex repos. + -- Secondly, configlist doesn't include information about + -- the remote's remotes. + sshscan = do + sshnote + v <- manualconfiglist + case v of + Nothing -> do + sshnote + configlist + ok -> return ok + + sshnote = do + showAction "sshing" + showOutput diff --git a/Command/Merge.hs b/Command/Merge.hs new file mode 100644 index 0000000000..0f46614971 --- /dev/null +++ b/Command/Merge.hs @@ -0,0 +1,31 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Merge where + +import Common.Annex +import Command +import qualified Annex.Branch + +def :: [Command] +def = [command "merge" paramNothing seek + "auto-merge remote changes into git-annex branch"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = do + showStart "merge" "." + next perform + +perform :: CommandPerform +perform = do + Annex.Branch.update + -- commit explicitly, in case no remote branches were merged + Annex.Branch.commit "update" + next $ return True diff --git a/Command/Migrate.hs b/Command/Migrate.hs new file mode 100644 index 0000000000..5374bc928a --- /dev/null +++ b/Command/Migrate.hs @@ -0,0 +1,71 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Migrate where + +import Common.Annex +import Command +import Backend +import qualified Types.Key +import qualified Types.Backend +import Types.KeySource +import Annex.Content +import qualified Command.ReKey +import qualified Command.Fsck + +def :: [Command] +def = [notDirect $ + command "migrate" paramPaths seek "switch data to different backend"] + +seek :: [CommandSeek] +seek = [withFilesInGit $ whenAnnexed start] + +start :: FilePath -> (Key, Backend) -> CommandStart +start file (key, oldbackend) = do + exists <- inAnnex key + newbackend <- choosebackend =<< chooseBackend file + if (newbackend /= oldbackend || upgradableKey oldbackend key) && exists + then do + showStart "migrate" file + next $ perform file key oldbackend newbackend + else stop + where + choosebackend Nothing = Prelude.head <$> orderedList + choosebackend (Just backend) = return backend + +{- Checks if a key is upgradable to a newer representation. + - + - Reasons for migration: + - - Ideally, all keys have file size metadata. Old keys may not. + - - Something has changed in the backend, such as a bug fix. + -} +upgradableKey :: Backend -> Key -> Bool +upgradableKey backend key = isNothing (Types.Key.keySize key) || backendupgradable + where + backendupgradable = maybe False (\a -> a key) + (Types.Backend.canUpgradeKey backend) + +{- Store the old backend's key in the new backend + - The old backend's key is not dropped from it, because there may + - be other files still pointing at that key. -} +perform :: FilePath -> Key -> Backend -> Backend -> CommandPerform +perform file oldkey oldbackend newbackend = do + ifM (Command.Fsck.checkBackend oldbackend oldkey) + ( maybe stop go =<< genkey + , stop + ) + where + go newkey = stopUnless (Command.ReKey.linkKey oldkey newkey) $ + next $ Command.ReKey.cleanup file oldkey newkey + genkey = do + content <- inRepo $ gitAnnexLocation oldkey + let source = KeySource + { keyFilename = file + , contentLocation = content + , inodeCache = Nothing + } + liftM fst <$> genKey source (Just newbackend) diff --git a/Command/Move.hs b/Command/Move.hs new file mode 100644 index 0000000000..6d087ecae5 --- /dev/null +++ b/Command/Move.hs @@ -0,0 +1,160 @@ +{- git-annex command + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Move where + +import Common.Annex +import Command +import qualified Command.Drop +import qualified Annex +import Annex.Content +import qualified Remote +import Annex.UUID +import qualified Option +import Logs.Presence +import Logs.Transfer + +def :: [Command] +def = [withOptions options $ command "move" paramPaths seek + "move content of files to/from another repository"] + +fromOption :: Option +fromOption = Option.field ['f'] "from" paramRemote "source remote" + +toOption :: Option +toOption = Option.field ['t'] "to" paramRemote "destination remote" + +options :: [Option] +options = [fromOption, toOption] + +seek :: [CommandSeek] +seek = [withField toOption Remote.byName $ \to -> + withField fromOption Remote.byName $ \from -> + withFilesInGit $ whenAnnexed $ start to from True] + +start :: Maybe Remote -> Maybe Remote -> Bool -> FilePath -> (Key, Backend) -> CommandStart +start to from move file (key, _) = do + noAuto + case (from, to) of + (Nothing, Nothing) -> error "specify either --from or --to" + (Nothing, Just dest) -> toStart dest move file key + (Just src, Nothing) -> fromStart src move file key + (_ , _) -> error "only one of --from or --to can be specified" + where + noAuto = when move $ whenM (Annex.getState Annex.auto) $ error + "--auto is not supported for move" + +showMoveAction :: Bool -> FilePath -> Annex () +showMoveAction True file = showStart "move" file +showMoveAction False file = showStart "copy" file + +{- Moves (or copies) the content of an annexed file to a remote. + - + - If the remote already has the content, it is still removed from + - the current repository. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. + -} +toStart :: Remote -> Bool -> FilePath -> Key -> CommandStart +toStart dest move file key = do + u <- getUUID + ishere <- inAnnex key + if not ishere || u == Remote.uuid dest + then stop -- not here, so nothing to do + else do + showMoveAction move file + next $ toPerform dest move key file +toPerform :: Remote -> Bool -> Key -> FilePath -> CommandPerform +toPerform dest move key file = moveLock move key $ do + -- Checking the remote is expensive, so not done in the start step. + -- In fast mode, location tracking is assumed to be correct, + -- and an explicit check is not done, when copying. When moving, + -- it has to be done, to avoid inaverdent data loss. + fast <- Annex.getState Annex.fast + let fastcheck = fast && not move && not (Remote.hasKeyCheap dest) + isthere <- if fastcheck + then Right <$> expectedpresent + else Remote.hasKey dest key + case isthere of + Left err -> do + showNote err + stop + Right False -> do + showAction $ "to " ++ Remote.name dest + ok <- upload (Remote.uuid dest) key (Just file) noRetry $ + Remote.storeKey dest key (Just file) + if ok + then do + Remote.logStatus dest key InfoPresent + finish + else do + when fastcheck $ + warning "This could have failed because --fast is enabled." + stop + Right True -> do + unlessM expectedpresent $ + Remote.logStatus dest key InfoPresent + finish + where + finish + | move = do + removeAnnex key + next $ Command.Drop.cleanupLocal key + | otherwise = next $ return True + expectedpresent = do + remotes <- Remote.keyPossibilities key + return $ dest `elem` remotes + +{- Moves (or copies) the content of an annexed file from a remote + - to the current repository. + - + - If the current repository already has the content, it is still removed + - from the remote. + -} +fromStart :: Remote -> Bool -> FilePath -> Key -> CommandStart +fromStart src move file key + | move = go + | otherwise = stopUnless (not <$> inAnnex key) go + where + go = stopUnless (fromOk src key) $ do + showMoveAction move file + next $ fromPerform src move key file + +fromOk :: Remote -> Key -> Annex Bool +fromOk src key + | Remote.hasKeyCheap src = + either (const expensive) return =<< Remote.hasKey src key + | otherwise = expensive + where + expensive = do + u <- getUUID + remotes <- Remote.keyPossibilities key + return $ u /= Remote.uuid src && elem src remotes + +fromPerform :: Remote -> Bool -> Key -> FilePath -> CommandPerform +fromPerform src move key file = moveLock move key $ + ifM (inAnnex key) + ( handle move True + , handle move =<< go + ) + where + go = download (Remote.uuid src) key (Just file) noRetry $ do + showAction $ "from " ++ Remote.name src + getViaTmp key $ Remote.retrieveKeyFile src key (Just file) + handle _ False = stop -- failed + handle False True = next $ return True -- copy complete + handle True True = do -- finish moving + ok <- Remote.removeKey src key + next $ Command.Drop.cleanupRemote key src ok + +{- Locks a key in order for it to be moved. + - No lock is needed when a key is being copied. -} +moveLock :: Bool -> Key -> Annex a -> Annex a +moveLock True key a = lockContent key a +moveLock False _ a = a diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs new file mode 100644 index 0000000000..23b6ecc0aa --- /dev/null +++ b/Command/PreCommit.hs @@ -0,0 +1,52 @@ +{- git-annex command + - + - Copyright 2010, 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.PreCommit where + +import Common.Annex +import Command +import qualified Command.Add +import qualified Command.Fix +import qualified Git.DiffTree +import Annex.CatFile +import Annex.Content.Direct +import Git.Sha + +def :: [Command] +def = [command "pre-commit" paramPaths seek "run by git pre-commit hook"] + +seek :: [CommandSeek] +seek = + -- fix symlinks to files being committed + [ whenNotDirect $ withFilesToBeCommitted $ whenAnnexed $ Command.Fix.start + -- inject unlocked files into the annex + , whenNotDirect $ withFilesUnlockedToBeCommitted startIndirect + -- update direct mode mappings for committed files + , whenDirect $ withWords startDirect + ] + +startIndirect :: FilePath -> CommandStart +startIndirect file = next $ do + unlessM (doCommand $ Command.Add.start file) $ + error $ "failed to add " ++ file ++ "; canceling commit" + next $ return True + +startDirect :: [String] -> CommandStart +startDirect _ = next $ do + (diffs, clean) <- inRepo $ Git.DiffTree.diffIndex + forM_ diffs go + next $ liftIO clean + where + go diff = do + withkey (Git.DiffTree.srcsha diff) removeAssociatedFile + withkey (Git.DiffTree.dstsha diff) addAssociatedFile + where + withkey sha a = when (sha /= nullSha) $ do + k <- catKey sha + case k of + Nothing -> noop + Just key -> void $ a key (Git.DiffTree.file diff) diff --git a/Command/ReKey.hs b/Command/ReKey.hs new file mode 100644 index 0000000000..54a345dcbb --- /dev/null +++ b/Command/ReKey.hs @@ -0,0 +1,73 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ReKey where + +import Common.Annex +import Command +import qualified Annex +import Types.Key +import Annex.Content +import qualified Command.Add +import Logs.Web +import Config +import Utility.CopyFile + +def :: [Command] +def = [notDirect $ command "rekey" + (paramOptional $ paramRepeating $ paramPair paramPath paramKey) + seek "change keys used for files"] + +seek :: [CommandSeek] +seek = [withPairs start] + +start :: (FilePath, String) -> CommandStart +start (file, keyname) = ifAnnexed file go stop + where + newkey = fromMaybe (error "bad key") $ file2key keyname + go (oldkey, _) + | oldkey == newkey = stop + | otherwise = do + showStart "rekey" file + next $ perform file oldkey newkey + +perform :: FilePath -> Key -> Key -> CommandPerform +perform file oldkey newkey = do + present <- inAnnex oldkey + _ <- if present + then linkKey oldkey newkey + else do + unlessM (Annex.getState Annex.force) $ + error $ file ++ " is not available (use --force to override)" + return True + next $ cleanup file oldkey newkey + +{- Make a hard link to the old key content, to avoid wasting disk space. -} +linkKey :: Key -> Key -> Annex Bool +linkKey oldkey newkey = getViaTmpUnchecked newkey $ \tmp -> do + src <- inRepo $ gitAnnexLocation oldkey + ifM (liftIO $ doesFileExist tmp) + ( return True + , ifM crippledFileSystem + ( liftIO $ copyFileExternal src tmp + , do + liftIO $ createLink src tmp + return True + ) + ) + +cleanup :: FilePath -> Key -> Key -> CommandCleanup +cleanup file oldkey newkey = do + -- If the old key had some associated urls, record them for + -- the new key as well. + urls <- getUrls oldkey + unless (null urls) $ + mapM_ (setUrlPresent newkey) urls + + -- Update symlink to use the new key. + liftIO $ removeFile file + Command.Add.cleanup file newkey True diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs new file mode 100644 index 0000000000..11a5fd5caa --- /dev/null +++ b/Command/RecvKey.hs @@ -0,0 +1,66 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.RecvKey where + +import Common.Annex +import Command +import CmdLine +import Annex.Content +import Utility.Rsync +import Logs.Transfer +import Command.SendKey (fieldTransfer) +import qualified Fields +import qualified Types.Key +import qualified Types.Backend +import qualified Backend + +def :: [Command] +def = [noCommit $ command "recvkey" paramKey seek + "runs rsync in server mode to receive content"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: Key -> CommandStart +start key = ifM (inAnnex key) + ( error "key is already present in annex" + , fieldTransfer Download key $ \_p -> do + ifM (getViaTmp key go) + ( do + -- forcibly quit after receiving one key, + -- and shutdown cleanly + _ <- shutdown True + return True + , return False + ) + ) + where + go tmp = ifM (liftIO $ rsyncServerReceive tmp) + ( ifM (isJust <$> Fields.getField Fields.direct) + ( directcheck tmp + , return True + ) + , return False + ) + {- If the sending repository uses direct mode, the file + - it sends could be modified as it's sending it. So check + - that the right size file was received, and that the key/value + - Backend is happy with it. -} + directcheck tmp = do + oksize <- case Types.Key.keySize key of + Nothing -> return True + Just size -> do + size' <- fromIntegral . fileSize + <$> liftIO (getFileStatus tmp) + return $ size == size' + if oksize + then case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of + Nothing -> return False + Just backend -> maybe (return True) (\a -> a key tmp) + (Types.Backend.fsckKey backend) + else return False diff --git a/Command/Reinject.hs b/Command/Reinject.hs new file mode 100644 index 0000000000..12657f7f45 --- /dev/null +++ b/Command/Reinject.hs @@ -0,0 +1,58 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Reinject where + +import Common.Annex +import Command +import Logs.Location +import Annex.Content +import qualified Command.Fsck + +def :: [Command] +def = [notDirect $ command "reinject" (paramPair "SRC" "DEST") seek + "sets content of annexed file"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [FilePath] -> CommandStart +start (src:dest:[]) + | src == dest = stop + | otherwise = + ifAnnexed src + (error $ "cannot used annexed file as src: " ++ src) + go + where + go = do + showStart "reinject" dest + next $ whenAnnexed (perform src) dest +start _ = error "specify a src file and a dest file" + +perform :: FilePath -> FilePath -> (Key, Backend) -> CommandPerform +perform src _dest (key, backend) = do + {- Check the content before accepting it. -} + ifM (Command.Fsck.checkKeySizeOr reject key src + <&&> Command.Fsck.checkBackendOr reject backend key src) + ( do + unlessM move $ error "mv failed!" + next $ cleanup key + , error "not reinjecting" + ) + where + -- the file might be on a different filesystem, + -- so mv is used rather than simply calling + -- moveToObjectDir; disk space is also + -- checked this way. + move = getViaTmp key $ \tmp -> + liftIO $ boolSystem "mv" [File src, File tmp] + reject = const $ return "wrong file?" + +cleanup :: Key -> CommandCleanup +cleanup key = do + logStatus key InfoPresent + return True diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs new file mode 100644 index 0000000000..f8c3062131 --- /dev/null +++ b/Command/Semitrust.hs @@ -0,0 +1,32 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Semitrust where + +import Common.Annex +import Command +import qualified Remote +import Logs.Trust + +def :: [Command] +def = [command "semitrust" (paramRepeating paramRemote) seek + "return repository to default trust level"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start ws = do + let name = unwords ws + showStart "semitrust" name + u <- Remote.nameToUUID name + next $ perform u + +perform :: UUID -> CommandPerform +perform uuid = do + trustSet uuid SemiTrusted + next $ return True diff --git a/Command/SendKey.hs b/Command/SendKey.hs new file mode 100644 index 0000000000..dfdec7f928 --- /dev/null +++ b/Command/SendKey.hs @@ -0,0 +1,46 @@ +{- git-annex command + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.SendKey where + +import Common.Annex +import Command +import Annex.Content +import Utility.Rsync +import Logs.Transfer +import qualified Fields + +def :: [Command] +def = [noCommit $ command "sendkey" paramKey seek + "runs rsync in server mode to send content"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: Key -> CommandStart +start key = ifM (inAnnex key) + ( fieldTransfer Upload key $ \_p -> + sendAnnex key rollback $ liftIO . rsyncServerSend + , do + warning "requested key is not present" + liftIO exitFailure + ) + where + {- No need to do any rollback; when sendAnnex fails, a nonzero + - exit will be propigated, and the remote will know the transfer + - failed. -} + rollback = noop + +fieldTransfer :: Direction -> Key -> (MeterUpdate -> Annex Bool) -> CommandStart +fieldTransfer direction key a = do + afile <- Fields.getField Fields.associatedFile + ok <- maybe (a $ const noop) + (\u -> runTransfer (Transfer direction (toUUID u) key) afile noRetry a) + =<< Fields.getField Fields.remoteUUID + if ok + then liftIO exitSuccess + else liftIO exitFailure diff --git a/Command/Status.hs b/Command/Status.hs new file mode 100644 index 0000000000..89ba55cfa1 --- /dev/null +++ b/Command/Status.hs @@ -0,0 +1,296 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE PackageImports, BangPatterns #-} + +module Command.Status where + +import "mtl" Control.Monad.State.Strict +import qualified Data.Map as M +import Text.JSON +import Data.Tuple + +import Common.Annex +import qualified Types.Backend as B +import qualified Types.Remote as R +import qualified Remote +import qualified Command.Unused +import qualified Git +import qualified Annex +import Command +import Utility.DataUnits +import Utility.DiskFree +import Annex.Content +import Types.Key +import Backend +import Logs.UUID +import Logs.Trust +import Remote +import Config +import Utility.Percentage +import Logs.Transfer +import Types.TrustLevel + +-- a named computation that produces a statistic +type Stat = StatState (Maybe (String, StatState String)) + +-- data about a set of keys +data KeyData = KeyData + { countKeys :: Integer + , sizeKeys :: Integer + , unknownSizeKeys :: Integer + , backendsKeys :: M.Map String Integer + } + +-- cached info that multiple Stats use +data StatInfo = StatInfo + { presentData :: Maybe KeyData + , referencedData :: Maybe KeyData + } + +-- a state monad for running Stats in +type StatState = StateT StatInfo Annex + +def :: [Command] +def = [command "status" paramNothing seek + "shows status information about the annex"] + +seek :: [CommandSeek] +seek = [withNothing start] + +{- Order is significant. Less expensive operations, and operations + - that share data go together. + -} +fast_stats :: [Stat] +fast_stats = + [ supported_backends + , supported_remote_types + , repository_mode + , remote_list Trusted + , remote_list SemiTrusted + , remote_list UnTrusted + , remote_list DeadTrusted + , transfer_list + , disk_size + ] +slow_stats :: [Stat] +slow_stats = + [ tmp_size + , bad_data_size + , local_annex_keys + , local_annex_size + , known_annex_keys + , known_annex_size + , bloom_info + , backend_usage + ] + +start :: CommandStart +start = do + fast <- Annex.getState Annex.fast + let stats = if fast then fast_stats else fast_stats ++ slow_stats + showCustom "status" $ do + evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing) + return True + stop + +stat :: String -> (String -> StatState String) -> Stat +stat desc a = return $ Just (desc, a desc) + +nostat :: Stat +nostat = return Nothing + +json :: JSON j => (j -> String) -> StatState j -> String -> StatState String +json serialize a desc = do + j <- a + lift $ maybeShowJSON [(desc, j)] + return $ serialize j + +nojson :: StatState String -> String -> StatState String +nojson a _ = a + +showStat :: Stat -> StatState () +showStat s = maybe noop calc =<< s + where + calc (desc, a) = do + (lift . showHeader) desc + lift . showRaw =<< a + +supported_backends :: Stat +supported_backends = stat "supported backends" $ json unwords $ + return $ map B.name Backend.list + +supported_remote_types :: Stat +supported_remote_types = stat "supported remote types" $ json unwords $ + return $ map R.typename Remote.remoteTypes + +repository_mode :: Stat +repository_mode = stat "repository mode" $ json id $ lift $ + ifM isDirect + ( return "direct", return "indirect" ) + +remote_list :: TrustLevel -> Stat +remote_list level = stat n $ nojson $ lift $ do + us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap Remote.name) + rs <- fst <$> trustPartition level us + s <- prettyPrintUUIDs n rs + return $ if null s then "0" else show (length rs) ++ "\n" ++ beginning s + where + n = showTrustLevel level ++ " repositories" + +local_annex_size :: Stat +local_annex_size = stat "local annex size" $ json id $ + showSizeKeys <$> cachedPresentData + +local_annex_keys :: Stat +local_annex_keys = stat "local annex keys" $ json show $ + countKeys <$> cachedPresentData + +known_annex_size :: Stat +known_annex_size = stat "known annex size" $ json id $ + showSizeKeys <$> cachedReferencedData + +known_annex_keys :: Stat +known_annex_keys = stat "known annex keys" $ json show $ + countKeys <$> cachedReferencedData + +tmp_size :: Stat +tmp_size = staleSize "temporary directory size" gitAnnexTmpDir + +bad_data_size :: Stat +bad_data_size = staleSize "bad keys size" gitAnnexBadDir + +bloom_info :: Stat +bloom_info = stat "bloom filter size" $ json id $ do + localkeys <- countKeys <$> cachedPresentData + capacity <- fromIntegral <$> lift Command.Unused.bloomCapacity + let note = aside $ + if localkeys >= capacity + then "appears too small for this repository; adjust annex.bloomcapacity" + else showPercentage 1 (percentage capacity localkeys) ++ " full" + + -- Two bloom filters are used at the same time, so double the size + -- of one. + size <- roughSize memoryUnits False . (* 2) . fromIntegral . fst <$> + lift Command.Unused.bloomBitsHashes + + return $ size ++ note + +transfer_list :: Stat +transfer_list = stat "transfers in progress" $ nojson $ lift $ do + uuidmap <- Remote.remoteMap id + ts <- getTransfers + if null ts + then return "none" + else return $ multiLine $ + map (\(t, i) -> line uuidmap t i) $ sort ts + where + line uuidmap t i = unwords + [ showLcDirection (transferDirection t) ++ "ing" + , fromMaybe (key2file $ transferKey t) (associatedFile i) + , if transferDirection t == Upload then "to" else "from" + , maybe (fromUUID $ transferUUID t) Remote.name $ + M.lookup (transferUUID t) uuidmap + ] + +disk_size :: Stat +disk_size = stat "available local disk space" $ json id $ lift $ + calcfree + <$> (annexDiskReserve <$> Annex.getGitConfig) + <*> inRepo (getDiskFree . gitAnnexDir) + where + calcfree reserve (Just have) = unwords + [ roughSize storageUnits False $ nonneg $ have - reserve + , "(+" ++ roughSize storageUnits False reserve + , "reserved)" + ] + calcfree _ _ = "unknown" + + nonneg x + | x >= 0 = x + | otherwise = 0 + +backend_usage :: Stat +backend_usage = stat "backend usage" $ nojson $ + calc + <$> (backendsKeys <$> cachedReferencedData) + <*> (backendsKeys <$> cachedPresentData) + where + calc x y = multiLine $ + map (\(n, b) -> b ++ ": " ++ show n) $ + reverse $ sort $ map swap $ M.toList $ + M.unionWith (+) x y + +cachedPresentData :: StatState KeyData +cachedPresentData = do + s <- get + case presentData s of + Just v -> return v + Nothing -> do + v <- foldKeys <$> lift getKeysPresent + put s { presentData = Just v } + return v + +cachedReferencedData :: StatState KeyData +cachedReferencedData = do + s <- get + case referencedData s of + Just v -> return v + Nothing -> do + !v <- lift $ Command.Unused.withKeysReferenced + emptyKeyData addKey + put s { referencedData = Just v } + return v + +emptyKeyData :: KeyData +emptyKeyData = KeyData 0 0 0 M.empty + +foldKeys :: [Key] -> KeyData +foldKeys = foldl' (flip addKey) emptyKeyData + +addKey :: Key -> KeyData -> KeyData +addKey key (KeyData count size unknownsize backends) = + KeyData count' size' unknownsize' backends' + where + {- All calculations strict to avoid thunks when repeatedly + - applied to many keys. -} + !count' = count + 1 + !backends' = M.insertWith' (+) (keyBackendName key) 1 backends + !size' = maybe size (+ size) ks + !unknownsize' = maybe (unknownsize + 1) (const unknownsize) ks + ks = keySize key + +showSizeKeys :: KeyData -> String +showSizeKeys d = total ++ missingnote + where + total = roughSize storageUnits False $ sizeKeys d + missingnote + | unknownSizeKeys d == 0 = "" + | otherwise = aside $ + "+ " ++ show (unknownSizeKeys d) ++ + " keys of unknown size" + +staleSize :: String -> (Git.Repo -> FilePath) -> Stat +staleSize label dirspec = go =<< lift (Command.Unused.staleKeys dirspec) + where + go [] = nostat + go keys = onsize =<< sum <$> keysizes keys + onsize 0 = nostat + onsize size = stat label $ + json (++ aside "clean up with git-annex unused") $ + return $ roughSize storageUnits False size + keysizes keys = map (fromIntegral . fileSize) <$> stats keys + stats keys = do + dir <- lift $ fromRepo dirspec + liftIO $ forM keys $ \k -> getFileStatus (dir keyFile k) + +aside :: String -> String +aside s = " (" ++ s ++ ")" + +multiLine :: [String] -> String +multiLine = concatMap (\l -> "\n\t" ++ l) + diff --git a/Command/Sync.hs b/Command/Sync.hs new file mode 100644 index 0000000000..39eda90f7b --- /dev/null +++ b/Command/Sync.hs @@ -0,0 +1,327 @@ +{- git-annex command + - + - Copyright 2011 Joachim Breitner + - Copyright 2011,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Sync where + +import Common.Annex +import Command +import qualified Remote +import qualified Annex +import qualified Annex.Branch +import qualified Annex.Queue +import Annex.Content +import Annex.Direct +import Annex.CatFile +import Annex.Link +import qualified Git.Command +import qualified Git.LsFiles as LsFiles +import qualified Git.Merge +import qualified Git.Branch +import qualified Git.Ref +import qualified Git +import Git.Types (BlobType(..)) +import qualified Types.Remote +import qualified Remote.Git +import Types.Key +import Config + +import Data.Hash.MD5 + +def :: [Command] +def = [command "sync" (paramOptional (paramRepeating paramRemote)) + [seek] "synchronize local repository with remotes"] + +-- syncing involves several operations, any of which can independently fail +seek :: CommandSeek +seek rs = do + branch <- fromMaybe nobranch <$> inRepo Git.Branch.current + remotes <- syncRemotes rs + return $ concat + [ [ commit ] + , [ mergeLocal branch ] + , [ pullRemote remote branch | remote <- remotes ] + , [ mergeAnnex ] + , [ pushLocal branch ] + , [ pushRemote remote branch | remote <- remotes ] + ] + where + nobranch = error "no branch is checked out" + +syncBranch :: Git.Ref -> Git.Ref +syncBranch = Git.Ref.under "refs/heads/synced/" + +remoteBranch :: Remote -> Git.Ref -> Git.Ref +remoteBranch remote = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote + +syncRemotes :: [String] -> Annex [Remote] +syncRemotes rs = ifM (Annex.getState Annex.fast) ( nub <$> pickfast , wanted ) + where + pickfast = (++) <$> listed <*> (good =<< fastest <$> available) + wanted + | null rs = good =<< concat . Remote.byCost <$> available + | otherwise = listed + listed = do + l <- catMaybes <$> mapM (Remote.byName . Just) rs + let s = filter Remote.specialRemote l + unless (null s) $ + error $ "cannot sync special remotes: " ++ + unwords (map Types.Remote.name s) + return l + available = filter (not . Remote.specialRemote) + . filter (remoteAnnexSync . Types.Remote.gitconfig) + <$> Remote.enabledRemoteList + good = filterM $ Remote.Git.repoAvail . Types.Remote.repo + fastest = fromMaybe [] . headMaybe . Remote.byCost + +commit :: CommandStart +commit = next $ next $ do + ifM isDirect + ( ifM stageDirect + ( runcommit [] , return True ) + , runcommit [Param "-a"] + ) + where + runcommit ps = do + showStart "commit" "" + showOutput + Annex.Branch.commit "update" + -- Commit will fail when the tree is clean, so ignore failure. + _ <- inRepo $ Git.Command.runBool $ (Param "commit") : ps ++ + [Param "-m", Param "git-annex automatic sync"] + return True + +mergeLocal :: Git.Ref -> CommandStart +mergeLocal branch = go =<< needmerge + where + syncbranch = syncBranch branch + needmerge = do + unlessM (inRepo $ Git.Ref.exists syncbranch) $ + inRepo $ updateBranch syncbranch + inRepo $ Git.Branch.changed branch syncbranch + go False = stop + go True = do + showStart "merge" $ Git.Ref.describe syncbranch + next $ next $ mergeFrom syncbranch + +pushLocal :: Git.Ref -> CommandStart +pushLocal branch = do + inRepo $ updateBranch $ syncBranch branch + stop + +updateBranch :: Git.Ref -> Git.Repo -> IO () +updateBranch syncbranch g = + unlessM go $ error $ "failed to update " ++ show syncbranch + where + go = Git.Command.runBool + [ Param "branch" + , Param "-f" + , Param $ show $ Git.Ref.base syncbranch + ] g + +pullRemote :: Remote -> Git.Ref -> CommandStart +pullRemote remote branch = do + showStart "pull" (Remote.name remote) + next $ do + showOutput + stopUnless fetch $ + next $ mergeRemote remote (Just branch) + where + fetch = inRepo $ Git.Command.runBool + [Param "fetch", Param $ Remote.name remote] + +{- The remote probably has both a master and a synced/master branch. + - Which to merge from? Well, the master has whatever latest changes + - were committed, while the synced/master may have changes that some + - other remote synced to this remote. So, merge them both. -} +mergeRemote :: Remote -> (Maybe Git.Ref) -> CommandCleanup +mergeRemote remote b = case b of + Nothing -> do + branch <- inRepo Git.Branch.currentUnsafe + all id <$> (mapM merge $ branchlist branch) + Just _ -> all id <$> (mapM merge =<< tomerge (branchlist b)) + where + merge = mergeFrom . remoteBranch remote + tomerge branches = filterM (changed remote) branches + branchlist Nothing = [] + branchlist (Just branch) = [branch, syncBranch branch] + +pushRemote :: Remote -> Git.Ref -> CommandStart +pushRemote remote branch = go =<< needpush + where + needpush = anyM (newer remote) [syncBranch branch, Annex.Branch.name] + go False = stop + go True = do + showStart "push" (Remote.name remote) + next $ next $ do + showOutput + inRepo $ pushBranch remote branch + +pushBranch :: Remote -> Git.Ref -> Git.Repo -> IO Bool +pushBranch remote branch g = + Git.Command.runBool + [ Param "push" + , Param $ Remote.name remote + , Param $ refspec Annex.Branch.name + , Param $ refspec branch + ] g + where + refspec b = concat + [ show $ Git.Ref.base b + , ":" + , show $ Git.Ref.base $ syncBranch b + ] + +mergeAnnex :: CommandStart +mergeAnnex = do + void $ Annex.Branch.forceUpdate + stop + +{- Merges from a branch into the current branch. -} +mergeFrom :: Git.Ref -> Annex Bool +mergeFrom branch = do + showOutput + ifM isDirect + ( maybe go godirect =<< inRepo Git.Branch.current + , go + ) + where + go = runmerge $ inRepo $ Git.Merge.mergeNonInteractive branch + godirect currbranch = do + old <- inRepo $ Git.Ref.sha currbranch + d <- fromRepo gitAnnexMergeDir + r <- runmerge $ inRepo $ mergeDirect d branch + new <- inRepo $ Git.Ref.sha currbranch + case (old, new) of + (Just oldsha, Just newsha) -> + mergeDirectCleanup d oldsha newsha + _ -> noop + return r + runmerge a = ifM (a) + ( return True + , resolveMerge + ) + +{- Resolves a conflicted merge. It's important that any conflicts be + - resolved in a way that itself avoids later merge conflicts, since + - multiple repositories may be doing this concurrently. + - + - Only annexed files are resolved; other files are left for the user to + - handle. + - + - This uses the Keys pointed to by the files to construct new + - filenames. So when both sides modified file foo, + - it will be deleted, and replaced with files foo.KEYA and foo.KEYB. + - + - On the other hand, when one side deleted foo, and the other modified it, + - it will be deleted, and the modified version stored as file + - foo.KEYA (or KEYB). + -} +resolveMerge :: Annex Bool +resolveMerge = do + top <- fromRepo Git.repoPath + (fs, cleanup) <- inRepo (LsFiles.unmerged [top]) + merged <- all id <$> mapM resolveMerge' fs + void $ liftIO cleanup + + (deleted, cleanup2) <- inRepo (LsFiles.deleted [top]) + unless (null deleted) $ + Annex.Queue.addCommand "rm" [Params "--quiet -f --"] deleted + void $ liftIO cleanup2 + + when merged $ do + Annex.Queue.flush + void $ inRepo $ Git.Command.runBool + [ Param "commit" + , Param "-m" + , Param "git-annex automatic merge conflict fix" + ] + return merged + +resolveMerge' :: LsFiles.Unmerged -> Annex Bool +resolveMerge' u + | issymlink LsFiles.valUs && issymlink LsFiles.valThem = + withKey LsFiles.valUs $ \keyUs -> + withKey LsFiles.valThem $ \keyThem -> do + go keyUs keyThem + | otherwise = return False + where + go keyUs keyThem + | keyUs == keyThem = do + makelink keyUs + return True + | otherwise = do + ifM isDirect + ( maybe noop (\k -> removeDirect k file) keyUs + , liftIO $ nukeFile file + ) + Annex.Queue.addCommand "rm" [Params "--quiet -f --"] [file] + makelink keyUs + makelink keyThem + return True + file = LsFiles.unmergedFile u + issymlink select = any (select (LsFiles.unmergedBlobType u) ==) + [Just SymlinkBlob, Nothing] + makelink (Just key) = do + let dest = mergeFile file key + l <- calcGitLink dest key + liftIO $ nukeFile dest + addAnnexLink l dest + whenM (isDirect) $ + toDirect key dest + makelink _ = noop + withKey select a = do + let msha = select $ LsFiles.unmergedSha u + case msha of + Nothing -> a Nothing + Just sha -> do + key <- catKey sha + maybe (return False) (a . Just) key + +{- The filename to use when resolving a conflicted merge of a file, + - that points to a key. + - + - Something derived from the key needs to be included in the filename, + - but rather than exposing the whole key to the user, a very weak hash + - is used. There is a very real, although still unlikely, chance of + - conflicts using this hash. + - + - In the event that there is a conflict with the filename generated + - for some other key, that conflict will itself be handled by the + - conflicted merge resolution code. That case is detected, and the full + - key is used in the filename. + -} +mergeFile :: FilePath -> Key -> FilePath +mergeFile file key + | doubleconflict = go $ key2file key + | otherwise = go $ shortHash $ key2file key + where + varmarker = ".variant-" + doubleconflict = varmarker `isSuffixOf` (dropExtension file) + go v = takeDirectory file + dropExtension (takeFileName file) + ++ varmarker ++ v + ++ takeExtension file + +shortHash :: String -> String +shortHash = take 4 . md5s . md5FilePath + +changed :: Remote -> Git.Ref -> Annex Bool +changed remote b = do + let r = remoteBranch remote b + ifM (inRepo $ Git.Ref.exists r) + ( inRepo $ Git.Branch.changed b r + , return False + ) + +newer :: Remote -> Git.Ref -> Annex Bool +newer remote b = do + let r = remoteBranch remote b + ifM (inRepo $ Git.Ref.exists r) + ( inRepo $ Git.Branch.changed r b + , return True + ) diff --git a/Command/Test.hs b/Command/Test.hs new file mode 100644 index 0000000000..7ff9f29631 --- /dev/null +++ b/Command/Test.hs @@ -0,0 +1,23 @@ +{- git-annex command + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Test where + +import Command + +def :: [Command] +def = [ dontCheck repoExists $ + command "test" paramNothing seek "run built-in test suite"] + +seek :: [CommandSeek] +seek = [withWords start] + +{- We don't actually run the test suite here because of a dependency loop. + - The main program notices when the command is test and runs it; this + - function is never run if that works. -} +start :: [String] -> CommandStart +start _ = error "Cannot specify any additional parameters when running test" diff --git a/Command/TransferInfo.hs b/Command/TransferInfo.hs new file mode 100644 index 0000000000..800b721691 --- /dev/null +++ b/Command/TransferInfo.hs @@ -0,0 +1,59 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.TransferInfo where + +import Common.Annex +import Command +import Annex.Content +import Logs.Transfer +import Types.Key +import qualified Fields + +def :: [Command] +def = [noCommit $ command "transferinfo" paramKey seek + "updates sender on number of bytes of content received"] + +seek :: [CommandSeek] +seek = [withWords start] + +{- Security: + - + - The transfer info file contains the user-supplied key, but + - the built-in guards prevent slashes in it from showing up in the filename. + - It also contains the UUID of the remote. But slashes are also filtered + - out of that when generating the filename. + - + - Checks that the key being transferred is inAnnex, to prevent + - malicious spamming of bogus keys. Does not check that a transfer + - of the key is actually in progress, because this could be started + - concurrently with sendkey, and win the race. + -} +start :: [String] -> CommandStart +start (k:[]) = do + case (file2key k) of + Nothing -> error "bad key" + (Just key) -> whenM (inAnnex key) $ do + file <- Fields.getField Fields.associatedFile + u <- maybe (error "missing remoteuuid") toUUID + <$> Fields.getField Fields.remoteUUID + let t = Transfer + { transferDirection = Upload + , transferUUID = u + , transferKey = key + } + info <- liftIO $ startTransferInfo file + (update, tfile, _) <- mkProgressUpdater t info + liftIO $ mapM_ void + [ tryIO $ forever $ do + bytes <- readish <$> getLine + maybe (error "transferinfo protocol error") update bytes + , tryIO $ removeFile tfile + , exitSuccess + ] + stop +start _ = error "wrong number of parameters" diff --git a/Command/TransferKey.hs b/Command/TransferKey.hs new file mode 100644 index 0000000000..28ace3cd22 --- /dev/null +++ b/Command/TransferKey.hs @@ -0,0 +1,58 @@ +{- git-annex command, used internally by assistant + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.TransferKey where + +import Common.Annex +import Command +import Annex.Content +import Logs.Location +import Logs.Transfer +import qualified Remote +import Types.Remote +import qualified Command.Move +import qualified Option + +def :: [Command] +def = [withOptions options $ + noCommit $ command "transferkey" paramKey seek + "transfers a key from or to a remote"] + +options :: [Option] +options = fileOption : Command.Move.options + +fileOption :: Option +fileOption = Option.field [] "file" paramFile "the associated file" + +seek :: [CommandSeek] +seek = [withField Command.Move.toOption Remote.byName $ \to -> + withField Command.Move.fromOption Remote.byName $ \from -> + withField fileOption return $ \file -> + withKeys $ start to from file] + +start :: Maybe Remote -> Maybe Remote -> AssociatedFile -> Key -> CommandStart +start to from file key = + case (from, to) of + (Nothing, Just dest) -> next $ toPerform dest key file + (Just src, Nothing) -> next $ fromPerform src key file + _ -> error "specify either --from or --to" + +toPerform :: Remote -> Key -> AssociatedFile -> CommandPerform +toPerform remote key file = go $ + upload (uuid remote) key file forwardRetry $ \p -> do + ok <- Remote.storeKey remote key file p + when ok $ + Remote.logStatus remote key InfoPresent + return ok + +fromPerform :: Remote -> Key -> AssociatedFile -> CommandPerform +fromPerform remote key file = go $ + download (uuid remote) key file forwardRetry $ + getViaTmp key $ Remote.retrieveKeyFile remote key file + +go :: Annex Bool -> CommandPerform +go a = ifM a ( liftIO exitSuccess, liftIO exitFailure) diff --git a/Command/Trust.hs b/Command/Trust.hs new file mode 100644 index 0000000000..d976b86a8f --- /dev/null +++ b/Command/Trust.hs @@ -0,0 +1,31 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Trust where + +import Common.Annex +import Command +import qualified Remote +import Logs.Trust + +def :: [Command] +def = [command "trust" (paramRepeating paramRemote) seek "trust a repository"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start ws = do + let name = unwords ws + showStart "trust" name + u <- Remote.nameToUUID name + next $ perform u + +perform :: UUID -> CommandPerform +perform uuid = do + trustSet uuid Trusted + next $ return True diff --git a/Command/Unannex.hs b/Command/Unannex.hs new file mode 100644 index 0000000000..d1f27e86a4 --- /dev/null +++ b/Command/Unannex.hs @@ -0,0 +1,70 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unannex where + +import Common.Annex +import Command +import qualified Annex +import Logs.Location +import Annex.Content +import qualified Git.Command +import qualified Git.LsFiles as LsFiles + +def :: [Command] +def = [notDirect $ + command "unannex" paramPaths seek "undo accidential add command"] + +seek :: [CommandSeek] +seek = [withFilesInGit $ whenAnnexed start] + +start :: FilePath -> (Key, Backend) -> CommandStart +start file (key, _) = stopUnless (inAnnex key) $ do + showStart "unannex" file + next $ perform file key + +perform :: FilePath -> Key -> CommandPerform +perform file key = next $ cleanup file key + +cleanup :: FilePath -> Key -> CommandCleanup +cleanup file key = do + liftIO $ removeFile file + -- git rm deletes empty directory without --cached + inRepo $ Git.Command.run [Params "rm --cached --quiet --", File file] + + -- If the file was already committed, it is now staged for removal. + -- Commit that removal now, to avoid later confusing the + -- pre-commit hook if this file is later added back to + -- git as a normal, non-annexed file. + (s, clean) <- inRepo $ LsFiles.staged [file] + when (not $ null s) $ do + inRepo $ Git.Command.run + [ Param "commit" + , Param "-q" + , Param "-m", Param "content removed from git annex" + , Param "--", File file + ] + void $ liftIO clean + + ifM (Annex.getState Annex.fast) + ( goFast + , go + ) + + return True + where + goFast = do + -- fast mode: hard link to content in annex + src <- inRepo $ gitAnnexLocation key + -- creating a hard link could fall; fall back to non fast mode + ifM (liftIO $ catchBoolIO $ createLink src file >> return True) + ( thawContent file + , go + ) + go = do + fromAnnex key file + logStatus key InfoMissing diff --git a/Command/Ungroup.hs b/Command/Ungroup.hs new file mode 100644 index 0000000000..ea5b8edb66 --- /dev/null +++ b/Command/Ungroup.hs @@ -0,0 +1,34 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Ungroup where + +import Common.Annex +import Command +import qualified Remote +import Logs.Group +import Types.Group + +import qualified Data.Set as S + +def :: [Command] +def = [command "ungroup" (paramPair paramRemote paramDesc) seek "remove a repository from a group"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start (name:g:[]) = do + showStart "ungroup" name + u <- Remote.nameToUUID name + next $ perform u g +start _ = error "Specify a repository and a group." + +perform :: UUID -> Group -> CommandPerform +perform uuid g = do + groupChange uuid (S.delete g) + next $ return True diff --git a/Command/Uninit.hs b/Command/Uninit.hs new file mode 100644 index 0000000000..2ba32a2a6f --- /dev/null +++ b/Command/Uninit.hs @@ -0,0 +1,72 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Uninit where + +import Common.Annex +import Command +import qualified Git +import qualified Git.Command +import qualified Annex +import qualified Command.Unannex +import Init +import qualified Annex.Branch +import Annex.Content + +def :: [Command] +def = [notDirect $ addCheck check $ command "uninit" paramPaths seek + "de-initialize git-annex and clean out repository"] + +check :: Annex () +check = do + b <- current_branch + when (b == Annex.Branch.name) $ error $ + "cannot uninit when the " ++ show b ++ " branch is checked out" + top <- fromRepo Git.repoPath + cwd <- liftIO getCurrentDirectory + whenM ((/=) <$> liftIO (absPath top) <*> liftIO (absPath cwd)) $ + error "can only run uninit from the top of the git repository" + where + current_branch = Git.Ref . Prelude.head . lines <$> revhead + revhead = inRepo $ Git.Command.pipeReadStrict + [Params "rev-parse --abbrev-ref HEAD"] + +seek :: [CommandSeek] +seek = + [ withFilesNotInGit $ whenAnnexed startCheckIncomplete + , withFilesInGit $ whenAnnexed startUnannex + , withNothing start + ] + +{- git annex symlinks that are not checked into git could be left by an + - interrupted add. -} +startCheckIncomplete :: FilePath -> (Key, Backend) -> CommandStart +startCheckIncomplete file _ = error $ unlines + [ file ++ " points to annexed content, but is not checked into git." + , "Perhaps this was left behind by an interrupted git annex add?" + , "Not continuing with uninit; either delete or git annex add the file and retry." + ] + +startUnannex :: FilePath -> (Key, Backend) -> CommandStart +startUnannex file info = do + -- Force fast mode before running unannex. This way, if multiple + -- files link to a key, it will be left in the annex and hardlinked + -- to by each. + Annex.changeState $ \s -> s { Annex.fast = True } + Command.Unannex.start file info + +start :: CommandStart +start = next $ next $ do + annexdir <- fromRepo gitAnnexDir + uninitialize + mapM_ removeAnnex =<< getKeysPresent + liftIO $ removeDirectoryRecursive annexdir + -- avoid normal shutdown + saveState False + inRepo $ Git.Command.run + [Param "branch", Param "-D", Param $ show Annex.Branch.name] + liftIO exitSuccess diff --git a/Command/Unlock.hs b/Command/Unlock.hs new file mode 100644 index 0000000000..422afcc555 --- /dev/null +++ b/Command/Unlock.hs @@ -0,0 +1,50 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unlock where + +import Common.Annex +import Command +import Annex.Content +import Utility.CopyFile + +def :: [Command] +def = + [ c "unlock" "unlock files for modification" + , c "edit" "same as unlock" + ] + where + c n = notDirect . command n paramPaths seek + +seek :: [CommandSeek] +seek = [withFilesInGit $ whenAnnexed start] + +{- The unlock subcommand replaces the symlink with a copy of the file's + - content. -} +start :: FilePath -> (Key, Backend) -> CommandStart +start file (key, _) = do + showStart "unlock" file + next $ perform file key + +perform :: FilePath -> Key -> CommandPerform +perform dest key = do + unlessM (inAnnex key) $ error "content not present" + unlessM (checkDiskSpace Nothing key 0) $ error "cannot unlock" + + src <- inRepo $ gitAnnexLocation key + tmpdest <- fromRepo $ gitAnnexTmpLocation key + liftIO $ createDirectoryIfMissing True (parentDir tmpdest) + showAction "copying" + ifM (liftIO $ copyFileExternal src tmpdest) + ( do + liftIO $ do + removeFile dest + moveFile tmpdest dest + thawContent dest + next $ return True + , error "copy failed!" + ) diff --git a/Command/Untrust.hs b/Command/Untrust.hs new file mode 100644 index 0000000000..e16040e6bb --- /dev/null +++ b/Command/Untrust.hs @@ -0,0 +1,32 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Untrust where + +import Common.Annex +import Command +import qualified Remote +import Logs.Trust + +def :: [Command] +def = [command "untrust" (paramRepeating paramRemote) seek + "do not trust a repository"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start ws = do + let name = unwords ws + showStart "untrust" name + u <- Remote.nameToUUID name + next $ perform u + +perform :: UUID -> CommandPerform +perform uuid = do + trustSet uuid UnTrusted + next $ return True diff --git a/Command/Unused.hs b/Command/Unused.hs new file mode 100644 index 0000000000..5f8af21854 --- /dev/null +++ b/Command/Unused.hs @@ -0,0 +1,307 @@ +{- git-annex command + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Command.Unused where + +import qualified Data.Set as S +import qualified Data.ByteString.Lazy as L +import Data.BloomFilter +import Data.BloomFilter.Easy +import Data.BloomFilter.Hash +import Control.Monad.ST + +import Common.Annex +import Command +import Logs.Unused +import Annex.Content +import Utility.FileMode +import Logs.Location +import qualified Annex +import qualified Git +import qualified Git.Command +import qualified Git.Ref +import qualified Git.LsFiles as LsFiles +import qualified Git.LsTree as LsTree +import qualified Backend +import qualified Remote +import qualified Annex.Branch +import qualified Option +import Annex.CatFile +import Types.Key + +def :: [Command] +def = [withOptions [fromOption] $ command "unused" paramNothing seek + "look for unused file content"] + +fromOption :: Option +fromOption = Option.field ['f'] "from" paramRemote "remote to check for unused content" + +seek :: [CommandSeek] +seek = [withNothing start] + +{- Finds unused content in the annex. -} +start :: CommandStart +start = do + from <- Annex.getField $ Option.name fromOption + let (name, action) = case from of + Nothing -> (".", checkUnused) + Just "." -> (".", checkUnused) + Just "here" -> (".", checkUnused) + Just n -> (n, checkRemoteUnused n) + showStart "unused" name + next action + +checkUnused :: CommandPerform +checkUnused = chain 0 + [ check "" unusedMsg $ findunused =<< Annex.getState Annex.fast + , check "bad" staleBadMsg $ staleKeysPrune gitAnnexBadDir + , check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpDir + ] + where + findunused True = do + showNote "fast mode enabled; only finding stale files" + return [] + findunused False = do + showAction "checking for unused data" + excludeReferenced =<< getKeysPresent + chain _ [] = next $ return True + chain v (a:as) = do + v' <- a v + chain v' as + +checkRemoteUnused :: String -> CommandPerform +checkRemoteUnused name = go =<< fromJust <$> Remote.byName (Just name) + where + go r = do + showAction "checking for unused data" + _ <- check "" (remoteUnusedMsg r) (remoteunused r) 0 + next $ return True + remoteunused r = excludeReferenced <=< loggedKeysFor $ Remote.uuid r + +check :: FilePath -> ([(Int, Key)] -> String) -> Annex [Key] -> Int -> Annex Int +check file msg a c = do + l <- a + let unusedlist = number c l + unless (null l) $ showLongNote $ msg unusedlist + writeUnusedLog file unusedlist + return $ c + length l + +number :: Int -> [a] -> [(Int, a)] +number _ [] = [] +number n (x:xs) = (n+1, x) : number (n+1) xs + +table :: [(Int, Key)] -> [String] +table l = " NUMBER KEY" : map cols l + where + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ key2file k + pad n s = s ++ replicate (n - length s) ' ' + +staleTmpMsg :: [(Int, Key)] -> String +staleTmpMsg t = unlines $ + ["Some partially transferred data exists in temporary files:"] + ++ table t ++ [dropMsg Nothing] + +staleBadMsg :: [(Int, Key)] -> String +staleBadMsg t = unlines $ + ["Some corrupted files have been preserved by fsck, just in case:"] + ++ table t ++ [dropMsg Nothing] + +unusedMsg :: [(Int, Key)] -> String +unusedMsg u = unusedMsg' u + ["Some annexed data is no longer used by any files:"] + [dropMsg Nothing] +unusedMsg' :: [(Int, Key)] -> [String] -> [String] -> String +unusedMsg' u header trailer = unlines $ + header ++ + table u ++ + ["(To see where data was previously used, try: git log --stat -S'KEY')"] ++ + trailer + +remoteUnusedMsg :: Remote -> [(Int, Key)] -> String +remoteUnusedMsg r u = unusedMsg' u + ["Some annexed data on " ++ name ++ " is not used by any files:"] + [dropMsg $ Just r] + where + name = Remote.name r + +dropMsg :: Maybe Remote -> String +dropMsg Nothing = dropMsg' "" +dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r +dropMsg' :: String -> String +dropMsg' s = "\nTo remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER\n" + +{- Finds keys in the list that are not referenced in the git repository. + - + - Strategy: + - + - * Build a bloom filter of all keys referenced by symlinks. This + - is the fastest one to build and will filter out most keys. + - * If keys remain, build a second bloom filter of keys referenced by + - all branches. + - * The list is streamed through these bloom filters lazily, so both will + - exist at the same time. This means that twice the memory is used, + - but they're relatively small, so the added complexity of using a + - mutable bloom filter does not seem worthwhile. + - * Generating the second bloom filter can take quite a while, since + - it needs enumerating all keys in all git branches. But, the common + - case, if the second filter is needed, is for some keys to be globally + - unused, and in that case, no short-circuit is possible. + - Short-circuiting if the first filter filters all the keys handles the + - other common case. + -} +excludeReferenced :: [Key] -> Annex [Key] +excludeReferenced ks = runfilter firstlevel ks >>= runfilter secondlevel + where + runfilter _ [] = return [] -- optimisation + runfilter a l = bloomFilter show l <$> genBloomFilter show a + firstlevel = withKeysReferencedM + secondlevel = withKeysReferencedInGit + +{- Finds items in the first, smaller list, that are not + - present in the second, larger list. + - + - Constructing a single set, of the list that tends to be + - smaller, appears more efficient in both memory and CPU + - than constructing and taking the S.difference of two sets. -} +exclude :: Ord a => [a] -> [a] -> [a] +exclude [] _ = [] -- optimisation +exclude smaller larger = S.toList $ remove larger $ S.fromList smaller + where + remove a b = foldl (flip S.delete) b a + +{- A bloom filter capable of holding half a million keys with a + - false positive rate of 1 in 1000 uses around 8 mb of memory, + - so will easily fit on even my lowest memory systems. + -} +bloomCapacity :: Annex Int +bloomCapacity = fromMaybe 500000 . annexBloomCapacity <$> Annex.getGitConfig +bloomAccuracy :: Annex Int +bloomAccuracy = fromMaybe 1000 . annexBloomAccuracy <$> Annex.getGitConfig +bloomBitsHashes :: Annex (Int, Int) +bloomBitsHashes = do + capacity <- bloomCapacity + accuracy <- bloomAccuracy + return $ suggestSizing capacity (1/ fromIntegral accuracy) + +{- Creates a bloom filter, and runs an action, such as withKeysReferenced, + - to populate it. + - + - The action is passed a callback that it can use to feed values into the + - bloom filter. + - + - Once the action completes, the mutable filter is frozen + - for later use. + -} +genBloomFilter :: Hashable t => (v -> t) -> ((v -> Annex ()) -> Annex b) -> Annex (Bloom t) +genBloomFilter convert populate = do + (numbits, numhashes) <- bloomBitsHashes + bloom <- lift $ newMB (cheapHashes numhashes) numbits + _ <- populate $ \v -> lift $ insertMB bloom (convert v) + lift $ unsafeFreezeMB bloom + where + lift = liftIO . stToIO + +bloomFilter :: Hashable t => (v -> t) -> [v] -> Bloom t -> [v] +bloomFilter convert l bloom = filter (\k -> convert k `notElemB` bloom) l + +{- Given an initial value, folds it with each key referenced by + - symlinks in the git repo. -} +withKeysReferenced :: v -> (Key -> v -> v) -> Annex v +withKeysReferenced initial a = withKeysReferenced' initial folda + where + folda k v = return $ a k v + +{- Runs an action on each referenced key in the git repo. -} +withKeysReferencedM :: (Key -> Annex ()) -> Annex () +withKeysReferencedM a = withKeysReferenced' () calla + where + calla k _ = a k + +withKeysReferenced' :: v -> (Key -> v -> Annex v) -> Annex v +withKeysReferenced' initial a = do + (files, clean) <- getfiles + r <- go initial files + liftIO $ void clean + return r + where + getfiles = ifM isBareRepo + ( return ([], return True) + , do + top <- fromRepo Git.repoPath + inRepo $ LsFiles.inRepo [top] + ) + go v [] = return v + go v (f:fs) = do + x <- Backend.lookupFile f + case x of + Nothing -> go v fs + Just (k, _) -> do + !v' <- a k v + go v' fs + +withKeysReferencedInGit :: (Key -> Annex ()) -> Annex () +withKeysReferencedInGit a = do + rs <- relevantrefs <$> showref + forM_ rs (withKeysReferencedInGitRef a) + where + showref = inRepo $ Git.Command.pipeReadStrict [Param "show-ref"] + relevantrefs = map (Git.Ref . snd) . + nubBy uniqref . + filter ourbranches . + map (separate (== ' ')) . lines + uniqref (x, _) (y, _) = x == y + ourbranchend = '/' : show Annex.Branch.name + ourbranches (_, b) = not (ourbranchend `isSuffixOf` b) + && not ("refs/synced/" `isPrefixOf` b) + +withKeysReferencedInGitRef :: (Key -> Annex ()) -> Git.Ref -> Annex () +withKeysReferencedInGitRef a ref = do + showAction $ "checking " ++ Git.Ref.describe ref + go <=< inRepo $ LsTree.lsTree ref + where + go [] = noop + go (l:ls) + | isSymLink (LsTree.mode l) = do + content <- encodeW8 . L.unpack + <$> catFile ref (LsTree.file l) + case fileKey (takeFileName content) of + Nothing -> go ls + Just k -> do + a k + go ls + | otherwise = go ls + +{- Looks in the specified directory for bad/tmp keys, and returns a list + - of those that might still have value, or might be stale and removable. + - + - Also, stale keys that can be proven to have no value are deleted. + -} +staleKeysPrune :: (Git.Repo -> FilePath) -> Annex [Key] +staleKeysPrune dirspec = do + contents <- staleKeys dirspec + + dups <- filterM inAnnex contents + let stale = contents `exclude` dups + + dir <- fromRepo dirspec + liftIO $ forM_ dups $ \t -> removeFile $ dir keyFile t + + return stale + +staleKeys :: (Git.Repo -> FilePath) -> Annex [Key] +staleKeys dirspec = do + dir <- fromRepo dirspec + ifM (liftIO $ doesDirectoryExist dir) + ( do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM doesFileExist $ + map (dir ) contents + return $ mapMaybe (fileKey . takeFileName) files + , return [] + ) diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs new file mode 100644 index 0000000000..d1c1eb3b07 --- /dev/null +++ b/Command/Upgrade.hs @@ -0,0 +1,27 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Upgrade where + +import Common.Annex +import Command +import Upgrade +import Annex.Version + +def :: [Command] +def = [dontCheck repoExists $ -- because an old version may not seem to exist + command "upgrade" paramNothing seek "upgrade repository layout"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = do + showStart "upgrade" "." + r <- upgrade + setVersion defaultVersion + next $ next $ return r diff --git a/Command/Version.hs b/Command/Version.hs new file mode 100644 index 0000000000..907811e750 --- /dev/null +++ b/Command/Version.hs @@ -0,0 +1,36 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Version where + +import Common.Annex +import Command +import qualified Build.SysConfig as SysConfig +import Annex.Version + +def :: [Command] +def = [noCommit $ noRepo showPackageVersion $ dontCheck repoExists $ + command "version" paramNothing seek "show version info"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = do + v <- getVersion + liftIO $ do + showPackageVersion + putStrLn $ "local repository version: " ++ fromMaybe "unknown" v + putStrLn $ "default repository version: " ++ defaultVersion + putStrLn $ "supported repository versions: " ++ vs supportedVersions + putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions + stop + where + vs = join " " + +showPackageVersion :: IO () +showPackageVersion = putStrLn $ "git-annex version: " ++ SysConfig.packageversion diff --git a/Command/Vicfg.hs b/Command/Vicfg.hs new file mode 100644 index 0000000000..8aefd86bbd --- /dev/null +++ b/Command/Vicfg.hs @@ -0,0 +1,190 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Vicfg where + +import qualified Data.Map as M +import qualified Data.Set as S +import System.Environment (getEnv) +import Data.Tuple (swap) +import Data.Char (isSpace) + +import Common.Annex +import Command +import Annex.Perms +import Types.TrustLevel +import Types.Group +import Logs.Trust +import Logs.Group +import Logs.PreferredContent +import Remote + +def :: [Command] +def = [command "vicfg" paramNothing seek + "edit git-annex's configuration"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = do + f <- fromRepo gitAnnexTmpCfgFile + createAnnexDirectory $ parentDir f + cfg <- getCfg + descs <- uuidDescriptions + liftIO $ writeFile f $ genCfg cfg descs + vicfg cfg f + stop + +vicfg :: Cfg -> FilePath -> Annex () +vicfg curcfg f = do + vi <- liftIO $ catchDefaultIO "vi" $ getEnv "EDITOR" + -- Allow EDITOR to be processed by the shell, so it can contain options. + unlessM (liftIO $ boolSystem "sh" [Param "-c", Param $ unwords [vi, shellEscape f]]) $ + error $ vi ++ " exited nonzero; aborting" + r <- parseCfg curcfg <$> liftIO (readFileStrict f) + liftIO $ nukeFile f + case r of + Left s -> do + liftIO $ writeFile f s + vicfg curcfg f + Right newcfg -> setCfg curcfg newcfg + +data Cfg = Cfg + { cfgTrustMap :: TrustMap + , cfgGroupMap :: M.Map UUID (S.Set Group) + , cfgPreferredContentMap :: M.Map UUID String + } + +getCfg :: Annex Cfg +getCfg = Cfg + <$> trustMapRaw -- without local trust overrides + <*> (groupsByUUID <$> groupMap) + <*> preferredContentMapRaw + +setCfg :: Cfg -> Cfg -> Annex () +setCfg curcfg newcfg = do + let (trustchanges, groupchanges, preferredcontentchanges) = diffCfg curcfg newcfg + mapM_ (uncurry trustSet) $ M.toList trustchanges + mapM_ (uncurry groupSet) $ M.toList groupchanges + mapM_ (uncurry preferredContentSet) $ M.toList preferredcontentchanges + +diffCfg :: Cfg -> Cfg -> (TrustMap, M.Map UUID (S.Set Group), M.Map UUID String) +diffCfg curcfg newcfg = (diff cfgTrustMap, diff cfgGroupMap, diff cfgPreferredContentMap) + where + diff f = M.differenceWith (\x y -> if x == y then Nothing else Just x) + (f newcfg) (f curcfg) + +genCfg :: Cfg -> M.Map UUID String -> String +genCfg cfg descs = unlines $ concat [intro, trust, groups, preferredcontent] + where + intro = + [ com "git-annex configuration" + , com "" + , com "Changes saved to this file will be recorded in the git-annex branch." + , com "" + , com "Lines in this file have the format:" + , com " setting uuid = value" + ] + + trust = settings cfgTrustMap + [ "" + , com "Repository trust configuration" + , com "(Valid trust levels: " ++ + unwords (map showTrustLevel [Trusted .. DeadTrusted]) ++ + ")" + ] + (\(t, u) -> line "trust" u $ showTrustLevel t) + (\u -> lcom $ line "trust" u $ showTrustLevel SemiTrusted) + + groups = settings cfgGroupMap + [ "" + , com "Repository groups" + , com "(Separate group names with spaces)" + ] + (\(s, u) -> line "group" u $ unwords $ S.toList s) + (\u -> lcom $ line "group" u "") + + preferredcontent = settings cfgPreferredContentMap + [ "" + , com "Repository preferred contents" + ] + (\(s, u) -> line "preferred-content" u s) + (\u -> line "preferred-content" u "") + + settings field desc showvals showdefaults = concat + [ desc + , concatMap showvals $ sort $ map swap $ M.toList $ field cfg + , concatMap (\u -> lcom $ showdefaults u) $ missing field + ] + + line setting u value = + [ com $ "(for " ++ (fromMaybe "" $ M.lookup u descs) ++ ")" + , unwords [setting, fromUUID u, "=", value] + ] + lcom = map (\l -> if "#" `isPrefixOf` l then l else "#" ++ l) + missing field = S.toList $ M.keysSet descs `S.difference` M.keysSet (field cfg) + +{- If there's a parse error, returns a new version of the file, + - with the problem lines noted. -} +parseCfg :: Cfg -> String -> Either String Cfg +parseCfg curcfg = go [] curcfg . lines + where + go c cfg [] + | null (catMaybes $ map fst c) = Right cfg + | otherwise = Left $ unlines $ + badheader ++ concatMap showerr (reverse c) + go c cfg (l:ls) = case parse (dropWhile isSpace l) cfg of + Left msg -> go ((Just msg, l):c) cfg ls + Right cfg' -> go ((Nothing, l):c) cfg' ls + + parse l cfg + | null l = Right cfg + | "#" `isPrefixOf` l = Right cfg + | null setting || null u = Left "missing repository uuid" + | otherwise = handle cfg (toUUID u) setting value' + where + (setting, rest) = separate isSpace l + (r, value) = separate (== '=') rest + value' = trimspace value + u = reverse $ trimspace $ reverse $ trimspace r + trimspace = dropWhile isSpace + + handle cfg u setting value + | setting == "trust" = case readTrustLevel value of + Nothing -> badval "trust value" value + Just t -> + let m = M.insert u t (cfgTrustMap cfg) + in Right $ cfg { cfgTrustMap = m } + | setting == "group" = + let m = M.insert u (S.fromList $ words value) (cfgGroupMap cfg) + in Right $ cfg { cfgGroupMap = m } + | setting == "preferred-content" = + case checkPreferredContentExpression value of + Just e -> Left e + Nothing -> + let m = M.insert u value (cfgPreferredContentMap cfg) + in Right $ cfg { cfgPreferredContentMap = m } + | otherwise = badval "setting" setting + + showerr (Just msg, l) = [parseerr ++ msg, l] + showerr (Nothing, l) + -- filter out the header and parse error lines + -- from any previous parse failure + | any (`isPrefixOf` l) (parseerr:badheader) = [] + | otherwise = [l] + + badval desc val = Left $ "unknown " ++ desc ++ " \"" ++ val ++ "\"" + badheader = + [ com "There was a problem parsing your input." + , com "Search for \"Parse error\" to find the bad lines." + , com "Either fix the bad lines, or delete them (to discard your changes)." + ] + parseerr = com "Parse error in next line: " + +com :: String -> String +com s = "# " ++ s diff --git a/Command/Watch.hs b/Command/Watch.hs new file mode 100644 index 0000000000..25b5c6bba6 --- /dev/null +++ b/Command/Watch.hs @@ -0,0 +1,35 @@ +{- git-annex watch command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Watch where + +import Common.Annex +import Assistant +import Command +import Option + +def :: [Command] +def = [notBareRepo $ withOptions [foregroundOption, stopOption] $ + command "watch" paramNothing seek "watch for changes"] + +seek :: [CommandSeek] +seek = [withFlag stopOption $ \stopdaemon -> + withFlag foregroundOption $ \foreground -> + withNothing $ start False foreground stopdaemon] + +foregroundOption :: Option +foregroundOption = Option.flag [] "foreground" "do not daemonize" + +stopOption :: Option +stopOption = Option.flag [] "stop" "stop daemon" + +start :: Bool -> Bool -> Bool -> CommandStart +start assistant foreground stopdaemon = do + if stopdaemon + then stopDaemon + else startDaemon assistant foreground Nothing -- does not return + stop diff --git a/Command/WebApp.hs b/Command/WebApp.hs new file mode 100644 index 0000000000..5e461ed218 --- /dev/null +++ b/Command/WebApp.hs @@ -0,0 +1,147 @@ +{- git-annex webapp launcher + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.WebApp where + +import Common.Annex +import Command +import Assistant +import Assistant.Common +import Assistant.NamedThread +import Assistant.Threads.WebApp +import Assistant.WebApp +import Assistant.Install +import Utility.WebApp +import Utility.Daemon (checkDaemon) +import Init +import qualified Git +import qualified Git.Config +import qualified Git.CurrentRepo +import qualified Annex +import Locations.UserConfig + +import System.Posix.Directory +import Control.Concurrent +import Control.Concurrent.STM +import System.Process (env, std_out, std_err) + +def :: [Command] +def = [noCommit $ noRepo startNoRepo $ dontCheck repoExists $ notBareRepo $ + command "webapp" paramNothing seek "launch webapp"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStart +start = start' True + +start' :: Bool -> CommandStart +start' allowauto = do + liftIO $ ensureInstalled + ifM isInitialized ( go , auto ) + stop + where + go = do + browser <- fromRepo webBrowser + f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim + ifM (checkpid <&&> checkshim f) + ( liftIO $ openBrowser browser f Nothing Nothing + , startDaemon True True $ Just $ + \origout origerr _url htmlshim -> + openBrowser browser htmlshim origout origerr + ) + auto + | allowauto = liftIO startNoRepo + | otherwise = do + d <- liftIO getCurrentDirectory + error $ "no git repository in " ++ d + checkpid = do + pidfile <- fromRepo gitAnnexPidFile + liftIO $ isJust <$> checkDaemon pidfile + checkshim f = liftIO $ doesFileExist f + +{- When run without a repo, start the first available listed repository in + - the autostart file. If not, it's our first time being run! -} +startNoRepo :: IO () +startNoRepo = do + dirs <- liftIO $ filterM doesDirectoryExist =<< readAutoStartFile + case dirs of + [] -> firstRun + (d:_) -> do + changeWorkingDirectory d + state <- Annex.new =<< Git.CurrentRepo.get + void $ Annex.eval state $ doCommand $ start' False + +{- Run the webapp without a repository, which prompts the user, makes one, + - changes to it, starts the regular assistant, and redirects the + - browser to its url. + - + - This is a very tricky dance -- The first webapp calls the signaler, + - which signals the main thread when it's ok to continue by writing to a + - MVar. The main thread starts the second webapp, and uses its callback + - to write its url back to the MVar, from where the signaler retrieves it, + - returning it to the first webapp, which does the redirect. + - + - Note that it's important that mainthread never terminates! Much + - of this complication is due to needing to keep the mainthread running. + -} +firstRun :: IO () +firstRun = do + {- Without a repository, we cannot have an Annex monad, so cannot + - get a ThreadState. Using undefined is only safe because the + - webapp checks its noAnnex field before accessing the + - threadstate. -} + let st = undefined + {- Get a DaemonStatus without running in the Annex monad. -} + dstatus <- atomically . newTMVar =<< newDaemonStatus + d <- newAssistantData st dstatus + urlrenderer <- newUrlRenderer + v <- newEmptyMVar + let callback a = Just $ a v + runAssistant d $ do + startNamedThread (Just urlrenderer) $ + webAppThread d urlrenderer True + (callback signaler) + (callback mainthread) + waitNamedThreads + where + signaler v = do + putMVar v "" + takeMVar v + mainthread v _url htmlshim = do + browser <- maybe Nothing webBrowser <$> Git.Config.global + openBrowser browser htmlshim Nothing Nothing + + _wait <- takeMVar v + + state <- Annex.new =<< Git.CurrentRepo.get + Annex.eval state $ + startDaemon True True $ Just $ sendurlback v + sendurlback v _origout _origerr url _htmlshim = putMVar v url + +openBrowser :: Maybe FilePath -> FilePath -> Maybe Handle -> Maybe Handle -> IO () +openBrowser cmd htmlshim outh errh = do + hPutStrLn (fromMaybe stdout outh) $ "Launching web browser on " ++ url + environ <- cleanEnvironment + (_, _, _, pid) <- createProcess p + { env = environ + , std_out = maybe Inherit UseHandle outh + , std_err = maybe Inherit UseHandle errh + } + exitcode <- waitForProcess pid + unless (exitcode == ExitSuccess) $ + hPutStrLn (fromMaybe stderr errh) "failed to start web browser" + where + url = fileUrl htmlshim + p = proc (fromMaybe browserCommand cmd) [htmlshim] + +{- web.browser is a generic git config setting for a web browser program -} +webBrowser :: Git.Repo -> Maybe FilePath +webBrowser = Git.Config.getMaybe "web.browser" + +fileUrl :: FilePath -> String +fileUrl file = "file://" ++ file diff --git a/Command/Whereis.hs b/Command/Whereis.hs new file mode 100644 index 0000000000..251c4ec7aa --- /dev/null +++ b/Command/Whereis.hs @@ -0,0 +1,54 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Whereis where + +import qualified Data.Map as M + +import Common.Annex +import Command +import Remote +import Logs.Trust + +def :: [Command] +def = [noCommit $ command "whereis" paramPaths seek + "lists repositories that have file content"] + +seek :: [CommandSeek] +seek = [withValue (remoteMap id) $ \m -> + withFilesInGit $ whenAnnexed $ start m] + +start :: M.Map UUID Remote -> FilePath -> (Key, Backend) -> CommandStart +start remotemap file (key, _) = do + showStart "whereis" file + next $ perform remotemap key + +perform :: M.Map UUID Remote -> Key -> CommandPerform +perform remotemap key = do + locations <- keyLocations key + (untrustedlocations, safelocations) <- trustPartition UnTrusted locations + let num = length safelocations + showNote $ show num ++ " " ++ copiesplural num + pp <- prettyPrintUUIDs "whereis" safelocations + unless (null safelocations) $ showLongNote pp + pp' <- prettyPrintUUIDs "untrusted" untrustedlocations + unless (null untrustedlocations) $ showLongNote $ untrustedheader ++ pp' + forM_ (mapMaybe (`M.lookup` remotemap) locations) $ + performRemote key + if null safelocations then stop else next $ return True + where + copiesplural 1 = "copy" + copiesplural _ = "copies" + untrustedheader = "The following untrusted locations may also have copies:\n" + +performRemote :: Key -> Remote -> Annex () +performRemote key remote = maybe noop go $ whereisKey remote + where + go a = do + ls <- a key + unless (null ls) $ showLongNote $ unlines $ + map (\l -> name remote ++ ": " ++ l) ls diff --git a/Command/XMPPGit.hs b/Command/XMPPGit.hs new file mode 100644 index 0000000000..c54d6a84a2 --- /dev/null +++ b/Command/XMPPGit.hs @@ -0,0 +1,42 @@ +{- git-annex command + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.XMPPGit where + +import Common.Annex +import Command +import Assistant.XMPP.Git + +def :: [Command] +def = [noCommit $ noRepo xmppGitRelay $ dontCheck repoExists $ + command "xmppgit" paramNothing seek "git to XMPP relay (internal use)"] + +seek :: [CommandSeek] +seek = [withWords start] + +start :: [String] -> CommandStart +start _ = do + liftIO gitRemoteHelper + liftIO xmppGitRelay + stop + +{- A basic implementation of the git-remote-helpers protocol. -} +gitRemoteHelper :: IO () +gitRemoteHelper = do + expect "capabilities" + respond ["connect"] + expect "connect git-receive-pack" + respond [] + where + expect s = do + cmd <- getLine + unless (cmd == s) $ + error $ "git-remote-helpers protocol error: expected: " ++ s ++ ", but got: " ++ cmd + respond l = do + mapM_ putStrLn l + putStrLn "" + hFlush stdout diff --git a/Common.hs b/Common.hs new file mode 100644 index 0000000000..3513425c0a --- /dev/null +++ b/Common.hs @@ -0,0 +1,33 @@ +{-# LANGUAGE PackageImports #-} + +module Common (module X) where + +import Control.Monad as X hiding (join) +import Control.Monad.IfElse as X +import Control.Applicative as X +import "mtl" Control.Monad.State.Strict as X (liftIO) +import Control.Exception.Extensible as X (IOException) + +import Data.Maybe as X +import Data.List as X hiding (head, tail, init, last) +import Data.String.Utils as X + +import "MissingH" System.Path as X +import System.FilePath as X +import System.Directory as X +import System.IO as X hiding (FilePath) +import System.Posix.Files as X +import System.Posix.IO as X +import System.Exit as X + +import Utility.Misc as X +import Utility.Exception as X +import Utility.SafeCommand as X +import Utility.Process as X +import Utility.Path as X +import Utility.Directory as X +import Utility.Monad as X +import Utility.Applicative as X +import Utility.FileSystemEncoding as X + +import Utility.PartialPrelude as X diff --git a/Common/Annex.hs b/Common/Annex.hs new file mode 100644 index 0000000000..e90825f0e9 --- /dev/null +++ b/Common/Annex.hs @@ -0,0 +1,8 @@ +module Common.Annex (module X) where + +import Common as X +import Types as X +import Types.UUID as X (toUUID, fromUUID) +import Annex as X (gitRepo, inRepo, fromRepo) +import Locations as X +import Messages as X diff --git a/Config.hs b/Config.hs new file mode 100644 index 0000000000..2eaf17bf80 --- /dev/null +++ b/Config.hs @@ -0,0 +1,103 @@ +{- Git configuration + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Config where + +import Common.Annex +import qualified Git +import qualified Git.Config +import qualified Git.Command +import qualified Annex + +type UnqualifiedConfigKey = String +data ConfigKey = ConfigKey String + +{- Looks up a setting in git config. -} +getConfig :: ConfigKey -> String -> Annex String +getConfig (ConfigKey key) def = fromRepo $ Git.Config.get key def + +{- Changes a git config setting in both internal state and .git/config -} +setConfig :: ConfigKey -> String -> Annex () +setConfig (ConfigKey key) value = do + inRepo $ Git.Command.run [Param "config", Param key, Param value] + Annex.changeGitRepo =<< inRepo Git.Config.reRead + +{- Unsets a git config setting. (Leaves it in state currently.) -} +unsetConfig :: ConfigKey -> Annex () +unsetConfig (ConfigKey key) = inRepo $ Git.Command.run + [Param "config", Param "--unset", Param key] + +{- A per-remote config setting in git config. -} +remoteConfig :: Git.Repo -> UnqualifiedConfigKey -> ConfigKey +remoteConfig r key = ConfigKey $ + "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" ++ key + +{- A global annex setting in git config. -} +annexConfig :: UnqualifiedConfigKey -> ConfigKey +annexConfig key = ConfigKey $ "annex." ++ key + +{- Calculates cost for a remote. Either the specific default, or as configured + - by remote..annex-cost, or if remote..annex-cost-command + - is set and prints a number, that is used. -} +remoteCost :: RemoteGitConfig -> Int -> Annex Int +remoteCost c def = case remoteAnnexCostCommand c of + Just cmd | not (null cmd) -> liftIO $ + (fromMaybe def . readish) <$> + readProcess "sh" ["-c", cmd] + _ -> return $ fromMaybe def $ remoteAnnexCost c + +cheapRemoteCost :: Int +cheapRemoteCost = 100 +semiCheapRemoteCost :: Int +semiCheapRemoteCost = 110 +expensiveRemoteCost :: Int +expensiveRemoteCost = 200 +veryExpensiveRemoteCost :: Int +veryExpensiveRemoteCost = 1000 + +{- Adjusts a remote's cost to reflect it being encrypted. -} +encryptedRemoteCostAdj :: Int +encryptedRemoteCostAdj = 50 + +{- Make sure the remote cost numbers work out. -} +prop_cost_sane :: Bool +prop_cost_sane = False `notElem` + [ expensiveRemoteCost > 0 + , cheapRemoteCost < semiCheapRemoteCost + , semiCheapRemoteCost < expensiveRemoteCost + , cheapRemoteCost + encryptedRemoteCostAdj > semiCheapRemoteCost + , cheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost + , semiCheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost + ] + +getNumCopies :: Maybe Int -> Annex Int +getNumCopies (Just v) = return v +getNumCopies Nothing = annexNumCopies <$> Annex.getGitConfig + +isDirect :: Annex Bool +isDirect = annexDirect <$> Annex.getGitConfig + +setDirect :: Bool -> Annex () +setDirect b = do + setConfig (annexConfig "direct") (Git.Config.boolConfig b) + Annex.changeGitConfig $ \c -> c { annexDirect = b } + +crippledFileSystem :: Annex Bool +crippledFileSystem = annexCrippledFileSystem <$> Annex.getGitConfig + +setCrippledFileSystem :: Bool -> Annex () +setCrippledFileSystem b = do + setConfig (annexConfig "crippledfilesystem") (Git.Config.boolConfig b) + Annex.changeGitConfig $ \c -> c { annexCrippledFileSystem = b } + +{- Gets the http headers to use. -} +getHttpHeaders :: Annex [String] +getHttpHeaders = do + v <- annexHttpHeadersCommand <$> Annex.getGitConfig + case v of + Just cmd -> lines <$> liftIO (readProcess "sh" ["-c", cmd]) + Nothing -> annexHttpHeaders <$> Annex.getGitConfig diff --git a/Creds.hs b/Creds.hs new file mode 100644 index 0000000000..06d3a52f90 --- /dev/null +++ b/Creds.hs @@ -0,0 +1,149 @@ +{- Credentials storage + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Creds where + +import Common.Annex +import Annex.Perms +import Utility.FileMode +import Crypto +import Types.Remote (RemoteConfig, RemoteConfigKey) +import Remote.Helper.Encryptable (remoteCipher, embedCreds) + +import System.Environment +import System.Posix.Env (setEnv) +import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.Map as M +import Utility.Base64 + +type Creds = String -- can be any data +type CredPair = (String, String) -- login, password + +{- A CredPair can be stored in a file, or in the environment, or perhaps + - in a remote's configuration. -} +data CredPairStorage = CredPairStorage + { credPairFile :: FilePath + , credPairEnvironment :: (String, String) + , credPairRemoteKey :: Maybe RemoteConfigKey + } + +{- Stores creds in a remote's configuration, if the remote allows + - that. Otherwise, caches them locally. -} +setRemoteCredPair :: RemoteConfig -> CredPairStorage -> Annex RemoteConfig +setRemoteCredPair c storage = go =<< getRemoteCredPair c storage + where + go (Just creds) + | embedCreds c = case credPairRemoteKey storage of + Nothing -> localcache creds + Just key -> storeconfig creds key =<< remoteCipher c + | otherwise = localcache creds + go Nothing = return c + + localcache creds = do + writeCacheCredPair creds storage + return c + + storeconfig creds key (Just cipher) = do + s <- liftIO $ encrypt cipher + (feedBytes $ L.pack $ encodeCredPair creds) + (readBytes $ return . L.unpack) + return $ M.insert key (toB64 s) c + storeconfig creds key Nothing = + return $ M.insert key (toB64 $ encodeCredPair creds) c + +{- Gets a remote's credpair, from the environment if set, otherwise + - from the cache in gitAnnexCredsDir, or failing that, from the + - value in RemoteConfig. -} +getRemoteCredPairFor :: String -> RemoteConfig -> CredPairStorage -> Annex (Maybe CredPair) +getRemoteCredPairFor this c storage = maybe missing (return . Just) =<< getRemoteCredPair c storage + where + (loginvar, passwordvar) = credPairEnvironment storage + missing = do + warning $ unwords + [ "Set both", loginvar + , "and", passwordvar + , "to use", this + ] + return Nothing + +getRemoteCredPair :: RemoteConfig -> CredPairStorage -> Annex (Maybe CredPair) +getRemoteCredPair c storage = maybe fromcache (return . Just) =<< fromenv + where + fromenv = liftIO $ getEnvCredPair storage + fromcache = maybe fromconfig (return . Just) =<< readCacheCredPair storage + fromconfig = case credPairRemoteKey storage of + Just key -> do + mcipher <- remoteCipher c + case (M.lookup key c, mcipher) of + (Nothing, _) -> return Nothing + (Just enccreds, Just cipher) -> do + creds <- liftIO $ decrypt cipher + (feedBytes $ L.pack $ fromB64 enccreds) + (readBytes $ return . L.unpack) + fromcreds creds + (Just bcreds, Nothing) -> + fromcreds $ fromB64 bcreds + Nothing -> return Nothing + fromcreds creds = case decodeCredPair creds of + Just credpair -> do + writeCacheCredPair credpair storage + return $ Just credpair + _ -> do error $ "bad creds" + +{- Gets a CredPair from the environment. -} +getEnvCredPair :: CredPairStorage -> IO (Maybe CredPair) +getEnvCredPair storage = liftM2 (,) + <$> get uenv + <*> get penv + where + (uenv, penv) = credPairEnvironment storage + get = catchMaybeIO . getEnv + +{- Stores a CredPair in the environment. -} +setEnvCredPair :: CredPair -> CredPairStorage -> IO () +setEnvCredPair (l, p) storage = do + set uenv l + set penv p + where + (uenv, penv) = credPairEnvironment storage + set var val = setEnv var val True + +writeCacheCredPair :: CredPair -> CredPairStorage -> Annex () +writeCacheCredPair credpair storage = + writeCacheCreds (encodeCredPair credpair) (credPairFile storage) + +{- Stores the creds in a file inside gitAnnexCredsDir that only the user + - can read. -} +writeCacheCreds :: Creds -> FilePath -> Annex () +writeCacheCreds creds file = do + d <- fromRepo gitAnnexCredsDir + createAnnexDirectory d + liftIO $ do + let f = d file + h <- openFile f WriteMode + modifyFileMode f $ removeModes + [groupReadMode, otherReadMode] + hPutStr h creds + hClose h + +readCacheCredPair :: CredPairStorage -> Annex (Maybe CredPair) +readCacheCredPair storage = maybe Nothing decodeCredPair + <$> readCacheCreds (credPairFile storage) + +readCacheCreds :: FilePath -> Annex (Maybe Creds) +readCacheCreds file = do + d <- fromRepo gitAnnexCredsDir + let f = d file + liftIO $ catchMaybeIO $ readFile f + +encodeCredPair :: CredPair -> Creds +encodeCredPair (l, p) = unlines [l, p] + +decodeCredPair :: Creds -> Maybe CredPair +decodeCredPair creds = case lines creds of + l:p:[] -> Just (l, p) + _ -> Nothing diff --git a/Crypto.hs b/Crypto.hs new file mode 100644 index 0000000000..ed489cdbc3 --- /dev/null +++ b/Crypto.hs @@ -0,0 +1,153 @@ +{- git-annex crypto + - + - Currently using gpg; could later be modified to support different + - crypto backends if neccessary. + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Crypto ( + Cipher, + KeyIds(..), + StorableCipher(..), + genEncryptedCipher, + genSharedCipher, + updateEncryptedCipher, + describeCipher, + decryptCipher, + encryptKey, + feedFile, + feedBytes, + readBytes, + encrypt, + decrypt, + + prop_hmacWithCipher_sane +) where + +import qualified Data.ByteString.Lazy as L +import Data.ByteString.Lazy.UTF8 (fromString) +import Data.Digest.Pure.SHA +import Control.Applicative + +import Common.Annex +import qualified Utility.Gpg as Gpg +import Types.Key +import Types.Crypto + +{- The beginning of a Cipher is used for HMAC; the remainder + - is used as the GPG symmetric encryption passphrase. + - + - HMAC SHA1 needs only 64 bytes. The rest of the HMAC key is for expansion, + - perhaps to HMAC SHA512, which needs 128 bytes (ideally). + - It also provides room the Cipher to contain data in a form like base64, + - which does not pack a full byte of entropy into a byte of data. + - + - 256 bytes is enough for gpg's symetric cipher; unlike weaker public key + - crypto, the key does not need to be too large. + -} +cipherBeginning :: Int +cipherBeginning = 256 + +cipherSize :: Int +cipherSize = 512 + +cipherPassphrase :: Cipher -> String +cipherPassphrase (Cipher c) = drop cipherBeginning c + +cipherHmac :: Cipher -> String +cipherHmac (Cipher c) = take cipherBeginning c + +{- Creates a new Cipher, encrypted to the specified key id. -} +genEncryptedCipher :: String -> IO StorableCipher +genEncryptedCipher keyid = do + ks <- Gpg.findPubKeys keyid + random <- Gpg.genRandom cipherSize + encryptCipher (Cipher random) ks + +{- Creates a new, shared Cipher. -} +genSharedCipher :: IO StorableCipher +genSharedCipher = SharedCipher <$> Gpg.genRandom cipherSize + +{- Updates an existing Cipher, re-encrypting it to add a keyid. -} +updateEncryptedCipher :: String -> StorableCipher -> IO StorableCipher +updateEncryptedCipher _ (SharedCipher _) = undefined +updateEncryptedCipher keyid encipher@(EncryptedCipher _ ks) = do + ks' <- Gpg.findPubKeys keyid + cipher <- decryptCipher encipher + encryptCipher cipher (merge ks ks') + where + merge (KeyIds a) (KeyIds b) = KeyIds $ a ++ b + +describeCipher :: StorableCipher -> String +describeCipher (SharedCipher _) = "shared cipher" +describeCipher (EncryptedCipher _ (KeyIds ks)) = + "with gpg " ++ keys ks ++ " " ++ unwords ks + where + keys [_] = "key" + keys _ = "keys" + +{- Encrypts a Cipher to the specified KeyIds. -} +encryptCipher :: Cipher -> KeyIds -> IO StorableCipher +encryptCipher (Cipher c) (KeyIds ks) = do + let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids + encipher <- Gpg.pipeStrict ([ Params "--encrypt" ] ++ recipients ks') c + return $ EncryptedCipher encipher (KeyIds ks') + where + recipients l = force_recipients : + concatMap (\k -> [Param "--recipient", Param k]) l + -- Force gpg to only encrypt to the specified + -- recipients, not configured defaults. + force_recipients = Params "--no-encrypt-to --no-default-recipient" + +{- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} +decryptCipher :: StorableCipher -> IO Cipher +decryptCipher (SharedCipher t) = return $ Cipher t +decryptCipher (EncryptedCipher t _) = + Cipher <$> Gpg.pipeStrict [ Param "--decrypt" ] t + +{- Generates an encrypted form of a Key. The encryption does not need to be + - reversable, nor does it need to be the same type of encryption used + - on content. It does need to be repeatable. -} +encryptKey :: Cipher -> Key -> Key +encryptKey c k = Key + { keyName = hmacWithCipher c (key2file k) + , keyBackendName = "GPGHMACSHA1" + , keySize = Nothing -- size and mtime omitted + , keyMtime = Nothing -- to avoid leaking data + } + +type Feeder = Handle -> IO () +type Reader a = Handle -> IO a + +feedFile :: FilePath -> Feeder +feedFile f h = L.hPut h =<< L.readFile f + +feedBytes :: L.ByteString -> Feeder +feedBytes = flip L.hPut + +readBytes :: (L.ByteString -> IO a) -> Reader a +readBytes a h = L.hGetContents h >>= a + +{- Runs a Feeder action, that generates content that is encrypted with the + - Cipher, and read by the Reader action. -} +encrypt :: Cipher -> Feeder -> Reader a -> IO a +encrypt = Gpg.feedRead [Params "--symmetric --force-mdc"] . cipherPassphrase + +{- Runs a Feeder action, that generates content that is decrypted with the + - Cipher, and read by the Reader action. -} +decrypt :: Cipher -> Feeder -> Reader a -> IO a +decrypt = Gpg.feedRead [Param "--decrypt"] . cipherPassphrase + +hmacWithCipher :: Cipher -> String -> String +hmacWithCipher c = hmacWithCipher' (cipherHmac c) +hmacWithCipher' :: String -> String -> String +hmacWithCipher' c s = showDigest $ hmacSha1 (fromString c) (fromString s) + +{- Ensure that hmacWithCipher' returns the same thing forevermore. -} +prop_hmacWithCipher_sane :: Bool +prop_hmacWithCipher_sane = known_good == hmacWithCipher' "foo" "bar" + where + known_good = "46b4ec586117154dacd49d664e5d63fdc88efb51" diff --git a/Fields.hs b/Fields.hs new file mode 100644 index 0000000000..ffd273be67 --- /dev/null +++ b/Fields.hs @@ -0,0 +1,35 @@ +{- git-annex fields + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Fields where + +import Common.Annex +import qualified Annex + +import Data.Char + +{- A field, stored in Annex state, with a value sanity checker. -} +data Field = Field + { fieldName :: String + , fieldCheck :: String -> Bool + } + +getField :: Field -> Annex (Maybe String) +getField = Annex.getField . fieldName + +remoteUUID :: Field +remoteUUID = Field "remoteuuid" $ + -- does it look like a UUID? + all (\c -> isAlphaNum c || c == '-') + +associatedFile :: Field +associatedFile = Field "associatedfile" $ \f -> + -- is the file a safe relative filename? + not (isAbsolute f) && not ("../" `isPrefixOf` f) + +direct :: Field +direct = Field "direct" $ \f -> f == "1" diff --git a/Git.hs b/Git.hs new file mode 100644 index 0000000000..46f995e77a --- /dev/null +++ b/Git.hs @@ -0,0 +1,130 @@ +{- git repository handling + - + - This is written to be completely independant of git-annex and should be + - suitable for other uses. + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git ( + Repo(..), + Ref(..), + Branch, + Sha, + Tag, + repoIsUrl, + repoIsSsh, + repoIsHttp, + repoIsLocal, + repoIsLocalBare, + repoIsLocalUnknown, + repoDescribe, + repoLocation, + repoPath, + localGitDir, + attributes, + hookPath, + assertLocal, +) where + +import Network.URI (uriPath, uriScheme, unEscapeString) +import System.Posix.Files + +import Common +import Git.Types +import Utility.FileMode + +{- User-visible description of a git repo. -} +repoDescribe :: Repo -> String +repoDescribe Repo { remoteName = Just name } = name +repoDescribe Repo { location = Url url } = show url +repoDescribe Repo { location = Local { worktree = Just dir } } = dir +repoDescribe Repo { location = Local { gitdir = dir } } = dir +repoDescribe Repo { location = LocalUnknown dir } = dir +repoDescribe Repo { location = Unknown } = "UNKNOWN" + +{- Location of the repo, either as a path or url. -} +repoLocation :: Repo -> String +repoLocation Repo { location = Url url } = show url +repoLocation Repo { location = Local { worktree = Just dir } } = dir +repoLocation Repo { location = Local { gitdir = dir } } = dir +repoLocation Repo { location = LocalUnknown dir } = dir +repoLocation Repo { location = Unknown } = undefined + +{- Path to a repository. For non-bare, this is the worktree, for bare, + - it's the gitdir, and for URL repositories, is the path on the remote + - host. -} +repoPath :: Repo -> FilePath +repoPath Repo { location = Url u } = unEscapeString $ uriPath u +repoPath Repo { location = Local { worktree = Just d } } = d +repoPath Repo { location = Local { gitdir = d } } = d +repoPath Repo { location = LocalUnknown dir } = dir +repoPath Repo { location = Unknown } = undefined + +{- Path to a local repository's .git directory. -} +localGitDir :: Repo -> FilePath +localGitDir Repo { location = Local { gitdir = d } } = d +localGitDir _ = undefined + +{- Some code needs to vary between URL and normal repos, + - or bare and non-bare, these functions help with that. -} +repoIsUrl :: Repo -> Bool +repoIsUrl Repo { location = Url _ } = True +repoIsUrl _ = False + +repoIsSsh :: Repo -> Bool +repoIsSsh Repo { location = Url url } + | scheme == "ssh:" = True + -- git treats these the same as ssh + | scheme == "git+ssh:" = True + | scheme == "ssh+git:" = True + | otherwise = False + where + scheme = uriScheme url +repoIsSsh _ = False + +repoIsHttp :: Repo -> Bool +repoIsHttp Repo { location = Url url } + | uriScheme url == "http:" = True + | uriScheme url == "https:" = True + | otherwise = False +repoIsHttp _ = False + +repoIsLocal :: Repo -> Bool +repoIsLocal Repo { location = Local { } } = True +repoIsLocal _ = False + +repoIsLocalBare :: Repo -> Bool +repoIsLocalBare Repo { location = Local { worktree = Nothing } } = True +repoIsLocalBare _ = False + +repoIsLocalUnknown :: Repo -> Bool +repoIsLocalUnknown Repo { location = LocalUnknown { } } = True +repoIsLocalUnknown _ = False + +assertLocal :: Repo -> a -> a +assertLocal repo action + | repoIsUrl repo = error $ unwords + [ "acting on non-local git repo" + , repoDescribe repo + , "not supported" + ] + | otherwise = action + +{- Path to a repository's gitattributes file. -} +attributes :: Repo -> FilePath +attributes repo + | repoIsLocalBare repo = repoPath repo ++ "/info/.gitattributes" + | otherwise = repoPath repo ++ "/.gitattributes" + +{- Path to a given hook script in a repository, only if the hook exists + - and is executable. -} +hookPath :: String -> Repo -> IO (Maybe FilePath) +hookPath script repo = do + let hook = localGitDir repo "hooks" script + ifM (catchBoolIO $ isexecutable hook) + ( return $ Just hook , return Nothing ) + where + isexecutable f = isExecutable . fileMode <$> getFileStatus f diff --git a/Git/AutoCorrect.hs b/Git/AutoCorrect.hs new file mode 100644 index 0000000000..325632de95 --- /dev/null +++ b/Git/AutoCorrect.hs @@ -0,0 +1,71 @@ +{- git autocorrection using Damerau-Levenshtein edit distance + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.AutoCorrect where + +import Common +import Git.Types +import qualified Git.Config + +import Text.EditDistance +import Control.Concurrent + +{- These are the same cost values as used in git. -} +gitEditCosts :: EditCosts +gitEditCosts = EditCosts + { deletionCosts = ConstantCost 4 + , insertionCosts = ConstantCost 1 + , substitutionCosts = ConstantCost 2 + , transpositionCosts = ConstantCost 0 + } + +{- Git's source calls this "an empirically derived magic number" -} +similarityFloor :: Int +similarityFloor = 7 + +{- Finds inexact matches for the input amoung the choices. + - Returns an ordered list of good enough matches, or an empty list if + - nothing matches well. -} +fuzzymatches :: String -> (c -> String) -> [c] -> [c] +fuzzymatches input showchoice choices = fst $ unzip $ + sortBy comparecost $ filter similarEnough $ zip choices costs + where + distance = restrictedDamerauLevenshteinDistance gitEditCosts input + costs = map (distance . showchoice) choices + comparecost a b = compare (snd a) (snd b) + similarEnough (_, cst) = cst < similarityFloor + +{- Takes action based on git's autocorrect configuration, in preparation for + - an autocorrected command being run. -} +prepare :: String -> (c -> String) -> [c] -> Repo -> IO () +prepare input showmatch matches r = + case readish $ Git.Config.get "help.autocorrect" "0" r of + Just n + | n == 0 -> list + | n < 0 -> warn + | otherwise -> sleep n + Nothing -> list + where + list = error $ unlines $ + [ "Unknown command '" ++ input ++ "'" + , "" + , "Did you mean one of these?" + ] ++ map (\m -> "\t" ++ showmatch m) matches + warn = + hPutStr stderr $ unlines + [ "WARNING: You called a command named '" ++ + input ++ "', which does not exist." + , "Continuing under the assumption that you meant '" ++ + showmatch (Prelude.head matches) ++ "'" + ] + sleep n = do + warn + hPutStrLn stderr $ unwords + [ "in" + , show (fromIntegral n / 10 :: Float) + , "seconds automatically..."] + threadDelay (n * 100000) -- deciseconds to microseconds diff --git a/Git/Branch.hs b/Git/Branch.hs new file mode 100644 index 0000000000..41ae2559eb --- /dev/null +++ b/Git/Branch.hs @@ -0,0 +1,102 @@ +{- git branch stuff + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Git.Branch where + +import Common +import Git +import Git.Sha +import Git.Command + +{- The currently checked out branch. + - + - In a just initialized git repo before the first commit, + - symbolic-ref will show the master branch, even though that + - branch is not created yet. So, this also looks at show-ref HEAD + - to double-check. + -} +current :: Repo -> IO (Maybe Git.Ref) +current r = do + v <- currentUnsafe r + case v of + Nothing -> return Nothing + Just branch -> + ifM (null <$> pipeReadStrict [Param "show-ref", Param $ show branch] r) + ( return Nothing + , return v + ) + +{- The current branch, which may not really exist yet. -} +currentUnsafe :: Repo -> IO (Maybe Git.Ref) +currentUnsafe r = parse . firstLine + <$> pipeReadStrict [Param "symbolic-ref", Param "HEAD"] r + where + parse l + | null l = Nothing + | otherwise = Just $ Git.Ref l + +{- Checks if the second branch has any commits not present on the first + - branch. -} +changed :: Branch -> Branch -> Repo -> IO Bool +changed origbranch newbranch repo + | origbranch == newbranch = return False + | otherwise = not . null <$> diffs + where + diffs = pipeReadStrict + [ Param "log" + , Param (show origbranch ++ ".." ++ show newbranch) + , Params "--oneline -n1" + ] repo + +{- Given a set of refs that are all known to have commits not + - on the branch, tries to update the branch by a fast-forward. + - + - In order for that to be possible, one of the refs must contain + - every commit present in all the other refs. + -} +fastForward :: Branch -> [Ref] -> Repo -> IO Bool +fastForward _ [] _ = return True +fastForward branch (first:rest) repo = + -- First, check that the branch does not contain any + -- new commits that are not in the first ref. If it does, + -- cannot fast-forward. + ifM (changed first branch repo) + ( no_ff + , maybe no_ff do_ff =<< findbest first rest + ) + where + no_ff = return False + do_ff to = do + run [Param "update-ref", Param $ show branch, Param $ show to] repo + return True + findbest c [] = return $ Just c + findbest c (r:rs) + | c == r = findbest c rs + | otherwise = do + better <- changed c r repo + worse <- changed r c repo + case (better, worse) of + (True, True) -> return Nothing -- divergent fail + (True, False) -> findbest r rs -- better + (False, True) -> findbest c rs -- worse + (False, False) -> findbest c rs -- same + +{- Commits the index into the specified branch (or other ref), + - with the specified parent refs, and returns the committed sha -} +commit :: String -> Branch -> [Ref] -> Repo -> IO Sha +commit message branch parentrefs repo = do + tree <- getSha "write-tree" $ + pipeReadStrict [Param "write-tree"] repo + sha <- getSha "commit-tree" $ pipeWriteRead + (map Param $ ["commit-tree", show tree] ++ ps) + message repo + run [Param "update-ref", Param $ show branch, Param $ show sha] repo + return sha + where + ps = concatMap (\r -> ["-p", show r]) parentrefs diff --git a/Git/CatFile.hs b/Git/CatFile.hs new file mode 100644 index 0000000000..704724211c --- /dev/null +++ b/Git/CatFile.hs @@ -0,0 +1,75 @@ +{- git cat-file interface + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.CatFile ( + CatFileHandle, + catFileStart, + catFileStop, + catFile, + catObject, + catObjectDetails, +) where + +import System.IO +import qualified Data.ByteString as S +import qualified Data.ByteString.Lazy as L + +import Common +import Git +import Git.Sha +import Git.Command +import Git.Types +import qualified Utility.CoProcess as CoProcess + +type CatFileHandle = CoProcess.CoProcessHandle + +catFileStart :: Repo -> IO CatFileHandle +catFileStart = gitCoProcessStart + [ Param "cat-file" + , Param "--batch" + ] + +catFileStop :: CatFileHandle -> IO () +catFileStop = CoProcess.stop + +{- Reads a file from a specified branch. -} +catFile :: CatFileHandle -> Branch -> FilePath -> IO L.ByteString +catFile h branch file = catObject h $ Ref $ show branch ++ ":" ++ file + +{- Uses a running git cat-file read the content of an object. + - Objects that do not exist will have "" returned. -} +catObject :: CatFileHandle -> Ref -> IO L.ByteString +catObject h object = maybe L.empty fst <$> catObjectDetails h object + +{- Gets both the content of an object, and its Sha. -} +catObjectDetails :: CatFileHandle -> Ref -> IO (Maybe (L.ByteString, Sha)) +catObjectDetails h object = CoProcess.query h send receive + where + send to = do + fileEncoding to + hPutStrLn to $ show object + receive from = do + fileEncoding from + header <- hGetLine from + case words header of + [sha, objtype, size] + | length sha == shaSize && + isJust (readObjectType objtype) -> + case reads size of + [(bytes, "")] -> readcontent bytes from sha + _ -> dne + | otherwise -> dne + _ + | header == show object ++ " missing" -> dne + | otherwise -> error $ "unknown response from git cat-file " ++ show (header, object) + readcontent bytes from sha = do + content <- S.hGet from bytes + c <- hGetChar from + when (c /= '\n') $ + error "missing newline from git cat-file" + return $ Just (L.fromChunks [content], Ref sha) + dne = return Nothing diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs new file mode 100644 index 0000000000..f9279d460f --- /dev/null +++ b/Git/CheckAttr.hs @@ -0,0 +1,67 @@ +{- git check-attr interface + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.CheckAttr where + +import Common +import Git +import Git.Command +import qualified Git.Version +import qualified Utility.CoProcess as CoProcess + +type CheckAttrHandle = (CoProcess.CoProcessHandle, [Attr], String) + +type Attr = String + +{- Starts git check-attr running to look up the specified gitattributes + - values and returns a handle. -} +checkAttrStart :: [Attr] -> Repo -> IO CheckAttrHandle +checkAttrStart attrs repo = do + cwd <- getCurrentDirectory + h <- gitCoProcessStart params repo + return (h, attrs, cwd) + where + params = + [ Param "check-attr" + , Params "-z --stdin" + ] ++ map Param attrs ++ + [ Param "--" ] + +checkAttrStop :: CheckAttrHandle -> IO () +checkAttrStop (h, _, _) = CoProcess.stop h + +{- Gets an attribute of a file. -} +checkAttr :: CheckAttrHandle -> Attr -> FilePath -> IO String +checkAttr (h, attrs, cwd) want file = do + pairs <- CoProcess.query h send receive + let vals = map snd $ filter (\(attr, _) -> attr == want) pairs + case vals of + [v] -> return v + _ -> error $ "unable to determine " ++ want ++ " attribute of " ++ file + where + send to = do + fileEncoding to + hPutStr to $ file' ++ "\0" + receive from = forM attrs $ \attr -> do + fileEncoding from + l <- hGetLine from + return (attr, attrvalue attr l) + {- Before git 1.7.7, git check-attr worked best with + - absolute filenames; using them worked around some bugs + - with relative filenames. + - + - With newer git, git check-attr chokes on some absolute + - filenames, and the bugs that necessitated them were fixed, + - so use relative filenames. -} + oldgit = Git.Version.older "1.7.7" + file' + | oldgit = absPathFrom cwd file + | otherwise = relPathDirToFile cwd $ absPathFrom cwd file + attrvalue attr l = end bits !! 0 + where + bits = split sep l + sep = ": " ++ attr ++ ": " diff --git a/Git/Command.hs b/Git/Command.hs new file mode 100644 index 0000000000..f3841c7fa6 --- /dev/null +++ b/Git/Command.hs @@ -0,0 +1,112 @@ +{- running git commands + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Command where + +import System.Process (std_out, env) + +import Common +import Git +import Git.Types +import qualified Utility.CoProcess as CoProcess + +{- Constructs a git command line operating on the specified repo. -} +gitCommandLine :: [CommandParam] -> Repo -> [CommandParam] +gitCommandLine params Repo { location = l@(Local _ _ ) } = setdir : settree ++ params + where + setdir = Param $ "--git-dir=" ++ gitdir l + settree = case worktree l of + Nothing -> [] + Just t -> [Param $ "--work-tree=" ++ t] +gitCommandLine _ repo = assertLocal repo $ error "internal" + +{- Runs git in the specified repo. -} +runBool :: [CommandParam] -> Repo -> IO Bool +runBool params repo = assertLocal repo $ + boolSystemEnv "git" + (gitCommandLine params repo) + (gitEnv repo) + +{- Runs git in the specified repo, throwing an error if it fails. -} +run :: [CommandParam] -> Repo -> IO () +run params repo = assertLocal repo $ + unlessM (runBool params repo) $ + error $ "git " ++ show params ++ " failed" + +{- Runs git and forces it to be quiet, throwing an error if it fails. -} +runQuiet :: [CommandParam] -> Repo -> IO () +runQuiet params repo = withQuietOutput createProcessSuccess $ + (proc "git" $ toCommand $ gitCommandLine (params) repo) + { env = gitEnv repo } + +{- Runs a git command and returns its output, lazily. + - + - Also returns an action that should be used when the output is all + - read (or no more is needed), that will wait on the command, and + - return True if it succeeded. Failure to wait will result in zombies. + -} +pipeReadLazy :: [CommandParam] -> Repo -> IO (String, IO Bool) +pipeReadLazy params repo = assertLocal repo $ do + (_, Just h, _, pid) <- createProcess p { std_out = CreatePipe } + fileEncoding h + c <- hGetContents h + return (c, checkSuccessProcess pid) + where + p = gitCreateProcess params repo + +{- Runs a git command, and returns its output, strictly. + - + - Nonzero exit status is ignored. + -} +pipeReadStrict :: [CommandParam] -> Repo -> IO String +pipeReadStrict params repo = assertLocal repo $ + withHandle StdoutHandle (createProcessChecked ignoreFailureProcess) p $ \h -> do + fileEncoding h + output <- hGetContentsStrict h + hClose h + return output + where + p = gitCreateProcess params repo + +{- Runs a git command, feeding it input, and returning its output, + - which is expected to be fairly small, since it's all read into memory + - strictly. -} +pipeWriteRead :: [CommandParam] -> String -> Repo -> IO String +pipeWriteRead params s repo = assertLocal repo $ + writeReadProcessEnv "git" (toCommand $ gitCommandLine params repo) + (gitEnv repo) s (Just fileEncoding) + +{- Runs a git command, feeding it input on a handle with an action. -} +pipeWrite :: [CommandParam] -> Repo -> (Handle -> IO ()) -> IO () +pipeWrite params repo = withHandle StdinHandle createProcessSuccess $ + gitCreateProcess params repo + +{- Reads null terminated output of a git command (as enabled by the -z + - parameter), and splits it. -} +pipeNullSplit :: [CommandParam] -> Repo -> IO ([String], IO Bool) +pipeNullSplit params repo = do + (s, cleanup) <- pipeReadLazy params repo + return (filter (not . null) $ split sep s, cleanup) + where + sep = "\0" + + +pipeNullSplitZombie :: [CommandParam] -> Repo -> IO [String] +pipeNullSplitZombie params repo = leaveZombie <$> pipeNullSplit params repo + +{- Doesn't run the cleanup action. A zombie results. -} +leaveZombie :: (a, IO Bool) -> a +leaveZombie = fst + +{- Runs a git command as a coprocess. -} +gitCoProcessStart :: [CommandParam] -> Repo -> IO CoProcess.CoProcessHandle +gitCoProcessStart params repo = CoProcess.start "git" (toCommand $ gitCommandLine params repo) (gitEnv repo) + +gitCreateProcess :: [CommandParam] -> Repo -> CreateProcess +gitCreateProcess params repo = + (proc "git" $ toCommand $ gitCommandLine params repo) + { env = gitEnv repo } diff --git a/Git/Config.hs b/Git/Config.hs new file mode 100644 index 0000000000..adc75a2085 --- /dev/null +++ b/Git/Config.hs @@ -0,0 +1,155 @@ +{- git repository configuration handling + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Config where + +import qualified Data.Map as M +import Data.Char +import System.Process (cwd, env) + +import Common +import Git +import Git.Types +import qualified Git.Construct +import Utility.UserInfo + +{- Returns a single git config setting, or a default value if not set. -} +get :: String -> String -> Repo -> String +get key defaultValue repo = M.findWithDefault defaultValue key (config repo) + +{- Returns a list with each line of a multiline config setting. -} +getList :: String -> Repo -> [String] +getList key repo = M.findWithDefault [] key (fullconfig repo) + +{- Returns a single git config setting, if set. -} +getMaybe :: String -> Repo -> Maybe String +getMaybe key repo = M.lookup key (config repo) + +{- Runs git config and populates a repo with its config. + - Avoids re-reading config when run repeatedly. -} +read :: Repo -> IO Repo +read repo@(Repo { config = c }) + | c == M.empty = read' repo + | otherwise = return repo + +{- Reads config even if it was read before. -} +reRead :: Repo -> IO Repo +reRead r = read' $ r + { config = M.empty + , fullconfig = M.empty + } + +{- Cannot use pipeRead because it relies on the config having been already + - read. Instead, chdir to the repo and run git config. + -} +read' :: Repo -> IO Repo +read' repo = go repo + where + go Repo { location = Local { gitdir = d } } = git_config d + go Repo { location = LocalUnknown d } = git_config d + go _ = assertLocal repo $ error "internal" + git_config d = withHandle StdoutHandle createProcessSuccess p $ + hRead repo + where + params = ["config", "--null", "--list"] + p = (proc "git" params) + { cwd = Just d + , env = gitEnv repo + } + +{- Gets the global git config, returning a dummy Repo containing it. -} +global :: IO (Maybe Repo) +global = do + home <- myHomeDir + ifM (doesFileExist $ home ".gitconfig") + ( do + repo <- Git.Construct.fromUnknown + repo' <- withHandle StdoutHandle createProcessSuccess p $ + hRead repo + return $ Just repo' + , return Nothing + ) + where + params = ["config", "--null", "--list", "--global"] + p = (proc "git" params) + +{- Reads git config from a handle and populates a repo with it. -} +hRead :: Repo -> Handle -> IO Repo +hRead repo h = do + -- We use the FileSystemEncoding when reading from git-config, + -- because it can contain arbitrary filepaths (and other strings) + -- in any encoding. + fileEncoding h + val <- hGetContentsStrict h + store val repo + +{- Stores a git config into a Repo, returning the new version of the Repo. + - The git config may be multiple lines, or a single line. + - Config settings can be updated incrementally. + -} +store :: String -> Repo -> IO Repo +store s repo = do + let c = parse s + repo' <- updateLocation $ repo + { config = (M.map Prelude.head c) `M.union` config repo + , fullconfig = M.unionWith (++) c (fullconfig repo) + } + rs <- Git.Construct.fromRemotes repo' + return $ repo' { remotes = rs } + +{- Updates the location of a repo, based on its configuration. + - + - Git.Construct makes LocalUknown repos, of which only a directory is + - known. Once the config is read, this can be fixed up to a Local repo, + - based on the core.bare and core.worktree settings. + -} +updateLocation :: Repo -> IO Repo +updateLocation r@(Repo { location = LocalUnknown d }) + | isBare r = updateLocation' r $ Local d Nothing + | otherwise = updateLocation' r $ Local (d ".git") (Just d) +updateLocation r@(Repo { location = l@(Local {}) }) = updateLocation' r l +updateLocation r = return r + +updateLocation' :: Repo -> RepoLocation -> IO Repo +updateLocation' r l = do + l' <- case getMaybe "core.worktree" r of + Nothing -> return l + Just d -> do + {- core.worktree is relative to the gitdir -} + top <- absPath $ gitdir l + return $ l { worktree = Just $ absPathFrom top d } + return $ r { location = l' } + +{- Parses git config --list or git config --null --list output into a + - config map. -} +parse :: String -> M.Map String [String] +parse [] = M.empty +parse s + -- --list output will have an = in the first line + | all ('=' `elem`) (take 1 ls) = sep '=' ls + -- --null --list output separates keys from values with newlines + | otherwise = sep '\n' $ split "\0" s + where + ls = lines s + sep c = M.fromListWith (++) . map (\(k,v) -> (k, [v])) . + map (separate (== c)) + +{- Checks if a string from git config is a true value. -} +isTrue :: String -> Maybe Bool +isTrue s + | s' == "true" = Just True + | s' == "false" = Just False + | otherwise = Nothing + where + s' = map toLower s + +boolConfig :: Bool -> String +boolConfig True = "true" +boolConfig False = "false" + +isBare :: Repo -> Bool +isBare r = fromMaybe False $ isTrue =<< getMaybe "core.bare" r diff --git a/Git/Construct.hs b/Git/Construct.hs new file mode 100644 index 0000000000..f9f4b464a9 --- /dev/null +++ b/Git/Construct.hs @@ -0,0 +1,253 @@ +{- Construction of Git Repo objects + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Construct ( + fromCwd, + fromAbsPath, + fromPath, + fromUrl, + fromUnknown, + localToUrl, + remoteNamed, + remoteNamedFromKey, + fromRemotes, + fromRemoteLocation, + repoAbsPath, + newFrom, +) where + +import System.Posix.User +import qualified Data.Map as M hiding (map, split) +import Network.URI + +import Common +import Git.Types +import Git +import qualified Git.Url as Url +import Utility.UserInfo + +{- Finds the git repository used for the cwd, which may be in a parent + - directory. -} +fromCwd :: IO (Maybe Repo) +fromCwd = getCurrentDirectory >>= seekUp + where + seekUp dir = do + r <- checkForRepo dir + case r of + Nothing -> case parentDir dir of + "" -> return Nothing + d -> seekUp d + Just loc -> Just <$> newFrom loc + +{- Local Repo constructor, accepts a relative or absolute path. -} +fromPath :: FilePath -> IO Repo +fromPath dir = fromAbsPath =<< absPath dir + +{- Local Repo constructor, requires an absolute path to the repo be + - specified. -} +fromAbsPath :: FilePath -> IO Repo +fromAbsPath dir + | "/" `isPrefixOf` dir = + ifM (doesDirectoryExist dir') ( ret dir' , hunt ) + | otherwise = + error $ "internal error, " ++ dir ++ " is not absolute" + where + ret = newFrom . LocalUnknown + {- Git always looks for "dir.git" in preference to + - to "dir", even if dir ends in a "/". -} + canondir = dropTrailingPathSeparator dir + dir' = canondir ++ ".git" + {- When dir == "foo/.git", git looks for "foo/.git/.git", + - and failing that, uses "foo" as the repository. -} + hunt + | "/.git" `isSuffixOf` canondir = + ifM (doesDirectoryExist $ dir ".git") + ( ret dir + , ret $ takeDirectory canondir + ) + | otherwise = ret dir + +{- Remote Repo constructor. Throws exception on invalid url. + - + - Git is somewhat forgiving about urls to repositories, allowing + - eg spaces that are not normally allowed unescaped in urls. + -} +fromUrl :: String -> IO Repo +fromUrl url + | not (isURI url) = fromUrlStrict $ escapeURIString isUnescapedInURI url + | otherwise = fromUrlStrict url + +fromUrlStrict :: String -> IO Repo +fromUrlStrict url + | startswith "file://" url = fromAbsPath $ uriPath u + | otherwise = newFrom $ Url u + where + u = fromMaybe bad $ parseURI url + bad = error $ "bad url " ++ url + +{- Creates a repo that has an unknown location. -} +fromUnknown :: IO Repo +fromUnknown = newFrom Unknown + +{- Converts a local Repo into a remote repo, using the reference repo + - which is assumed to be on the same host. -} +localToUrl :: Repo -> Repo -> Repo +localToUrl reference r + | not $ repoIsUrl reference = error "internal error; reference repo not url" + | repoIsUrl r = r + | otherwise = r { location = Url $ fromJust $ parseURI absurl } + where + absurl = concat + [ Url.scheme reference + , "//" + , Url.authority reference + , repoPath r + ] + +{- Calculates a list of a repo's configured remotes, by parsing its config. -} +fromRemotes :: Repo -> IO [Repo] +fromRemotes repo = mapM construct remotepairs + where + filterconfig f = filter f $ M.toList $ config repo + filterkeys f = filterconfig (\(k,_) -> f k) + remotepairs = filterkeys isremote + isremote k = startswith "remote." k && endswith ".url" k + construct (k,v) = remoteNamedFromKey k $ fromRemoteLocation v repo + +{- Sets the name of a remote when constructing the Repo to represent it. -} +remoteNamed :: String -> IO Repo -> IO Repo +remoteNamed n constructor = do + r <- constructor + return $ r { remoteName = Just n } + +{- Sets the name of a remote based on the git config key, such as + - "remote.foo.url". -} +remoteNamedFromKey :: String -> IO Repo -> IO Repo +remoteNamedFromKey k = remoteNamed basename + where + basename = join "." $ reverse $ drop 1 $ reverse $ drop 1 $ split "." k + +{- Constructs a new Repo for one of a Repo's remotes using a given + - location (ie, an url). -} +fromRemoteLocation :: String -> Repo -> IO Repo +fromRemoteLocation s repo = gen $ calcloc s + where + gen v + | scpstyle v = fromUrl $ scptourl v + | urlstyle v = fromUrl v + | otherwise = fromRemotePath v repo + -- insteadof config can rewrite remote location + calcloc l + | null insteadofs = l + | otherwise = replacement ++ drop (length bestvalue) l + where + replacement = drop (length prefix) $ + take (length bestkey - length suffix) bestkey + (bestkey, bestvalue) = maximumBy longestvalue insteadofs + longestvalue (_, a) (_, b) = compare b a + insteadofs = filterconfig $ \(k, v) -> + startswith prefix k && + endswith suffix k && + startswith v l + filterconfig f = filter f $ + concatMap splitconfigs $ M.toList $ fullconfig repo + splitconfigs (k, vs) = map (\v -> (k, v)) vs + (prefix, suffix) = ("url." , ".insteadof") + urlstyle v = isURI v || ":" `isInfixOf` v && "//" `isInfixOf` v + -- git remotes can be written scp style -- [user@]host:dir + -- but foo::bar is a git-remote-helper location instead + scpstyle v = ":" `isInfixOf` v + && not ("//" `isInfixOf` v) + && not ("::" `isInfixOf` v) + scptourl v = "ssh://" ++ host ++ slash dir + where + (host, dir) = separate (== ':') v + slash d | d == "" = "/~/" ++ d + | "/" `isPrefixOf` d = d + | "~" `isPrefixOf` d = '/':d + | otherwise = "/~/" ++ d + +{- Constructs a Repo from the path specified in the git remotes of + - another Repo. -} +fromRemotePath :: FilePath -> Repo -> IO Repo +fromRemotePath dir repo = do + dir' <- expandTilde dir + fromAbsPath $ repoPath repo dir' + +{- Git remotes can have a directory that is specified relative + - to the user's home directory, or that contains tilde expansions. + - This converts such a directory to an absolute path. + - Note that it has to run on the system where the remote is. + -} +repoAbsPath :: FilePath -> IO FilePath +repoAbsPath d = do + d' <- expandTilde d + h <- myHomeDir + return $ h d' + +expandTilde :: FilePath -> IO FilePath +expandTilde = expandt True + where + expandt _ [] = return "" + expandt _ ('/':cs) = do + v <- expandt True cs + return ('/':v) + expandt True ('~':'/':cs) = do + h <- myHomeDir + return $ h cs + expandt True ('~':cs) = do + let (name, rest) = findname "" cs + u <- getUserEntryForName name + return $ homeDirectory u rest + expandt _ (c:cs) = do + v <- expandt False cs + return (c:v) + findname n [] = (n, "") + findname n (c:cs) + | c == '/' = (n, cs) + | otherwise = findname (n++[c]) cs + +checkForRepo :: FilePath -> IO (Maybe RepoLocation) +checkForRepo dir = + check isRepo $ + check gitDirFile $ + check isBareRepo $ + return Nothing + where + check test cont = maybe cont (return . Just) =<< test + checkdir c = ifM c + ( return $ Just $ LocalUnknown dir + , return Nothing + ) + isRepo = checkdir $ gitSignature $ ".git" "config" + isBareRepo = checkdir $ gitSignature "config" + <&&> doesDirectoryExist (dir "objects") + gitDirFile = do + c <- firstLine <$> + catchDefaultIO "" (readFile $ dir ".git") + return $ if gitdirprefix `isPrefixOf` c + then Just $ Local + { gitdir = absPathFrom dir $ + drop (length gitdirprefix) c + , worktree = Just dir + } + else Nothing + where + gitdirprefix = "gitdir: " + gitSignature file = doesFileExist $ dir file + +newFrom :: RepoLocation -> IO Repo +newFrom l = return Repo + { location = l + , config = M.empty + , fullconfig = M.empty + , remotes = [] + , remoteName = Nothing + , gitEnv = Nothing + } + + diff --git a/Git/CurrentRepo.hs b/Git/CurrentRepo.hs new file mode 100644 index 0000000000..482873960a --- /dev/null +++ b/Git/CurrentRepo.hs @@ -0,0 +1,61 @@ +{- The current git repository. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.CurrentRepo where + +import System.Posix.Directory (changeWorkingDirectory) +import System.Posix.Env (getEnv, unsetEnv) + +import Common +import Git.Types +import Git.Construct +import qualified Git.Config + +{- Gets the current git repository. + - + - Honors GIT_DIR and GIT_WORK_TREE. + - Both environment variables are unset, to avoid confusing other git + - commands that also look at them. Instead, the Git module passes + - --work-tree and --git-dir to git commands it runs. + - + - When GIT_WORK_TREE or core.worktree are set, changes the working + - directory if necessary to ensure it is within the repository's work + - tree. While not needed for git commands, this is useful for anything + - else that looks for files in the worktree. + -} +get :: IO Repo +get = do + gd <- pathenv "GIT_DIR" + r <- configure gd =<< fromCwd + wt <- maybe (worktree $ location r) Just <$> pathenv "GIT_WORK_TREE" + case wt of + Nothing -> return r + Just d -> do + cwd <- getCurrentDirectory + unless (d `dirContains` cwd) $ + changeWorkingDirectory d + return $ addworktree wt r + where + pathenv s = do + v <- getEnv s + case v of + Just d -> do + unsetEnv s + Just <$> absPath d + Nothing -> return Nothing + + configure Nothing (Just r) = Git.Config.read r + configure (Just d) _ = do + absd <- absPath d + cwd <- getCurrentDirectory + r <- newFrom $ Local { gitdir = absd, worktree = Just cwd } + Git.Config.read r + configure Nothing Nothing = error "Not in a git repository." + + addworktree w r = changelocation r $ + Local { gitdir = gitdir (location r), worktree = w } + changelocation r l = r { location = l } diff --git a/Git/DiffTree.hs b/Git/DiffTree.hs new file mode 100644 index 0000000000..f122a4fb5b --- /dev/null +++ b/Git/DiffTree.hs @@ -0,0 +1,86 @@ +{- git diff-tree interface + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.DiffTree ( + DiffTreeItem(..), + diffTree, + diffTreeRecursive, + diffIndex, +) where + +import Numeric +import System.Posix.Types + +import Common +import Git +import Git.Sha +import Git.Command +import qualified Git.Filename +import qualified Git.Ref + +data DiffTreeItem = DiffTreeItem + { srcmode :: FileMode + , dstmode :: FileMode + , srcsha :: Sha -- nullSha if file was added + , dstsha :: Sha -- nullSha if file was deleted + , status :: String + , file :: FilePath + } deriving Show + +{- Diffs two tree Refs. -} +diffTree :: Ref -> Ref -> Repo -> IO ([DiffTreeItem], IO Bool) +diffTree src dst = getdiff (Param "diff-tree") + [Param (show src), Param (show dst)] + +{- Diffs two tree Refs, recursing into sub-trees -} +diffTreeRecursive :: Ref -> Ref -> Repo -> IO ([DiffTreeItem], IO Bool) +diffTreeRecursive src dst = getdiff (Param "diff-tree") + [Param "-r", Param (show src), Param (show dst)] + +{- Diffs between the repository and index. Does nothing if there is not + - yet a commit in the repository. -} +diffIndex :: Repo -> IO ([DiffTreeItem], IO Bool) +diffIndex repo = do + ifM (Git.Ref.headExists repo) + ( getdiff (Param "diff-index") [Param "--cached", Param "HEAD"] repo + , return ([], return True) + ) + +getdiff :: CommandParam -> [CommandParam] -> Repo -> IO ([DiffTreeItem], IO Bool) +getdiff command params repo = do + (diff, cleanup) <- pipeNullSplit ps repo + return (parseDiffTree diff, cleanup) + where + ps = command : Params "-z --raw --no-renames -l0" : params + +{- Parses diff-tree output. -} +parseDiffTree :: [String] -> [DiffTreeItem] +parseDiffTree l = go l [] + where + go [] c = c + go (info:f:rest) c = go rest (mk info f : c) + go (s:[]) _ = error $ "diff-tree parse error " ++ s + + mk info f = DiffTreeItem + { srcmode = readmode srcm + , dstmode = readmode dstm + , srcsha = fromMaybe (error "bad srcsha") $ extractSha ssha + , dstsha = fromMaybe (error "bad dstsha") $ extractSha dsha + , status = s + , file = Git.Filename.decode f + } + where + readmode = fst . Prelude.head . readOct + + -- info = : SP SP SP SP + -- All fields are fixed, so we can pull them out of + -- specific positions in the line. + (srcm, past_srcm) = splitAt 7 $ drop 1 info + (dstm, past_dstm) = splitAt 7 past_srcm + (ssha, past_ssha) = splitAt shaSize past_dstm + (dsha, past_dsha) = splitAt shaSize $ drop 1 past_ssha + s = drop 1 past_dsha diff --git a/Git/FilePath.hs b/Git/FilePath.hs new file mode 100644 index 0000000000..6344353d6f --- /dev/null +++ b/Git/FilePath.hs @@ -0,0 +1,34 @@ +{- git FilePath library + - + - Different git commands use different types of FilePaths to refer to + - files in the repository. Some commands use paths relative to the + - top of the repository even when run in a subdirectory. Adding some + - types helps keep that straight. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.FilePath ( + TopFilePath, + getTopFilePath, + toTopFilePath, + asTopFilePath, +) where + +import Common +import Git + +{- A FilePath, relative to the top of the git repository. -} +newtype TopFilePath = TopFilePath { getTopFilePath :: FilePath } + +{- The input FilePath can be absolute, or relative to the CWD. -} +toTopFilePath :: FilePath -> Git.Repo -> IO TopFilePath +toTopFilePath file repo = TopFilePath <$> + relPathDirToFile (repoPath repo) <$> absPath file + +{- The input FilePath must already be relative to the top of the git + - repository -} +asTopFilePath :: FilePath -> TopFilePath +asTopFilePath file = TopFilePath file diff --git a/Git/Filename.hs b/Git/Filename.hs new file mode 100644 index 0000000000..5e076d3b5a --- /dev/null +++ b/Git/Filename.hs @@ -0,0 +1,28 @@ +{- Some git commands output encoded filenames, in a rather annoyingly complex + - C-style encoding. + - + - Copyright 2010, 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Filename where + +import Utility.Format (decode_c, encode_c) + +import Common + +decode :: String -> FilePath +decode [] = [] +decode f@(c:s) + -- encoded strings will be inside double quotes + | c == '"' && end s == ['"'] = decode_c $ beginning s + | otherwise = f + +{- Should not need to use this, except for testing decode. -} +encode :: FilePath -> String +encode s = "\"" ++ encode_c s ++ "\"" + +{- for quickcheck -} +prop_idempotent_deencode :: String -> Bool +prop_idempotent_deencode s = s == decode (encode s) diff --git a/Git/HashObject.hs b/Git/HashObject.hs new file mode 100644 index 0000000000..b4a32ef1cd --- /dev/null +++ b/Git/HashObject.hs @@ -0,0 +1,45 @@ +{- git hash-object interface + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.HashObject where + +import Common +import Git +import Git.Sha +import Git.Command +import Git.Types +import qualified Utility.CoProcess as CoProcess + +type HashObjectHandle = CoProcess.CoProcessHandle + +hashObjectStart :: Repo -> IO HashObjectHandle +hashObjectStart = gitCoProcessStart + [ Param "hash-object" + , Param "-w" + , Param "--stdin-paths" + ] + +hashObjectStop :: HashObjectHandle -> IO () +hashObjectStop = CoProcess.stop + +{- Injects a file into git, returning the Sha of the object. -} +hashFile :: HashObjectHandle -> FilePath -> IO Sha +hashFile h file = CoProcess.query h send receive + where + send to = do + fileEncoding to + hPutStrLn to file + receive from = getSha "hash-object" $ hGetLine from + +{- Injects some content into git, returning its Sha. -} +hashObject :: ObjectType -> String -> Repo -> IO Sha +hashObject objtype content repo = getSha subcmd $ do + s <- pipeWriteRead (map Param params) content repo + return s + where + subcmd = "hash-object" + params = [subcmd, "-t", show objtype, "-w", "--stdin"] diff --git a/Git/Index.hs b/Git/Index.hs new file mode 100644 index 0000000000..80196ef787 --- /dev/null +++ b/Git/Index.hs @@ -0,0 +1,27 @@ +{- git index file stuff + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Index where + +import System.Posix.Env (setEnv, unsetEnv, getEnv) + +{- Forces git to use the specified index file. + - + - Returns an action that will reset back to the default + - index file. + - + - Warning: Not thread safe. + -} +override :: FilePath -> IO (IO ()) +override index = do + res <- getEnv var + setEnv var index True + return $ reset res + where + var = "GIT_INDEX_FILE" + reset (Just v) = setEnv var v True + reset _ = unsetEnv var diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs new file mode 100644 index 0000000000..ca08fe2618 --- /dev/null +++ b/Git/LsFiles.hs @@ -0,0 +1,184 @@ +{- git ls-files interface + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.LsFiles ( + inRepo, + notInRepo, + deleted, + modified, + staged, + stagedNotDeleted, + stagedDetails, + typeChanged, + typeChangedStaged, + Conflicting(..), + Unmerged(..), + unmerged, +) where + +import Common +import Git +import Git.Command +import Git.Types +import Git.Sha + +{- Scans for files that are checked into git at the specified locations. -} +inRepo :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +inRepo l = pipeNullSplit $ Params "ls-files --cached -z --" : map File l + +{- Scans for files at the specified locations that are not checked into git. -} +notInRepo :: Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +notInRepo include_ignored l repo = pipeNullSplit params repo + where + params = [Params "ls-files --others"] ++ exclude ++ + [Params "-z --"] ++ map File l + exclude + | include_ignored = [] + | otherwise = [Param "--exclude-standard"] + +{- Returns a list of files in the specified locations that have been + - deleted. -} +deleted :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +deleted l repo = pipeNullSplit params repo + where + params = [Params "ls-files --deleted -z --"] ++ map File l + +{- Returns a list of files in the specified locations that have been + - modified. -} +modified :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +modified l repo = pipeNullSplit params repo + where + params = [Params "ls-files --modified -z --"] ++ map File l + +{- Returns a list of all files that are staged for commit. -} +staged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +staged = staged' [] + +{- Returns a list of the files, staged for commit, that are being added, + - moved, or changed (but not deleted), from the specified locations. -} +stagedNotDeleted :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +stagedNotDeleted = staged' [Param "--diff-filter=ACMRT"] + +staged' :: [CommandParam] -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +staged' ps l = pipeNullSplit $ prefix ++ ps ++ suffix + where + prefix = [Params "diff --cached --name-only -z"] + suffix = Param "--" : map File l + +{- Returns details about files that are staged in the index + - (including the Sha of their staged contents), + - as well as files not yet in git. -} +stagedDetails :: [FilePath] -> Repo -> IO ([(FilePath, Maybe Sha)], IO Bool) +stagedDetails l repo = do + (ls, cleanup) <- pipeNullSplit params repo + return (map parse ls, cleanup) + where + params = [Params "ls-files --others --exclude-standard --stage -z --"] ++ + map File l + parse s + | null file = (s, Nothing) + | otherwise = (file, extractSha $ take shaSize $ drop 7 metadata) + where + (metadata, file) = separate (== '\t') s + +{- Returns a list of the files in the specified locations that are staged + - for commit, and whose type has changed. -} +typeChangedStaged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChangedStaged = typeChanged' [Param "--cached"] + +{- Returns a list of the files in the specified locations whose type has + - changed. Files only staged for commit will not be included. -} +typeChanged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChanged = typeChanged' [] + +typeChanged' :: [CommandParam] -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChanged' ps l repo = do + (fs, cleanup) <- pipeNullSplit (prefix ++ ps ++ suffix) repo + -- git diff returns filenames relative to the top of the git repo; + -- convert to filenames relative to the cwd, like git ls-files. + let top = repoPath repo + cwd <- getCurrentDirectory + return (map (\f -> relPathDirToFile cwd $ top f) fs, cleanup) + where + prefix = [Params "diff --name-only --diff-filter=T -z"] + suffix = Param "--" : map File l + +{- A item in conflict has two possible values. + - Either can be Nothing, when that side deleted the file. -} +data Conflicting v = Conflicting + { valUs :: Maybe v + , valThem :: Maybe v + } deriving (Show) + +data Unmerged = Unmerged + { unmergedFile :: FilePath + , unmergedBlobType :: Conflicting BlobType + , unmergedSha :: Conflicting Sha + } deriving (Show) + +{- Returns a list of the files in the specified locations that have + - unresolved merge conflicts. + - + - ls-files outputs multiple lines per conflicting file, each with its own + - stage number: + - 1 = old version, can be ignored + - 2 = us + - 3 = them + - If a line is omitted, that side removed the file. + -} +unmerged :: [FilePath] -> Repo -> IO ([Unmerged], IO Bool) +unmerged l repo = do + (fs, cleanup) <- pipeNullSplit params repo + return (reduceUnmerged [] $ catMaybes $ map parseUnmerged fs, cleanup) + where + params = Params "ls-files --unmerged -z --" : map File l + +data InternalUnmerged = InternalUnmerged + { isus :: Bool + , ifile :: FilePath + , iblobtype :: Maybe BlobType + , isha :: Maybe Sha + } deriving (Show) + +parseUnmerged :: String -> Maybe InternalUnmerged +parseUnmerged s + | null file = Nothing + | otherwise = case words metadata of + (rawblobtype:rawsha:rawstage:_) -> do + stage <- readish rawstage :: Maybe Int + unless (stage == 2 || stage == 3) $ + fail undefined -- skip stage 1 + blobtype <- readBlobType rawblobtype + sha <- extractSha rawsha + return $ InternalUnmerged (stage == 2) file + (Just blobtype) (Just sha) + _ -> Nothing + where + (metadata, file) = separate (== '\t') s + +reduceUnmerged :: [Unmerged] -> [InternalUnmerged] -> [Unmerged] +reduceUnmerged c [] = c +reduceUnmerged c (i:is) = reduceUnmerged (new:c) rest + where + (rest, sibi) = findsib i is + (blobtypeA, blobtypeB, shaA, shaB) + | isus i = (iblobtype i, iblobtype sibi, isha i, isha sibi) + | otherwise = (iblobtype sibi, iblobtype i, isha sibi, isha i) + new = Unmerged + { unmergedFile = ifile i + , unmergedBlobType = Conflicting blobtypeA blobtypeB + , unmergedSha = Conflicting shaA shaB + } + findsib templatei [] = ([], removed templatei) + findsib templatei (l:ls) + | ifile l == ifile templatei = (ls, l) + | otherwise = (l:ls, removed templatei) + removed templatei = templatei + { isus = not (isus templatei) + , iblobtype = Nothing + , isha = Nothing + } diff --git a/Git/LsTree.hs b/Git/LsTree.hs new file mode 100644 index 0000000000..c61ae7faba --- /dev/null +++ b/Git/LsTree.hs @@ -0,0 +1,58 @@ +{- git ls-tree interface + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.LsTree ( + TreeItem(..), + lsTree, + lsTreeFiles, + parseLsTree +) where + +import Numeric +import Control.Applicative +import System.Posix.Types + +import Common +import Git +import Git.Command +import Git.Sha +import qualified Git.Filename + +data TreeItem = TreeItem + { mode :: FileMode + , typeobj :: String + , sha :: String + , file :: FilePath + } deriving Show + +{- Lists the complete contents of a tree. -} +lsTree :: Ref -> Repo -> IO [TreeItem] +lsTree t repo = map parseLsTree <$> + pipeNullSplitZombie [Params "ls-tree --full-tree -z -r --", File $ show t] repo + +{- Lists specified files in a tree. -} +lsTreeFiles :: Ref -> [FilePath] -> Repo -> IO [TreeItem] +lsTreeFiles t fs repo = map parseLsTree <$> + pipeNullSplitZombie ([Params "ls-tree -z --", File $ show t] ++ map File fs) repo + +{- Parses a line of ls-tree output. + - (The --long format is not currently supported.) -} +parseLsTree :: String -> TreeItem +parseLsTree l = TreeItem + { mode = fst $ Prelude.head $ readOct m + , typeobj = t + , sha = s + , file = Git.Filename.decode f + } + where + -- l = SP SP TAB + -- All fields are fixed, so we can pull them out of + -- specific positions in the line. + (m, past_m) = splitAt 7 l + (t, past_t) = splitAt 4 past_m + (s, past_s) = splitAt shaSize $ Prelude.tail past_t + f = Prelude.tail past_s diff --git a/Git/Merge.hs b/Git/Merge.hs new file mode 100644 index 0000000000..e70a71d64a --- /dev/null +++ b/Git/Merge.hs @@ -0,0 +1,21 @@ +{- git merging + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Merge where + +import Common +import Git +import Git.Command +import Git.Version + +{- Avoids recent git's interactive merge. -} +mergeNonInteractive :: Ref -> Repo -> IO Bool +mergeNonInteractive branch + | older "1.7.7.6" = merge [Param $ show branch] + | otherwise = merge [Param "--no-edit", Param $ show branch] + where + merge ps = runBool $ Param "merge" : ps diff --git a/Git/Queue.hs b/Git/Queue.hs new file mode 100644 index 0000000000..712d476cd6 --- /dev/null +++ b/Git/Queue.hs @@ -0,0 +1,160 @@ +{- git repository command queue + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Git.Queue ( + Queue, + new, + addCommand, + addUpdateIndex, + size, + full, + flush, +) where + +import qualified Data.Map as M +import System.IO +import System.Process +import Data.String.Utils + +import Utility.SafeCommand +import Common +import Git +import Git.Command +import qualified Git.UpdateIndex + +{- Queable actions that can be performed in a git repository. + -} +data Action + {- Updating the index file, using a list of streamers that can + - be added to as the queue grows. -} + = UpdateIndexAction + { getStreamers :: [Git.UpdateIndex.Streamer] -- in reverse order + } + {- A git command to run, on a list of files that can be added to + - as the queue grows. -} + | CommandAction + { getSubcommand :: String + , getParams :: [CommandParam] + , getFiles :: [FilePath] + } + +{- A key that can uniquely represent an action in a Map. -} +data ActionKey = UpdateIndexActionKey | CommandActionKey String + deriving (Eq, Ord) + +actionKey :: Action -> ActionKey +actionKey (UpdateIndexAction _) = UpdateIndexActionKey +actionKey CommandAction { getSubcommand = s } = CommandActionKey s + +{- A queue of actions to perform (in any order) on a git repository, + - with lists of files to perform them on. This allows coalescing + - similar git commands. -} +data Queue = Queue + { size :: Int + , _limit :: Int + , items :: M.Map ActionKey Action + } + +{- A recommended maximum size for the queue, after which it should be + - run. + - + - 10240 is semi-arbitrary. If we assume git filenames are between 10 and + - 255 characters long, then the queue will build up between 100kb and + - 2550kb long commands. The max command line length on linux is somewhere + - above 20k, so this is a fairly good balance -- the queue will buffer + - only a few megabytes of stuff and a minimal number of commands will be + - run by xargs. -} +defaultLimit :: Int +defaultLimit = 10240 + +{- Constructor for empty queue. -} +new :: Maybe Int -> Queue +new lim = Queue 0 (fromMaybe defaultLimit lim) M.empty + +{- Adds an git command to the queue. + - + - Git commands with the same subcommand but different parameters are + - assumed to be equivilant enough to perform in any order with the same + - result. + -} +addCommand :: String -> [CommandParam] -> [FilePath] -> Queue -> Repo -> IO Queue +addCommand subcommand params files q repo = + updateQueue action different (length newfiles) q repo + where + key = actionKey action + action = CommandAction + { getSubcommand = subcommand + , getParams = params + , getFiles = newfiles + } + newfiles = files ++ maybe [] getFiles (M.lookup key $ items q) + + different (CommandAction { getSubcommand = s }) = s /= subcommand + different _ = True + +{- Adds an update-index streamer to the queue. -} +addUpdateIndex :: Git.UpdateIndex.Streamer -> Queue -> Repo -> IO Queue +addUpdateIndex streamer q repo = + updateQueue action different 1 q repo + where + key = actionKey action + -- the list is built in reverse order + action = UpdateIndexAction $ streamer : streamers + streamers = maybe [] getStreamers $ M.lookup key $ items q + + different (UpdateIndexAction _) = False + different _ = True + +{- Updates or adds an action in the queue. If the queue already contains a + - different action, it will be flushed; this is to ensure that conflicting + - actions, like add and rm, are run in the right order.-} +updateQueue :: Action -> (Action -> Bool) -> Int -> Queue -> Repo -> IO Queue +updateQueue !action different sizeincrease q repo + | null (filter different (M.elems (items q))) = return $ go q + | otherwise = go <$> flush q repo + where + go q' = newq + where + !newq = q' + { size = newsize + , items = newitems + } + !newsize = size q' + sizeincrease + !newitems = M.insertWith' const (actionKey action) action (items q') + +{- Is a queue large enough that it should be flushed? -} +full :: Queue -> Bool +full (Queue cur lim _) = cur > lim + +{- Runs a queue on a git repository. -} +flush :: Queue -> Repo -> IO Queue +flush (Queue _ lim m) repo = do + forM_ (M.elems m) $ runAction repo + return $ Queue 0 lim M.empty + +{- Runs an Action on a list of files in a git repository. + - + - Complicated by commandline length limits. + - + - Intentionally runs the command even if the list of files is empty; + - this allows queueing commands that do not need a list of files. -} +runAction :: Repo -> Action -> IO () +runAction repo (UpdateIndexAction streamers) = + -- list is stored in reverse order + Git.UpdateIndex.streamUpdateIndex repo $ reverse streamers +runAction repo action@(CommandAction {}) = + withHandle StdinHandle createProcessSuccess p $ \h -> do + fileEncoding h + hPutStr h $ join "\0" $ getFiles action + hClose h + where + p = (proc "xargs" params) { env = gitEnv repo } + params = "-0":"git":baseparams + baseparams = toCommand $ gitCommandLine + (Param (getSubcommand action):getParams action) repo diff --git a/Git/Ref.hs b/Git/Ref.hs new file mode 100644 index 0000000000..26a1fc8bf1 --- /dev/null +++ b/Git/Ref.hs @@ -0,0 +1,97 @@ +{- git ref stuff + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Ref where + +import Common +import Git +import Git.Command + +import Data.Char (chr) + +{- Converts a fully qualified git ref into a user-visible string. -} +describe :: Ref -> String +describe = show . base + +{- Often git refs are fully qualified (eg: refs/heads/master). + - Converts such a fully qualified ref into a base ref (eg: master). -} +base :: Ref -> Ref +base = Ref . remove "refs/heads/" . remove "refs/remotes/" . show + where + remove prefix s + | prefix `isPrefixOf` s = drop (length prefix) s + | otherwise = s + +{- Given a directory such as "refs/remotes/origin", and a ref such as + - refs/heads/master, yields a version of that ref under the directory, + - such as refs/remotes/origin/master. -} +under :: String -> Ref -> Ref +under dir r = Ref $ dir show (base r) + +{- Checks if a ref exists. -} +exists :: Ref -> Repo -> IO Bool +exists ref = runBool + [Param "show-ref", Param "show-ref", Param "--verify", Param "-q", Param $ show ref] + +{- Checks if HEAD exists. It generally will, except for in a repository + - that was just created. -} +headExists :: Repo -> IO Bool +headExists repo = do + ls <- lines <$> pipeReadStrict [Param "show-ref", Param "--head"] repo + return $ any (" HEAD" `isSuffixOf`) ls + +{- Get the sha of a fully qualified git ref, if it exists. -} +sha :: Branch -> Repo -> IO (Maybe Sha) +sha branch repo = process <$> showref repo + where + showref = pipeReadStrict [Param "show-ref", + Param "--hash", -- get the hash + Param $ show branch] + process [] = Nothing + process s = Just $ Ref $ firstLine s + +{- List of (refs, branches) matching a given ref spec. -} +matching :: Ref -> Repo -> IO [(Ref, Branch)] +matching ref repo = map gen . lines <$> + pipeReadStrict [Param "show-ref", Param $ show ref] repo + where + gen l = let (r, b) = separate (== ' ') l + in (Ref r, Ref b) + +{- List of (refs, branches) matching a given ref spec. + - Duplicate refs are filtered out. -} +matchingUniq :: Ref -> Repo -> IO [(Ref, Branch)] +matchingUniq ref repo = nubBy uniqref <$> matching ref repo + where + uniqref (a, _) (b, _) = a == b + +{- Checks if a String is a legal git ref name. + - + - The rules for this are complex; see git-check-ref-format(1) -} +legal :: Bool -> String -> Bool +legal allowonelevel s = all (== False) illegal + where + illegal = + [ any ("." `isPrefixOf`) pathbits + , any (".lock" `isSuffixOf`) pathbits + , not allowonelevel && length pathbits < 2 + , contains ".." + , any (\c -> contains [c]) illegalchars + , begins "/" + , ends "/" + , contains "//" + , ends "." + , contains "@{" + , null s + ] + contains v = v `isInfixOf` s + ends v = v `isSuffixOf` s + begins v = v `isPrefixOf` s + + pathbits = split "/" s + illegalchars = " ~^:?*[\\" ++ controlchars + controlchars = chr 0o177 : [chr 0 .. chr (0o40-1)] diff --git a/Git/Remote.hs b/Git/Remote.hs new file mode 100644 index 0000000000..5640e9ff27 --- /dev/null +++ b/Git/Remote.hs @@ -0,0 +1,33 @@ +{- git remote stuff + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Remote where + +import Common +import Data.Char + +{- Construct a legal git remote name out of an arbitrary input string. + - + - There seems to be no formal definition of this in the git source, + - just some ad-hoc checks, and some other things that fail with certian + - types of names (like ones starting with '-'). + -} +makeLegalName :: String -> String +makeLegalName s = case filter legal $ replace "/" "_" s of + -- it can't be empty + [] -> "unnamed" + -- it can't start with / or - or . + '.':s' -> makeLegalName s' + '/':s' -> makeLegalName s' + '-':s' -> makeLegalName s' + s' -> s' + where + {- Only alphanumerics, and a few common bits of punctuation common + - in hostnames. -} + legal '_' = True + legal '.' = True + legal c = isAlphaNum c diff --git a/Git/Sha.hs b/Git/Sha.hs new file mode 100644 index 0000000000..ee1b6d6691 --- /dev/null +++ b/Git/Sha.hs @@ -0,0 +1,39 @@ +{- git SHA stuff + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Sha where + +import Common +import Git.Types + +{- Runs an action that causes a git subcommand to emit a Sha, and strips + - any trailing newline, returning the sha. -} +getSha :: String -> IO String -> IO Sha +getSha subcommand a = maybe bad return =<< extractSha <$> a + where + bad = error $ "failed to read sha from git " ++ subcommand + +{- Extracts the Sha from a string. There can be a trailing newline after + - it, but nothing else. -} +extractSha :: String -> Maybe Sha +extractSha s + | len == shaSize = val s + | len == shaSize + 1 && length s' == shaSize = val s' + | otherwise = Nothing + where + len = length s + s' = firstLine s + val v + | all (`elem` "1234567890ABCDEFabcdef") v = Just $ Ref v + | otherwise = Nothing + +{- Size of a git sha. -} +shaSize :: Int +shaSize = 40 + +nullSha :: Ref +nullSha = Ref $ replicate shaSize '0' diff --git a/Git/SharedRepository.hs b/Git/SharedRepository.hs new file mode 100644 index 0000000000..f3efa8fde9 --- /dev/null +++ b/Git/SharedRepository.hs @@ -0,0 +1,27 @@ +{- git core.sharedRepository handling + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.SharedRepository where + +import Data.Char + +import Common +import Git +import qualified Git.Config + +data SharedRepository = UnShared | GroupShared | AllShared | UmaskShared Int + +getSharedRepository :: Repo -> SharedRepository +getSharedRepository r = + case map toLower $ Git.Config.get "core.sharedrepository" "" r of + "1" -> GroupShared + "group" -> GroupShared + "true" -> GroupShared + "all" -> AllShared + "world" -> AllShared + "everybody" -> AllShared + v -> maybe UnShared UmaskShared (readish v) diff --git a/Git/Types.hs b/Git/Types.hs new file mode 100644 index 0000000000..57e5ca6e2d --- /dev/null +++ b/Git/Types.hs @@ -0,0 +1,83 @@ +{- git data types + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Types where + +import Network.URI +import qualified Data.Map as M + +{- Support repositories on local disk, and repositories accessed via an URL. + - + - Repos on local disk have a git directory, and unless bare, a worktree. + - + - A local repo may not have had its config read yet, in which case all + - that's known about it is its path. + - + - Finally, an Unknown repository may be known to exist, but nothing + - else known about it. + -} +data RepoLocation + = Local { gitdir :: FilePath, worktree :: Maybe FilePath } + | LocalUnknown FilePath + | Url URI + | Unknown + deriving (Show, Eq) + +data Repo = Repo + { location :: RepoLocation + , config :: M.Map String String + -- a given git config key can actually have multiple values + , fullconfig :: M.Map String [String] + , remotes :: [Repo] + -- remoteName holds the name used for this repo in remotes + , remoteName :: Maybe String + -- alternate environment to use when running git commands + , gitEnv :: Maybe [(String, String)] + } deriving (Show, Eq) + +{- A git ref. Can be a sha1, or a branch or tag name. -} +newtype Ref = Ref String + deriving (Eq) + +instance Show Ref where + show (Ref v) = v + +{- Aliases for Ref. -} +type Branch = Ref +type Sha = Ref +type Tag = Ref + +{- Types of objects that can be stored in git. -} +data ObjectType = BlobObject | CommitObject | TreeObject + deriving (Eq) + +instance Show ObjectType where + show BlobObject = "blob" + show CommitObject = "commit" + show TreeObject = "tree" + +readObjectType :: String -> Maybe ObjectType +readObjectType "blob" = Just BlobObject +readObjectType "commit" = Just CommitObject +readObjectType "tree" = Just TreeObject +readObjectType _ = Nothing + +{- Types of blobs. -} +data BlobType = FileBlob | ExecutableBlob | SymlinkBlob + deriving (Eq) + +{- Git uses magic numbers to denote the type of a blob. -} +instance Show BlobType where + show FileBlob = "100644" + show ExecutableBlob = "100755" + show SymlinkBlob = "120000" + +readBlobType :: String -> Maybe BlobType +readBlobType "100644" = Just FileBlob +readBlobType "100755" = Just ExecutableBlob +readBlobType "120000" = Just SymlinkBlob +readBlobType _ = Nothing diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs new file mode 100644 index 0000000000..464200af4f --- /dev/null +++ b/Git/UnionMerge.hs @@ -0,0 +1,110 @@ +{- git-union-merge library + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.UnionMerge ( + merge, + mergeIndex +) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Set as S + +import Common +import Git +import Git.Sha +import Git.CatFile +import Git.Command +import Git.UpdateIndex +import Git.HashObject +import Git.Types +import Git.FilePath + +{- Performs a union merge between two branches, staging it in the index. + - Any previously staged changes in the index will be lost. + - + - Should be run with a temporary index file configured by useIndex. + -} +merge :: Ref -> Ref -> Repo -> IO () +merge x y repo = do + h <- catFileStart repo + streamUpdateIndex repo + [ lsTree x repo + , mergeTrees x y h repo + ] + catFileStop h + +{- Merges a list of branches into the index. Previously staged changes in + - the index are preserved (and participate in the merge). + - + - update-index is run once per ref in turn, so that each ref is merged on + - top of the merge for the previous ref. It would be more efficient, but + - harder to calculate a single union merge involving all the refs, as well + - as the index. + -} +mergeIndex :: CatFileHandle -> Repo -> [Ref] -> IO () +mergeIndex h repo bs = forM_ bs $ \b -> + streamUpdateIndex repo [mergeTreeIndex b h repo] + +{- For merging two trees. -} +mergeTrees :: Ref -> Ref -> CatFileHandle -> Repo -> Streamer +mergeTrees (Ref x) (Ref y) h = doMerge h $ "diff-tree":diffOpts ++ [x, y] + +{- For merging a single tree into the index. -} +mergeTreeIndex :: Ref -> CatFileHandle -> Repo -> Streamer +mergeTreeIndex (Ref x) h = doMerge h $ + "diff-index" : diffOpts ++ ["--cached", x] + +diffOpts :: [String] +diffOpts = ["--raw", "-z", "-r", "--no-renames", "-l0"] + +{- Streams update-index changes to perform a merge, + - using git to get a raw diff. -} +doMerge :: CatFileHandle -> [String] -> Repo -> Streamer +doMerge ch differ repo streamer = do + (diff, cleanup) <- pipeNullSplit (map Param differ) repo + go diff + void $ cleanup + where + go [] = noop + go (info:file:rest) = mergeFile info file ch repo >>= + maybe (go rest) (\l -> streamer l >> go rest) + go (_:[]) = error $ "parse error " ++ show differ + +{- Given an info line from a git raw diff, and the filename, generates + - a line suitable for update-index that union merges the two sides of the + - diff. -} +mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String) +mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of + [] -> return Nothing + (sha:[]) -> use sha + shas -> use + =<< either return (\s -> hashObject BlobObject (unlines s) repo) + =<< calcMerge . zip shas <$> mapM getcontents shas + where + [_colonmode, _bmode, asha, bsha, _status] = words info + use sha = return $ Just $ + updateIndexLine sha FileBlob $ asTopFilePath file + -- We don't know how the file is encoded, but need to + -- split it into lines to union merge. Using the + -- FileSystemEncoding for this is a hack, but ensures there + -- are no decoding errors. Note that this works because + -- hashObject sets fileEncoding on its write handle. + getcontents s = lines . encodeW8 . L.unpack <$> catObject h s + +{- Calculates a union merge between a list of refs, with contents. + - + - When possible, reuses the content of an existing ref, rather than + - generating new content. + -} +calcMerge :: [(Ref, [String])] -> Either Ref [String] +calcMerge shacontents + | null reuseable = Right $ new + | otherwise = Left $ fst $ Prelude.head reuseable + where + reuseable = filter (\c -> sorteduniq (snd c) == new) shacontents + new = sorteduniq $ concat $ map snd shacontents + sorteduniq = S.toList . S.fromList diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs new file mode 100644 index 0000000000..aa65b4429d --- /dev/null +++ b/Git/UpdateIndex.hs @@ -0,0 +1,77 @@ +{- git-update-index library + - + - Copyright 2011, 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns #-} + +module Git.UpdateIndex ( + Streamer, + pureStreamer, + streamUpdateIndex, + lsTree, + updateIndexLine, + unstageFile, + stageSymlink +) where + +import Common +import Git +import Git.Types +import Git.Command +import Git.FilePath +import Git.Sha + +{- Streamers are passed a callback and should feed it lines in the form + - read by update-index, and generated by ls-tree. -} +type Streamer = (String -> IO ()) -> IO () + +{- A streamer with a precalculated value. -} +pureStreamer :: String -> Streamer +pureStreamer !s = \streamer -> streamer s + +{- Streams content into update-index from a list of Streamers. -} +streamUpdateIndex :: Repo -> [Streamer] -> IO () +streamUpdateIndex repo as = pipeWrite params repo $ \h -> do + fileEncoding h + forM_ as (stream h) + hClose h + where + params = map Param ["update-index", "-z", "--index-info"] + stream h a = a (streamer h) + streamer h s = do + hPutStr h s + hPutStr h "\0" + +{- A streamer that adds the current tree for a ref. Useful for eg, copying + - and modifying branches. -} +lsTree :: Ref -> Repo -> Streamer +lsTree (Ref x) repo streamer = do + (s, cleanup) <- pipeNullSplit params repo + mapM_ streamer s + void $ cleanup + where + params = map Param ["ls-tree", "-z", "-r", "--full-tree", x] + +{- Generates a line suitable to be fed into update-index, to add + - a given file with a given sha. -} +updateIndexLine :: Sha -> BlobType -> TopFilePath -> String +updateIndexLine sha filetype file = + show filetype ++ " blob " ++ show sha ++ "\t" ++ getTopFilePath file + +{- A streamer that removes a file from the index. -} +unstageFile :: FilePath -> Repo -> IO Streamer +unstageFile file repo = do + p <- toTopFilePath file repo + return $ pureStreamer $ "0 " ++ show nullSha ++ "\t" ++ getTopFilePath p + +{- A streamer that adds a symlink to the index. -} +stageSymlink :: FilePath -> Sha -> Repo -> IO Streamer +stageSymlink file sha repo = do + !line <- updateIndexLine + <$> pure sha + <*> pure SymlinkBlob + <*> toTopFilePath file repo + return $ pureStreamer line diff --git a/Git/Url.hs b/Git/Url.hs new file mode 100644 index 0000000000..7befc46690 --- /dev/null +++ b/Git/Url.hs @@ -0,0 +1,70 @@ +{- git repository urls + - + - Copyright 2010, 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Url ( + scheme, + host, + port, + hostuser, + authority, +) where + +import Network.URI hiding (scheme, authority) + +import Common +import Git.Types +import Git + +{- Scheme of an URL repo. -} +scheme :: Repo -> String +scheme Repo { location = Url u } = uriScheme u +scheme repo = notUrl repo + +{- Work around a bug in the real uriRegName + - -} +uriRegName' :: URIAuth -> String +uriRegName' a = fixup $ uriRegName a + where + fixup x@('[':rest) + | rest !! len == ']' = take len rest + | otherwise = x + where + len = length rest - 1 + fixup x = x + +{- Hostname of an URL repo. -} +host :: Repo -> String +host = authpart uriRegName' + +{- Port of an URL repo, if it has a nonstandard one. -} +port :: Repo -> Maybe Integer +port r = + case authpart uriPort r of + ":" -> Nothing + (':':p) -> readish p + _ -> Nothing + +{- Hostname of an URL repo, including any username (ie, "user@host") -} +hostuser :: Repo -> String +hostuser r = authpart uriUserInfo r ++ authpart uriRegName' r + +{- The full authority portion an URL repo. (ie, "user@host:port") -} +authority :: Repo -> String +authority = authpart assemble + where + assemble a = uriUserInfo a ++ uriRegName' a ++ uriPort a + +{- Applies a function to extract part of the uriAuthority of an URL repo. -} +authpart :: (URIAuth -> a) -> Repo -> a +authpart a Repo { location = Url u } = a auth + where + auth = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) +authpart _ repo = notUrl repo + +notUrl :: Repo -> a +notUrl repo = error $ + "acting on local git repo " ++ repoDescribe repo ++ " not supported" diff --git a/Git/Version.hs b/Git/Version.hs new file mode 100644 index 0000000000..44385d9b83 --- /dev/null +++ b/Git/Version.hs @@ -0,0 +1,38 @@ +{- git version checking + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Version where + +import Common +import qualified Build.SysConfig + +{- Using the version it was configured for avoids running git to check its + - version, at the cost that upgrading git won't be noticed. + - This is only acceptable because it's rare that git's version influences + - code's behavior. -} +version :: String +version = Build.SysConfig.gitversion + +older :: String -> Bool +older v = normalize version < normalize v + +{- To compare dotted versions like 1.7.7 and 1.8, they are normalized to + - a somewhat arbitrary integer representation. -} +normalize :: String -> Integer +normalize = sum . mult 1 . reverse . + extend precision . take precision . + map readi . split "." + where + extend n l = l ++ replicate (n - length l) 0 + mult _ [] = [] + mult n (x:xs) = (n*x) : mult (n*10^width) xs + readi :: String -> Integer + readi s = case reads s of + ((x,_):_) -> x + _ -> 0 + precision = 10 -- number of segments of the version to compare + width = length "yyyymmddhhmmss" -- maximum width of a segment diff --git a/GitAnnex.hs b/GitAnnex.hs new file mode 100644 index 0000000000..0b4a7f8d9e --- /dev/null +++ b/GitAnnex.hs @@ -0,0 +1,188 @@ +{- git-annex main program + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module GitAnnex where + +import System.Console.GetOpt + +import Common.Annex +import qualified Git.Config +import qualified Git.CurrentRepo +import CmdLine +import Command +import Types.TrustLevel +import qualified Annex +import qualified Remote +import qualified Limit +import qualified Option + +import qualified Command.Add +import qualified Command.Unannex +import qualified Command.Drop +import qualified Command.Move +import qualified Command.Copy +import qualified Command.Get +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.TransferKey +import qualified Command.ReKey +import qualified Command.Reinject +import qualified Command.Fix +import qualified Command.Init +import qualified Command.Describe +import qualified Command.InitRemote +import qualified Command.Fsck +import qualified Command.Unused +import qualified Command.DropUnused +import qualified Command.AddUnused +import qualified Command.Unlock +import qualified Command.Lock +import qualified Command.PreCommit +import qualified Command.Find +import qualified Command.Whereis +import qualified Command.Log +import qualified Command.Merge +import qualified Command.Status +import qualified Command.Migrate +import qualified Command.Uninit +import qualified Command.Trust +import qualified Command.Untrust +import qualified Command.Semitrust +import qualified Command.Dead +import qualified Command.Group +import qualified Command.Ungroup +import qualified Command.Vicfg +import qualified Command.Sync +import qualified Command.AddUrl +import qualified Command.Import +import qualified Command.Map +import qualified Command.Direct +import qualified Command.Indirect +import qualified Command.Upgrade +import qualified Command.Version +import qualified Command.Help +#ifdef WITH_ASSISTANT +import qualified Command.Watch +import qualified Command.Assistant +#ifdef WITH_WEBAPP +import qualified Command.WebApp +#endif +#ifdef WITH_XMPP +import qualified Command.XMPPGit +#endif +#endif +#ifdef WITH_TESTSUITE +import qualified Command.Test +#endif + +cmds :: [Command] +cmds = concat + [ Command.Add.def + , Command.Get.def + , Command.Drop.def + , Command.Move.def + , Command.Copy.def + , Command.Unlock.def + , Command.Lock.def + , Command.Sync.def + , Command.AddUrl.def + , Command.Import.def + , Command.Init.def + , Command.Describe.def + , Command.InitRemote.def + , Command.Reinject.def + , Command.Unannex.def + , Command.Uninit.def + , Command.PreCommit.def + , Command.Trust.def + , Command.Untrust.def + , Command.Semitrust.def + , Command.Dead.def + , Command.Group.def + , Command.Ungroup.def + , Command.Vicfg.def + , Command.FromKey.def + , Command.DropKey.def + , Command.TransferKey.def + , Command.ReKey.def + , Command.Fix.def + , Command.Fsck.def + , Command.Unused.def + , Command.DropUnused.def + , Command.AddUnused.def + , Command.Find.def + , Command.Whereis.def + , Command.Log.def + , Command.Merge.def + , Command.Status.def + , Command.Migrate.def + , Command.Map.def + , Command.Direct.def + , Command.Indirect.def + , Command.Upgrade.def + , Command.Version.def + , Command.Help.def +#ifdef WITH_ASSISTANT + , Command.Watch.def + , Command.Assistant.def +#ifdef WITH_WEBAPP + , Command.WebApp.def +#endif +#ifdef WITH_XMPP + , Command.XMPPGit.def +#endif +#endif +#ifdef WITH_TESTSUITE + , Command.Test.def +#endif + ] + +options :: [Option] +options = Option.common ++ + [ Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber) + "override default number of copies" + , Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote) + "override trust setting" + , Option [] ["semitrust"] (ReqArg (Remote.forceTrust SemiTrusted) paramRemote) + "override trust setting back to default" + , Option [] ["untrust"] (ReqArg (Remote.forceTrust UnTrusted) paramRemote) + "override trust setting to untrusted" + , Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE") + "override git configuration setting" + , Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob) + "skip files matching the glob pattern" + , Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob) + "don't skip files matching the glob pattern" + , Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote) + "skip files not present in a remote" + , Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber) + "skip files with fewer copies" + , Option ['B'] ["inbackend"] (ReqArg Limit.addInBackend paramName) + "skip files not using a key-value backend" + , Option [] ["inallgroup"] (ReqArg Limit.addInAllGroup paramGroup) + "skip files not present in all remotes in a group" + , Option [] ["largerthan"] (ReqArg Limit.addLargerThan paramSize) + "skip files larger than a size" + , Option [] ["smallerthan"] (ReqArg Limit.addSmallerThan paramSize) + "skip files smaller than a size" + , Option ['T'] ["time-limit"] (ReqArg Limit.addTimeLimit paramTime) + "stop after the specified amount of time" + , Option [] ["trust-glacier"] (NoArg (Annex.setFlag "trustglacier")) "Trust Amazon Glacier inventory" + ] ++ Option.matcher + where + setnumcopies v = maybe noop + (\n -> Annex.changeGitConfig $ \c -> c { annexNumCopies = n }) + (readish v) + setgitconfig v = Annex.changeGitRepo =<< inRepo (Git.Config.store v) + +header :: String +header = "Usage: git-annex command [option ..]" + +run :: [String] -> IO () +run args = dispatch True args cmds options [] header Git.CurrentRepo.get diff --git a/GitAnnexShell.hs b/GitAnnexShell.hs new file mode 100644 index 0000000000..fca36cfc5d --- /dev/null +++ b/GitAnnexShell.hs @@ -0,0 +1,175 @@ +{- git-annex-shell main program + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module GitAnnexShell where + +import System.Posix.Env +import System.Console.GetOpt + +import Common.Annex +import qualified Git.Construct +import CmdLine +import Command +import Annex.UUID +import qualified Option +import Fields +import Utility.UserInfo + +import qualified Command.ConfigList +import qualified Command.InAnnex +import qualified Command.DropKey +import qualified Command.RecvKey +import qualified Command.SendKey +import qualified Command.TransferInfo +import qualified Command.Commit + +cmds_readonly :: [Command] +cmds_readonly = concat + [ Command.ConfigList.def + , Command.InAnnex.def + , Command.SendKey.def + , Command.TransferInfo.def + ] + +cmds_notreadonly :: [Command] +cmds_notreadonly = concat + [ Command.RecvKey.def + , Command.DropKey.def + , Command.Commit.def + ] + +cmds :: [Command] +cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly + where + adddirparam c = c { cmdparamdesc = "DIRECTORY " ++ cmdparamdesc c } + +options :: [OptDescr (Annex ())] +options = Option.common ++ + [ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "local repository uuid" + ] + where + checkuuid expected = getUUID >>= check + where + check u | u == toUUID expected = noop + check NoUUID = unexpected "uninitialized repository" + check u = unexpected $ "UUID " ++ fromUUID u + unexpected s = error $ + "expected repository UUID " ++ + expected ++ " but found " ++ s + +header :: String +header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]" + +run :: [String] -> IO () +run [] = failure +-- skip leading -c options, passed by eg, ssh +run ("-c":p) = run p +-- a command can be either a builtin or something to pass to git-shell +run c@(cmd:dir:params) + | cmd `elem` builtins = builtin cmd dir params + | otherwise = external c +run c@(cmd:_) + -- Handle the case of being the user's login shell. It will be passed + -- a single string containing all the real parameters. + | "git-annex-shell " `isPrefixOf` cmd = run $ drop 1 $ shellUnEscape cmd + | cmd `elem` builtins = failure + | otherwise = external c + +builtins :: [String] +builtins = map cmdname cmds + +builtin :: String -> String -> [String] -> IO () +builtin cmd dir params = do + checkNotReadOnly cmd + checkDirectory $ Just dir + let (params', fieldparams) = partitionParams params + let fields = filter checkField $ parseFields fieldparams + dispatch False (cmd : params') cmds options fields header $ + Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath + +external :: [String] -> IO () +external params = do + {- Normal git-shell commands all have the directory as their last + - parameter. -} + let lastparam = lastMaybe =<< shellUnEscape <$> lastMaybe params + checkDirectory lastparam + checkNotLimited + unlessM (boolSystem "git-shell" $ map Param $ "-c":fst (partitionParams params)) $ + error "git-shell failed" + +{- Parameters between two -- markers are field settings, in the form: + - field=value field=value + - + - Parameters after the last -- are ignored, these tend to be passed by + - rsync and not be useful. + -} +partitionParams :: [String] -> ([String], [String]) +partitionParams ps = case segment (== "--") ps of + params:fieldparams:_ -> (params, fieldparams) + [params] -> (params, []) + _ -> ([], []) + +parseFields :: [String] -> [(String, String)] +parseFields = map (separate (== '=')) + +{- Only allow known fields to be set, ignore others. + - Make sure that field values make sense. -} +checkField :: (String, String) -> Bool +checkField (field, value) + | field == fieldName remoteUUID = fieldCheck remoteUUID value + | field == fieldName associatedFile = fieldCheck associatedFile value + | field == fieldName direct = fieldCheck direct value + | otherwise = False + +failure :: IO () +failure = error $ "bad parameters\n\n" ++ usage header cmds options + +checkNotLimited :: IO () +checkNotLimited = checkEnv "GIT_ANNEX_SHELL_LIMITED" + +checkNotReadOnly :: String -> IO () +checkNotReadOnly cmd + | cmd `elem` map cmdname cmds_readonly = noop + | otherwise = checkEnv "GIT_ANNEX_SHELL_READONLY" + +checkDirectory :: Maybe FilePath -> IO () +checkDirectory mdir = do + v <- getEnv "GIT_ANNEX_SHELL_DIRECTORY" + case (v, mdir) of + (Nothing, _) -> noop + (Just d, Nothing) -> req d Nothing + (Just d, Just dir) + | d `equalFilePath` dir -> noop + | otherwise -> do + home <- myHomeDir + d' <- canondir home d + dir' <- canondir home dir + if d' `equalFilePath` dir' + then noop + else req d' (Just dir') + where + req d mdir' = error $ unwords + [ "Only allowed to access" + , d + , maybe "and could not determine directory from command line" ("not " ++) mdir' + ] + + {- A directory may start with ~/ or in some cases, even /~/, + - or could just be relative to home, or of course could + - be absolute. -} + canondir home d + | "~/" `isPrefixOf` d = return d + | "/~/" `isPrefixOf` d = return $ drop 1 d + | otherwise = relHome $ absPathFrom home d + +checkEnv :: String -> IO () +checkEnv var = do + v <- getEnv var + case v of + Nothing -> noop + Just "" -> noop + Just _ -> error $ "Action blocked by " ++ var diff --git a/INSTALL b/INSTALL new file mode 120000 index 0000000000..67566818f0 --- /dev/null +++ b/INSTALL @@ -0,0 +1 @@ +doc/install.mdwn \ No newline at end of file diff --git a/Init.hs b/Init.hs new file mode 100644 index 0000000000..f925900126 --- /dev/null +++ b/Init.hs @@ -0,0 +1,145 @@ +{- git-annex repository initialization + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Init ( + ensureInitialized, + isInitialized, + initialize, + uninitialize, + probeCrippledFileSystem +) where + +import Common.Annex +import Utility.TempFile +import Utility.Network +import qualified Git +import qualified Git.LsFiles +import qualified Annex.Branch +import Logs.UUID +import Annex.Version +import Annex.UUID +import Utility.UserInfo +import Utility.Shell +import Utility.FileMode +import Config +import Annex.Direct +import Annex.Content.Direct +import Backend + +genDescription :: Maybe String -> Annex String +genDescription (Just d) = return d +genDescription Nothing = do + hostname <- maybe "" id <$> liftIO getHostname + let at = if null hostname then "" else "@" + username <- liftIO myUserName + reldir <- liftIO . relHome =<< fromRepo Git.repoPath + return $ concat [username, at, hostname, ":", reldir] + +initialize :: Maybe String -> Annex () +initialize mdescription = do + u <- ensureUUID + setVersion defaultVersion + checkCrippledFileSystem + Annex.Branch.create + gitPreCommitHookWrite + createInodeSentinalFile + describeUUID u =<< genDescription mdescription + +uninitialize :: Annex () +uninitialize = do + gitPreCommitHookUnWrite + removeRepoUUID + removeVersion + +{- Will automatically initialize if there is already a git-annex + - branch from somewhere. Otherwise, require a manual init + - to avoid git-annex accidentially being run in git + - repos that did not intend to use it. -} +ensureInitialized :: Annex () +ensureInitialized = getVersion >>= maybe needsinit checkVersion + where + needsinit = ifM Annex.Branch.hasSibling + ( initialize Nothing + , error "First run: git-annex init" + ) + +{- Checks if a repository is initialized. Does not check version for ugrade. -} +isInitialized :: Annex Bool +isInitialized = maybe Annex.Branch.hasSibling (const $ return True) =<< getVersion + +{- set up a git pre-commit hook, if one is not already present -} +gitPreCommitHookWrite :: Annex () +gitPreCommitHookWrite = unlessBare $ do + hook <- preCommitHook + ifM (liftIO $ doesFileExist hook) + ( warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" + , unlessM crippledFileSystem $ + liftIO $ do + viaTmp writeFile hook preCommitScript + p <- getPermissions hook + setPermissions hook $ p {executable = True} + ) + +gitPreCommitHookUnWrite :: Annex () +gitPreCommitHookUnWrite = unlessBare $ do + hook <- preCommitHook + whenM (liftIO $ doesFileExist hook) $ + ifM (liftIO $ (==) preCommitScript <$> readFile hook) + ( liftIO $ removeFile hook + , warning $ "pre-commit hook (" ++ hook ++ + ") contents modified; not deleting." ++ + " Edit it to remove call to git annex." + ) + +unlessBare :: Annex () -> Annex () +unlessBare = unlessM $ fromRepo Git.repoIsLocalBare + +preCommitHook :: Annex FilePath +preCommitHook = () <$> fromRepo Git.localGitDir <*> pure "hooks/pre-commit" + +preCommitScript :: String +preCommitScript = unlines + [ shebang + , "# automatically configured by git-annex" + , "git annex pre-commit ." + ] + +probeCrippledFileSystem :: Annex Bool +probeCrippledFileSystem = do + tmp <- fromRepo gitAnnexTmpDir + let f = tmp "gaprobe" + liftIO $ do + createDirectoryIfMissing True tmp + writeFile f "" + uncrippled <- liftIO $ probe f + liftIO $ removeFile f + return $ not uncrippled + where + probe f = catchBoolIO $ do + let f2 = f ++ "2" + nukeFile f2 + createLink f f2 + nukeFile f2 + createSymbolicLink f f2 + nukeFile f2 + preventWrite f + allowWrite f + return True + +checkCrippledFileSystem :: Annex () +checkCrippledFileSystem = whenM (probeCrippledFileSystem) $ do + warning "Detected a crippled filesystem." + setCrippledFileSystem True + unlessM isDirect $ do + warning "Enabling direct mode." + top <- fromRepo Git.repoPath + (l, clean) <- inRepo $ Git.LsFiles.inRepo [top] + forM_ l $ \f -> + maybe noop (`toDirect` f) =<< isAnnexLink f + void $ liftIO clean + setDirect True + setVersion directModeVersion diff --git a/Limit.hs b/Limit.hs new file mode 100644 index 0000000000..1d02decbe2 --- /dev/null +++ b/Limit.hs @@ -0,0 +1,219 @@ +{- user-specified limits on files to act on + - + - Copyright 2011,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE PackageImports, CPP #-} + +module Limit where + +import Data.Time.Clock.POSIX +import qualified Data.Set as S +import qualified Data.Map as M +import System.Path.WildMatch +import Text.Regex + +import Common.Annex +import qualified Annex +import qualified Utility.Matcher +import qualified Remote +import qualified Backend +import Annex.Content +import Annex.UUID +import Logs.Trust +import Types.TrustLevel +import Types.Key +import Types.Group +import Logs.Group +import Utility.HumanTime +import Utility.DataUnits + +type MatchFiles = AssumeNotPresent -> Annex.FileInfo -> Annex Bool +type MkLimit = String -> Either String MatchFiles +type AssumeNotPresent = S.Set UUID + +{- Checks if there are user-specified limits. -} +limited :: Annex Bool +limited = (not . Utility.Matcher.isEmpty) <$> getMatcher' + +{- Gets a matcher for the user-specified limits. The matcher is cached for + - speed; once it's obtained the user-specified limits can't change. -} +getMatcher :: Annex (Annex.FileInfo -> Annex Bool) +getMatcher = Utility.Matcher.matchM <$> getMatcher' + +getMatcher' :: Annex (Utility.Matcher.Matcher (Annex.FileInfo -> Annex Bool)) +getMatcher' = do + m <- Annex.getState Annex.limit + case m of + Right r -> return r + Left l -> do + let matcher = Utility.Matcher.generate (reverse l) + Annex.changeState $ \s -> s { Annex.limit = Right matcher } + return matcher + +{- Adds something to the limit list, which is built up reversed. -} +add :: Utility.Matcher.Token (Annex.FileInfo -> Annex Bool) -> Annex () +add l = Annex.changeState $ \s -> s { Annex.limit = prepend $ Annex.limit s } + where + prepend (Left ls) = Left $ l:ls + prepend _ = error "internal" + +{- Adds a new token. -} +addToken :: String -> Annex () +addToken = add . Utility.Matcher.token + +{- Adds a new limit. -} +addLimit :: Either String MatchFiles -> Annex () +addLimit = either error (\l -> add $ Utility.Matcher.Operation $ l S.empty) + +{- Add a limit to skip files that do not match the glob. -} +addInclude :: String -> Annex () +addInclude = addLimit . limitInclude + +limitInclude :: MkLimit +limitInclude glob = Right $ const $ return . matchglob glob + +{- Add a limit to skip files that match the glob. -} +addExclude :: String -> Annex () +addExclude = addLimit . limitExclude + +limitExclude :: MkLimit +limitExclude glob = Right $ const $ return . not . matchglob glob + +{- Could just use wildCheckCase, but this way the regex is only compiled + - once. -} +matchglob :: String -> Annex.FileInfo -> Bool +matchglob glob (Annex.FileInfo { Annex.matchFile = f }) = + isJust $ matchRegex cregex f + where + cregex = mkRegex regex + regex = '^':wildToRegex glob + +{- Adds a limit to skip files not believed to be present + - in a specfied repository. -} +addIn :: String -> Annex () +addIn = addLimit . limitIn + +limitIn :: MkLimit +limitIn name = Right $ \notpresent -> check $ + if name == "." + then inhere notpresent + else inremote notpresent + where + check a = lookupFile >=> handle a + handle _ Nothing = return False + handle a (Just (key, _)) = a key + inremote notpresent key = do + u <- Remote.nameToUUID name + us <- Remote.keyLocations key + return $ u `elem` us && u `S.notMember` notpresent + inhere notpresent key + | S.null notpresent = inAnnex key + | otherwise = do + u <- getUUID + if u `S.member` notpresent + then return False + else inAnnex key + +{- Limit to content that is currently present on a uuid. -} +limitPresent :: Maybe UUID -> MkLimit +limitPresent u _ = Right $ const $ check $ \key -> do + hereu <- getUUID + if u == Just hereu || u == Nothing + then inAnnex key + else do + us <- Remote.keyLocations key + return $ maybe False (`elem` us) u + where + check a = lookupFile >=> handle a + handle _ Nothing = return False + handle a (Just (key, _)) = a key + +{- Adds a limit to skip files not believed to have the specified number + - of copies. -} +addCopies :: String -> Annex () +addCopies = addLimit . limitCopies + +limitCopies :: MkLimit +limitCopies want = case split ":" want of + [v, n] -> case readTrustLevel v of + Just trust -> go n $ checktrust trust + Nothing -> go n $ checkgroup v + [n] -> go n $ const $ return True + _ -> Left "bad value for copies" + where + go num good = case readish num of + Nothing -> Left "bad number for copies" + Just n -> Right $ \notpresent f -> + lookupFile f >>= handle n good notpresent + handle _ _ _ Nothing = return False + handle n good notpresent (Just (key, _)) = do + us <- filter (`S.notMember` notpresent) + <$> (filterM good =<< Remote.keyLocations key) + return $ length us >= n + checktrust t u = (== t) <$> lookupTrust u + checkgroup g u = S.member g <$> lookupGroups u + +{- Adds a limit to skip files not believed to be present in all + - repositories in the specified group. -} +addInAllGroup :: String -> Annex () +addInAllGroup groupname = do + m <- groupMap + addLimit $ limitInAllGroup m groupname + +limitInAllGroup :: GroupMap -> MkLimit +limitInAllGroup m groupname + | S.null want = Right $ const $ const $ return True + | otherwise = Right $ \notpresent -> lookupFile >=> check notpresent + where + want = fromMaybe S.empty $ M.lookup groupname $ uuidsByGroup m + check _ Nothing = return False + check notpresent (Just (key, _)) + -- optimisation: Check if a wanted uuid is notpresent. + | not (S.null (S.intersection want notpresent)) = return False + | otherwise = do + present <- S.fromList <$> Remote.keyLocations key + return $ S.null $ want `S.difference` present + +{- Adds a limit to skip files not using a specified key-value backend. -} +addInBackend :: String -> Annex () +addInBackend = addLimit . limitInBackend + +limitInBackend :: MkLimit +limitInBackend name = Right $ const $ lookupFile >=> check + where + wanted = Backend.lookupBackendName name + check = return . maybe False ((==) wanted . snd) + +{- Adds a limit to skip files that are too large or too small -} +addLargerThan :: String -> Annex () +addLargerThan = addLimit . limitSize (>) + +addSmallerThan :: String -> Annex () +addSmallerThan = addLimit . limitSize (<) + +limitSize :: (Maybe Integer -> Maybe Integer -> Bool) -> MkLimit +limitSize vs s = case readSize dataUnits s of + Nothing -> Left "bad size" + Just sz -> Right $ const $ lookupFile >=> check sz + where + check _ Nothing = return False + check sz (Just (key, _)) = return $ keySize key `vs` Just sz + +addTimeLimit :: String -> Annex () +addTimeLimit s = do + let seconds = fromMaybe (error "bad time-limit") $ parseDuration s + start <- liftIO getPOSIXTime + let cutoff = start + seconds + addLimit $ Right $ const $ const $ do + now <- liftIO getPOSIXTime + if now > cutoff + then do + warning $ "Time limit (" ++ s ++ ") reached!" + liftIO $ exitWith $ ExitFailure 101 + else return True + +lookupFile :: Annex.FileInfo -> Annex (Maybe (Key, Backend)) +lookupFile = Backend.lookupFile . Annex.relFile diff --git a/Locations.hs b/Locations.hs new file mode 100644 index 0000000000..9f892a8f3c --- /dev/null +++ b/Locations.hs @@ -0,0 +1,328 @@ +{- git-annex file locations + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Locations ( + keyFile, + fileKey, + keyPaths, + keyPath, + gitAnnexLocation, + gitAnnexMapping, + gitAnnexInodeCache, + gitAnnexInodeSentinal, + gitAnnexInodeSentinalCache, + annexLocations, + annexLocation, + gitAnnexDir, + gitAnnexObjectDir, + gitAnnexTmpDir, + gitAnnexTmpLocation, + gitAnnexBadDir, + gitAnnexBadLocation, + gitAnnexUnusedLog, + gitAnnexFsckState, + gitAnnexTransferDir, + gitAnnexCredsDir, + gitAnnexMergeDir, + gitAnnexJournalDir, + gitAnnexJournalLock, + gitAnnexIndex, + gitAnnexIndexLock, + gitAnnexPidFile, + gitAnnexDaemonStatusFile, + gitAnnexLogFile, + gitAnnexHtmlShim, + gitAnnexUrlFile, + gitAnnexTmpCfgFile, + gitAnnexSshDir, + gitAnnexRemotesDir, + gitAnnexAssistantDefaultDir, + isLinkToAnnex, + annexHashes, + hashDirMixed, + hashDirLower, + + prop_idempotent_fileKey +) where + +import Data.Bits +import Data.Word +import Data.Hash.MD5 + +import Common +import Types +import Types.Key +import qualified Git + +{- Conventions: + - + - Functions ending in "Dir" should always return values ending with a + - trailing path separator. Most code does not rely on that, but a few + - things do. + - + - Everything else should not end in a trailing path sepatator. + - + - Only functions (with names starting with "git") that build a path + - based on a git repository should return an absolute path. + - Everything else should use relative paths. + -} + +{- The directory git annex uses for local state, relative to the .git + - directory -} +annexDir :: FilePath +annexDir = addTrailingPathSeparator "annex" + +{- The directory git annex uses for locally available object content, + - relative to the .git directory -} +objectDir :: FilePath +objectDir = addTrailingPathSeparator $ annexDir "objects" + +{- Annexed file's possible locations relative to the .git directory. + - There are two different possibilities, using different hashes. -} +annexLocations :: Key -> [FilePath] +annexLocations key = map (annexLocation key) annexHashes +annexLocation :: Key -> Hasher -> FilePath +annexLocation key hasher = objectDir keyPath key hasher + +{- Annexed file's absolute location in a repository. + - + - When there are multiple possible locations, returns the one where the + - file is actually present. + - + - When the file is not present, returns the location where the file should + - be stored. + - + - This does not take direct mode into account, so in direct mode it is not + - the actual location of the file's content. + -} +gitAnnexLocation :: Key -> Git.Repo -> IO FilePath +gitAnnexLocation key r + | Git.repoIsLocalBare r = + {- Bare repositories default to hashDirLower for new + - content, as it's more portable. -} + check $ map inrepo $ annexLocations key + | otherwise = + {- Non-bare repositories only use hashDirMixed, so + - don't need to do any work to check if the file is + - present. -} + return $ inrepo $ annexLocation key hashDirMixed + where + inrepo d = Git.localGitDir r d + check locs@(l:_) = fromMaybe l <$> firstM doesFileExist locs + check [] = error "internal" + +{- File that maps from a key to the file(s) in the git repository. + - Used in direct mode. -} +gitAnnexMapping :: Key -> Git.Repo -> IO FilePath +gitAnnexMapping key r = do + loc <- gitAnnexLocation key r + return $ loc ++ ".map" + +{- File that caches information about a key's content, used to determine + - if a file has changed. + - Used in direct mode. -} +gitAnnexInodeCache :: Key -> Git.Repo -> IO FilePath +gitAnnexInodeCache key r = do + loc <- gitAnnexLocation key r + return $ loc ++ ".cache" + +gitAnnexInodeSentinal :: Git.Repo -> FilePath +gitAnnexInodeSentinal r = gitAnnexDir r "sentinal" + +gitAnnexInodeSentinalCache :: Git.Repo -> FilePath +gitAnnexInodeSentinalCache r = gitAnnexInodeSentinal r ++ ".cache" + +{- The annex directory of a repository. -} +gitAnnexDir :: Git.Repo -> FilePath +gitAnnexDir r = addTrailingPathSeparator $ Git.localGitDir r annexDir + +{- The part of the annex directory where file contents are stored. -} +gitAnnexObjectDir :: Git.Repo -> FilePath +gitAnnexObjectDir r = addTrailingPathSeparator $ Git.localGitDir r objectDir + +{- .git/annex/tmp/ is used for temp files -} +gitAnnexTmpDir :: Git.Repo -> FilePath +gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r "tmp" + +{- The temp file to use for a given key. -} +gitAnnexTmpLocation :: Key -> Git.Repo -> FilePath +gitAnnexTmpLocation key r = gitAnnexTmpDir r keyFile key + +{- .git/annex/bad/ is used for bad files found during fsck -} +gitAnnexBadDir :: Git.Repo -> FilePath +gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r "bad" + +{- The bad file to use for a given key. -} +gitAnnexBadLocation :: Key -> Git.Repo -> FilePath +gitAnnexBadLocation key r = gitAnnexBadDir r keyFile key + +{- .git/annex/foounused is used to number possibly unused keys -} +gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath +gitAnnexUnusedLog prefix r = gitAnnexDir r (prefix ++ "unused") + +{- .git/annex/fsckstate is used to store information about incremental fscks. -} +gitAnnexFsckState :: Git.Repo -> FilePath +gitAnnexFsckState r = gitAnnexDir r "fsckstate" + +{- .git/annex/creds/ is used to store credentials to access some special + - remotes. -} +gitAnnexCredsDir :: Git.Repo -> FilePath +gitAnnexCredsDir r = addTrailingPathSeparator $ gitAnnexDir r "creds" + +{- .git/annex/merge/ is used for direct mode merges. -} +gitAnnexMergeDir :: Git.Repo -> FilePath +gitAnnexMergeDir r = addTrailingPathSeparator $ gitAnnexDir r "merge" + +{- .git/annex/transfer/ is used to record keys currently + - being transferred, and other transfer bookkeeping info. -} +gitAnnexTransferDir :: Git.Repo -> FilePath +gitAnnexTransferDir r = addTrailingPathSeparator $ gitAnnexDir r "transfer" + +{- .git/annex/journal/ is used to journal changes made to the git-annex + - branch -} +gitAnnexJournalDir :: Git.Repo -> FilePath +gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r "journal" + +{- Lock file for the journal. -} +gitAnnexJournalLock :: Git.Repo -> FilePath +gitAnnexJournalLock r = gitAnnexDir r "journal.lck" + +{- .git/annex/index is used to stage changes to the git-annex branch -} +gitAnnexIndex :: Git.Repo -> FilePath +gitAnnexIndex r = gitAnnexDir r "index" + +{- Lock file for .git/annex/index. -} +gitAnnexIndexLock :: Git.Repo -> FilePath +gitAnnexIndexLock r = gitAnnexDir r "index.lck" + +{- Pid file for daemon mode. -} +gitAnnexPidFile :: Git.Repo -> FilePath +gitAnnexPidFile r = gitAnnexDir r "daemon.pid" + +{- Status file for daemon mode. -} +gitAnnexDaemonStatusFile :: Git.Repo -> FilePath +gitAnnexDaemonStatusFile r = gitAnnexDir r "daemon.status" + +{- Log file for daemon mode. -} +gitAnnexLogFile :: Git.Repo -> FilePath +gitAnnexLogFile r = gitAnnexDir r "daemon.log" + +{- Html shim file used to launch the webapp. -} +gitAnnexHtmlShim :: Git.Repo -> FilePath +gitAnnexHtmlShim r = gitAnnexDir r "webapp.html" + +{- File containing the url to the webapp. -} +gitAnnexUrlFile :: Git.Repo -> FilePath +gitAnnexUrlFile r = gitAnnexDir r "url" + +{- Temporary file used to edit configuriation from the git-annex branch. -} +gitAnnexTmpCfgFile :: Git.Repo -> FilePath +gitAnnexTmpCfgFile r = gitAnnexDir r "config.tmp" + +{- .git/annex/ssh/ is used for ssh connection caching -} +gitAnnexSshDir :: Git.Repo -> FilePath +gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r "ssh" + +{- .git/annex/remotes/ is used for remote-specific state. -} +gitAnnexRemotesDir :: Git.Repo -> FilePath +gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r "remotes" + +{- This is the base directory name used by the assistant when making + - repositories, by default. -} +gitAnnexAssistantDefaultDir :: FilePath +gitAnnexAssistantDefaultDir = "annex" + +{- Checks a symlink target to see if it appears to point to annexed content. + - + - We only look at paths inside the .git directory, and not at the .git + - directory itself, because GIT_DIR may cause a directory name other + - than .git to be used. + -} +isLinkToAnnex :: FilePath -> Bool +isLinkToAnnex s = ('/':objectDir) `isInfixOf` s + +{- Converts a key into a filename fragment without any directory. + - + - Escape "/" in the key name, to keep a flat tree of files and avoid + - issues with keys containing "/../" or ending with "/" etc. + - + - "/" is escaped to "%" because it's short and rarely used, and resembles + - a slash + - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping + - is one to one. + - ":" is escaped to "&c", because despite it being 2011, people still care + - about FAT. + -} +keyFile :: Key -> FilePath +keyFile key = replace "/" "%" $ replace ":" "&c" $ + replace "%" "&s" $ replace "&" "&a" $ key2file key + +{- A location to store a key on the filesystem. A directory hash is used, + - to protect against filesystems that dislike having many items in a + - single directory. + - + - The file is put in a directory with the same name, this allows + - write-protecting the directory to avoid accidental deletion of the file. + -} +keyPath :: Key -> Hasher -> FilePath +keyPath key hasher = hasher key f f + where + f = keyFile key + +{- All possibile locations to store a key using different directory hashes. -} +keyPaths :: Key -> [FilePath] +keyPaths key = map (keyPath key) annexHashes + +{- Reverses keyFile, converting a filename fragment (ie, the basename of + - the symlink target) into a key. -} +fileKey :: FilePath -> Maybe Key +fileKey file = file2key $ + replace "&a" "&" $ replace "&s" "%" $ + replace "&c" ":" $ replace "%" "/" file + +{- for quickcheck -} +prop_idempotent_fileKey :: String -> Bool +prop_idempotent_fileKey s = Just k == fileKey (keyFile k) + where + k = stubKey { keyName = s, keyBackendName = "test" } + +{- Two different directory hashes may be used. The mixed case hash + - came first, and is fine, except for the problem of case-strict + - filesystems such as Linux VFAT (mounted with shortname=mixed), + - which do not allow using a directory "XX" when "xx" already exists. + - To support that, most repositories use the lower case hash for new data. -} +type Hasher = Key -> FilePath +annexHashes :: [Hasher] +annexHashes = [hashDirLower, hashDirMixed] + +hashDirMixed :: Hasher +hashDirMixed k = addTrailingPathSeparator $ take 2 dir drop 2 dir + where + dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d] + ABCD (a,b,c,d) = md5 $ md5FilePath $ key2file k + +hashDirLower :: Hasher +hashDirLower k = addTrailingPathSeparator $ take 3 dir drop 3 dir + where + dir = take 6 $ md5s $ md5FilePath $ key2file k + +{- modified version of display_32bits_as_hex from Data.Hash.MD5 + - Copyright (C) 2001 Ian Lynagh + - License: Either BSD or GPL + -} +display_32bits_as_dir :: Word32 -> String +display_32bits_as_dir w = trim $ swap_pairs cs + where + -- Need 32 characters to use. To avoid inaverdently making + -- a real word, use letters that appear less frequently. + chars = ['0'..'9'] ++ "zqjxkmvwgpfZQJXKMVWGPF" + cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7] + getc n = chars !! fromIntegral n + swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs + swap_pairs _ = [] + -- Last 2 will always be 00, so omit. + trim = take 6 diff --git a/Locations/UserConfig.hs b/Locations/UserConfig.hs new file mode 100644 index 0000000000..429ed8fd55 --- /dev/null +++ b/Locations/UserConfig.hs @@ -0,0 +1,57 @@ +{- git-annex user config files + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Locations.UserConfig where + +import Common +import Utility.TempFile +import Utility.FreeDesktop + +{- ~/.config/git-annex/file -} +userConfigFile :: FilePath -> IO FilePath +userConfigFile file = do + dir <- userConfigDir + return $ dir "git-annex" file + +autoStartFile :: IO FilePath +autoStartFile = userConfigFile "autostart" + +{- Returns anything listed in the autostart file (which may not exist). -} +readAutoStartFile :: IO [FilePath] +readAutoStartFile = do + f <- autoStartFile + nub . lines <$> catchDefaultIO "" (readFile f) + +{- Adds a directory to the autostart file. -} +addAutoStartFile :: FilePath -> IO () +addAutoStartFile path = do + dirs <- readAutoStartFile + when (path `notElem` dirs) $ do + f <- autoStartFile + createDirectoryIfMissing True (parentDir f) + viaTmp writeFile f $ unlines $ dirs ++ [path] + +{- Removes a directory from the autostart file. -} +removeAutoStartFile :: FilePath -> IO () +removeAutoStartFile path = do + dirs <- readAutoStartFile + when (path `elem` dirs) $ do + f <- autoStartFile + createDirectoryIfMissing True (parentDir f) + viaTmp writeFile f $ unlines $ + filter (not . equalFilePath path) dirs + +{- The path to git-annex is written here; which is useful when cabal + - has installed it to some aweful non-PATH location. -} +programFile :: IO FilePath +programFile = userConfigFile "program" + +{- Returns a command to run for git-annex. -} +readProgramFile :: IO FilePath +readProgramFile = do + programfile <- programFile + catchDefaultIO "git-annex" $ readFile programfile diff --git a/Logs/Group.hs b/Logs/Group.hs new file mode 100644 index 0000000000..8ccb6b7f25 --- /dev/null +++ b/Logs/Group.hs @@ -0,0 +1,75 @@ +{- git-annex group log + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Group ( + groupLog, + groupChange, + groupSet, + lookupGroups, + groupMap, + groupMapLoad, + getStandardGroup, +) where + +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Time.Clock.POSIX + +import Common.Annex +import qualified Annex.Branch +import qualified Annex +import Logs.UUIDBased +import Types.Group +import Types.StandardGroups + +{- Filename of group.log. -} +groupLog :: FilePath +groupLog = "group.log" + +{- Returns the groups of a given repo UUID. -} +lookupGroups :: UUID -> Annex (S.Set Group) +lookupGroups u = (fromMaybe S.empty . M.lookup u) . groupsByUUID <$> groupMap + +{- Applies a set modifier to change the groups for a uuid in the groupLog. -} +groupChange :: UUID -> (S.Set Group -> S.Set Group) -> Annex () +groupChange uuid modifier = do + curr <- lookupGroups uuid + ts <- liftIO getPOSIXTime + Annex.Branch.change groupLog $ + showLog (unwords . S.toList) . + changeLog ts uuid (modifier curr) . + parseLog (Just . S.fromList . words) + Annex.changeState $ \s -> s { Annex.groupmap = Nothing } + +groupSet :: UUID -> S.Set Group -> Annex () +groupSet u g = groupChange u (const g) + +{- The map is cached for speed. -} +groupMap :: Annex GroupMap +groupMap = maybe groupMapLoad return =<< Annex.getState Annex.groupmap + +{- Loads the map, updating the cache. -} +groupMapLoad :: Annex GroupMap +groupMapLoad = do + m <- makeGroupMap . simpleMap . + parseLog (Just . S.fromList . words) <$> + Annex.Branch.get groupLog + Annex.changeState $ \s -> s { Annex.groupmap = Just m } + return m + +makeGroupMap :: M.Map UUID (S.Set Group) -> GroupMap +makeGroupMap byuuid = GroupMap byuuid bygroup + where + bygroup = M.fromListWith S.union $ + concat $ map explode $ M.toList byuuid + explode (u, s) = map (\g -> (g, S.singleton u)) (S.toList s) + +{- If a repository is in exactly one standard group, returns it. -} +getStandardGroup :: S.Set Group -> Maybe StandardGroup +getStandardGroup s = case catMaybes $ map toStandardGroup $ S.toList s of + [g] -> Just g + _ -> Nothing diff --git a/Logs/Location.hs b/Logs/Location.hs new file mode 100644 index 0000000000..8bab5cd558 --- /dev/null +++ b/Logs/Location.hs @@ -0,0 +1,73 @@ +{-# LANGUAGE BangPatterns #-} + +{- git-annex location log + - + - git-annex keeps track of which repositories have the contents of annexed + - files. + - + - Repositories record their UUID and the date when they --get or --drop + - a value. + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Location ( + LogStatus(..), + logStatus, + logChange, + loggedLocations, + loggedKeys, + loggedKeysFor, + logFile, + logFileKey +) where + +import Common.Annex +import qualified Annex.Branch +import Logs.Presence +import Annex.UUID + +{- Log a change in the presence of a key's value in current repository. -} +logStatus :: Key -> LogStatus -> Annex () +logStatus key status = maybe noop (\u -> logChange key u status) =<< getUUID + +{- Log a change in the presence of a key's value in a repository. -} +logChange :: Key -> UUID -> LogStatus -> Annex () +logChange key u s = addLog (logFile key) =<< logNow s (fromUUID u) + +{- Returns a list of repository UUIDs that, according to the log, have + - the value of a key. + -} +loggedLocations :: Key -> Annex [UUID] +loggedLocations key = mapMaybe toUUID <$> (currentLog . logFile) key + +{- Finds all keys that have location log information. + - (There may be duplicate keys in the list.) -} +loggedKeys :: Annex [Key] +loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Annex.Branch.files + +{- Finds all keys that have location log information indicating + - they are present for the specified repository. -} +loggedKeysFor :: UUID -> Annex [Key] +loggedKeysFor u = filterM isthere =<< loggedKeys + where + {- This should run strictly to avoid the filterM + - building many thunks containing keyLocations data. -} + isthere k = do + us <- loggedLocations k + let !there = u `elem` us + return there + +{- The filename of the log file for a given key. -} +logFile :: Key -> String +logFile key = hashDirLower key ++ keyFile key ++ ".log" + +{- Converts a log filename into a key. -} +logFileKey :: FilePath -> Maybe Key +logFileKey file + | ext == ".log" = fileKey base + | otherwise = Nothing + where + (base, ext) = splitAt (length file - 4) file diff --git a/Logs/PreferredContent.hs b/Logs/PreferredContent.hs new file mode 100644 index 0000000000..3340cf5ef7 --- /dev/null +++ b/Logs/PreferredContent.hs @@ -0,0 +1,148 @@ +{- git-annex preferred content matcher configuration + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.PreferredContent ( + preferredContentLog, + preferredContentSet, + isPreferredContent, + preferredContentMap, + preferredContentMapLoad, + preferredContentMapRaw, + checkPreferredContentExpression, + setStandardGroup, +) where + +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Either +import Data.Time.Clock.POSIX + +import Common.Annex +import qualified Annex.Branch +import qualified Annex +import Logs.UUIDBased +import Limit +import qualified Utility.Matcher +import Annex.UUID +import Git.FilePath +import Types.Group +import Logs.Group +import Types.StandardGroups + +{- Filename of preferred-content.log. -} +preferredContentLog :: FilePath +preferredContentLog = "preferred-content.log" + +{- Changes the preferred content configuration of a remote. -} +preferredContentSet :: UUID -> String -> Annex () +preferredContentSet uuid@(UUID _) val = do + ts <- liftIO getPOSIXTime + Annex.Branch.change preferredContentLog $ + showLog id . changeLog ts uuid val . parseLog Just + Annex.changeState $ \s -> s { Annex.groupmap = Nothing } +preferredContentSet NoUUID _ = error "unknown UUID; cannot modify" + +{- Checks if a file is preferred content for the specified repository + - (or the current repository if none is specified). -} +isPreferredContent :: Maybe UUID -> AssumeNotPresent -> FilePath -> Bool -> Annex Bool +isPreferredContent mu notpresent file def = do + matchfile <- getTopFilePath <$> inRepo (toTopFilePath file) + let fi = Annex.FileInfo + { Annex.matchFile = matchfile + , Annex.relFile = file + } + u <- maybe getUUID return mu + m <- preferredContentMap + case M.lookup u m of + Nothing -> return def + Just matcher + | Utility.Matcher.isEmpty matcher -> return def + | otherwise -> Utility.Matcher.matchMrun matcher $ + \a -> a notpresent fi + +{- The map is cached for speed. -} +preferredContentMap :: Annex Annex.PreferredContentMap +preferredContentMap = maybe preferredContentMapLoad return + =<< Annex.getState Annex.preferredcontentmap + +{- Loads the map, updating the cache. -} +preferredContentMapLoad :: Annex Annex.PreferredContentMap +preferredContentMapLoad = do + groupmap <- groupMap + m <- simpleMap + . parseLogWithUUID ((Just .) . makeMatcher groupmap) + <$> Annex.Branch.get preferredContentLog + Annex.changeState $ \s -> s { Annex.preferredcontentmap = Just m } + return m + +preferredContentMapRaw :: Annex (M.Map UUID String) +preferredContentMapRaw = simpleMap . parseLog Just + <$> Annex.Branch.get preferredContentLog + +{- This intentionally never fails, even on unparsable expressions, + - because the configuration is shared amoung repositories and newer + - versions of git-annex may add new features. Instead, parse errors + - result in a Matcher that will always succeed. -} +makeMatcher :: GroupMap -> UUID -> String -> Utility.Matcher.Matcher MatchFiles +makeMatcher groupmap u s + | s == "standard" = standardMatcher groupmap u + | null (lefts tokens) = Utility.Matcher.generate $ rights tokens + | otherwise = matchAll + where + tokens = map (parseToken (Just u) groupmap) (tokenizeMatcher s) + +{- Standard matchers are pre-defined for some groups. If none is defined, + - or a repository is in multiple groups with standard matchers, match all. -} +standardMatcher :: GroupMap -> UUID -> Utility.Matcher.Matcher MatchFiles +standardMatcher m u = maybe matchAll (makeMatcher m u . preferredContent) $ + getStandardGroup =<< u `M.lookup` groupsByUUID m + +matchAll :: Utility.Matcher.Matcher MatchFiles +matchAll = Utility.Matcher.generate [] + +{- Checks if an expression can be parsed, if not returns Just error -} +checkPreferredContentExpression :: String -> Maybe String +checkPreferredContentExpression s + | s == "standard" = Nothing + | otherwise = case lefts $ map (parseToken Nothing emptyGroupMap) (tokenizeMatcher s) of + [] -> Nothing + l -> Just $ unwords $ map ("Parse failure: " ++) l + +parseToken :: (Maybe UUID) -> GroupMap -> String -> Either String (Utility.Matcher.Token MatchFiles) +parseToken mu groupmap t + | any (== t) Utility.Matcher.tokens = Right $ Utility.Matcher.token t + | t == "present" = use $ limitPresent mu + | otherwise = maybe (Left $ "near " ++ show t) use $ M.lookup k $ + M.fromList + [ ("include", limitInclude) + , ("exclude", limitExclude) + , ("copies", limitCopies) + , ("inbackend", limitInBackend) + , ("largerthan", limitSize (>)) + , ("smallerthan", limitSize (<)) + , ("inallgroup", limitInAllGroup groupmap) + ] + where + (k, v) = separate (== '=') t + use a = Utility.Matcher.Operation <$> a v + +{- This is really dumb tokenization; there's no support for quoted values. + - Open and close parens are always treated as standalone tokens; + - otherwise tokens must be separated by whitespace. -} +tokenizeMatcher :: String -> [String] +tokenizeMatcher = filter (not . null ) . concatMap splitparens . words + where + splitparens = segmentDelim (`elem` "()") + +{- Puts a UUID in a standard group, and sets its preferred content to use + - the standard expression for that group, unless something is already set. -} +setStandardGroup :: UUID -> StandardGroup -> Annex () +setStandardGroup u g = do + groupSet u $ S.singleton $ fromStandardGroup g + m <- preferredContentMap + unless (isJust $ M.lookup u m) $ + preferredContentSet u "standard" diff --git a/Logs/Presence.hs b/Logs/Presence.hs new file mode 100644 index 0000000000..ec5cec209a --- /dev/null +++ b/Logs/Presence.hs @@ -0,0 +1,122 @@ +{- git-annex presence log + - + - This is used to store presence information in the git-annex branch in + - a way that can be union merged. + - + - A line of the log will look like: "date N INFO" + - Where N=1 when the INFO is present, and 0 otherwise. + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Presence ( + LogStatus(..), + LogLine(LogLine), + addLog, + readLog, + getLog, + parseLog, + showLog, + logNow, + compactLog, + currentLog, + prop_parse_show_log, +) where + +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale +import qualified Data.Map as M + +import Common.Annex +import qualified Annex.Branch +import Utility.QuickCheck + +data LogLine = LogLine { + date :: POSIXTime, + status :: LogStatus, + info :: String +} deriving (Eq, Show) + +data LogStatus = InfoPresent | InfoMissing + deriving (Eq, Show, Bounded, Enum) + +addLog :: FilePath -> LogLine -> Annex () +addLog file line = Annex.Branch.change file $ \s -> + showLog $ compactLog (line : parseLog s) + +{- Reads a log file. + - Note that the LogLines returned may be in any order. -} +readLog :: FilePath -> Annex [LogLine] +readLog = parseLog <$$> Annex.Branch.get + +{- Parses a log file. Unparseable lines are ignored. -} +parseLog :: String -> [LogLine] +parseLog = mapMaybe parseline . lines + where + parseline l = LogLine + <$> (utcTimeToPOSIXSeconds <$> parseTime defaultTimeLocale "%s%Qs" d) + <*> parsestatus s + <*> pure rest + where + (d, pastd) = separate (== ' ') l + (s, rest) = separate (== ' ') pastd + parsestatus "1" = Just InfoPresent + parsestatus "0" = Just InfoMissing + parsestatus _ = Nothing + +{- Generates a log file. -} +showLog :: [LogLine] -> String +showLog = unlines . map genline + where + genline (LogLine d s i) = unwords [show d, genstatus s, i] + genstatus InfoPresent = "1" + genstatus InfoMissing = "0" + +{- Generates a new LogLine with the current date. -} +logNow :: LogStatus -> String -> Annex LogLine +logNow s i = do + now <- liftIO getPOSIXTime + return $ LogLine now s i + +{- Reads a log and returns only the info that is still in effect. -} +currentLog :: FilePath -> Annex [String] +currentLog file = map info . filterPresent <$> readLog file + +{- Given a log, returns only the info that is are still in effect. -} +getLog :: String -> [String] +getLog = map info . filterPresent . parseLog + +{- Returns the info from LogLines that are in effect. -} +filterPresent :: [LogLine] -> [LogLine] +filterPresent = filter (\l -> InfoPresent == status l) . compactLog + +{- Compacts a set of logs, returning a subset that contains the current + - status. -} +compactLog :: [LogLine] -> [LogLine] +compactLog = M.elems . foldr mapLog M.empty + +type LogMap = M.Map String LogLine + +{- Inserts a log into a map of logs, if the log has better (ie, newer) + - information than the other logs in the map -} +mapLog :: LogLine -> LogMap -> LogMap +mapLog l m + | better = M.insert i l m + | otherwise = m + where + better = maybe True newer $ M.lookup i m + newer l' = date l' <= date l + i = info l + +instance Arbitrary LogLine where + arbitrary = LogLine + <$> arbitrary + <*> elements [minBound..maxBound] + <*> arbitrary `suchThat` ('\n' `notElem`) + +prop_parse_show_log :: [LogLine] -> Bool +prop_parse_show_log l = parseLog (showLog l) == l + diff --git a/Logs/Remote.hs b/Logs/Remote.hs new file mode 100644 index 0000000000..55fb40f4b1 --- /dev/null +++ b/Logs/Remote.hs @@ -0,0 +1,100 @@ +{- git-annex remote log + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Remote ( + remoteLog, + readRemoteLog, + configSet, + keyValToConfig, + configToKeyVal, + showConfig, + parseConfig, + + prop_idempotent_configEscape, + prop_parse_show_Config, +) where + +import qualified Data.Map as M +import Data.Time.Clock.POSIX +import Data.Char + +import Common.Annex +import qualified Annex.Branch +import Types.Remote +import Logs.UUIDBased + +{- Filename of remote.log. -} +remoteLog :: FilePath +remoteLog = "remote.log" + +{- Adds or updates a remote's config in the log. -} +configSet :: UUID -> RemoteConfig -> Annex () +configSet u c = do + ts <- liftIO getPOSIXTime + Annex.Branch.change remoteLog $ + showLog showConfig . changeLog ts u c . parseLog parseConfig + +{- Map of remotes by uuid containing key/value config maps. -} +readRemoteLog :: Annex (M.Map UUID RemoteConfig) +readRemoteLog = simpleMap . parseLog parseConfig <$> Annex.Branch.get remoteLog + +parseConfig :: String -> Maybe RemoteConfig +parseConfig = Just . keyValToConfig . words + +showConfig :: RemoteConfig -> String +showConfig = unwords . configToKeyVal + +{- Given Strings like "key=value", generates a RemoteConfig. -} +keyValToConfig :: [String] -> RemoteConfig +keyValToConfig ws = M.fromList $ map (/=/) ws + where + (/=/) s = (k, v) + where + k = takeWhile (/= '=') s + v = configUnEscape $ drop (1 + length k) s + +configToKeyVal :: M.Map String String -> [String] +configToKeyVal m = map toword $ sort $ M.toList m + where + toword (k, v) = k ++ "=" ++ configEscape v + +configEscape :: String -> String +configEscape = concatMap escape + where + escape c + | isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";" + | otherwise = [c] + +configUnEscape :: String -> String +configUnEscape = unescape + where + unescape [] = [] + unescape (c:rest) + | c == '&' = entity rest + | otherwise = c : unescape rest + entity s + | not (null num) && ";" `isPrefixOf` r = + chr (Prelude.read num) : unescape rest + | otherwise = + '&' : unescape s + where + num = takeWhile isNumber s + r = drop (length num) s + rest = drop 1 r + +{- for quickcheck -} +prop_idempotent_configEscape :: String -> Bool +prop_idempotent_configEscape s = s == (configUnEscape . configEscape) s + +prop_parse_show_Config :: RemoteConfig -> Bool +prop_parse_show_Config c + -- whitespace and '=' are not supported in keys + | any (\k -> any isSpace k || any (== '=') k) (M.keys c) = True + | otherwise = parseConfig (showConfig c) ~~ Just c + where + normalize v = sort . M.toList <$> v + a ~~ b = normalize a == normalize b diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs new file mode 100644 index 0000000000..97a6484b18 --- /dev/null +++ b/Logs/Transfer.hs @@ -0,0 +1,351 @@ +{- git-annex transfer information files and lock files + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Transfer where + +import Common.Annex +import Annex.Perms +import Annex.Exception +import qualified Git +import Types.Remote +import Types.Key +import Utility.Percentage +import Utility.QuickCheck + +import System.Posix.Types +import Data.Time.Clock +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale +import Control.Concurrent + +{- Enough information to uniquely identify a transfer, used as the filename + - of the transfer information file. -} +data Transfer = Transfer + { transferDirection :: Direction + , transferUUID :: UUID + , transferKey :: Key + } + deriving (Eq, Ord, Read, Show) + +{- Information about a Transfer, stored in the transfer information file. + - + - Note that the associatedFile may not correspond to a file in the local + - git repository. It's some file, possibly relative to some directory, + - of some repository, that was acted on to initiate the transfer. + -} +data TransferInfo = TransferInfo + { startedTime :: Maybe POSIXTime + , transferPid :: Maybe ProcessID + , transferTid :: Maybe ThreadId + , transferRemote :: Maybe Remote + , bytesComplete :: Maybe Integer + , associatedFile :: Maybe FilePath + , transferPaused :: Bool + } + deriving (Show, Eq, Ord) + +stubTransferInfo :: TransferInfo +stubTransferInfo = TransferInfo Nothing Nothing Nothing Nothing Nothing Nothing False + +data Direction = Upload | Download + deriving (Eq, Ord, Read, Show) + +showLcDirection :: Direction -> String +showLcDirection Upload = "upload" +showLcDirection Download = "download" + +readLcDirection :: String -> Maybe Direction +readLcDirection "upload" = Just Upload +readLcDirection "download" = Just Download +readLcDirection _ = Nothing + +describeTransfer :: Transfer -> TransferInfo -> String +describeTransfer t info = unwords + [ show $ transferDirection t + , show $ transferUUID t + , fromMaybe (key2file $ transferKey t) (associatedFile info) + ] + +{- Transfers that will accomplish the same task. -} +equivilantTransfer :: Transfer -> Transfer -> Bool +equivilantTransfer t1 t2 + | transferDirection t1 == Download && transferDirection t2 == Download && + transferKey t1 == transferKey t2 = True + | otherwise = t1 == t2 + +percentComplete :: Transfer -> TransferInfo -> Maybe Percentage +percentComplete (Transfer { transferKey = key }) info = + percentage <$> keySize key <*> Just (fromMaybe 0 $ bytesComplete info) + +type RetryDecider = TransferInfo -> TransferInfo -> Bool + +noRetry :: RetryDecider +noRetry _ _ = False + +{- Retries a transfer when it fails, as long as the failed transfer managed + - to send some data. -} +forwardRetry :: RetryDecider +forwardRetry old new = bytesComplete old < bytesComplete new + +upload :: UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex Bool) -> Annex Bool +upload u key = runTransfer (Transfer Upload u key) + +download :: UUID -> Key -> AssociatedFile -> RetryDecider -> Annex Bool -> Annex Bool +download u key file shouldretry a = runTransfer (Transfer Download u key) file shouldretry (const a) + +{- Runs a transfer action. Creates and locks the lock file while the + - action is running, and stores info in the transfer information + - file. Will throw an error if the transfer is already in progress. + - + - If the transfer action returns False, the transfer info is + - left in the failedTransferDir. + - + - An upload can be run from a read-only filesystem, and in this case + - no transfer information or lock file is used. + -} +runTransfer :: Transfer -> Maybe FilePath -> RetryDecider -> (MeterUpdate -> Annex Bool) -> Annex Bool +runTransfer t file shouldretry a = do + info <- liftIO $ startTransferInfo file + (meter, tfile, metervar) <- mkProgressUpdater t info + mode <- annexFileMode + ok <- retry info metervar $ + bracketIO (prep tfile mode info) (cleanup tfile) (a meter) + unless ok $ failed info + return ok + where + prep tfile mode info = catchMaybeIO $ do + fd <- openFd (transferLockFile tfile) ReadWrite (Just mode) + defaultFileFlags { trunc = True } + locked <- catchMaybeIO $ + setLock fd (WriteLock, AbsoluteSeek, 0, 0) + when (locked == Nothing) $ + error $ "transfer already in progress" + writeTransferInfoFile info tfile + return fd + cleanup _ Nothing = noop + cleanup tfile (Just fd) = do + void $ tryIO $ removeFile tfile + void $ tryIO $ removeFile $ transferLockFile tfile + closeFd fd + failed info = do + failedtfile <- fromRepo $ failedTransferFile t + createAnnexDirectory $ takeDirectory failedtfile + liftIO $ writeTransferInfoFile info failedtfile + retry oldinfo metervar run = do + v <- tryAnnex run + case v of + Right b -> return b + Left _ -> do + b <- getbytescomplete metervar + let newinfo = oldinfo { bytesComplete = Just b } + if shouldretry oldinfo newinfo + then retry newinfo metervar run + else return False + getbytescomplete metervar + | transferDirection t == Upload = + liftIO $ readMVar metervar + | otherwise = do + f <- fromRepo $ gitAnnexTmpLocation (transferKey t) + liftIO $ catchDefaultIO 0 $ + fromIntegral . fileSize <$> getFileStatus f + +{- Generates a callback that can be called as transfer progresses to update + - the transfer info file. Also returns the file it'll be updating, and a + - MVar that can be used to read the number of bytesComplete. -} +mkProgressUpdater :: Transfer -> TransferInfo -> Annex (MeterUpdate, FilePath, MVar Integer) +mkProgressUpdater t info = do + tfile <- fromRepo $ transferFile t + _ <- tryAnnex $ createAnnexDirectory $ takeDirectory tfile + mvar <- liftIO $ newMVar 0 + return (liftIO . updater tfile mvar, tfile, mvar) + where + updater tfile mvar bytes = modifyMVar_ mvar $ \oldbytes -> do + if (bytes - oldbytes >= mindelta) + then do + let info' = info { bytesComplete = Just bytes } + _ <- tryIO $ writeTransferInfoFile info' tfile + return bytes + else return oldbytes + {- The minimum change in bytesComplete that is worth + - updating a transfer info file for is 1% of the total + - keySize, rounded down. -} + mindelta = case keySize (transferKey t) of + Just sz -> sz `div` 100 + Nothing -> 100 * 1024 -- arbitrarily, 100 kb + +startTransferInfo :: Maybe FilePath -> IO TransferInfo +startTransferInfo file = TransferInfo + <$> (Just . utcTimeToPOSIXSeconds <$> getCurrentTime) + <*> pure Nothing -- pid not stored in file, so omitted for speed + <*> pure Nothing -- tid ditto + <*> pure Nothing -- not 0; transfer may be resuming + <*> pure Nothing + <*> pure file + <*> pure False + +{- If a transfer is still running, returns its TransferInfo. -} +checkTransfer :: Transfer -> Annex (Maybe TransferInfo) +checkTransfer t = do + mode <- annexFileMode + tfile <- fromRepo $ transferFile t + mfd <- liftIO $ catchMaybeIO $ + openFd (transferLockFile tfile) ReadOnly (Just mode) defaultFileFlags + case mfd of + Nothing -> return Nothing -- failed to open file; not running + Just fd -> do + locked <- liftIO $ + getLock fd (WriteLock, AbsoluteSeek, 0, 0) + liftIO $ closeFd fd + case locked of + Nothing -> return Nothing + Just (pid, _) -> liftIO $ catchDefaultIO Nothing $ + readTransferInfoFile (Just pid) tfile + +{- Gets all currently running transfers. -} +getTransfers :: Annex [(Transfer, TransferInfo)] +getTransfers = do + transfers <- catMaybes . map parseTransferFile . concat <$> findfiles + infos <- mapM checkTransfer transfers + return $ map (\(t, Just i) -> (t, i)) $ + filter running $ zip transfers infos + where + findfiles = liftIO . mapM dirContentsRecursive + =<< mapM (fromRepo . transferDir) [Download, Upload] + running (_, i) = isJust i + +{- Gets failed transfers for a given remote UUID. -} +getFailedTransfers :: UUID -> Annex [(Transfer, TransferInfo)] +getFailedTransfers u = catMaybes <$> (liftIO . getpairs =<< concat <$> findfiles) + where + getpairs = mapM $ \f -> do + let mt = parseTransferFile f + mi <- readTransferInfoFile Nothing f + return $ case (mt, mi) of + (Just t, Just i) -> Just (t, i) + _ -> Nothing + findfiles = liftIO . mapM dirContentsRecursive + =<< mapM (fromRepo . failedTransferDir u) [Download, Upload] + +removeFailedTransfer :: Transfer -> Annex () +removeFailedTransfer t = do + f <- fromRepo $ failedTransferFile t + liftIO $ void $ tryIO $ removeFile f + +{- The transfer information file to use for a given Transfer. -} +transferFile :: Transfer -> Git.Repo -> FilePath +transferFile (Transfer direction u key) r = transferDir direction r + filter (/= '/') (fromUUID u) + keyFile key + +{- The transfer information file to use to record a failed Transfer -} +failedTransferFile :: Transfer -> Git.Repo -> FilePath +failedTransferFile (Transfer direction u key) r = failedTransferDir u direction r + keyFile key + +{- The transfer lock file corresponding to a given transfer info file. -} +transferLockFile :: FilePath -> FilePath +transferLockFile infofile = let (d,f) = splitFileName infofile in + combine d ("lck." ++ f) + +{- Parses a transfer information filename to a Transfer. -} +parseTransferFile :: FilePath -> Maybe Transfer +parseTransferFile file + | "lck." `isPrefixOf` (takeFileName file) = Nothing + | otherwise = case drop (length bits - 3) bits of + [direction, u, key] -> Transfer + <$> readLcDirection direction + <*> toUUID u + <*> fileKey key + _ -> Nothing + where + bits = splitDirectories file + +writeTransferInfoFile :: TransferInfo -> FilePath -> IO () +writeTransferInfoFile info tfile = do + h <- openFile tfile WriteMode + fileEncoding h + hPutStr h $ writeTransferInfo info + hClose h + +{- File format is a header line containing the startedTime and any + - bytesComplete value. Followed by a newline and the associatedFile. + - + - The transferPid is not included; instead it is obtained by looking + - at the process that locks the file. + -} +writeTransferInfo :: TransferInfo -> String +writeTransferInfo info = unlines + [ (maybe "" show $ startedTime info) ++ + (maybe "" (\b -> " " ++ show b) $ bytesComplete info) + , fromMaybe "" $ associatedFile info -- comes last; arbitrary content + ] + +readTransferInfoFile :: (Maybe ProcessID) -> FilePath -> IO (Maybe TransferInfo) +readTransferInfoFile mpid tfile = catchDefaultIO Nothing $ do + h <- openFile tfile ReadMode + fileEncoding h + hClose h `after` (readTransferInfo mpid <$> hGetContentsStrict h) + +readTransferInfo :: (Maybe ProcessID) -> String -> Maybe TransferInfo +readTransferInfo mpid s = TransferInfo + <$> time + <*> pure mpid + <*> pure Nothing + <*> pure Nothing + <*> bytes + <*> pure (if null filename then Nothing else Just filename) + <*> pure False + where + (firstline, rest) = separate (== '\n') s + filename + | end rest == "\n" = beginning rest + | otherwise = rest + bits = split " " firstline + numbits = length bits + time = if numbits > 0 + then Just <$> parsePOSIXTime =<< headMaybe bits + else pure Nothing -- not failure + bytes = if numbits > 1 + then Just <$> readish =<< headMaybe (drop 1 bits) + else pure Nothing -- not failure + +parsePOSIXTime :: String -> Maybe POSIXTime +parsePOSIXTime s = utcTimeToPOSIXSeconds + <$> parseTime defaultTimeLocale "%s%Qs" s + +{- The directory holding transfer information files for a given Direction. -} +transferDir :: Direction -> Git.Repo -> FilePath +transferDir direction r = gitAnnexTransferDir r showLcDirection direction + +{- The directory holding failed transfer information files for a given + - Direction and UUID -} +failedTransferDir :: UUID -> Direction -> Git.Repo -> FilePath +failedTransferDir u direction r = gitAnnexTransferDir r + "failed" + showLcDirection direction + filter (/= '/') (fromUUID u) + +instance Arbitrary TransferInfo where + arbitrary = TransferInfo + <$> arbitrary + <*> arbitrary + <*> pure Nothing -- cannot generate a ThreadID + <*> pure Nothing -- remote not needed + <*> arbitrary + -- associated file cannot be empty (but can be Nothing) + <*> arbitrary `suchThat` (/= Just "") + <*> arbitrary + +prop_read_write_transferinfo :: TransferInfo -> Bool +prop_read_write_transferinfo info + | transferRemote info /= Nothing = True -- remote not stored + | transferTid info /= Nothing = True -- tid not stored + | otherwise = Just (info { transferPaused = False }) == info' + where + info' = readTransferInfo (transferPid info) (writeTransferInfo info) + diff --git a/Logs/Trust.hs b/Logs/Trust.hs new file mode 100644 index 0000000000..d61e1f960e --- /dev/null +++ b/Logs/Trust.hs @@ -0,0 +1,122 @@ +{- git-annex trust log + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Trust ( + trustLog, + TrustLevel(..), + trustGet, + trustSet, + trustPartition, + trustExclude, + lookupTrust, + trustMapLoad, + trustMapRaw, + + prop_parse_show_TrustLog, +) where + +import qualified Data.Map as M +import Data.Time.Clock.POSIX + +import Common.Annex +import Types.TrustLevel +import qualified Annex.Branch +import qualified Annex +import Logs.UUIDBased +import Remote.List +import qualified Types.Remote + +{- Filename of trust.log. -} +trustLog :: FilePath +trustLog = "trust.log" + +{- Returns a list of UUIDs that the trustLog indicates have the + - specified trust level. + - Note that the list can be incomplete for SemiTrusted, since that's + - the default. -} +trustGet :: TrustLevel -> Annex [UUID] +trustGet level = M.keys . M.filter (== level) <$> trustMap + +{- Changes the trust level for a uuid in the trustLog. -} +trustSet :: UUID -> TrustLevel -> Annex () +trustSet uuid level = do + ts <- liftIO getPOSIXTime + Annex.Branch.change trustLog $ + showLog showTrustLog . + changeLog ts uuid level . + parseLog (Just . parseTrustLog) + Annex.changeState $ \s -> s { Annex.trustmap = Nothing } + +{- Returns the TrustLevel of a given repo UUID. -} +lookupTrust :: UUID -> Annex TrustLevel +lookupTrust u = (fromMaybe SemiTrusted . M.lookup u) <$> trustMap + +{- Partitions a list of UUIDs to those matching a TrustLevel and not. -} +trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID]) +trustPartition level ls + | level == SemiTrusted = do + t <- trustGet Trusted + u <- trustGet UnTrusted + d <- trustGet DeadTrusted + let uncandidates = t ++ u ++ d + return $ partition (`notElem` uncandidates) ls + | otherwise = do + candidates <- trustGet level + return $ partition (`elem` candidates) ls + +{- Filters UUIDs to those not matching a TrustLevel. -} +trustExclude :: TrustLevel -> [UUID] -> Annex ([UUID]) +trustExclude level ls = snd <$> trustPartition level ls + +{- trustLog in a map, overridden with any values from forcetrust or + - the git config. The map is cached for speed. -} +trustMap :: Annex TrustMap +trustMap = maybe trustMapLoad return =<< Annex.getState Annex.trustmap + +{- Loads the map, updating the cache, -} +trustMapLoad :: Annex TrustMap +trustMapLoad = do + overrides <- Annex.getState Annex.forcetrust + logged <- trustMapRaw + configured <- M.fromList . catMaybes + <$> (map configuredtrust <$> remoteList) + let m = M.union overrides $ M.union configured logged + Annex.changeState $ \s -> s { Annex.trustmap = Just m } + return m + where + configuredtrust r = case Types.Remote.uuid r of + Just u -> (\l -> Just (u, l)) + =<< readTrustLevel + =<< remoteAnnexTrustLevel (Types.Remote.gitconfig r) + Nothing -> Nothing + +{- Does not include forcetrust or git config values, just those from the + - log file. -} +trustMapRaw :: Annex TrustMap +trustMapRaw = simpleMap . parseLog (Just . parseTrustLog) + <$> Annex.Branch.get trustLog + +{- The trust.log used to only list trusted repos, without a field for the + - trust status, which is why this defaults to Trusted. -} +parseTrustLog :: String -> TrustLevel +parseTrustLog s = maybe Trusted parse $ headMaybe $ words s + where + parse "1" = Trusted + parse "0" = UnTrusted + parse "X" = DeadTrusted + parse _ = SemiTrusted + +showTrustLog :: TrustLevel -> String +showTrustLog Trusted = "1" +showTrustLog UnTrusted = "0" +showTrustLog DeadTrusted = "X" +showTrustLog SemiTrusted = "?" + +prop_parse_show_TrustLog :: Bool +prop_parse_show_TrustLog = all check [minBound .. maxBound] + where + check l = parseTrustLog (showTrustLog l) == l diff --git a/Logs/UUID.hs b/Logs/UUID.hs new file mode 100644 index 0000000000..cfdc3151fc --- /dev/null +++ b/Logs/UUID.hs @@ -0,0 +1,71 @@ +{- git-annex uuids + - + - Each git repository used by git-annex has an annex.uuid setting that + - uniquely identifies that repository. + - + - UUIDs of remotes are cached in git config, using keys named + - remote..annex-uuid + - + - uuid.log stores a list of known uuids, and their descriptions. + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.UUID ( + uuidLog, + describeUUID, + recordUUID, + uuidMap, + uuidMapLoad +) where + +import qualified Data.Map as M +import Data.Time.Clock.POSIX + +import Types.UUID +import Common.Annex +import qualified Annex +import qualified Annex.Branch +import Logs.UUIDBased +import qualified Annex.UUID + +{- Filename of uuid.log. -} +uuidLog :: FilePath +uuidLog = "uuid.log" + +{- Records a description for a uuid in the log. -} +describeUUID :: UUID -> String -> Annex () +describeUUID uuid desc = do + ts <- liftIO getPOSIXTime + Annex.Branch.change uuidLog $ + showLog id . changeLog ts uuid desc . parseLog Just + +{- Records the uuid in the log, if it's not already there. -} +recordUUID :: UUID -> Annex () +recordUUID u = go . M.lookup u =<< uuidMap + where + go (Just "") = set + go Nothing = set + go _ = noop + set = describeUUID u "" + +{- The map is cached for speed. -} +uuidMap :: Annex UUIDMap +uuidMap = maybe uuidMapLoad return =<< Annex.getState Annex.uuidmap + +{- Read the uuidLog into a simple Map. + - + - The UUID of the current repository is included explicitly, since + - it may not have been described and so otherwise would not appear. -} +uuidMapLoad :: Annex UUIDMap +uuidMapLoad = do + m <- (simpleMap . parseLog Just) + <$> Annex.Branch.get uuidLog + m' <- maybe m (\u -> M.insertWith' preferold u "" m) + <$> Annex.UUID.getUUID + Annex.changeState $ \s -> s { Annex.uuidmap = Just m' } + return m' + where + preferold = flip const diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs new file mode 100644 index 0000000000..63ee00e639 --- /dev/null +++ b/Logs/UUIDBased.hs @@ -0,0 +1,118 @@ +{- git-annex uuid-based logs + - + - This is used to store information about a UUID in a way that can + - be union merged. + - + - A line of the log will look like: "UUID[ INFO[ timestamp=foo]]" + - The timestamp is last for backwards compatability reasons, + - and may not be present on old log lines. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.UUIDBased ( + Log, + LogEntry(..), + TimeStamp(..), + parseLog, + parseLogWithUUID, + showLog, + changeLog, + addLog, + simpleMap, + + prop_TimeStamp_sane, + prop_addLog_sane, +) where + +import qualified Data.Map as M +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale + +import Common +import Types.UUID + +data TimeStamp = Unknown | Date POSIXTime + deriving (Eq, Ord, Show) + +data LogEntry a = LogEntry + { changed :: TimeStamp + , value :: a + } deriving (Eq, Show) + +type Log a = M.Map UUID (LogEntry a) + +tskey :: String +tskey = "timestamp=" + +showLog :: (a -> String) -> Log a -> String +showLog shower = unlines . map showpair . M.toList + where + showpair (k, LogEntry (Date p) v) = + unwords [fromUUID k, shower v, tskey ++ show p] + showpair (k, LogEntry Unknown v) = + unwords [fromUUID k, shower v] + +parseLog :: (String -> Maybe a) -> String -> Log a +parseLog = parseLogWithUUID . const + +parseLogWithUUID :: (UUID -> String -> Maybe a) -> String -> Log a +parseLogWithUUID parser = M.fromListWith best . mapMaybe parse . lines + where + parse line + | null ws = Nothing + | otherwise = case toUUID $ Prelude.head ws of + Just u -> parser u (unwords info) >>= makepair u + Nothing -> Nothing + where + makepair u v = Just (u, LogEntry ts v) + ws = words line + t = Prelude.last ws + ts + | tskey `isPrefixOf` t = + pdate $ drop 1 $ dropWhile (/= '=') t + | otherwise = Unknown + info + | ts == Unknown = drop 1 ws + | otherwise = drop 1 $ beginning ws + pdate s = case parseTime defaultTimeLocale "%s%Qs" s of + Nothing -> Unknown + Just d -> Date $ utcTimeToPOSIXSeconds d + +changeLog :: POSIXTime -> UUID -> a -> Log a -> Log a +changeLog t u v = M.insert u $ LogEntry (Date t) v + +{- Only add an LogEntry if it's newer (or at least as new as) than any + - existing LogEntry for a UUID. -} +addLog :: UUID -> LogEntry a -> Log a -> Log a +addLog = M.insertWith' best + +{- Converts a Log into a simple Map without the timestamp information. + - This is a one-way trip, but useful for code that never needs to change + - the log. -} +simpleMap :: Log a -> M.Map UUID a +simpleMap = M.map value + +best :: LogEntry a -> LogEntry a -> LogEntry a +best new old + | changed old > changed new = old + | otherwise = new + +-- Unknown is oldest. +prop_TimeStamp_sane :: Bool +prop_TimeStamp_sane = Unknown < Date 1 + +prop_addLog_sane :: Bool +prop_addLog_sane = isJust mu && newWins && newestWins + where + newWins = addLog (u) (LogEntry (Date 1) "new") l == l2 + newestWins = addLog (u) (LogEntry (Date 1) "newest") l2 /= l2 + + l = M.fromList [(u, LogEntry (Date 0) "old")] + l2 = M.fromList [(u, LogEntry (Date 1) "new")] + + mu = toUUID "foo" + u = fromMaybe (error "failed to make uuid") mu diff --git a/Logs/Unused.hs b/Logs/Unused.hs new file mode 100644 index 0000000000..bef78a992a --- /dev/null +++ b/Logs/Unused.hs @@ -0,0 +1,91 @@ +{- git-annex unused log file + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Unused ( + UnusedMap, + UnusedMaps(..), + writeUnusedLog, + readUnusedLog, + withUnusedMaps, + startUnused, +) where + +import qualified Data.Map as M + +import Common.Annex +import Command +import Types.Key +import Utility.TempFile + +writeUnusedLog :: FilePath -> [(Int, Key)] -> Annex () +writeUnusedLog prefix l = do + logfile <- fromRepo $ gitAnnexUnusedLog prefix + liftIO $ viaTmp writeFile logfile $ + unlines $ map (\(n, k) -> show n ++ " " ++ key2file k) l + +readUnusedLog :: FilePath -> Annex UnusedMap +readUnusedLog prefix = do + f <- fromRepo $ gitAnnexUnusedLog prefix + ifM (liftIO $ doesFileExist f) + ( M.fromList . catMaybes . map parse . lines + <$> liftIO (readFile f) + , return M.empty + ) + where + parse line = case (readish tag, file2key rest) of + (Just num, Just key) -> Just (num, key) + _ -> Nothing + where + (tag, rest) = separate (== ' ') line + +type UnusedMap = M.Map Int Key + +data UnusedMaps = UnusedMaps + { unusedMap :: UnusedMap + , unusedBadMap :: UnusedMap + , unusedTmpMap :: UnusedMap + } + +{- Read unused logs once, and pass the maps to each start action. -} +withUnusedMaps :: (UnusedMaps -> Int -> CommandStart) -> CommandSeek +withUnusedMaps a params = do + unused <- readUnusedLog "" + unusedbad <- readUnusedLog "bad" + unusedtmp <- readUnusedLog "tmp" + return $ map (a $ UnusedMaps unused unusedbad unusedtmp) $ + concatMap unusedSpec params + +unusedSpec :: String -> [Int] +unusedSpec spec + | "-" `isInfixOf` spec = range $ separate (== '-') spec + | otherwise = maybe badspec (: []) (readish spec) + where + range (a, b) = case (readish a, readish b) of + (Just x, Just y) -> [x..y] + _ -> badspec + badspec = error $ "Expected number or range, not \"" ++ spec ++ "\"" + +{- Start action for unused content. Finds the number in the maps, and + - calls either of 3 actions, depending on the type of unused file. -} +startUnused :: String + -> (Key -> CommandPerform) + -> (Key -> CommandPerform) + -> (Key -> CommandPerform) + -> UnusedMaps -> Int -> CommandStart +startUnused message unused badunused tmpunused maps n = search + [ (unusedMap maps, unused) + , (unusedBadMap maps, badunused) + , (unusedTmpMap maps, tmpunused) + ] + where + search [] = stop + search ((m, a):rest) = + case M.lookup n m of + Nothing -> search rest + Just key -> do + showStart message (show n) + next $ a key diff --git a/Logs/Web.hs b/Logs/Web.hs new file mode 100644 index 0000000000..8d2349e540 --- /dev/null +++ b/Logs/Web.hs @@ -0,0 +1,58 @@ +{- Web url logs. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Web ( + URLString, + webUUID, + getUrls, + setUrlPresent, + setUrlMissing, +) where + +import Common.Annex +import Logs.Presence +import Logs.Location +import Types.Key + +type URLString = String + +-- Dummy uuid for the whole web. Do not alter. +webUUID :: UUID +webUUID = fromMaybe (error "failed to construct webUUID") + (toUUID "00000000-0000-0000-0000-000000000001") + +urlLog :: Key -> FilePath +urlLog key = hashDirLower key keyFile key ++ ".log.web" + +{- Used to store the urls elsewhere. -} +oldurlLogs :: Key -> [FilePath] +oldurlLogs key = + [ "remote/web" hashDirLower key key2file key ++ ".log" + , "remote/web" hashDirLower key keyFile key ++ ".log" + ] + +{- Gets all urls that a key might be available from. -} +getUrls :: Key -> Annex [URLString] +getUrls key = go $ urlLog key : oldurlLogs key + where + go [] = return [] + go (l:ls) = do + us <- currentLog l + if null us + then go ls + else return us + +setUrlPresent :: Key -> URLString -> Annex () +setUrlPresent key url = do + us <- getUrls key + unless (url `elem` us) $ do + addLog (urlLog key) =<< logNow InfoPresent url + -- update location log to indicate that the web has the key + logChange key webUUID InfoPresent + +setUrlMissing :: Key -> URLString -> Annex () +setUrlMissing key url = addLog (urlLog key) =<< logNow InfoMissing url diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..4c49f8dca5 --- /dev/null +++ b/Makefile @@ -0,0 +1,169 @@ +mans=git-annex.1 git-annex-shell.1 +all=git-annex $(mans) docs + +GHC?=ghc +GHCMAKE=$(GHC) $(GHCFLAGS) --make +PREFIX=/usr +CABAL=runhaskell Setup.hs + +# Am I typing :make in vim? Do a fast build. +ifdef VIM +all=fast +endif + +build: $(all) + +# We bypass cabal, and only run the main ghc --make command for a +# fast development built. Note: Does not rebuild C libraries. +fast: dist/caballog + @$$(grep 'ghc --make' dist/caballog | head -n 1) + @ln -sf dist/build/git-annex/git-annex git-annex + +dist/caballog: + $(CABAL) configure -f"-Production" -O0 + $(CABAL) build -v2 | tee $@ + +Build/SysConfig.hs: configure.hs Build/TestConfig.hs Build/Configure.hs + $(CABAL) configure + +git-annex: Build/SysConfig.hs + $(CABAL) build + ln -sf dist/build/git-annex/git-annex git-annex + +git-annex.1: doc/git-annex.mdwn + ./Build/mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 +git-annex-shell.1: doc/git-annex-shell.mdwn + ./Build/mdwn2man git-annex-shell 1 doc/git-annex-shell.mdwn > git-annex-shell.1 +git-union-merge.1: doc/git-union-merge.mdwn + ./Build/mdwn2man git-union-merge 1 doc/git-union-merge.mdwn > git-union-merge.1 + +install-mans: $(mans) + install -d $(DESTDIR)$(PREFIX)/share/man/man1 + install -m 0644 $(mans) $(DESTDIR)$(PREFIX)/share/man/man1 + +install-docs: docs install-mans + install -d $(DESTDIR)$(PREFIX)/share/doc/git-annex + if [ -d html ]; then \ + rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \ + fi + +install: build install-docs + install -d $(DESTDIR)$(PREFIX)/bin + install git-annex $(DESTDIR)$(PREFIX)/bin + ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell + runghc Build/InstallDesktopFile.hs $(PREFIX)/bin/git-annex || true + +test: git-annex + ./git-annex test + +# hothasktags chokes on some tempolate haskell etc, so ignore errors +tags: + find . | grep -v /.git/ | grep -v /doc/ | egrep '\.hs$$' | xargs hothasktags > tags 2>/dev/null + +# If ikiwiki is available, build static html docs suitable for being +# shipped in the software package. +ifeq ($(shell which ikiwiki),) +IKIWIKI=@echo "** ikiwiki not found, skipping building docs" >&2; true +else +IKIWIKI=ikiwiki +endif + +docs: $(mans) + $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ + --no-usedirs --disable-plugin=openid --plugin=sidebar \ + --underlaydir=/dev/null --disable-plugin=shortcut \ + --disable-plugin=smiley \ + --plugin=comments --set comments_pagespec="*" \ + --exclude='news/.*' --exclude='design/assistant/blog/*' \ + --exclude='bugs/*' --exclude='todo/*' --exclude='forum/*' + +clean: + rm -rf tmp dist git-annex $(mans) configure *.tix .hpc \ + doc/.ikiwiki html dist tags Build/SysConfig.hs + +sdist: clean $(mans) + ./Build/make-sdist.sh + +# Upload to hackage. +hackage: sdist + @cabal upload dist/*.tar.gz + +LINUXSTANDALONE_DEST=tmp/git-annex.linux +linuxstandalone: + $(MAKE) git-annex + + rm -rf "$(LINUXSTANDALONE_DEST)" + mkdir -p tmp + cp -R standalone/linux "$(LINUXSTANDALONE_DEST)" + + install -d "$(LINUXSTANDALONE_DEST)/bin" + cp git-annex "$(LINUXSTANDALONE_DEST)/bin/" + strip "$(LINUXSTANDALONE_DEST)/bin/git-annex" + ln -sf git-annex "$(LINUXSTANDALONE_DEST)/bin/git-annex-shell" + zcat standalone/licences.gz > $(LINUXSTANDALONE_DEST)/LICENSE + + runghc Build/Standalone.hs "$(LINUXSTANDALONE_DEST)" + + install -d "$(LINUXSTANDALONE_DEST)/git-core" + (cd "$(shell git --exec-path)" && tar c .) | (cd "$(LINUXSTANDALONE_DEST)"/git-core && tar x) + install -d "$(LINUXSTANDALONE_DEST)/templates" + + touch "$(LINUXSTANDALONE_DEST)/libdirs.tmp" + for lib in $$(ldd "$(LINUXSTANDALONE_DEST)"/bin/* $$(find "$(LINUXSTANDALONE_DEST)"/git-core/ -type f) | grep -v -f standalone/linux/glibc-libs | grep -v "not a dynamic executable" | egrep '^ ' | sed 's/^\t//' | sed 's/.*=> //' | cut -d ' ' -f 1 | sort | uniq); do \ + dir=$$(dirname "$$lib"); \ + install -d "$(LINUXSTANDALONE_DEST)/$$dir"; \ + echo "$$dir" >> "$(LINUXSTANDALONE_DEST)/libdirs.tmp"; \ + cp "$$lib" "$(LINUXSTANDALONE_DEST)/$$dir"; \ + if [ -L "$lib" ]; then \ + link=$$(readlink -f "$$lib"); \ + cp "$$link" "$(LINUXSTANDALONE_DEST)/$$(dirname "$$link")"; \ + fi; \ + done + sort "$(LINUXSTANDALONE_DEST)/libdirs.tmp" | uniq > "$(LINUXSTANDALONE_DEST)/libdirs" + rm -f "$(LINUXSTANDALONE_DEST)/libdirs.tmp" + + cd tmp && tar czf git-annex-standalone-$(shell dpkg --print-architecture).tar.gz git-annex.linux + +OSXAPP_DEST=tmp/build-dmg/git-annex.app +OSXAPP_BASE=$(OSXAPP_DEST)/Contents/MacOS +osxapp: + $(MAKE) git-annex + + rm -rf "$(OSXAPP_DEST)" + install -d tmp/build-dmg + cp -R standalone/osx/git-annex.app "$(OSXAPP_DEST)" + + install -d "$(OSXAPP_BASE)" + cp git-annex "$(OSXAPP_BASE)" + strip "$(OSXAPP_BASE)/git-annex" + ln -sf git-annex "$(OSXAPP_BASE)/git-annex-shell" + gzcat standalone/licences.gz > $(OSXAPP_BASE)/LICENSE + cp $(OSXAPP_BASE)/LICENSE tmp/build-dmg/LICENSE.txt + + runghc Build/Standalone.hs $(OSXAPP_BASE) + + (cd "$(shell git --exec-path)" && tar c .) | (cd "$(OSXAPP_BASE)" && tar x) + install -d "$(OSXAPP_BASE)/templates" + + runghc Build/OSXMkLibs.hs $(OSXAPP_BASE) + rm -f tmp/git-annex.dmg + hdiutil create -size 640m -format UDRW -srcfolder tmp/build-dmg \ + -volname git-annex -o tmp/git-annex.dmg + rm -f tmp/git-annex.dmg.bz2 + bzip2 --fast tmp/git-annex.dmg + +# Cross compile for Android. +# Uses https://github.com/neurocyte/ghc-android +android: + $(CABAL) configure +# cabal cannot cross compile with custom build type, so workaround + sed -i 's/Build-type: Custom/Build-type: Simple/' git-annex.cabal + $$HOME/.ghc/android-14/arm-linux-androideabi-4.7/arm-linux-androideabi/bin/cabal configure -f'Android Assistant -Pairing -Webapp' + $(MAKE) git-annex + sed -i 's/Build-type: Simple/Build-type: Custom/' git-annex.cabal + +androidapp: + $(MAKE) android + $(MAKE) -C standalone/android + +.PHONY: git-annex tags diff --git a/Messages.hs b/Messages.hs new file mode 100644 index 0000000000..d79c91aa03 --- /dev/null +++ b/Messages.hs @@ -0,0 +1,237 @@ +{- git-annex output messages + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Messages ( + showStart, + showNote, + showAction, + showProgress, + metered, + meteredBytes, + showSideAction, + doSideAction, + doQuietSideAction, + showStoringStateAction, + showOutput, + showLongNote, + showEndOk, + showEndFail, + showEndResult, + showErr, + warning, + warningIO, + fileNotFound, + indent, + maybeShowJSON, + showFullJSON, + showCustom, + showHeader, + showRaw, + + setupConsole +) where + +import Text.JSON +import Data.Progress.Meter +import Data.Progress.Tracker +import Data.Quantity +import System.Log.Logger +import System.Log.Formatter +import System.Log.Handler (setFormatter, LogHandler) +import System.Log.Handler.Simple + +import Common +import Types +import Types.Messages +import Types.Key +import qualified Annex +import qualified Messages.JSON as JSON +import qualified Data.Set as S + +showStart :: String -> String -> Annex () +showStart command file = handle (JSON.start command $ Just file) $ + flushed $ putStr $ command ++ " " ++ file ++ " " + +showNote :: String -> Annex () +showNote s = handle (JSON.note s) $ + flushed $ putStr $ "(" ++ s ++ ") " + +showAction :: String -> Annex () +showAction s = showNote $ s ++ "..." + +{- Progress dots. -} +showProgress :: Annex () +showProgress = handle q $ + flushed $ putStr "." + +{- Shows a progress meter while performing a transfer of a key. + - The action is passed a callback to use to update the meter. -} +metered :: (Maybe MeterUpdate) -> Key -> (MeterUpdate -> Annex a) -> Annex a +metered combinemeterupdate key a = go (keySize key) + where + go (Just size) = meteredBytes combinemeterupdate size a + go _ = a (const noop) + +{- Shows a progress meter while performing an action on a given number + - of bytes. -} +meteredBytes :: (Maybe MeterUpdate) -> Integer -> (MeterUpdate -> Annex a) -> Annex a +meteredBytes combinemeterupdate size a = withOutputType go + where + go NormalOutput = do + progress <- liftIO $ newProgress "" size + meter <- liftIO $ newMeter progress "B" 25 (renderNums binaryOpts 1) + showOutput + r <- a $ \n -> liftIO $ do + incrP progress n + displayMeter stdout meter + maybe noop (\m -> m n) combinemeterupdate + liftIO $ clearMeter stdout meter + return r + go _ = a (const noop) + +showSideAction :: String -> Annex () +showSideAction m = Annex.getState Annex.output >>= go + where + go st + | sideActionBlock st == StartBlock = do + p + let st' = st { sideActionBlock = InBlock } + Annex.changeState $ \s -> s { Annex.output = st' } + | sideActionBlock st == InBlock = return () + | otherwise = p + p = handle q $ putStrLn $ "(" ++ m ++ "...)" + +showStoringStateAction :: Annex () +showStoringStateAction = showSideAction "Recording state in git" + +{- Performs an action, supressing showSideAction messages. -} +doQuietSideAction :: Annex a -> Annex a +doQuietSideAction = doSideAction' InBlock + +{- Performs an action, that may call showSideAction multiple times. + - Only the first will be displayed. -} +doSideAction :: Annex a -> Annex a +doSideAction = doSideAction' StartBlock + +doSideAction' :: SideActionBlock -> Annex a -> Annex a +doSideAction' b a = do + o <- Annex.getState Annex.output + set $ o { sideActionBlock = b } + set o `after` a + where + set o = Annex.changeState $ \s -> s { Annex.output = o } + +showOutput :: Annex () +showOutput = handle q $ + putStr "\n" + +showLongNote :: String -> Annex () +showLongNote s = handle (JSON.note s) $ + putStrLn $ '\n' : indent s + +showEndOk :: Annex () +showEndOk = showEndResult True + +showEndFail :: Annex () +showEndFail = showEndResult False + +showEndResult :: Bool -> Annex () +showEndResult ok = handle (JSON.end ok) $ putStrLn msg + where + msg + | ok = "ok" + | otherwise = "failed" + +showErr :: (Show a) => a -> Annex () +showErr e = warning' $ "git-annex: " ++ show e + +warning :: String -> Annex () +warning = warning' . indent + +warning' :: String -> Annex () +warning' w = do + handle q $ putStr "\n" + liftIO $ do + hFlush stdout + hPutStrLn stderr w + +warningIO :: String -> IO () +warningIO w = do + putStr "\n" + hFlush stdout + hPutStrLn stderr w + +{- Displays a warning one time about a file the user specified not existing. -} +fileNotFound :: FilePath -> Annex () +fileNotFound file = do + st <- Annex.getState Annex.output + let shown = fileNotFoundShown st + when (S.notMember file shown) $ do + let shown' = S.insert file shown + let st' = st { fileNotFoundShown = shown' } + Annex.changeState $ \s -> s { Annex.output = st' } + liftIO $ hPutStrLn stderr $ unwords + [ "git-annex:", file, "not found" ] + +indent :: String -> String +indent = join "\n" . map (\l -> " " ++ l) . lines + +{- Shows a JSON fragment only when in json mode. -} +maybeShowJSON :: JSON a => [(String, a)] -> Annex () +maybeShowJSON v = handle (JSON.add v) q + +{- Shows a complete JSON value, only when in json mode. -} +showFullJSON :: JSON a => [(String, a)] -> Annex Bool +showFullJSON v = withOutputType $ liftIO . go + where + go JSONOutput = JSON.complete v >> return True + go _ = return False + +{- Performs an action that outputs nonstandard/customized output, and + - in JSON mode wraps its output in JSON.start and JSON.end, so it's + - a complete JSON document. + - This is only needed when showStart and showEndOk is not used. -} +showCustom :: String -> Annex Bool -> Annex () +showCustom command a = do + handle (JSON.start command Nothing) q + r <- a + handle (JSON.end r) q + +showHeader :: String -> Annex () +showHeader h = handle q $ + flushed $ putStr $ h ++ ": " + +showRaw :: String -> Annex () +showRaw s = handle q $ putStrLn s + +setupConsole :: IO () +setupConsole = do + s <- setFormatter + <$> streamHandler stderr DEBUG + <*> pure (simpleLogFormatter "[$time] $msg") + updateGlobalLogger rootLoggerName (setLevel NOTICE . setHandlers [s]) + {- This avoids ghc's output layer crashing on + - invalid encoded characters in + - filenames when printing them out. -} + fileEncoding stdout + fileEncoding stderr + +handle :: IO () -> IO () -> Annex () +handle json normal = withOutputType go + where + go NormalOutput = liftIO normal + go QuietOutput = q + go JSONOutput = liftIO $ flushed json + +q :: Monad m => m () +q = noop + +flushed :: IO () -> IO () +flushed a = a >> hFlush stdout + +withOutputType :: (OutputType -> Annex a) -> Annex a +withOutputType a = outputType <$> Annex.getState Annex.output >>= a diff --git a/Messages/JSON.hs b/Messages/JSON.hs new file mode 100644 index 0000000000..e262192a84 --- /dev/null +++ b/Messages/JSON.hs @@ -0,0 +1,40 @@ +{- git-annex JSON output + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Messages.JSON ( + start, + end, + note, + add, + complete +) where + +import Text.JSON + +import qualified Utility.JSONStream as Stream + +start :: String -> Maybe String -> IO () +start command file = + putStr $ Stream.start $ ("command", command) : filepart file + where + filepart Nothing = [] + filepart (Just f) = [("file", f)] + +end :: Bool -> IO () +end b = putStr $ Stream.add [("success", b)] ++ Stream.end + +note :: String -> IO () +note s = add [("note", s)] + +add :: JSON a => [(String, a)] -> IO () +add v = putStr $ Stream.add v + +complete :: JSON a => [(String, a)] -> IO () +complete v = putStr $ concat + [ Stream.start v + , Stream.end + ] diff --git a/Meters.hs b/Meters.hs new file mode 100644 index 0000000000..378e570a2a --- /dev/null +++ b/Meters.hs @@ -0,0 +1,40 @@ +{- git-annex meters + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Meters where + +import Common +import Types.Meters +import Utility.Observed + +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S + +{- Sends the content of a file to an action, updating the meter as it's + - consumed. -} +withMeteredFile :: FilePath -> MeterUpdate -> (L.ByteString -> IO a) -> IO a +withMeteredFile f meterupdate a = withBinaryFile f ReadMode $ \h -> + hGetContentsObserved h (meterupdate . toInteger) >>= a + +{- Sends the content of a file to a Handle, updating the meter as it's + - written. -} +streamMeteredFile :: FilePath -> MeterUpdate -> Handle -> IO () +streamMeteredFile f meterupdate h = withMeteredFile f meterupdate $ L.hPut h + +{- Writes a ByteString to a Handle, updating a meter as it's written. -} +meteredWrite :: MeterUpdate -> Handle -> L.ByteString -> IO () +meteredWrite meterupdate h = go . L.toChunks + where + go [] = return () + go (c:cs) = do + S.hPut h c + meterupdate $ toInteger $ S.length c + go cs + +meteredWriteFile :: MeterUpdate -> FilePath -> L.ByteString -> IO () +meteredWriteFile meterupdate f b = withBinaryFile f WriteMode $ \h -> + meteredWrite meterupdate h b diff --git a/NEWS b/NEWS new file mode 120000 index 0000000000..798088bec2 --- /dev/null +++ b/NEWS @@ -0,0 +1 @@ +debian/NEWS \ No newline at end of file diff --git a/Option.hs b/Option.hs new file mode 100644 index 0000000000..78fc43438b --- /dev/null +++ b/Option.hs @@ -0,0 +1,77 @@ +{- git-annex command-line options + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Option ( + common, + matcher, + flag, + field, + name, + ArgDescr(..), + OptDescr(..), +) where + +import System.Console.GetOpt +import System.Log.Logger + +import Common.Annex +import qualified Annex +import Types.Messages +import Limit +import Usage + +common :: [Option] +common = + [ Option [] ["force"] (NoArg (setforce True)) + "allow actions that may lose annexed data" + , Option ['F'] ["fast"] (NoArg (setfast True)) + "avoid slow operations" + , Option ['a'] ["auto"] (NoArg (setauto True)) + "automatic mode" + , Option ['q'] ["quiet"] (NoArg (Annex.setOutput QuietOutput)) + "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (Annex.setOutput NormalOutput)) + "allow verbose output (default)" + , Option ['j'] ["json"] (NoArg (Annex.setOutput JSONOutput)) + "enable JSON output" + , Option ['d'] ["debug"] (NoArg setdebug) + "show debug messages" + , Option ['b'] ["backend"] (ReqArg setforcebackend paramName) + "specify key-value backend to use" + ] + where + setforce v = Annex.changeState $ \s -> s { Annex.force = v } + setfast v = Annex.changeState $ \s -> s { Annex.fast = v } + setauto v = Annex.changeState $ \s -> s { Annex.auto = v } + setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v } + setdebug = liftIO $ updateGlobalLogger rootLoggerName $ setLevel DEBUG + +matcher :: [Option] +matcher = + [ longopt "not" "negate next option" + , longopt "and" "both previous and next option must match" + , longopt "or" "either previous or next option must match" + , shortopt "(" "open group of options" + , shortopt ")" "close group of options" + ] + where + longopt o = Option [] [o] $ NoArg $ addToken o + shortopt o = Option o [] $ NoArg $ addToken o + +{- An option that sets a flag. -} +flag :: String -> String -> String -> Option +flag short opt description = + Option short [opt] (NoArg (Annex.setFlag opt)) description + +{- An option that sets a field. -} +field :: String -> String -> String -> String -> Option +field short opt paramdesc description = + Option short [opt] (ReqArg (Annex.setField opt) paramdesc) description + +{- The flag or field name used for an option. -} +name :: Option -> String +name (Option _ o _ _) = Prelude.head o diff --git a/README b/README new file mode 100644 index 0000000000..ce67d68166 --- /dev/null +++ b/README @@ -0,0 +1,6 @@ +git-annex allows managing files with git, without checking the file +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. + +For documentation, see doc/ or diff --git a/Remote.hs b/Remote.hs new file mode 100644 index 0000000000..3f8f5f04ec --- /dev/null +++ b/Remote.hs @@ -0,0 +1,248 @@ +{- git-annex remotes + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote ( + Remote, + uuid, + name, + storeKey, + retrieveKeyFile, + retrieveKeyFileCheap, + removeKey, + hasKey, + hasKeyCheap, + whereisKey, + + remoteTypes, + remoteList, + enabledRemoteList, + specialRemote, + remoteMap, + uuidDescriptions, + byName, + byCost, + prettyPrintUUIDs, + prettyListUUIDs, + remoteFromUUID, + remotesWithUUID, + remotesWithoutUUID, + keyLocations, + keyPossibilities, + keyPossibilitiesTrusted, + nameToUUID, + showTriedRemotes, + showLocations, + forceTrust, + logStatus +) where + +import qualified Data.Map as M +import Text.JSON +import Text.JSON.Generic +import Data.Tuple + +import Common.Annex +import Types.Remote +import qualified Annex +import Annex.UUID +import Logs.UUID +import Logs.Trust +import Logs.Location hiding (logStatus) +import Remote.List + +{- Map from UUIDs of Remotes to a calculated value. -} +remoteMap :: (Remote -> a) -> Annex (M.Map UUID a) +remoteMap c = M.fromList . mapMaybe topair <$> remoteList + where + topair r = maybe Nothing (\u -> Just (u, c r)) (uuid r) + +{- Map of UUIDs of remotes and their descriptions. + - The names of Remotes are added to suppliment any description that has + - been set for a repository. -} +uuidDescriptions :: Annex (M.Map UUID String) +uuidDescriptions = M.unionWith addName <$> uuidMap <*> remoteMap name + +addName :: String -> String -> String +addName desc n + | desc == n = desc + | null desc = n + | otherwise = n ++ " (" ++ desc ++ ")" + +{- When a name is specified, looks up the remote matching that name. + - (Or it can be a UUID.) Only finds currently configured git remotes. -} +byName :: Maybe String -> Annex (Maybe Remote) +byName Nothing = return Nothing +byName (Just n) = either error Just <$> byName' n +byName' :: String -> Annex (Either String Remote) +byName' "" = return $ Left "no remote specified" +byName' n = handle . filter matching <$> remoteList + where + handle [] = Left $ "there is no available git remote named \"" ++ n ++ "\"" + handle (match:_) + | isNothing (uuid match) = Left $ "cannot determine uuid for " ++ name match + | otherwise = Right match + matching r = n == name r || toUUID n == uuid r + +{- Looks up a remote by name (or by UUID, or even by description), + - and returns its UUID. Finds even remotes that are not configured in + - .git/config. -} +nameToUUID :: String -> Annex UUID +nameToUUID "." = ensureUUID -- special case for current repo +nameToUUID "here" = ensureUUID +nameToUUID "" = error "no remote specified" +nameToUUID n = byName' n >>= go + where + go (Right r) = return $ fromMaybe (error $ "cannot determine uuid for " ++ name r) (uuid r) + go (Left e) = fromMaybe (error e) <$> bydescription + bydescription = do + m <- uuidMap + case M.lookup n $ transform swap m of + Just u -> return $ Just u + Nothing -> return $ byuuid m + byuuid m = case toUUID n of + Just u -> M.lookup u $ transform double m + Nothing -> Nothing + transform a = M.fromList . map a . M.toList + double (a, _) = (a, a) + +{- Pretty-prints a list of UUIDs of remotes, for human display. + - + - When JSON is enabled, also generates a machine-readable description + - of the UUIDs. -} +prettyPrintUUIDs :: String -> [UUID] -> Annex String +prettyPrintUUIDs desc uuids = do + hereu <- ensureUUID + m <- uuidDescriptions + maybeShowJSON [(desc, map (jsonify m hereu) uuids)] + return $ unwords $ map (\u -> "\t" ++ prettify m hereu u ++ "\n") uuids + where + finddescription m u = M.findWithDefault "" u m + prettify m hereu u + | not (null d) = fromUUID u ++ " -- " ++ d + | otherwise = fromUUID u + where + ishere = hereu == u + n = finddescription m u + d + | null n && ishere = "here" + | ishere = addName n "here" + | otherwise = n + jsonify m hereu u = toJSObject + [ ("uuid", toJSON $ fromUUID u) + , ("description", toJSON $ finddescription m u) + , ("here", toJSON $ hereu == u) + ] + +{- List of remote names and/or descriptions, for human display. -} +prettyListUUIDs :: [UUID] -> Annex [String] +prettyListUUIDs uuids = do + hereu <- ensureUUID + m <- uuidDescriptions + return $ map (\u -> prettify m hereu u) uuids + where + finddescription m u = M.findWithDefault "" u m + prettify m hereu u + | u == hereu = addName n "here" + | otherwise = n + where + n = finddescription m u + +{- Gets the remote associated with a UUID. + - There's no associated remote when this is the UUID of the local repo. -} +remoteFromUUID :: UUID -> Annex (Maybe Remote) +remoteFromUUID u = ifM ((==) u <$> getUUID) + ( return Nothing + , Just . fromMaybe (error "Unknown UUID") . M.lookup u <$> remoteMap id + ) + +{- Filters a list of remotes to ones that have the listed uuids. -} +remotesWithUUID :: [Remote] -> [UUID] -> [Remote] +remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs + +{- Filters a list of remotes to ones that do not have the listed uuids. -} +remotesWithoutUUID :: [Remote] -> [UUID] -> [Remote] +remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs + +{- List of repository UUIDs that the location log indicates may have a key. + - Dead repositories are excluded. -} +keyLocations :: Key -> Annex [UUID] +keyLocations key = trustExclude DeadTrusted =<< loggedLocations key + +{- Cost ordered lists of remotes that the location log indicates + - may have a key. + -} +keyPossibilities :: Key -> Annex [Remote] +keyPossibilities key = fst <$> keyPossibilities' key [] + +{- Cost ordered lists of remotes that the location log indicates + - may have a key. + - + - Also returns a list of UUIDs that are trusted to have the key + - (some may not have configured remotes). + -} +keyPossibilitiesTrusted :: Key -> Annex ([Remote], [UUID]) +keyPossibilitiesTrusted key = keyPossibilities' key =<< trustGet Trusted + +keyPossibilities' :: Key -> [UUID] -> Annex ([Remote], [UUID]) +keyPossibilities' key trusted = do + u <- getUUID + + -- uuids of all remotes that are recorded to have the key + validuuids <- filter (/= u) <$> keyLocations key + + -- note that validuuids is assumed to not have dups + let validtrusteduuids = validuuids `intersect` trusted + + -- remotes that match uuids that have the key + allremotes <- enabledRemoteList + let validremotes = remotesWithUUID allremotes validuuids + + return (sort validremotes, validtrusteduuids) + +{- Displays known locations of a key. -} +showLocations :: Key -> [UUID] -> String -> Annex () +showLocations key exclude nolocmsg = do + u <- getUUID + uuids <- keyLocations key + untrusteduuids <- trustGet UnTrusted + let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) + let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted) + ppuuidswanted <- Remote.prettyPrintUUIDs "wanted" uuidswanted + ppuuidsskipped <- Remote.prettyPrintUUIDs "skipped" uuidsskipped + showLongNote $ message ppuuidswanted ppuuidsskipped + where + filteruuids l x = filter (`notElem` x) l + message [] [] = nolocmsg + message rs [] = "Try making some of these repositories available:\n" ++ rs + message [] us = "Also these untrusted repositories may contain the file:\n" ++ us + message rs us = message rs [] ++ message [] us + +showTriedRemotes :: [Remote] -> Annex () +showTriedRemotes [] = noop +showTriedRemotes remotes = + showLongNote $ "Unable to access these remotes: " ++ + join ", " (map name remotes) + +forceTrust :: TrustLevel -> String -> Annex () +forceTrust level remotename = do + u <- nameToUUID remotename + Annex.changeState $ \s -> + s { Annex.forcetrust = M.insert u level (Annex.forcetrust s) } + +{- Used to log a change in a remote's having a key. The change is logged + - in the local repo, not on the remote. The process of transferring the + - key to the remote, or removing the key from it *may* log the change + - on the remote, but this cannot always be relied on. -} +logStatus :: Remote -> Key -> LogStatus -> Annex () +logStatus remote key = logChange key (uuid remote) + +{- Orders remotes by cost, with ones with the lowest cost grouped together. -} +byCost :: [Remote] -> [[Remote]] +byCost = map snd . sort . M.toList . costmap + where + costmap = M.fromListWith (++) . map costpair + costpair r = (cost r, [r]) diff --git a/Remote/Bup.hs b/Remote/Bup.hs new file mode 100644 index 0000000000..602bda70c7 --- /dev/null +++ b/Remote/Bup.hs @@ -0,0 +1,276 @@ +{- Using bup as a remote. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Bup (remote) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M +import System.Process + +import Common.Annex +import Types.Remote +import Types.Key +import qualified Git +import qualified Git.Command +import qualified Git.Config +import qualified Git.Construct +import qualified Git.Ref +import Config +import Remote.Helper.Ssh +import Remote.Helper.Special +import Remote.Helper.Encryptable +import Crypto +import Data.ByteString.Lazy.UTF8 (fromString) +import Data.Digest.Pure.SHA +import Utility.UserInfo +import Annex.Content + +type BupRepo = String + +remote :: RemoteType +remote = RemoteType { + typename = "bup", + enumerate = findSpecialRemotes "buprepo", + generate = gen, + setup = bupSetup +} + +gen :: Git.Repo -> Maybe UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r mu c gc = do + bupr <- liftIO $ bup2GitRemote buprepo + cst <- remoteCost gc $ + if bupLocal buprepo + then semiCheapRemoteCost + else expensiveRemoteCost + (mu', bupr') <- getBupUUID bupr mu + + let new = Remote + { uuid = mu' + , cost = cst + , name = Git.repoDescribe r + , storeKey = store new buprepo + , retrieveKeyFile = retrieve buprepo + , retrieveKeyFileCheap = retrieveCheap buprepo + , removeKey = remove + , hasKey = checkPresent r bupr' + , hasKeyCheap = bupLocal buprepo + , whereisKey = Nothing + , config = c + , repo = r + , gitconfig = gc + , localpath = if bupLocal buprepo && not (null buprepo) + then Just buprepo + else Nothing + , remotetype = remote + , readonly = False + } + return $ encryptableRemote c + (storeEncrypted new buprepo) + (retrieveEncrypted buprepo) + new + where + buprepo = fromMaybe (error "missing buprepo") $ remoteAnnexBupRepo gc + +bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +bupSetup u c = do + -- verify configuration is sane + let buprepo = fromMaybe (error "Specify buprepo=") $ + M.lookup "buprepo" c + c' <- encryptionSetup c + + -- bup init will create the repository. + -- (If the repository already exists, bup init again appears safe.) + showAction "bup init" + unlessM (bup "init" buprepo []) $ error "bup init failed" + + storeBupUUID u buprepo + + -- The buprepo is stored in git config, as well as this repo's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' "buprepo" buprepo + + return c' + +bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam] +bupParams command buprepo params = + Param command : [Param "-r", Param buprepo] ++ params + +bup :: String -> BupRepo -> [CommandParam] -> Annex Bool +bup command buprepo params = do + showOutput -- make way for bup output + liftIO $ boolSystem "bup" $ bupParams command buprepo params + +pipeBup :: [CommandParam] -> Maybe Handle -> Maybe Handle -> IO Bool +pipeBup params inh outh = do + p <- runProcess "bup" (toCommand params) + Nothing Nothing inh outh Nothing + ok <- waitForProcess p + case ok of + ExitSuccess -> return True + _ -> return False + +bupSplitParams :: Remote -> BupRepo -> Key -> [CommandParam] -> Annex [CommandParam] +bupSplitParams r buprepo k src = do + let os = map Param $ remoteAnnexBupSplitOptions $ gitconfig r + showOutput -- make way for bup output + return $ bupParams "split" buprepo + (os ++ [Param "-n", Param (bupRef k)] ++ src) + +store :: Remote -> BupRepo -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store r buprepo k _f _p = sendAnnex k (rollback k buprepo) $ \src -> do + params <- bupSplitParams r buprepo k [File src] + liftIO $ boolSystem "bup" params + +storeEncrypted :: Remote -> BupRepo -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool +storeEncrypted r buprepo (cipher, enck) k _p = + sendAnnex k (rollback enck buprepo) $ \src -> do + params <- bupSplitParams r buprepo enck [] + liftIO $ catchBoolIO $ + encrypt cipher (feedFile src) $ \h -> + pipeBup params (Just h) Nothing + +retrieve :: BupRepo -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve buprepo k _f d = do + let params = bupParams "join" buprepo [Param $ bupRef k] + liftIO $ catchBoolIO $ do + tofile <- openFile d WriteMode + pipeBup params Nothing (Just tofile) + +retrieveCheap :: BupRepo -> Key -> FilePath -> Annex Bool +retrieveCheap _ _ _ = return False + +retrieveEncrypted :: BupRepo -> (Cipher, Key) -> Key -> FilePath -> Annex Bool +retrieveEncrypted buprepo (cipher, enck) _ f = liftIO $ catchBoolIO $ + withHandle StdoutHandle createProcessSuccess p $ \h -> do + decrypt cipher (\toh -> L.hPut toh =<< L.hGetContents h) $ + readBytes $ L.writeFile f + return True + where + params = bupParams "join" buprepo [Param $ bupRef enck] + p = proc "bup" $ toCommand params + +remove :: Key -> Annex Bool +remove _ = do + warning "content cannot be removed from bup remote" + return False + +{- Cannot revert having stored a key in bup, but at least the data for the + - key will be used for deltaing data of other keys stored later. + - + - We can, however, remove the git branch that bup created for the key. + -} +rollback :: Key -> BupRepo -> Annex () +rollback k bupr = go =<< liftIO (bup2GitRemote bupr) + where + go r + | Git.repoIsUrl r = void $ onBupRemote r boolSystem "git" params + | otherwise = void $ liftIO $ catchMaybeIO $ + boolSystem "git" $ Git.Command.gitCommandLine params r + params = [ Params "branch -D", Param (bupRef k) ] + +{- Bup does not provide a way to tell if a given dataset is present + - in a bup repository. One way it to check if the git repository has + - a branch matching the name (as created by bup split -n). + -} +checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either String Bool) +checkPresent r bupr k + | Git.repoIsUrl bupr = do + showAction $ "checking " ++ Git.repoDescribe r + ok <- onBupRemote bupr boolSystem "git" params + return $ Right ok + | otherwise = liftIO $ catchMsgIO $ + boolSystem "git" $ Git.Command.gitCommandLine params bupr + where + params = + [ Params "show-ref --quiet --verify" + , Param $ "refs/heads/" ++ bupRef k + ] + +{- Store UUID in the annex.uuid setting of the bup repository. -} +storeBupUUID :: UUID -> BupRepo -> Annex () +storeBupUUID u buprepo = do + r <- liftIO $ bup2GitRemote buprepo + if Git.repoIsUrl r + then do + showAction "storing uuid" + unlessM (onBupRemote r boolSystem "git" + [Params $ "config annex.uuid " ++ v]) $ + error "ssh failed" + else liftIO $ do + r' <- Git.Config.read r + let olduuid = Git.Config.get "annex.uuid" "" r' + when (olduuid == "") $ + Git.Command.run + [ Param "config" + , Param "annex.uuid" + , Param v + ] r' + where + v = fromUUID u + +onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a +onBupRemote r a command params = do + let dir = shellEscape (Git.repoPath r) + sshparams <- sshToRepo r [Param $ + "cd " ++ dir ++ " && " ++ unwords (command : toCommand params)] + liftIO $ a "ssh" sshparams + +{- Allow for bup repositories on removable media by checking + - local bup repositories to see if they are available, and getting their + - uuid (which may be different from the stored uuid for the bup remote). + - + - If a bup repository is not available, returns Nothing. + - This will cause checkPresent to indicate nothing from the bup remote + - is known to be present. + - + - Also, returns a version of the repo with config read, if it is local. + -} +getBupUUID :: Git.Repo -> Maybe UUID -> Annex (Maybe UUID, Git.Repo) +getBupUUID r u + | Git.repoIsUrl r = return (u, r) + | otherwise = liftIO $ do + ret <- tryIO $ Git.Config.read r + case ret of + Right r' -> return (toUUID $ Git.Config.get "annex.uuid" "" r', r') + Left _ -> return (Nothing, r) + +{- Converts a bup remote path spec into a Git.Repo. There are some + - differences in path representation between git and bup. -} +bup2GitRemote :: BupRepo -> IO Git.Repo +bup2GitRemote "" = do + -- bup -r "" operates on ~/.bup + h <- myHomeDir + Git.Construct.fromAbsPath $ h ".bup" +bup2GitRemote r + | bupLocal r = + if "/" `isPrefixOf` r + then Git.Construct.fromAbsPath r + else error "please specify an absolute path" + | otherwise = Git.Construct.fromUrl $ "ssh://" ++ host ++ slash dir + where + bits = split ":" r + host = Prelude.head bits + dir = join ":" $ drop 1 bits + -- "host:~user/dir" is not supported specially by bup; + -- "host:dir" is relative to the home directory; + -- "host:" goes in ~/.bup + slash d + | null d = "/~/.bup" + | "/" `isPrefixOf` d = d + | otherwise = "/~/" ++ d + +{- Converts a key into a git ref name, which bup-split -n will use to point + - to it. -} +bupRef :: Key -> String +bupRef k + | Git.Ref.legal True shown = shown + | otherwise = "git-annex-" ++ showDigest (sha256 (fromString shown)) + where + shown = key2file k + +bupLocal :: BupRepo -> Bool +bupLocal = notElem ':' diff --git a/Remote/Directory.hs b/Remote/Directory.hs new file mode 100644 index 0000000000..1bc4b8cfaa --- /dev/null +++ b/Remote/Directory.hs @@ -0,0 +1,230 @@ +{- A "remote" that is just a filesystem directory. + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Directory (remote) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S +import qualified Data.Map as M +import qualified Control.Exception as E +import Data.Int + +import Common.Annex +import Types.Remote +import qualified Git +import Config +import Utility.FileMode +import Remote.Helper.Special +import Remote.Helper.Encryptable +import Remote.Helper.Chunked +import Crypto +import Annex.Content +import Meters + +remote :: RemoteType +remote = RemoteType { + typename = "directory", + enumerate = findSpecialRemotes "directory", + generate = gen, + setup = directorySetup +} + +gen :: Git.Repo -> (Maybe UUID) -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r mu c gc = do + cst <- remoteCost gc cheapRemoteCost + let chunksize = chunkSize c + return $ encryptableRemote c + (storeEncrypted dir chunksize) + (retrieveEncrypted dir chunksize) + Remote { + uuid = mu, + cost = cst, + name = Git.repoDescribe r, + storeKey = store dir chunksize, + retrieveKeyFile = retrieve dir chunksize, + retrieveKeyFileCheap = retrieveCheap dir chunksize, + removeKey = remove dir, + hasKey = checkPresent dir chunksize, + hasKeyCheap = True, + whereisKey = Nothing, + config = M.empty, + repo = r, + gitconfig = gc, + localpath = Just dir, + readonly = False, + remotetype = remote + } + where + dir = fromMaybe (error "missing directory") $ remoteAnnexDirectory gc + +directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig +directorySetup u c = do + -- verify configuration is sane + let dir = fromMaybe (error "Specify directory=") $ + M.lookup "directory" c + liftIO $ unlessM (doesDirectoryExist dir) $ + error $ "Directory does not exist: " ++ dir + c' <- encryptionSetup c + + -- The directory is stored in git config, not in this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' "directory" dir + return $ M.delete "directory" c' + +{- Locations to try to access a given Key in the Directory. + - We try more than since we used to write to different hash directories. -} +locations :: FilePath -> Key -> [FilePath] +locations d k = map (d ) (keyPaths k) + +{- Directory where the file(s) for a key are stored. -} +storeDir :: FilePath -> Key -> FilePath +storeDir d k = addTrailingPathSeparator $ d hashDirLower k keyFile k + +{- Where we store temporary data for a key as it's being uploaded. -} +tmpDir :: FilePath -> Key -> FilePath +tmpDir d k = addTrailingPathSeparator $ d "tmp" keyFile k + +withCheckedFiles :: (FilePath -> IO Bool) -> ChunkSize -> FilePath -> Key -> ([FilePath] -> IO Bool) -> IO Bool +withCheckedFiles _ _ [] _ _ = return False +withCheckedFiles check Nothing d k a = go $ locations d k + where + go [] = return False + go (f:fs) = ifM (check f) ( a [f] , go fs ) +withCheckedFiles check (Just _) d k a = go $ locations d k + where + go [] = return False + go (f:fs) = do + let chunkcount = f ++ chunkCount + ifM (check chunkcount) + ( do + chunks <- listChunks f <$> readFile chunkcount + ifM (all id <$> mapM check chunks) + ( a chunks , return False ) + , go fs + ) + +withStoredFiles :: ChunkSize -> FilePath -> Key -> ([FilePath] -> IO Bool) -> IO Bool +withStoredFiles = withCheckedFiles doesFileExist + +store :: FilePath -> ChunkSize -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store d chunksize k _f p = sendAnnex k (void $ remove d k) $ \src -> + metered (Just p) k $ \meterupdate -> + storeHelper d chunksize k $ \dests -> + case chunksize of + Nothing -> do + let dest = Prelude.head dests + meteredWriteFile meterupdate dest + =<< L.readFile src + return [dest] + Just _ -> + storeSplit meterupdate chunksize dests + =<< L.readFile src + +storeEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool +storeEncrypted d chunksize (cipher, enck) k p = sendAnnex k (void $ remove d enck) $ \src -> + metered (Just p) k $ \meterupdate -> + storeHelper d chunksize enck $ \dests -> + encrypt cipher (feedFile src) $ readBytes $ \b -> + case chunksize of + Nothing -> do + let dest = Prelude.head dests + meteredWriteFile meterupdate dest b + return [dest] + Just _ -> storeSplit meterupdate chunksize dests b + +{- Splits a ByteString into chunks and writes to dests, obeying configured + - chunk size (not to be confused with the L.ByteString chunk size). + - Note: Must always write at least one file, even for empty ByteString. -} +storeSplit :: MeterUpdate -> ChunkSize -> [FilePath] -> L.ByteString -> IO [FilePath] +storeSplit _ Nothing _ _ = error "bad storeSplit call" +storeSplit _ _ [] _ = error "bad storeSplit call" +storeSplit meterupdate (Just chunksize) alldests@(firstdest:_) b + | L.null b = do + -- must always write at least one file, even for empty + L.writeFile firstdest b + return [firstdest] + | otherwise = storeSplit' meterupdate chunksize alldests (L.toChunks b) [] +storeSplit' :: MeterUpdate -> Int64 -> [FilePath] -> [S.ByteString] -> [FilePath] -> IO [FilePath] +storeSplit' _ _ [] _ _ = error "ran out of dests" +storeSplit' _ _ _ [] c = return $ reverse c +storeSplit' meterupdate chunksize (d:dests) bs c = do + bs' <- E.bracket (openFile d WriteMode) hClose (feed chunksize bs) + storeSplit' meterupdate chunksize dests bs' (d:c) + where + feed _ [] _ = return [] + feed sz (l:ls) h = do + let s = fromIntegral $ S.length l + if s <= sz || sz == chunksize + then do + S.hPut h l + meterupdate $ toInteger s + feed (sz - s) ls h + else return (l:ls) + +storeHelper :: FilePath -> ChunkSize -> Key -> ([FilePath] -> IO [FilePath]) -> Annex Bool +storeHelper d chunksize key storer = check <&&> go + where + tmpdir = tmpDir d key + destdir = storeDir d key + {- The size is not exactly known when encrypting the key; + - this assumes that at least the size of the key is + - needed as free space. -} + check = do + liftIO $ createDirectoryIfMissing True tmpdir + checkDiskSpace (Just tmpdir) key 0 + go = liftIO $ catchBoolIO $ + storeChunks key tmpdir destdir chunksize storer recorder finalizer + finalizer tmp dest = do + void $ tryIO $ allowWrite dest -- may already exist + void $ tryIO $ removeDirectoryRecursive dest -- or not exist + createDirectoryIfMissing True (parentDir dest) + renameDirectory tmp dest + -- may fail on some filesystems + void $ tryIO $ do + mapM_ preventWrite =<< dirContents dest + preventWrite dest + recorder f s = do + void $ tryIO $ allowWrite f + writeFile f s + void $ tryIO $ preventWrite f + +retrieve :: FilePath -> ChunkSize -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve d chunksize k _ f = metered Nothing k $ \meterupdate -> + liftIO $ withStoredFiles chunksize d k $ \files -> + catchBoolIO $ do + meteredWriteFileChunks meterupdate f files $ L.readFile + return True + +retrieveEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> FilePath -> Annex Bool +retrieveEncrypted d chunksize (cipher, enck) k f = metered Nothing k $ \meterupdate -> + liftIO $ withStoredFiles chunksize d enck $ \files -> + catchBoolIO $ do + decrypt cipher (feeder files) $ + readBytes $ meteredWriteFile meterupdate f + return True + where + feeder files h = forM_ files $ \file -> L.hPut h =<< L.readFile file + +retrieveCheap :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool +retrieveCheap _ (Just _) _ _ = return False -- no cheap retrieval for chunks +retrieveCheap d _ k f = liftIO $ withStoredFiles Nothing d k go + where + go [file] = catchBoolIO $ createSymbolicLink file f >> return True + go _files = return False + +remove :: FilePath -> Key -> Annex Bool +remove d k = liftIO $ do + void $ tryIO $ allowWrite dir + catchBoolIO $ do + removeDirectoryRecursive dir + return True + where + dir = storeDir d k + +checkPresent :: FilePath -> ChunkSize -> Key -> Annex (Either String Bool) +checkPresent d chunksize k = liftIO $ catchMsgIO $ withStoredFiles chunksize d k $ + const $ return True -- withStoredFiles checked that it exists diff --git a/Remote/Git.hs b/Remote/Git.hs new file mode 100644 index 0000000000..82fff1a32d --- /dev/null +++ b/Remote/Git.hs @@ -0,0 +1,458 @@ +{- Standard git remotes. + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Git ( + remote, + configRead, + repoAvail, +) where + +import qualified Data.Map as M +import Control.Exception.Extensible + +import Common.Annex +import Utility.CopyFile +import Utility.Rsync +import Remote.Helper.Ssh +import Types.Remote +import Types.GitConfig +import qualified Git +import qualified Git.Config +import qualified Git.Construct +import qualified Git.Command +import qualified Annex +import Logs.Presence +import Logs.Transfer +import Annex.UUID +import Annex.Exception +import qualified Annex.Content +import qualified Annex.BranchState +import qualified Annex.Branch +import qualified Utility.Url as Url +import Utility.TempFile +import Config +import Init +import Types.Key +import qualified Fields +import Logs.Location + +import Control.Concurrent +import Control.Concurrent.MSampleVar +import System.Process (std_in, std_err) + +remote :: RemoteType +remote = RemoteType { + typename = "git", + enumerate = list, + generate = gen, + setup = error "not supported" +} + +list :: Annex [Git.Repo] +list = do + c <- fromRepo Git.config + rs <- mapM (tweakurl c) =<< fromRepo Git.remotes + mapM configRead rs + where + annexurl n = "remote." ++ n ++ ".annexurl" + tweakurl c r = do + let n = fromJust $ Git.remoteName r + case M.lookup (annexurl n) c of + Nothing -> return r + Just url -> inRepo $ \g -> + Git.Construct.remoteNamed n $ + Git.Construct.fromRemoteLocation url g + +{- It's assumed to be cheap to read the config of non-URL remotes, so this is + - done each time git-annex is run in a way that uses remotes. + - + - Conversely, the config of an URL remote is only read when there is no + - cached UUID value. -} +configRead :: Git.Repo -> Annex Git.Repo +configRead r = do + g <- fromRepo id + let c = extractRemoteGitConfig g (Git.repoDescribe r) + mu <- getRepoUUID r + case (repoCheap r, remoteAnnexIgnore c, mu) of + (_, True, _) -> return r + (True, _, _) -> tryGitConfigRead r + (False, _, Nothing) -> tryGitConfigRead r + _ -> return r + +repoCheap :: Git.Repo -> Bool +repoCheap = not . Git.repoIsUrl + +gen :: Git.Repo -> Maybe UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r mu _ gc = go <$> remoteCost gc defcst + where + defcst = if repoCheap r then cheapRemoteCost else expensiveRemoteCost + go cst = new + where + new = Remote + { uuid = mu + , cost = cst + , name = Git.repoDescribe r + , storeKey = copyToRemote new + , retrieveKeyFile = copyFromRemote new + , retrieveKeyFileCheap = copyFromRemoteCheap new + , removeKey = dropKey new + , hasKey = inAnnex r + , hasKeyCheap = repoCheap r + , whereisKey = Nothing + , config = M.empty + , localpath = if Git.repoIsLocal r || Git.repoIsLocalUnknown r + then Just $ Git.repoPath r + else Nothing + , repo = r + , gitconfig = gc + , readonly = Git.repoIsHttp r + , remotetype = remote + } + +{- Checks relatively inexpensively if a repository is available for use. -} +repoAvail :: Git.Repo -> Annex Bool +repoAvail r + | Git.repoIsHttp r = return True + | Git.repoIsUrl r = return True + | Git.repoIsLocalUnknown r = return False + | otherwise = liftIO $ catchBoolIO $ onLocal r $ return True + +{- Avoids performing an action on a local repository that's not usable. + - Does not check that the repository is still available on disk. -} +guardUsable :: Git.Repo -> a -> Annex a -> Annex a +guardUsable r onerr a + | Git.repoIsLocalUnknown r = return onerr + | otherwise = a + +{- Tries to read the config for a specified remote, updates state, and + - returns the updated repo. -} +tryGitConfigRead :: Git.Repo -> Annex Git.Repo +tryGitConfigRead r + | not $ M.null $ Git.config r = return r -- already read + | Git.repoIsSsh r = store $ do + v <- onRemote r (pipedsshconfig, Left undefined) "configlist" [] [] + case (v, Git.remoteName r) of + (Right r', _) -> return r' + (Left _, Just n) -> do + {- Is this remote just not available, or does + - it not have git-annex-shell? + - Find out by trying to fetch from the remote. -} + whenM (inRepo $ Git.Command.runBool [Param "fetch", Param "--quiet", Param n]) $ do + let k = "remote." ++ n ++ ".annex-ignore" + warning $ "Remote " ++ n ++ " does not have git-annex installed; setting " ++ k + inRepo $ Git.Command.run [Param "config", Param k, Param "true"] + return r + _ -> return r + | Git.repoIsHttp r = do + headers <- getHttpHeaders + store $ safely $ geturlconfig headers + | Git.repoIsUrl r = return r + | otherwise = store $ safely $ onLocal r $ do + ensureInitialized + Annex.getState Annex.repo + where + -- Reading config can fail due to IO error or + -- for other reasons; catch all possible exceptions. + safely a = either (const $ return r) return + =<< liftIO (try a :: IO (Either SomeException Git.Repo)) + + pipedconfig cmd params = + withHandle StdoutHandle createProcessSuccess p $ \h -> do + fileEncoding h + val <- hGetContentsStrict h + r' <- Git.Config.store val r + when (isNothing (getUncachedUUID r') && not (null val)) $ do + warningIO $ "Failed to get annex.uuid configuration of repository " ++ Git.repoDescribe r + warningIO $ "Instead, got: " ++ show val + warningIO $ "This is unexpected; please check the network transport!" + return r' + where + p = proc cmd $ toCommand params + + pipedsshconfig cmd params = + liftIO (try (pipedconfig cmd params) :: IO (Either SomeException Git.Repo)) + + geturlconfig headers = do + s <- Url.get (Git.repoLocation r ++ "/config") headers + withTempFile "git-annex.tmp" $ \tmpfile h -> do + hPutStr h s + hClose h + safely $ pipedconfig "git" [Param "config", Param "--null", Param "--list", Param "--file", File tmpfile] + + store = observe $ \r' -> do + g <- gitRepo + let l = Git.remotes g + let g' = g { Git.remotes = exchange l r' } + Annex.changeState $ \s -> s { Annex.repo = g' } + + exchange [] _ = [] + exchange (old:ls) new + | Git.remoteName old == Git.remoteName new = + new : exchange ls new + | otherwise = + old : exchange ls new + +{- Checks if a given remote has the content for a key inAnnex. + - If the remote cannot be accessed, or if it cannot determine + - whether it has the content, returns a Left error message. + -} +inAnnex :: Git.Repo -> Key -> Annex (Either String Bool) +inAnnex r key + | Git.repoIsHttp r = checkhttp =<< getHttpHeaders + | Git.repoIsUrl r = checkremote + | otherwise = checklocal + where + checkhttp headers = liftIO $ go undefined $ keyUrls r key + where + go e [] = return $ Left e + go _ (u:us) = do + res <- catchMsgIO $ + Url.check u headers (keySize key) + case res of + Left e -> go e us + v -> return v + checkremote = do + showAction $ "checking " ++ Git.repoDescribe r + onRemote r (check, unknown) "inannex" [Param (key2file key)] [] + where + check c p = dispatch <$> safeSystem c p + dispatch ExitSuccess = Right True + dispatch (ExitFailure 1) = Right False + dispatch _ = unknown + checklocal = guardUsable r unknown $ dispatch <$> check + where + check = liftIO $ catchMsgIO $ onLocal r $ + Annex.Content.inAnnexSafe key + dispatch (Left e) = Left e + dispatch (Right (Just b)) = Right b + dispatch (Right Nothing) = unknown + unknown = Left $ "unable to check " ++ Git.repoDescribe r + +{- Runs an action on a local repository inexpensively, by making an annex + - monad using that repository. -} +onLocal :: Git.Repo -> Annex a -> IO a +onLocal r a = do + s <- Annex.new r + Annex.eval s $ do + -- No need to update the branch; its data is not used + -- for anything onLocal is used to do. + Annex.BranchState.disableUpdate + a + +keyUrls :: Git.Repo -> Key -> [String] +keyUrls r key = map tourl (annexLocations key) + where + tourl l = Git.repoLocation r ++ "/" ++ l + +dropKey :: Remote -> Key -> Annex Bool +dropKey r key + | not $ Git.repoIsUrl (repo r) = + guardUsable (repo r) False $ commitOnCleanup r $ liftIO $ onLocal (repo r) $ do + ensureInitialized + whenM (Annex.Content.inAnnex key) $ do + Annex.Content.lockContent key $ + Annex.Content.removeAnnex key + logStatus key InfoMissing + Annex.Content.saveState True + return True + | Git.repoIsHttp (repo r) = error "dropping from http repo not supported" + | otherwise = commitOnCleanup r $ onRemote (repo r) (boolSystem, False) "dropkey" + [ Params "--quiet --force" + , Param $ key2file key + ] + [] + +{- Tries to copy a key's content from a remote's annex to a file. -} +copyFromRemote :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool +copyFromRemote r key file dest + | not $ Git.repoIsUrl (repo r) = guardUsable (repo r) False $ do + let params = rsyncParams r + u <- ensureUUID + -- run copy from perspective of remote + liftIO $ onLocal (repo r) $ do + ensureInitialized + v <- Annex.Content.prepSendAnnex key + case v of + Nothing -> return False + Just (object, checksuccess) -> + upload u key file noRetry + (rsyncOrCopyFile params object dest) + <&&> checksuccess + | Git.repoIsSsh (repo r) = feedprogressback $ \feeder -> + rsyncHelper (Just feeder) + =<< rsyncParamsRemote r Download key dest file + | Git.repoIsHttp (repo r) = Annex.Content.downloadUrl (keyUrls (repo r) key) dest + | otherwise = error "copying from non-ssh, non-http repo not supported" + where + {- Feed local rsync's progress info back to the remote, + - by forking a feeder thread that runs + - git-annex-shell transferinfo at the same time + - git-annex-shell sendkey is running. + - + - Note that it actually waits for rsync to indicate + - progress before starting transferinfo, in order + - to ensure ssh connection caching works and reuses + - the connection set up for the sendkey. + - + - Also note that older git-annex-shell does not support + - transferinfo, so stderr is dropped and failure ignored. + -} + feedprogressback a = do + u <- ensureUUID + let fields = (Fields.remoteUUID, fromUUID u) + : maybe [] (\f -> [(Fields.associatedFile, f)]) file + Just (cmd, params) <- git_annex_shell (repo r) "transferinfo" + [Param $ key2file key] fields + v <- liftIO $ newEmptySV + tid <- liftIO $ forkIO $ void $ tryIO $ do + bytes <- readSV v + p <- createProcess $ + (proc cmd (toCommand params)) + { std_in = CreatePipe + , std_err = CreatePipe + } + hClose $ stderrHandle p + let h = stdinHandle p + let send b = do + hPutStrLn h $ show b + hFlush h + send bytes + forever $ + send =<< readSV v + let feeder = writeSV v + bracketIO noop (const $ tryIO $ killThread tid) (a feeder) + +copyFromRemoteCheap :: Remote -> Key -> FilePath -> Annex Bool +copyFromRemoteCheap r key file + | not $ Git.repoIsUrl (repo r) = guardUsable (repo r) False $ do + loc <- liftIO $ gitAnnexLocation key (repo r) + liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True + | Git.repoIsSsh (repo r) = + ifM (Annex.Content.preseedTmp key file) + ( copyFromRemote r key Nothing file + , return False + ) + | otherwise = return False + +{- Tries to copy a key's content to a remote's annex. -} +copyToRemote :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +copyToRemote r key file p + | not $ Git.repoIsUrl (repo r) = + guardUsable (repo r) False $ commitOnCleanup r $ + copylocal =<< Annex.Content.prepSendAnnex key + | Git.repoIsSsh (repo r) = commitOnCleanup r $ + Annex.Content.sendAnnex key noop $ \object -> + rsyncHelper (Just p) =<< rsyncParamsRemote r Upload key object file + | otherwise = error "copying to non-ssh repo not supported" + where + copylocal Nothing = return False + copylocal (Just (object, checksuccess)) = do + let params = rsyncParams r + u <- ensureUUID + -- run copy from perspective of remote + liftIO $ onLocal (repo r) $ ifM (Annex.Content.inAnnex key) + ( return False + , do + ensureInitialized + download u key file noRetry $ + Annex.Content.saveState True `after` + Annex.Content.getViaTmpChecked checksuccess key + (\d -> rsyncOrCopyFile params object d p) + ) + +rsyncHelper :: Maybe MeterUpdate -> [CommandParam] -> Annex Bool +rsyncHelper callback params = do + showOutput -- make way for progress bar + ifM (liftIO $ (maybe rsync rsyncProgress callback) params) + ( return True + , do + showLongNote "rsync failed -- run git annex again to resume file transfer" + return False + ) + +{- Copys a file with rsync unless both locations are on the same + - filesystem. Then cp could be faster. -} +rsyncOrCopyFile :: [CommandParam] -> FilePath -> FilePath -> MeterUpdate -> Annex Bool +rsyncOrCopyFile rsyncparams src dest p = + ifM (sameDeviceIds src dest) (docopy, dorsync) + where + sameDeviceIds a b = (==) <$> (getDeviceId a) <*> (getDeviceId b) + getDeviceId f = deviceID <$> liftIO (getFileStatus $ parentDir f) + dorsync = rsyncHelper (Just p) $ + rsyncparams ++ [Param src, Param dest] + docopy = liftIO $ bracket + (forkIO $ watchfilesize 0) + (void . tryIO . killThread) + (const $ copyFileExternal src dest) + watchfilesize oldsz = do + threadDelay 500000 -- 0.5 seconds + v <- catchMaybeIO $ + fromIntegral . fileSize + <$> getFileStatus dest + case v of + Just sz + | sz /= oldsz -> do + p sz + watchfilesize sz + _ -> watchfilesize oldsz + +{- Generates rsync parameters that ssh to the remote and asks it + - to either receive or send the key's content. -} +rsyncParamsRemote :: Remote -> Direction -> Key -> FilePath -> AssociatedFile -> Annex [CommandParam] +rsyncParamsRemote r direction key file afile = do + u <- ensureUUID + direct <- isDirect + let fields = (Fields.remoteUUID, fromUUID u) + : (Fields.direct, if direct then "1" else "") + : maybe [] (\f -> [(Fields.associatedFile, f)]) afile + Just (shellcmd, shellparams) <- git_annex_shell (repo r) + (if direction == Download then "sendkey" else "recvkey") + [ Param $ key2file key ] + fields + -- Convert the ssh command into rsync command line. + let eparam = rsyncShell (Param shellcmd:shellparams) + let o = rsyncParams r + if direction == Download + then return $ o ++ rsyncopts eparam dummy (File file) + else return $ o ++ rsyncopts eparam (File file) dummy + where + rsyncopts ps source dest + | end ps == [dashdash] = ps ++ [source, dest] + | otherwise = ps ++ [dashdash, source, dest] + dashdash = Param "--" + {- The rsync shell parameter controls where rsync + - goes, so the source/dest parameter can be a dummy value, + - that just enables remote rsync mode. + - For maximum compatability with some patched rsyncs, + - the dummy value needs to still contain a hostname, + - even though this hostname will never be used. -} + dummy = Param "dummy:" + +-- --inplace to resume partial files +rsyncParams :: Remote -> [CommandParam] +rsyncParams r = [Params "--progress --inplace"] ++ + map Param (remoteAnnexRsyncOptions $ gitconfig r) + +commitOnCleanup :: Remote -> Annex a -> Annex a +commitOnCleanup r a = go `after` a + where + go = Annex.addCleanup (Git.repoLocation $ repo r) cleanup + cleanup + | not $ Git.repoIsUrl (repo r) = liftIO $ onLocal (repo r) $ + doQuietSideAction $ + Annex.Branch.commit "update" + | otherwise = void $ do + Just (shellcmd, shellparams) <- + git_annex_shell (repo r) "commit" [] [] + + -- Throw away stderr, since the remote may not + -- have a new enough git-annex shell to + -- support committing. + liftIO $ catchMaybeIO $ do + withQuietOutput createProcessSuccess $ + proc shellcmd $ + toCommand shellparams diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs new file mode 100644 index 0000000000..f476967770 --- /dev/null +++ b/Remote/Glacier.hs @@ -0,0 +1,295 @@ +{- Amazon Glacier remotes. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Glacier (remote, jobList) where + +import qualified Data.Map as M +import qualified Data.Text as T +import System.Environment + +import Common.Annex +import Types.Remote +import Types.Key +import qualified Git +import Config +import Remote.Helper.Special +import Remote.Helper.Encryptable +import qualified Remote.Helper.AWS as AWS +import Crypto +import Creds +import Meters +import qualified Annex +import Annex.Content + +import System.Process + +type Vault = String +type Archive = FilePath + +remote :: RemoteType +remote = RemoteType { + typename = "glacier", + enumerate = findSpecialRemotes "glacier", + generate = gen, + setup = glacierSetup +} + +gen :: Git.Repo -> Maybe UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r mu c gc = new <$> remoteCost gc veryExpensiveRemoteCost + where + new cst = encryptableRemote c + (storeEncrypted this) + (retrieveEncrypted this) + this + where + this = Remote { + uuid = mu, + cost = cst, + name = Git.repoDescribe r, + storeKey = store this, + retrieveKeyFile = retrieve this, + retrieveKeyFileCheap = retrieveCheap this, + removeKey = remove this, + hasKey = checkPresent this, + hasKeyCheap = False, + whereisKey = Nothing, + config = c, + repo = r, + gitconfig = gc, + localpath = Nothing, + readonly = False, + remotetype = remote + } + +glacierSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +glacierSetup u c = do + c' <- encryptionSetup c + let fullconfig = c' `M.union` defaults + genVault fullconfig u + gitConfigSpecialRemote u fullconfig "glacier" "true" + setRemoteCredPair fullconfig (AWS.creds u) + where + remotename = fromJust (M.lookup "name" c) + defvault = remotename ++ "-" ++ fromUUID u + defaults = M.fromList + [ ("datacenter", T.unpack $ AWS.defaultRegion AWS.Glacier) + , ("vault", defvault) + ] + +store :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store r k _f m + | keySize k == Just 0 = do + warning "Cannot store empty files in Glacier." + return False + | otherwise = sendAnnex k (void $ remove r k) $ \src -> + metered (Just m) k $ \meterupdate -> + storeHelper r k $ streamMeteredFile src meterupdate + +storeEncrypted :: Remote -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool +storeEncrypted r (cipher, enck) k m = sendAnnex k (void $ remove r enck) $ \src -> do + metered (Just m) k $ \meterupdate -> + storeHelper r enck $ \h -> + encrypt cipher (feedFile src) + (readBytes $ meteredWrite meterupdate h) + +retrieve :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve r k _f d = metered Nothing k $ \meterupdate -> + retrieveHelper r k $ + readBytes $ meteredWriteFile meterupdate d + +retrieveCheap :: Remote -> Key -> FilePath -> Annex Bool +retrieveCheap _ _ _ = return False + +retrieveEncrypted :: Remote -> (Cipher, Key) -> Key -> FilePath -> Annex Bool +retrieveEncrypted r (cipher, enck) k d = metered Nothing k $ \meterupdate -> + retrieveHelper r enck $ readBytes $ \b -> + decrypt cipher (feedBytes b) $ + readBytes $ meteredWriteFile meterupdate d + +storeHelper :: Remote -> Key -> (Handle -> IO ()) -> Annex Bool +storeHelper r k feeder = go =<< glacierEnv c u + where + c = config r + u = uuid r + params = glacierParams c + [ Param "archive" + , Param "upload" + , Param "--name", Param $ archive r k + , Param $ getVault $ config r + , Param "-" + ] + go Nothing = return False + go (Just e) = do + let p = (proc "glacier" (toCommand params)) { env = Just e } + liftIO $ catchBoolIO $ + withHandle StdinHandle createProcessSuccess p $ \h -> do + feeder h + return True + +retrieveHelper :: Remote -> Key -> (Handle -> IO ()) -> Annex Bool +retrieveHelper r k reader = go =<< glacierEnv c u + where + c = config r + u = uuid r + params = glacierParams c + [ Param "archive" + , Param "retrieve" + , Param "-o-" + , Param $ getVault $ config r + , Param $ archive r k + ] + go Nothing = return False + go (Just e) = do + let p = (proc "glacier" (toCommand params)) { env = Just e } + ok <- liftIO $ catchBoolIO $ + withHandle StdoutHandle createProcessSuccess p $ \h -> + ifM (hIsEOF h) + ( return False + , do + reader h + return True + ) + unless ok later + return ok + later = showLongNote "Recommend you wait up to 4 hours, and then run this command again." + +remove :: Remote -> Key -> Annex Bool +remove r k = glacierAction r + [ Param "archive" + , Param "delete" + , Param $ getVault $ config r + , Param $ archive r k + ] + +checkPresent :: Remote -> Key -> Annex (Either String Bool) +checkPresent r k = do + showAction $ "checking " ++ name r + go =<< glacierEnv (config r) (uuid r) + where + go Nothing = return $ Left "cannot check glacier" + go (Just e) = do + {- glacier checkpresent outputs the archive name to stdout if + - it's present. -} + v <- liftIO $ catchMsgIO $ + readProcessEnv "glacier" (toCommand params) (Just e) + case v of + Right s -> do + let probablypresent = key2file k `elem` lines s + if probablypresent + then ifM (Annex.getFlag "trustglacier") + ( return $ Right True, untrusted ) + else return $ Right False + Left err -> return $ Left err + + params = + [ Param "archive" + , Param "checkpresent" + , Param $ getVault $ config r + , Param "--quiet" + , Param $ archive r k + ] + + untrusted = do + showLongNote $ unlines + [ "Glacier's inventory says it has a copy." + , "However, the inventory could be out of date, if it was recently removed." + , "(Use --trust-glacier if you're sure it's still in Glacier.)" + , "" + ] + return $ Right False + +glacierAction :: Remote -> [CommandParam] -> Annex Bool +glacierAction r params = runGlacier (config r) (uuid r) params + +runGlacier :: RemoteConfig -> Maybe UUID -> [CommandParam] -> Annex Bool +runGlacier c mu params = go =<< glacierEnv c mu + where + go Nothing = return False + go (Just e) = liftIO $ + boolSystemEnv "glacier" (glacierParams c params) (Just e) + +glacierParams :: RemoteConfig -> [CommandParam] -> [CommandParam] +glacierParams c params = datacenter:params + where + datacenter = Param $ "--region=" ++ + (fromJust $ M.lookup "datacenter" c) + +glacierEnv :: RemoteConfig -> Maybe UUID -> Annex (Maybe [(String, String)]) +glacierEnv _ Nothing = return Nothing +glacierEnv c (Just u) = go =<< getRemoteCredPairFor "glacier" c creds + where + go Nothing = return Nothing + go (Just (user, pass)) = do + e <- liftIO getEnvironment + return $ Just $ (uk, user):(pk, pass):e + + creds = AWS.creds u + (uk, pk) = credPairEnvironment creds + +getVault :: RemoteConfig -> Vault +getVault = fromJust . M.lookup "vault" + +archive :: Remote -> Key -> Archive +archive r k = fileprefix ++ key2file k + where + fileprefix = M.findWithDefault "" "fileprefix" $ config r + +-- glacier vault create will succeed even if the vault already exists. +genVault :: RemoteConfig -> UUID -> Annex () +genVault c u = unlessM (runGlacier c (Just u) params) $ + error "Failed creating glacier vault." + where + params = + [ Param "vault" + , Param "create" + , Param $ getVault c + ] + +{- Partitions the input list of keys into ones which have + - glacier retieval jobs that have succeeded, or failed. + - + - A complication is that `glacier job list` will display the encrypted + - keys when the remote is encrypted. + -} +jobList :: Remote -> [Key] -> Annex ([Key], [Key]) +jobList r keys = go =<< glacierEnv (config r) (uuid r) + where + params = [ Param "job", Param "list" ] + nada = ([], []) + myvault = getVault $ config r + + go Nothing = return nada + go (Just e) = do + v <- liftIO $ catchMaybeIO $ + readProcessEnv "glacier" (toCommand params) (Just e) + maybe (return nada) extract v + + extract s = do + let result@(succeeded, failed) = + parse nada $ (map words . lines) s + if result == nada + then return nada + else do + enckeys <- forM keys $ \k -> + maybe k snd <$> cipherKey (config r) k + let keymap = M.fromList $ zip enckeys keys + let convert = catMaybes . map (`M.lookup` keymap) + return (convert succeeded, convert failed) + + parse c [] = c + parse c@(succeeded, failed) ((status:_date:vault:key:[]):rest) + | vault == myvault = + case file2key key of + Nothing -> parse c rest + Just k + | "a/d" `isPrefixOf` status -> + parse (k:succeeded, failed) rest + | "a/e" `isPrefixOf` status -> + parse (succeeded, k:failed) rest + | otherwise -> + parse c rest + parse c (_:rest) = parse c rest diff --git a/Remote/Helper/AWS.hs b/Remote/Helper/AWS.hs new file mode 100644 index 0000000000..1d80ff1b4f --- /dev/null +++ b/Remote/Helper/AWS.hs @@ -0,0 +1,66 @@ +{- Amazon Web Services common infrastructure. + - + - Copyright 2011,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings, TupleSections #-} + +module Remote.Helper.AWS where + +import Common.Annex +import Creds + +import qualified Data.Map as M +import Data.Text (Text) + +creds :: UUID -> CredPairStorage +creds u = CredPairStorage + { credPairFile = fromUUID u + , credPairEnvironment = ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY") + , credPairRemoteKey = Just "s3creds" + } + +setCredsEnv :: CredPair -> IO () +setCredsEnv p = setEnvCredPair p $ creds undefined + +data Service = S3 | Glacier + deriving (Eq) + +type Region = Text + +regionMap :: Service -> M.Map Text Region +regionMap = M.fromList . regionInfo + +defaultRegion :: Service -> Region +defaultRegion = snd . Prelude.head . regionInfo + +{- S3 and Glacier use different names for some regions. Ie, "us-east-1" + - cannot be used with S3, while "US" cannot be used with Glacier. Dunno why. + - Also, Glacier is not yet available in all regions. -} +regionInfo :: Service -> [(Text, Region)] +regionInfo service = map (\(t, r) -> (t, fromServiceRegion r)) $ + filter (matchingService . snd) $ + concatMap (\(t, l) -> map (t,) l) regions + where + regions = + [ ("US East (N. Virginia)", [S3Region "US", GlacierRegion "us-east-1"]) + , ("US West (Oregon)", [BothRegion "us-west-2"]) + , ("US West (N. California)", [BothRegion "us-west-1"]) + , ("EU (Ireland)", [S3Region "EU", GlacierRegion "eu-west-1"]) + , ("Asia Pacific (Singapore)", [S3Region "ap-southeast-1"]) + , ("Asia Pacific (Tokyo)", [BothRegion "ap-northeast-1"]) + , ("Asia Pacific (Sydney)", [S3Region "ap-southeast-2"]) + , ("South America (São Paulo)", [S3Region "sa-east-1"]) + ] + + fromServiceRegion (BothRegion s) = s + fromServiceRegion (S3Region s) = s + fromServiceRegion (GlacierRegion s) = s + + matchingService (BothRegion _) = True + matchingService (S3Region _) = service == S3 + matchingService (GlacierRegion _) = service == Glacier + +data ServiceRegion = BothRegion Region | S3Region Region | GlacierRegion Region diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs new file mode 100644 index 0000000000..11b86042e8 --- /dev/null +++ b/Remote/Helper/Chunked.hs @@ -0,0 +1,127 @@ +{- git-annex chunked remotes + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Chunked where + +import Common.Annex +import Utility.DataUnits +import Types.Remote +import Meters + +import qualified Data.Map as M +import qualified Data.ByteString.Lazy as L +import Data.Int +import qualified Control.Exception as E + +type ChunkSize = Maybe Int64 + +{- Gets a remote's configured chunk size. -} +chunkSize :: RemoteConfig -> ChunkSize +chunkSize m = + case M.lookup "chunksize" m of + Nothing -> Nothing + Just v -> case readSize dataUnits v of + Nothing -> error "bad chunksize" + Just size + | size <= 0 -> error "bad chunksize" + | otherwise -> Just $ fromInteger size + +{- This is an extension that's added to the usual file (or whatever) + - where the remote stores a key. -} +type ChunkExt = String + +{- A record of the number of chunks used. + - + - While this can be guessed at based on the size of the key, encryption + - makes that larger. Also, using this helps deal with changes to chunksize + - over the life of a remote. + -} +chunkCount :: ChunkExt +chunkCount = ".chunkcount" + +{- Parses the String from the chunkCount file, and returns the files that + - are used to store the chunks. -} +listChunks :: FilePath -> String -> [FilePath] +listChunks basedest chunkcount = take count $ map (basedest ++) chunkStream + where + count = fromMaybe 0 $ readish chunkcount + +{- An infinite stream of extensions to use for chunks. -} +chunkStream :: [ChunkExt] +chunkStream = map (\n -> ".chunk" ++ show n) [1 :: Integer ..] + +{- Given the base destination to use to store a value, + - generates a stream of temporary destinations (just one when not chunking) + - and passes it to an action, which should chunk and store the data, + - and return the destinations it stored to, or [] on error. Then + - calls the storer to write the chunk count (if chunking). Finally, the + - finalizer is called to rename the tmp into the dest + - (and do any other cleanup). + -} +storeChunks :: Key -> FilePath -> FilePath -> ChunkSize -> ([FilePath] -> IO [FilePath]) -> (FilePath -> String -> IO ()) -> (FilePath -> FilePath -> IO ()) -> IO Bool +storeChunks key tmp dest chunksize storer recorder finalizer = either onerr return + =<< (E.try go :: IO (Either E.SomeException Bool)) + where + go = do + stored <- storer tmpdests + when (chunksize /= Nothing) $ do + let chunkcount = basef ++ chunkCount + recorder chunkcount (show $ length stored) + finalizer tmp dest + return (not $ null stored) + onerr e = do + print e + return False + + basef = tmp ++ keyFile key + tmpdests + | chunksize == Nothing = [basef] + | otherwise = map (basef ++ ) chunkStream + +{- Given a list of destinations to use, chunks the data according to the + - ChunkSize, and runs the storer action to store each chunk. Returns + - the destinations where data was stored, or [] on error. + - + - This buffers each chunk in memory. + - More optimal versions of this can be written, that rely + - on L.toChunks to split the lazy bytestring into chunks (typically + - smaller than the ChunkSize), and eg, write those chunks to a Handle. + - But this is the best that can be done with the storer interface that + - writes a whole L.ByteString at a time. + -} +storeChunked :: ChunkSize -> [FilePath] -> (FilePath -> L.ByteString -> IO ()) -> L.ByteString -> IO [FilePath] +storeChunked chunksize dests storer content = either onerr return + =<< (E.try (go chunksize dests) :: IO (Either E.SomeException [FilePath])) + where + go _ [] = return [] -- no dests!? + go Nothing (d:_) = do + storer d content + return [d] + go (Just sz) _ + -- always write a chunk, even if the data is 0 bytes + | L.null content = go Nothing dests + | otherwise = storechunks sz [] dests content + + onerr e = do + print e + return [] + + storechunks _ _ [] _ = return [] -- ran out of dests + storechunks sz useddests (d:ds) b + | L.null b = return $ reverse useddests + | otherwise = do + let (chunk, b') = L.splitAt sz b + storer d chunk + storechunks sz (d:useddests) ds b' + +{- Writes a series of chunks to a file. The feeder is called to get + - each chunk. -} +meteredWriteFileChunks :: MeterUpdate -> FilePath -> [v] -> (v -> IO L.ByteString) -> IO () +meteredWriteFileChunks meterupdate dest chunks feeder = + withBinaryFile dest WriteMode $ \h -> + forM_ chunks $ \c -> + meteredWrite meterupdate h =<< feeder c diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs new file mode 100644 index 0000000000..d322a5cf8d --- /dev/null +++ b/Remote/Helper/Encryptable.hs @@ -0,0 +1,127 @@ +{- common functions for encryptable remotes + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Encryptable where + +import qualified Data.Map as M + +import Common.Annex +import Types.Remote +import Crypto +import qualified Annex +import Config +import Utility.Base64 + +{- Encryption setup for a remote. The user must specify whether to use + - an encryption key, or not encrypt. An encrypted cipher is created, or is + - updated to be accessible to an additional encryption key. Or the user + - could opt to use a shared cipher, which is stored unencrypted. -} +encryptionSetup :: RemoteConfig -> Annex RemoteConfig +encryptionSetup c = case (M.lookup "encryption" c, extractCipher c) of + (Nothing, Nothing) -> error "Specify encryption=key or encryption=none or encryption=shared" + (Just "none", Nothing) -> return c + (Nothing, Just _) -> return c + (Just "shared", Just (SharedCipher _)) -> return c + (Just "none", Just _) -> cannotchange + (Just "shared", Just (EncryptedCipher _ _)) -> cannotchange + (Just _, Just (SharedCipher _)) -> cannotchange + (Just "shared", Nothing) -> use "encryption setup" $ genSharedCipher + (Just keyid, Nothing) -> use "encryption setup" $ genEncryptedCipher keyid + (Just keyid, Just v) -> use "encryption updated" $ updateEncryptedCipher keyid v + where + cannotchange = error "Cannot change encryption type of existing remote." + use m a = do + cipher <- liftIO a + showNote $ m ++ " " ++ describeCipher cipher + return $ M.delete "encryption" $ storeCipher c cipher + +{- Modifies a Remote to support encryption. + - + - Two additional functions must be provided by the remote, + - to support storing and retrieving encrypted content. -} +encryptableRemote + :: RemoteConfig + -> ((Cipher, Key) -> Key -> MeterUpdate -> Annex Bool) + -> ((Cipher, Key) -> Key -> FilePath -> Annex Bool) + -> Remote + -> Remote +encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = + r { + storeKey = store, + retrieveKeyFile = retrieve, + retrieveKeyFileCheap = retrieveCheap, + removeKey = withkey $ removeKey r, + hasKey = withkey $ hasKey r, + cost = cost r + encryptedRemoteCostAdj + } + where + store k f p = cip k >>= maybe + (storeKey r k f p) + (\enck -> storeKeyEncrypted enck k p) + retrieve k f d = cip k >>= maybe + (retrieveKeyFile r k f d) + (\enck -> retrieveKeyFileEncrypted enck k d) + retrieveCheap k d = cip k >>= maybe + (retrieveKeyFileCheap r k d) + (\_ -> return False) + withkey a k = cip k >>= maybe (a k) (a . snd) + cip = cipherKey c + +{- Gets encryption Cipher. The decrypted Ciphers are cached in the Annex + - state. -} +remoteCipher :: RemoteConfig -> Annex (Maybe Cipher) +remoteCipher c = go $ extractCipher c + where + go Nothing = return Nothing + go (Just encipher) = do + cache <- Annex.getState Annex.ciphers + case M.lookup encipher cache of + Just cipher -> return $ Just cipher + Nothing -> do + showNote "gpg" + cipher <- liftIO $ decryptCipher encipher + Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache }) + return $ Just cipher + +{- Checks if the remote's config allows storing creds in the remote's config. + - + - embedcreds=yes allows this, and embedcreds=no prevents it. + - + - If not set, the default is to only store creds when it's surely safe: + - When gpg encryption is used, in which case the creds will be encrypted + - using it. Not when a shared cipher is used. + -} +embedCreds :: RemoteConfig -> Bool +embedCreds c + | M.lookup "embedcreds" c == Just "yes" = True + | M.lookup "embedcreds" c == Just "no" = False + | isJust (M.lookup "cipherkeys" c) && isJust (M.lookup "cipher" c) = True + | otherwise = False + +{- Gets encryption Cipher, and encrypted version of Key. -} +cipherKey :: RemoteConfig -> Key -> Annex (Maybe (Cipher, Key)) +cipherKey c k = maybe Nothing make <$> remoteCipher c + where + make ciphertext = Just (ciphertext, encryptKey ciphertext k) + +{- Stores an StorableCipher in a remote's configuration. -} +storeCipher :: RemoteConfig -> StorableCipher -> RemoteConfig +storeCipher c (SharedCipher t) = M.insert "cipher" (toB64 t) c +storeCipher c (EncryptedCipher t ks) = + M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (showkeys ks) c + where + showkeys (KeyIds l) = join "," l + +{- Extracts an StorableCipher from a remote's configuration. -} +extractCipher :: RemoteConfig -> Maybe StorableCipher +extractCipher c = + case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of + (Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (readkeys ks) + (Just t, Nothing) -> Just $ SharedCipher (fromB64 t) + _ -> Nothing + where + readkeys = KeyIds . split "," diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs new file mode 100644 index 0000000000..bdeb653ebd --- /dev/null +++ b/Remote/Helper/Hooks.hs @@ -0,0 +1,86 @@ +{- Adds hooks to remotes. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Hooks (addHooks) where + +import qualified Data.Map as M + +import Common.Annex +import Types.Remote +import qualified Annex +import Annex.LockPool +import Annex.Perms + +{- Modifies a remote's access functions to first run the + - annex-start-command hook, and trigger annex-stop-command on shutdown. + - This way, the hooks are only run when a remote is actively being used. + -} +addHooks :: Remote -> Remote +addHooks r = addHooks' r + (remoteAnnexStartCommand $ gitconfig r) + (remoteAnnexStopCommand $ gitconfig r) +addHooks' :: Remote -> Maybe String -> Maybe String -> Remote +addHooks' r Nothing Nothing = r +addHooks' r starthook stophook = r' + where + r' = r + { storeKey = \k f p -> wrapper $ storeKey r k f p + , retrieveKeyFile = \k f d -> wrapper $ retrieveKeyFile r k f d + , retrieveKeyFileCheap = \k f -> wrapper $ retrieveKeyFileCheap r k f + , removeKey = \k -> wrapper $ removeKey r k + , hasKey = \k -> wrapper $ hasKey r k + } + where + wrapper = runHooks r' starthook stophook + +runHooks :: Remote -> Maybe String -> Maybe String -> Annex a -> Annex a +runHooks r starthook stophook a = do + dir <- fromRepo gitAnnexRemotesDir + let lck = dir remoteid ++ ".lck" + whenM (not . any (== lck) . M.keys <$> getPool) $ do + liftIO $ createDirectoryIfMissing True dir + firstrun lck + a + where + remoteid = show (uuid r) + run Nothing = noop + run (Just command) = void $ liftIO $ + boolSystem "sh" [Param "-c", Param command] + firstrun lck = do + -- Take a shared lock; This indicates that git-annex + -- is using the remote, and prevents other instances + -- of it from running the stophook. If another + -- instance is shutting down right now, this + -- will block waiting for its exclusive lock to clear. + lockFile lck + + -- The starthook is run even if some other git-annex + -- is already running, and ran it before. + -- It would be difficult to use locking to ensure + -- it's only run once, and it's also possible for + -- git-annex to be interrupted before it can run the + -- stophook, in which case the starthook + -- would be run again by the next git-annex. + -- So, requiring idempotency is the right approach. + run starthook + + Annex.addCleanup (remoteid ++ "-stop-command") $ runstop lck + runstop lck = do + -- Drop any shared lock we have, and take an + -- exclusive lock, without blocking. If the lock + -- succeeds, we're the only process using this remote, + -- so can stop it. + unlockFile lck + mode <- annexFileMode + fd <- liftIO $ noUmask mode $ + openFd lck ReadWrite (Just mode) defaultFileFlags + v <- liftIO $ tryIO $ + setLock fd (WriteLock, AbsoluteSeek, 0, 0) + case v of + Left _ -> noop + Right _ -> run stophook + liftIO $ closeFd fd diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs new file mode 100644 index 0000000000..7fc421f46f --- /dev/null +++ b/Remote/Helper/Special.hs @@ -0,0 +1,40 @@ +{- common functions for special remotes + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Special where + +import qualified Data.Map as M + +import Common.Annex +import Types.Remote +import qualified Git +import qualified Git.Command +import qualified Git.Construct + +{- Special remotes don't have a configured url, so Git.Repo does not + - automatically generate remotes for them. This looks for a different + - configuration key instead. + -} +findSpecialRemotes :: String -> Annex [Git.Repo] +findSpecialRemotes s = do + m <- fromRepo Git.config + liftIO $ mapM construct $ remotepairs m + where + remotepairs = M.toList . M.filterWithKey match + construct (k,_) = Git.Construct.remoteNamedFromKey k Git.Construct.fromUnknown + match k _ = startswith "remote." k && endswith (".annex-"++s) k + +{- Sets up configuration for a special remote in .git/config. -} +gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex () +gitConfigSpecialRemote u c k v = do + set ("annex-"++k) v + set ("annex-uuid") (fromUUID u) + where + set a b = inRepo $ Git.Command.run + [Param "config", Param (configsetting a), Param b] + remotename = fromJust (M.lookup "name" c) + configsetting s = "remote." ++ remotename ++ "." ++ s diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs new file mode 100644 index 0000000000..adedd1b177 --- /dev/null +++ b/Remote/Helper/Ssh.hs @@ -0,0 +1,72 @@ +{- git-annex remote access with ssh + - + - Copyright 2011,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Helper.Ssh where + +import Common.Annex +import qualified Git +import qualified Git.Url +import Annex.UUID +import Annex.Ssh +import Fields +import Types.GitConfig + +{- Generates parameters to ssh to a repository's host and run a command. + - Caller is responsible for doing any neccessary shellEscaping of the + - passed command. -} +sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam] +sshToRepo repo sshcmd = do + g <- fromRepo id + let c = extractRemoteGitConfig g (Git.repoDescribe repo) + let opts = map Param $ remoteAnnexSshOptions c + params <- sshParams (Git.Url.hostuser repo, Git.Url.port repo) opts + return $ params ++ sshcmd + +{- Generates parameters to run a git-annex-shell command on a remote + - repository. -} +git_annex_shell :: Git.Repo -> String -> [CommandParam] -> [(Field, String)] -> Annex (Maybe (FilePath, [CommandParam])) +git_annex_shell r command params fields + | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts ++ fieldopts) + | Git.repoIsSsh r = do + mu <- getRepoUUID r + sshparams <- sshToRepo r [Param $ sshcmd mu ] + return $ Just ("ssh", sshparams) + | otherwise = return Nothing + where + dir = Git.repoPath r + shellcmd = "git-annex-shell" + shellopts = Param command : File dir : params + sshcmd mu = unwords $ + shellcmd : map shellEscape (toCommand shellopts) ++ + uuidcheck mu ++ + map shellEscape (toCommand fieldopts) + uuidcheck (Just u) = ["--uuid", fromUUID u] + uuidcheck Nothing = [] + fieldopts + | null fields = [] + | otherwise = fieldsep : map fieldopt fields ++ [fieldsep] + fieldsep = Param "--" + fieldopt (field, value) = Param $ + fieldName field ++ "=" ++ value + +{- Uses a supplied function (such as boolSystem) to run a git-annex-shell + - command on a remote. + - + - Or, if the remote does not support running remote commands, returns + - a specified error value. -} +onRemote + :: Git.Repo + -> (FilePath -> [CommandParam] -> IO a, a) + -> String + -> [CommandParam] + -> [(Field, String)] + -> Annex a +onRemote r (with, errorval) command params fields = do + s <- git_annex_shell r command params fields + case s of + Just (c, ps) -> liftIO $ with c ps + Nothing -> return errorval diff --git a/Remote/Hook.hs b/Remote/Hook.hs new file mode 100644 index 0000000000..77dc107938 --- /dev/null +++ b/Remote/Hook.hs @@ -0,0 +1,142 @@ +{- A remote that provides hooks to run shell commands. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Hook (remote) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M +import System.Environment + +import Common.Annex +import Types.Remote +import Types.Key +import qualified Git +import Config +import Annex.Content +import Remote.Helper.Special +import Remote.Helper.Encryptable +import Crypto + +remote :: RemoteType +remote = RemoteType { + typename = "hook", + enumerate = findSpecialRemotes "hooktype", + generate = gen, + setup = hookSetup +} + +gen :: Git.Repo -> Maybe UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r mu c gc = do + cst <- remoteCost gc expensiveRemoteCost + return $ encryptableRemote c + (storeEncrypted hooktype) + (retrieveEncrypted hooktype) + Remote { + uuid = mu, + cost = cst, + name = Git.repoDescribe r, + storeKey = store hooktype, + retrieveKeyFile = retrieve hooktype, + retrieveKeyFileCheap = retrieveCheap hooktype, + removeKey = remove hooktype, + hasKey = checkPresent r hooktype, + hasKeyCheap = False, + whereisKey = Nothing, + config = M.empty, + localpath = Nothing, + repo = r, + gitconfig = gc, + readonly = False, + remotetype = remote + } + where + hooktype = fromMaybe (error "missing hooktype") $ remoteAnnexHookType gc + +hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +hookSetup u c = do + let hooktype = fromMaybe (error "Specify hooktype=") $ + M.lookup "hooktype" c + c' <- encryptionSetup c + gitConfigSpecialRemote u c' "hooktype" hooktype + return c' + +hookEnv :: Key -> Maybe FilePath -> IO (Maybe [(String, String)]) +hookEnv k f = Just <$> mergeenv (fileenv f ++ keyenv) + where + mergeenv l = M.toList . M.union (M.fromList l) + <$> M.fromList <$> getEnvironment + env s v = ("ANNEX_" ++ s, v) + keyenv = catMaybes + [ Just $ env "KEY" (key2file k) + , env "HASH_1" <$> headMaybe hashbits + , env "HASH_2" <$> headMaybe (drop 1 hashbits) + ] + fileenv Nothing = [] + fileenv (Just file) = [env "FILE" file] + hashbits = map takeDirectory $ splitPath $ hashDirMixed k + +lookupHook :: String -> String -> Annex (Maybe String) +lookupHook hooktype hook =do + command <- getConfig (annexConfig hookname) "" + if null command + then do + warning $ "missing configuration for " ++ hookname + return Nothing + else return $ Just command + where + hookname = hooktype ++ "-" ++ hook ++ "-hook" + +runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool +runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype hook + where + run command = do + showOutput -- make way for hook output + ifM (liftIO $ boolSystemEnv "sh" [Param "-c", Param command] =<< hookEnv k f) + ( a + , do + warning $ hook ++ " hook exited nonzero!" + return False + ) + +store :: String -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store h k _f _p = sendAnnex k (void $ remove h k) $ \src -> + runHook h "store" k (Just src) $ return True + +storeEncrypted :: String -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool +storeEncrypted h (cipher, enck) k _p = withTmp enck $ \tmp -> + sendAnnex k (void $ remove h enck) $ \src -> do + liftIO $ encrypt cipher (feedFile src) $ + readBytes $ L.writeFile tmp + runHook h "store" enck (Just tmp) $ return True + +retrieve :: String -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve h k _f d = runHook h "retrieve" k (Just d) $ return True + +retrieveCheap :: String -> Key -> FilePath -> Annex Bool +retrieveCheap _ _ _ = return False + +retrieveEncrypted :: String -> (Cipher, Key) -> Key -> FilePath -> Annex Bool +retrieveEncrypted h (cipher, enck) _ f = withTmp enck $ \tmp -> + runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBoolIO $ do + decrypt cipher (feedFile tmp) $ + readBytes $ L.writeFile f + return True + +remove :: String -> Key -> Annex Bool +remove h k = runHook h "remove" k Nothing $ return True + +checkPresent :: Git.Repo -> String -> Key -> Annex (Either String Bool) +checkPresent r h k = do + showAction $ "checking " ++ Git.repoDescribe r + v <- lookupHook h "checkpresent" + liftIO $ catchMsgIO $ check v + where + findkey s = key2file k `elem` lines s + check Nothing = error "checkpresent hook misconfigured" + check (Just hook) = do + env <- hookEnv k Nothing + findkey <$> readProcessEnv "sh" ["-c", hook] env diff --git a/Remote/List.hs b/Remote/List.hs new file mode 100644 index 0000000000..b0ad3c72b0 --- /dev/null +++ b/Remote/List.hs @@ -0,0 +1,107 @@ +{-# LANGUAGE CPP #-} + +{- git-annex remote list + - + - Copyright 2011,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.List where + +import qualified Data.Map as M + +import Common.Annex +import qualified Annex +import Logs.Remote +import Types.Remote +import Types.GitConfig +import Annex.UUID +import Remote.Helper.Hooks +import qualified Git +import qualified Git.Config + +import qualified Remote.Git +#ifdef WITH_S3 +import qualified Remote.S3 +#endif +import qualified Remote.Bup +import qualified Remote.Directory +import qualified Remote.Rsync +import qualified Remote.Web +#ifdef WITH_WEBDAV +import qualified Remote.WebDAV +#endif +import qualified Remote.Glacier +import qualified Remote.Hook + +remoteTypes :: [RemoteType] +remoteTypes = + [ Remote.Git.remote +#ifdef WITH_S3 + , Remote.S3.remote +#endif + , Remote.Bup.remote + , Remote.Directory.remote + , Remote.Rsync.remote + , Remote.Web.remote +#ifdef WITH_WEBDAV + , Remote.WebDAV.remote +#endif + , Remote.Glacier.remote + , Remote.Hook.remote + ] + +{- Builds a list of all available Remotes. + - Since doing so can be expensive, the list is cached. -} +remoteList :: Annex [Remote] +remoteList = do + rs <- Annex.getState Annex.remotes + if null rs + then do + m <- readRemoteLog + rs' <- concat <$> mapM (process m) remoteTypes + Annex.changeState $ \s -> s { Annex.remotes = rs' } + return rs' + else return rs + where + process m t = enumerate t >>= mapM (remoteGen m t) + +{- Forces the remoteList to be re-generated, re-reading the git config. -} +remoteListRefresh :: Annex [Remote] +remoteListRefresh = do + newg <- inRepo Git.Config.reRead + Annex.changeState $ \s -> s + { Annex.remotes = [] + , Annex.repo = newg + } + remoteList + +{- Generates a Remote. -} +remoteGen :: (M.Map UUID RemoteConfig) -> RemoteType -> Git.Repo -> Annex Remote +remoteGen m t r = do + mu <- getRepoUUID r + g <- fromRepo id + let gc = extractRemoteGitConfig g (Git.repoDescribe r) + let c = fromMaybe M.empty $ maybe Nothing (\u -> M.lookup u m) mu + addHooks <$> generate t r mu c gc + +{- Updates a local git Remote, re-reading its git config. -} +updateRemote :: Remote -> Annex Remote +updateRemote remote = do + m <- readRemoteLog + remote' <- updaterepo $ repo remote + remoteGen m (remotetype remote) remote' + where + updaterepo r + | Git.repoIsLocal r || Git.repoIsLocalUnknown r = + Remote.Git.configRead r + | otherwise = return r + +{- All remotes that are not ignored. -} +enabledRemoteList :: Annex [Remote] +enabledRemoteList = filter (not . remoteAnnexIgnore . gitconfig) <$> remoteList + +{- Checks if a remote is a special remote -} +specialRemote :: Remote -> Bool +specialRemote r = remotetype r /= Remote.Git.remote diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs new file mode 100644 index 0000000000..705e2f531e --- /dev/null +++ b/Remote/Rsync.hs @@ -0,0 +1,241 @@ +{- A remote that is only accessible by rsync. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Rsync (remote) where + +import qualified Data.ByteString.Lazy as L +import qualified Data.Map as M +import System.Posix.Process (getProcessID) + +import Common.Annex +import Types.Remote +import qualified Git +import Config +import Annex.Content +import Remote.Helper.Special +import Remote.Helper.Encryptable +import Crypto +import Utility.Rsync +import Utility.CopyFile +import Annex.Perms + +type RsyncUrl = String + +data RsyncOpts = RsyncOpts + { rsyncUrl :: RsyncUrl + , rsyncOptions :: [CommandParam] + , rsyncShellEscape :: Bool +} + +remote :: RemoteType +remote = RemoteType { + typename = "rsync", + enumerate = findSpecialRemotes "rsyncurl", + generate = gen, + setup = rsyncSetup +} + +gen :: Git.Repo -> Maybe UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r mu c gc = do + cst <- remoteCost gc expensiveRemoteCost + return $ encryptableRemote c + (storeEncrypted o) + (retrieveEncrypted o) + Remote + { uuid = mu + , cost = cst + , name = Git.repoDescribe r + , storeKey = store o + , retrieveKeyFile = retrieve o + , retrieveKeyFileCheap = retrieveCheap o + , removeKey = remove o + , hasKey = checkPresent r o + , hasKeyCheap = False + , whereisKey = Nothing + , config = M.empty + , repo = r + , gitconfig = gc + , localpath = if rsyncUrlIsPath $ rsyncUrl o + then Just $ rsyncUrl o + else Nothing + , readonly = False + , remotetype = remote + } + where + o = RsyncOpts url opts escape + url = fromMaybe (error "missing rsyncurl") $ remoteAnnexRsyncUrl gc + opts = map Param $ filter safe $ remoteAnnexRsyncOptions gc + escape = M.lookup "shellescape" c /= Just "no" + safe opt + -- Don't allow user to pass --delete to rsync; + -- that could cause it to delete other keys + -- in the same hash bucket as a key it sends. + | opt == "--delete" = False + | opt == "--delete-excluded" = False + | otherwise = True + +rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +rsyncSetup u c = do + -- verify configuration is sane + let url = fromMaybe (error "Specify rsyncurl=") $ + M.lookup "rsyncurl" c + c' <- encryptionSetup c + + -- The rsyncurl is stored in git config, not only in this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' "rsyncurl" url + return c' + +rsyncEscape :: RsyncOpts -> String -> String +rsyncEscape o s + | rsyncShellEscape o && rsyncUrlIsShell (rsyncUrl o) = shellEscape s + | otherwise = s + +rsyncUrls :: RsyncOpts -> Key -> [String] +rsyncUrls o k = map use annexHashes + where + use h = rsyncUrl o h k rsyncEscape o (f f) + f = keyFile k + +store :: RsyncOpts -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store o k _f p = sendAnnex k (void $ remove o k) $ rsyncSend o p k False + +storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool +storeEncrypted o (cipher, enck) k p = withTmp enck $ \tmp -> + sendAnnex k (void $ remove o enck) $ \src -> do + liftIO $ encrypt cipher (feedFile src) $ + readBytes $ L.writeFile tmp + rsyncSend o p enck True tmp + +retrieve :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve o k _ f = untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o Nothing + -- use inplace when retrieving to support resuming + [ Param "--inplace" + , Param u + , Param f + ] + +retrieveCheap :: RsyncOpts -> Key -> FilePath -> Annex Bool +retrieveCheap o k f = ifM (preseedTmp k f) ( retrieve o k undefined f , return False ) + +retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> FilePath -> Annex Bool +retrieveEncrypted o (cipher, enck) _ f = withTmp enck $ \tmp -> do + ifM (retrieve o enck undefined tmp) + ( liftIO $ catchBoolIO $ do + decrypt cipher (feedFile tmp) $ + readBytes $ L.writeFile f + return True + , return False + ) + +remove :: RsyncOpts -> Key -> Annex Bool +remove o k = withRsyncScratchDir $ \tmp -> liftIO $ do + {- Send an empty directory to rysnc to make it delete. -} + let dummy = tmp keyFile k + createDirectoryIfMissing True dummy + rsync $ rsyncOptions o ++ + map (\s -> Param $ "--include=" ++ s) includes ++ + [ Param "--exclude=*" -- exclude everything else + , Params "--quiet --delete --recursive" + , partialParams + , Param $ addTrailingPathSeparator dummy + , Param $ rsyncUrl o + ] + where + {- Specify include rules to match the directories where the + - content could be. Note that the parent directories have + - to also be explicitly included, due to how rsync + - traverses directories. -} + includes = concatMap use annexHashes + use h = let dir = h k in + [ parentDir dir + , dir + -- match content directory and anything in it + , dir keyFile k "***" + ] + +checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool) +checkPresent r o k = do + showAction $ "checking " ++ Git.repoDescribe r + -- note: Does not currently differentiate between rsync failing + -- to connect, and the file not being present. + Right <$> check + where + check = untilTrue (rsyncUrls o k) $ \u -> + liftIO $ catchBoolIO $ do + withQuietOutput createProcessSuccess $ + proc "rsync" $ toCommand $ + rsyncOptions o ++ [Param u] + return True + +{- Rsync params to enable resumes of sending files safely, + - ensure that files are only moved into place once complete + -} +partialParams :: CommandParam +partialParams = Params "--partial --partial-dir=.rsync-partial" + +{- Runs an action in an empty scratch directory that can be used to build + - up trees for rsync. -} +withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool +withRsyncScratchDir a = do + pid <- liftIO getProcessID + t <- fromRepo gitAnnexTmpDir + createAnnexDirectory t + let tmp = t "rsynctmp" show pid + nuke tmp + liftIO $ createDirectoryIfMissing True tmp + nuke tmp `after` a tmp + where + nuke d = liftIO $ whenM (doesDirectoryExist d) $ + removeDirectoryRecursive d + +rsyncRemote :: RsyncOpts -> (Maybe MeterUpdate) -> [CommandParam] -> Annex Bool +rsyncRemote o callback params = do + showOutput -- make way for progress bar + ifM (liftIO $ (maybe rsync rsyncProgress callback) ps) + ( return True + , do + showLongNote "rsync failed -- run git annex again to resume file transfer" + return False + ) + where + defaultParams = [Params "--progress"] + ps = rsyncOptions o ++ defaultParams ++ params + +{- To send a single key is slightly tricky; need to build up a temporary + - directory structure to pass to rsync so it can create the hash + - directories. + - + - This would not be necessary if the hash directory structure used locally + - was always the same as that used on the rsync remote. So if that's ever + - unified, this gets nicer. Especially in the crippled filesystem case. + - (When we have the right hash directory structure, we can just + - pass --include=X --include=X/Y --include=X/Y/file --exclude=*) + -} +rsyncSend :: RsyncOpts -> MeterUpdate -> Key -> Bool -> FilePath -> Annex Bool +rsyncSend o callback k canrename src = withRsyncScratchDir $ \tmp -> do + let dest = tmp Prelude.head (keyPaths k) + liftIO $ createDirectoryIfMissing True $ parentDir dest + ok <- if canrename + then do + liftIO $ renameFile src dest + return True + else ifM crippledFileSystem + ( liftIO $ copyFileExternal src dest + , do + liftIO $ createLink src dest + return True + ) + if ok + then rsyncRemote o (Just callback) + [ Param "--recursive" + , partialParams + -- tmp/ to send contents of tmp dir + , Param $ addTrailingPathSeparator tmp + , Param $ rsyncUrl o + ] + else return False diff --git a/Remote/S3.hs b/Remote/S3.hs new file mode 100644 index 0000000000..c90a746eb1 --- /dev/null +++ b/Remote/S3.hs @@ -0,0 +1,272 @@ +{- Amazon S3 remotes. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.S3 (remote) where + +import Network.AWS.AWSConnection +import Network.AWS.S3Object +import Network.AWS.S3Bucket hiding (size) +import Network.AWS.AWSResult +import qualified Data.Text as T +import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.Map as M +import Data.Char + +import Common.Annex +import Types.Remote +import Types.Key +import qualified Git +import Config +import Remote.Helper.Special +import Remote.Helper.Encryptable +import qualified Remote.Helper.AWS as AWS +import Crypto +import Creds +import Meters +import Annex.Content + +remote :: RemoteType +remote = RemoteType { + typename = "S3", + enumerate = findSpecialRemotes "s3", + generate = gen, + setup = s3Setup +} + +gen :: Git.Repo -> Maybe UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r mu c gc = new <$> remoteCost gc expensiveRemoteCost + where + new cst = encryptableRemote c + (storeEncrypted this) + (retrieveEncrypted this) + this + where + this = Remote { + uuid = mu, + cost = cst, + name = Git.repoDescribe r, + storeKey = store this, + retrieveKeyFile = retrieve this, + retrieveKeyFileCheap = retrieveCheap this, + removeKey = remove this, + hasKey = checkPresent this, + hasKeyCheap = False, + whereisKey = Nothing, + config = c, + repo = r, + gitconfig = gc, + localpath = Nothing, + readonly = False, + remotetype = remote + } + +s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig +s3Setup u c = handlehost $ M.lookup "host" c + where + remotename = fromJust (M.lookup "name" c) + defbucket = remotename ++ "-" ++ fromUUID u + defaults = M.fromList + [ ("datacenter", T.unpack $ AWS.defaultRegion AWS.S3) + , ("storageclass", "STANDARD") + , ("host", defaultAmazonS3Host) + , ("port", show defaultAmazonS3Port) + , ("bucket", defbucket) + ] + + handlehost Nothing = defaulthost + handlehost (Just h) + | ".archive.org" `isSuffixOf` map toLower h = archiveorg + | otherwise = defaulthost + + use fullconfig = do + gitConfigSpecialRemote u fullconfig "s3" "true" + setRemoteCredPair fullconfig (AWS.creds u) + + defaulthost = do + c' <- encryptionSetup c + let fullconfig = c' `M.union` defaults + genBucket fullconfig u + use fullconfig + + archiveorg = do + showNote "Internet Archive mode" + maybe (error "specify bucket=") (const noop) $ + M.lookup "bucket" archiveconfig + use archiveconfig + where + archiveconfig = + -- hS3 does not pass through x-archive-* headers + M.mapKeys (replace "x-archive-" "x-amz-") $ + -- encryption does not make sense here + M.insert "encryption" "none" $ + M.union c $ + -- special constraints on key names + M.insert "mungekeys" "ia" $ + -- bucket created only when files are uploaded + M.insert "x-amz-auto-make-bucket" "1" $ + -- no default bucket name; should be human-readable + M.delete "bucket" defaults + +store :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store r k _f p = s3Action r False $ \(conn, bucket) -> + sendAnnex k (void $ remove r k) $ \src -> do + res <- storeHelper (conn, bucket) r k p src + s3Bool res + +storeEncrypted :: Remote -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool +storeEncrypted r (cipher, enck) k p = s3Action r False $ \(conn, bucket) -> + -- To get file size of the encrypted content, have to use a temp file. + -- (An alternative would be chunking to to a constant size.) + withTmp enck $ \tmp -> sendAnnex k (void $ remove r enck) $ \src -> do + liftIO $ encrypt cipher (feedFile src) $ + readBytes $ L.writeFile tmp + res <- storeHelper (conn, bucket) r enck p tmp + s3Bool res + +storeHelper :: (AWSConnection, String) -> Remote -> Key -> MeterUpdate -> FilePath -> Annex (AWSResult ()) +storeHelper (conn, bucket) r k p file = do + size <- maybe getsize (return . fromIntegral) $ keySize k + meteredBytes (Just p) size $ \meterupdate -> + liftIO $ withMeteredFile file meterupdate $ \content -> do + -- size is provided to S3 so the whole content + -- does not need to be buffered to calculate it + let object = setStorageClass storageclass $ S3Object + bucket (bucketFile r k) "" + (("Content-Length", show size) : xheaders) + content + sendObject conn object + where + storageclass = + case fromJust $ M.lookup "storageclass" $ config r of + "REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY + _ -> STANDARD + + getsize = liftIO $ fromIntegral . fileSize <$> getFileStatus file + + xheaders = filter isxheader $ M.assocs $ config r + isxheader (h, _) = "x-amz-" `isPrefixOf` h + +retrieve :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve r k _f d = s3Action r False $ \(conn, bucket) -> + metered Nothing k $ \meterupdate -> do + res <- liftIO $ getObject conn $ bucketKey r bucket k + case res of + Right o -> do + liftIO $ meteredWriteFile meterupdate d $ + obj_data o + return True + Left e -> s3Warning e + +retrieveCheap :: Remote -> Key -> FilePath -> Annex Bool +retrieveCheap _ _ _ = return False + +retrieveEncrypted :: Remote -> (Cipher, Key) -> Key -> FilePath -> Annex Bool +retrieveEncrypted r (cipher, enck) k d = s3Action r False $ \(conn, bucket) -> + metered Nothing k $ \meterupdate -> do + res <- liftIO $ getObject conn $ bucketKey r bucket enck + case res of + Right o -> liftIO $ decrypt cipher (\h -> meteredWrite meterupdate h $ obj_data o) $ + readBytes $ \content -> do + L.writeFile d content + return True + Left e -> s3Warning e + +remove :: Remote -> Key -> Annex Bool +remove r k = s3Action r False $ \(conn, bucket) -> do + res <- liftIO $ deleteObject conn $ bucketKey r bucket k + s3Bool res + +checkPresent :: Remote -> Key -> Annex (Either String Bool) +checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do + showAction $ "checking " ++ name r + res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k + case res of + Right _ -> return $ Right True + Left (AWSError _ _) -> return $ Right False + Left e -> return $ Left (s3Error e) + where + noconn = Left $ error "S3 not configured" + +s3Warning :: ReqError -> Annex Bool +s3Warning e = do + warning $ prettyReqError e + return False + +s3Error :: ReqError -> a +s3Error e = error $ prettyReqError e + +s3Bool :: AWSResult () -> Annex Bool +s3Bool (Right _) = return True +s3Bool (Left e) = s3Warning e + +s3Action :: Remote -> a -> ((AWSConnection, String) -> Annex a) -> Annex a +s3Action r noconn action = do + let bucket = M.lookup "bucket" $ config r + conn <- case uuid r of + Nothing -> return Nothing + Just u -> s3Connection (config r) u + case (bucket, conn) of + (Just b, Just c) -> action (c, b) + _ -> return noconn + +bucketFile :: Remote -> Key -> FilePath +bucketFile r = munge . key2file + where + munge s = case M.lookup "mungekeys" c of + Just "ia" -> iaMunge $ fileprefix ++ s + _ -> fileprefix ++ s + fileprefix = M.findWithDefault "" "fileprefix" c + c = config r + +bucketKey :: Remote -> String -> Key -> S3Object +bucketKey r bucket k = S3Object bucket (bucketFile r k) "" [] L.empty + +{- Internet Archive limits filenames to a subset of ascii, + - with no whitespace. Other characters are xml entity + - encoded. -} +iaMunge :: String -> String +iaMunge = (>>= munge) + where + munge c + | isAsciiUpper c || isAsciiLower c || isNumber c = [c] + | c `elem` "_-.\"" = [c] + | isSpace c = [] + | otherwise = "&" ++ show (ord c) ++ ";" + +genBucket :: RemoteConfig -> UUID -> Annex () +genBucket c u = do + conn <- s3ConnectionRequired c u + showAction "checking bucket" + loc <- liftIO $ getBucketLocation conn bucket + case loc of + Right _ -> noop + Left err@(NetworkError _) -> s3Error err + Left (AWSError _ _) -> do + showAction $ "creating bucket in " ++ datacenter + res <- liftIO $ createBucketIn conn bucket datacenter + case res of + Right _ -> noop + Left err -> s3Error err + where + bucket = fromJust $ M.lookup "bucket" c + datacenter = fromJust $ M.lookup "datacenter" c + +s3ConnectionRequired :: RemoteConfig -> UUID -> Annex AWSConnection +s3ConnectionRequired c u = + maybe (error "Cannot connect to S3") return =<< s3Connection c u + +s3Connection :: RemoteConfig -> UUID -> Annex (Maybe AWSConnection) +s3Connection c u = go =<< getRemoteCredPairFor "S3" c (AWS.creds u) + where + go Nothing = return Nothing + go (Just (ak, sk)) = return $ Just $ AWSConnection host port ak sk + + host = fromJust $ M.lookup "host" c + port = let s = fromJust $ M.lookup "port" c in + case reads s of + [(p, _)] -> p + _ -> error $ "bad S3 port value: " ++ s diff --git a/Remote/Web.hs b/Remote/Web.hs new file mode 100644 index 0000000000..3fed619ba7 --- /dev/null +++ b/Remote/Web.hs @@ -0,0 +1,92 @@ +{- Web remotes. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Web (remote) where + +import Common.Annex +import Types.Remote +import qualified Git +import qualified Git.Construct +import Annex.Content +import Config +import Logs.Web +import qualified Utility.Url as Url +import Types.Key + +import qualified Data.Map as M + +remote :: RemoteType +remote = RemoteType { + typename = "web", + enumerate = list, + generate = gen, + setup = error "not supported" +} + +-- There is only one web remote, and it always exists. +-- (If the web should cease to exist, remove this module and redistribute +-- a new release to the survivors by carrier pigeon.) +list :: Annex [Git.Repo] +list = do + r <- liftIO $ Git.Construct.remoteNamed "web" Git.Construct.fromUnknown + return [r] + +gen :: Git.Repo -> Maybe UUID -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r _ _ gc = + return Remote { + uuid = Just webUUID, + cost = expensiveRemoteCost, + name = Git.repoDescribe r, + storeKey = uploadKey, + retrieveKeyFile = downloadKey, + retrieveKeyFileCheap = downloadKeyCheap, + removeKey = dropKey, + hasKey = checkKey, + hasKeyCheap = False, + whereisKey = Just getUrls, + config = M.empty, + gitconfig = gc, + localpath = Nothing, + repo = r, + readonly = True, + remotetype = remote + } + +downloadKey :: Key -> AssociatedFile -> FilePath -> Annex Bool +downloadKey key _file dest = get =<< getUrls key + where + get [] = do + warning "no known url" + return False + get urls = do + showOutput -- make way for download progress bar + downloadUrl urls dest + +downloadKeyCheap :: Key -> FilePath -> Annex Bool +downloadKeyCheap _ _ = return False + +uploadKey :: Key -> AssociatedFile -> MeterUpdate -> Annex Bool +uploadKey _ _ _ = do + warning "upload to web not supported" + return False + +dropKey :: Key -> Annex Bool +dropKey k = do + mapM_ (setUrlMissing k) =<< getUrls k + return True + +checkKey :: Key -> Annex (Either String Bool) +checkKey key = do + us <- getUrls key + if null us + then return $ Right False + else return . Right =<< checkKey' key us +checkKey' :: Key -> [URLString] -> Annex Bool +checkKey' key us = untilTrue us $ \u -> do + showAction $ "checking " ++ u + headers <- getHttpHeaders + liftIO $ Url.check u headers (keySize key) diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs new file mode 100644 index 0000000000..c439dd6b23 --- /dev/null +++ b/Remote/WebDAV.hs @@ -0,0 +1,342 @@ +{- WebDAV remotes. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE ScopedTypeVariables, CPP #-} + +#if defined VERSION_http_conduit +#if ! MIN_VERSION_http_conduit(1,9,0) +#define WITH_OLD_HTTP_CONDUIT +#endif +#endif + +module Remote.WebDAV (remote, davCreds, setCredsEnv) where + +import Network.Protocol.HTTP.DAV +import qualified Data.Map as M +import qualified Data.ByteString.UTF8 as B8 +import qualified Data.ByteString.Lazy.UTF8 as L8 +import qualified Data.ByteString.Lazy as L +import Network.URI (normalizePathSegments) +import qualified Control.Exception as E +import Network.HTTP.Conduit (HttpException(..)) +import Network.HTTP.Types +import System.IO.Error + +import Common.Annex +import Types.Remote +import qualified Git +import Config +import Remote.Helper.Special +import Remote.Helper.Encryptable +import Remote.Helper.Chunked +import Crypto +import Creds +import Meters +import Annex.Content + +type DavUrl = String +type DavUser = B8.ByteString +type DavPass = B8.ByteString + +remote :: RemoteType +remote = RemoteType { + typename = "webdav", + enumerate = findSpecialRemotes "webdav", + generate = gen, + setup = webdavSetup +} + +gen :: Git.Repo -> (Maybe UUID) -> RemoteConfig -> RemoteGitConfig -> Annex Remote +gen r mu c gc = new <$> remoteCost gc expensiveRemoteCost + where + new cst = encryptableRemote c + (storeEncrypted this) + (retrieveEncrypted this) + this + where + this = Remote { + uuid = mu, + cost = cst, + name = Git.repoDescribe r, + storeKey = store this, + retrieveKeyFile = retrieve this, + retrieveKeyFileCheap = retrieveCheap this, + removeKey = remove this, + hasKey = checkPresent this, + hasKeyCheap = False, + whereisKey = Nothing, + config = c, + repo = r, + gitconfig = gc, + localpath = Nothing, + readonly = False, + remotetype = remote + } + +webdavSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +webdavSetup u c = do + let url = fromMaybe (error "Specify url=") $ + M.lookup "url" c + c' <- encryptionSetup c + creds <- getCreds c' u + testDav url creds + gitConfigSpecialRemote u c' "webdav" "true" + setRemoteCredPair c' (davCreds u) + +store :: Remote -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool +store r k _f p = metered (Just p) k $ \meterupdate -> + davAction r False $ \(baseurl, user, pass) -> + sendAnnex k (void $ remove r k) $ \src -> + liftIO $ withMeteredFile src meterupdate $ + storeHelper r k baseurl user pass + +storeEncrypted :: Remote -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool +storeEncrypted r (cipher, enck) k p = metered (Just p) k $ \meterupdate -> + davAction r False $ \(baseurl, user, pass) -> + sendAnnex k (void $ remove r enck) $ \src -> + liftIO $ encrypt cipher (streamMeteredFile src meterupdate) $ + readBytes $ storeHelper r enck baseurl user pass + +storeHelper :: Remote -> Key -> DavUrl -> DavUser -> DavPass -> L.ByteString -> IO Bool +storeHelper r k baseurl user pass b = catchBoolIO $ do + davMkdir tmpurl user pass + storeChunks k tmpurl keyurl chunksize storer recorder finalizer + where + tmpurl = tmpLocation baseurl k + keyurl = davLocation baseurl k + chunksize = chunkSize $ config r + storer urls = storeChunked chunksize urls storehttp b + recorder url s = storehttp url (L8.fromString s) + finalizer srcurl desturl = do + void $ catchMaybeHttp (deleteContent desturl user pass) + davMkdir (urlParent desturl) user pass + moveContent srcurl (B8.fromString desturl) user pass + storehttp url v = putContent url user pass + (contentType, v) + +retrieveCheap :: Remote -> Key -> FilePath -> Annex Bool +retrieveCheap _ _ _ = return False + +retrieve :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve r k _f d = metered Nothing k $ \meterupdate -> + davAction r False $ \(baseurl, user, pass) -> liftIO $ catchBoolIO $ + withStoredFiles r k baseurl user pass onerr $ \urls -> do + meteredWriteFileChunks meterupdate d urls $ \url -> do + mb <- davGetUrlContent url user pass + case mb of + Nothing -> throwIO "download failed" + Just b -> return b + return True + where + onerr _ = return False + +retrieveEncrypted :: Remote -> (Cipher, Key) -> Key -> FilePath -> Annex Bool +retrieveEncrypted r (cipher, enck) k d = metered Nothing k $ \meterupdate -> + davAction r False $ \(baseurl, user, pass) -> liftIO $ catchBoolIO $ + withStoredFiles r enck baseurl user pass onerr $ \urls -> do + decrypt cipher (feeder user pass urls) $ + readBytes $ meteredWriteFile meterupdate d + return True + where + onerr _ = return False + + feeder _ _ [] _ = noop + feeder user pass (url:urls) h = do + mb <- davGetUrlContent url user pass + case mb of + Nothing -> throwIO "download failed" + Just b -> do + L.hPut h b + feeder user pass urls h + +remove :: Remote -> Key -> Annex Bool +remove r k = davAction r False $ \(baseurl, user, pass) -> liftIO $ do + -- Delete the key's whole directory, including any chunked + -- files, etc, in a single action. + let url = davLocation baseurl k + isJust <$> catchMaybeHttp (deleteContent url user pass) + +checkPresent :: Remote -> Key -> Annex (Either String Bool) +checkPresent r k = davAction r noconn go + where + noconn = Left $ error $ name r ++ " not configured" + + go (baseurl, user, pass) = do + showAction $ "checking " ++ name r + liftIO $ withStoredFiles r k baseurl user pass onerr check + where + check [] = return $ Right True + check (url:urls) = do + v <- davUrlExists url user pass + if v == Right True + then check urls + else return v + + {- Failed to read the chunkcount file; see if it's missing, + - or if there's a problem accessing it, + - or perhaps this was an intermittent error. -} + onerr url = do + v <- davUrlExists url user pass + if v == Right True + then return $ Left $ "failed to read " ++ url + else return v + +withStoredFiles + :: Remote + -> Key + -> DavUrl + -> DavUser + -> DavPass + -> (DavUrl -> IO a) + -> ([DavUrl] -> IO a) + -> IO a +withStoredFiles r k baseurl user pass onerr a + | isJust $ chunkSize $ config r = do + let chunkcount = keyurl ++ chunkCount + maybe (onerr chunkcount) (a . listChunks keyurl . L8.toString) + =<< davGetUrlContent chunkcount user pass + | otherwise = a [keyurl] + where + keyurl = davLocation baseurl k ++ keyFile k + +davAction :: Remote -> a -> ((DavUrl, DavUser, DavPass) -> Annex a) -> Annex a +davAction r unconfigured action = do + mcreds <- case uuid r of + Nothing -> return Nothing + Just u -> getCreds (config r) u + case (mcreds, M.lookup "url" $ config r) of + (Just (user, pass), Just url) -> + action (url, toDavUser user, toDavPass pass) + _ -> return unconfigured + +toDavUser :: String -> DavUser +toDavUser = B8.fromString + +toDavPass :: String -> DavPass +toDavPass = B8.fromString + +{- The directory where files(s) for a key are stored. -} +davLocation :: DavUrl -> Key -> DavUrl +davLocation baseurl k = addTrailingPathSeparator $ + davUrl baseurl $ hashDirLower k keyFile k + +{- Where we store temporary data for a key as it's being uploaded. -} +tmpLocation :: DavUrl -> Key -> DavUrl +tmpLocation baseurl k = addTrailingPathSeparator $ + davUrl baseurl $ "tmp" keyFile k + +davUrl :: DavUrl -> FilePath -> DavUrl +davUrl baseurl file = baseurl file + +davUrlExists :: DavUrl -> DavUser -> DavPass -> IO (Either String Bool) +davUrlExists url user pass = decode <$> catchHttp (getProps url user pass) + where + decode (Right _) = Right True +#ifdef WITH_OLD_HTTP_CONDUIT + decode (Left (Left (StatusCodeException status _))) +#else + decode (Left (Left (StatusCodeException status _ _))) +#endif + | statusCode status == statusCode notFound404 = Right False + decode (Left e) = Left $ showEitherException e + +davGetUrlContent :: DavUrl -> DavUser -> DavPass -> IO (Maybe L.ByteString) +davGetUrlContent url user pass = fmap (snd . snd) <$> + catchMaybeHttp (getPropsAndContent url user pass) + +{- Creates a directory in WebDAV, if not already present; also creating + - any missing parent directories. -} +davMkdir :: DavUrl -> DavUser -> DavPass -> IO () +davMkdir url user pass = go url + where + make u = makeCollection u user pass + + go u = do + r <- E.try (make u) :: IO (Either E.SomeException Bool) + case r of + {- Parent directory is missing. Recurse to create + - it, and try once more to create the directory. -} + Right False -> do + go (urlParent u) + void $ make u + {- Directory created successfully -} + Right True -> return () + {- Directory already exists, or some other error + - occurred. In the latter case, whatever wanted + - to use this directory will fail. -} + Left _ -> return () + +{- Catches HTTP and IO exceptions. -} +catchMaybeHttp :: IO a -> IO (Maybe a) +catchMaybeHttp a = (Just <$> a) `E.catches` + [ E.Handler $ \(_e :: HttpException) -> return Nothing + , E.Handler $ \(_e :: E.IOException) -> return Nothing + ] + +{- Catches HTTP and IO exceptions -} +catchHttp :: IO a -> IO (Either EitherException a) +catchHttp a = (Right <$> a) `E.catches` + [ E.Handler $ \(e :: HttpException) -> return $ Left $ Left e + , E.Handler $ \(e :: E.IOException) -> return $ Left $ Right e + ] + +type EitherException = Either HttpException E.IOException + +showEitherException :: EitherException -> String +#ifdef WITH_OLD_HTTP_CONDUIT +showEitherException (Left (StatusCodeException status _)) = +#else +showEitherException (Left (StatusCodeException status _ _)) = +#endif + show $ statusMessage status +showEitherException (Left httpexception) = show httpexception +showEitherException (Right ioexception) = show ioexception + +throwIO :: String -> IO a +throwIO msg = ioError $ mkIOError userErrorType msg Nothing Nothing + +urlParent :: DavUrl -> DavUrl +urlParent url = dropTrailingPathSeparator $ + normalizePathSegments (dropTrailingPathSeparator url ++ "/..") + where + +{- Test if a WebDAV store is usable, by writing to a test file, and then + - deleting the file. Exits with an IO error if not. -} +testDav :: String -> Maybe CredPair -> Annex () +testDav baseurl (Just (u, p)) = do + showSideAction "testing WebDAV server" + test "make directory" $ davMkdir baseurl user pass + test "write file" $ putContent testurl user pass + (contentType, L.empty) + test "delete file" $ deleteContent testurl user pass + where + test desc a = liftIO $ + either (\e -> throwIO $ "WebDAV failed to " ++ desc ++ ": " ++ showEitherException e) + (const noop) + =<< catchHttp a + + user = toDavUser u + pass = toDavPass p + testurl = davUrl baseurl "git-annex-test" +testDav _ Nothing = error "Need to configure webdav username and password." + +{- Content-Type to use for files uploaded to WebDAV. -} +contentType :: Maybe B8.ByteString +contentType = Just $ B8.fromString "application/octet-stream" + +getCreds :: RemoteConfig -> UUID -> Annex (Maybe CredPair) +getCreds c u = getRemoteCredPairFor "webdav" c (davCreds u) + +davCreds :: UUID -> CredPairStorage +davCreds u = CredPairStorage + { credPairFile = fromUUID u + , credPairEnvironment = ("WEBDAV_USERNAME", "WEBDAV_PASSWORD") + , credPairRemoteKey = Just "davcreds" + } + +setCredsEnv :: (String, String) -> IO () +setCredsEnv creds = setEnvCredPair creds $ davCreds undefined diff --git a/Seek.hs b/Seek.hs new file mode 100644 index 0000000000..6f87e8e6c6 --- /dev/null +++ b/Seek.hs @@ -0,0 +1,137 @@ +{- git-annex command seeking + - + - These functions find appropriate files or other things based on + - the values a user passes to a command, and prepare actions operating + - on them. + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Seek where + +import Common.Annex +import Types.Command +import Types.Key +import qualified Annex +import qualified Git +import qualified Git.Command +import qualified Git.LsFiles as LsFiles +import qualified Limit +import qualified Option +import Config + +seekHelper :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> [FilePath] -> Annex [FilePath] +seekHelper a params = do + ll <- inRepo $ \g -> + runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g) params + {- Show warnings only for files/directories that do not exist. -} + forM_ (map fst $ filter (null . snd) $ zip params ll) $ \p -> + unlessM (isJust <$> (liftIO $ catchMaybeIO $ getSymbolicLinkStatus p)) $ + fileNotFound p + return $ concat ll + +withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek +withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params + +withFilesNotInGit :: (FilePath -> CommandStart) -> CommandSeek +withFilesNotInGit a params = do + {- dotfiles are not acted on unless explicitly listed -} + files <- filter (not . dotfile) <$> + seekunless (null ps && not (null params)) ps + dotfiles <- seekunless (null dotps) dotps + prepFiltered a $ return $ concat $ segmentPaths params (files++dotfiles) + where + (dotps, ps) = partition dotfile params + seekunless True _ = return [] + seekunless _ l = do + force <- Annex.getState Annex.force + g <- gitRepo + liftIO $ Git.Command.leaveZombie <$> LsFiles.notInRepo force l g + +withPathContents :: ((FilePath, FilePath) -> CommandStart) -> CommandSeek +withPathContents a params = map a . concat <$> liftIO (mapM get params) + where + get p = ifM (isDirectory <$> getFileStatus p) + ( map (\f -> (f, makeRelative p f)) <$> dirContentsRecursive p + , return [(p, takeFileName p)] + ) + +withWords :: ([String] -> CommandStart) -> CommandSeek +withWords a params = return [a params] + +withStrings :: (String -> CommandStart) -> CommandSeek +withStrings a params = return $ map a params + +withPairs :: ((String, String) -> CommandStart) -> CommandSeek +withPairs a params = return $ map a $ pairs [] params + where + pairs c [] = reverse c + pairs c (x:y:xs) = pairs ((x,y):c) xs + pairs _ _ = error "expected pairs" + +withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek +withFilesToBeCommitted a params = prepFiltered a $ + seekHelper LsFiles.stagedNotDeleted params + +withFilesUnlocked :: (FilePath -> CommandStart) -> CommandSeek +withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged + +withFilesUnlockedToBeCommitted :: (FilePath -> CommandStart) -> CommandSeek +withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged + +withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> (FilePath -> CommandStart) -> CommandSeek +withFilesUnlocked' typechanged a params = do + -- unlocked files have changed type from a symlink to a regular file + typechangedfiles <- seekHelper typechanged params + let unlockedfiles = liftIO $ filterM notSymlink typechangedfiles + prepFiltered a unlockedfiles + +{- Finds files that may be modified. -} +withFilesMaybeModified :: (FilePath -> CommandStart) -> CommandSeek +withFilesMaybeModified a params = + prepFiltered a $ seekHelper LsFiles.modified params + +withKeys :: (Key -> CommandStart) -> CommandSeek +withKeys a params = return $ map (a . parse) params + where + parse p = fromMaybe (error "bad key") $ file2key p + +withValue :: Annex v -> (v -> CommandSeek) -> CommandSeek +withValue v a params = do + r <- v + a r params + +{- Modifies a seek action using the value of a field option, which is fed into + - a conversion function, and then is passed into the seek action. + - This ensures that the conversion function only runs once. + -} +withField :: Option -> (Maybe String -> Annex a) -> (a -> CommandSeek) -> CommandSeek +withField option converter = withValue $ + converter <=< Annex.getField $ Option.name option + +withFlag :: Option -> (Bool -> CommandSeek) -> CommandSeek +withFlag option = withValue $ Annex.getFlag (Option.name option) + +withNothing :: CommandStart -> CommandSeek +withNothing a [] = return [a] +withNothing _ _ = error "This command takes no parameters." + + +prepFiltered :: (FilePath -> CommandStart) -> Annex [FilePath] -> Annex [CommandStart] +prepFiltered a fs = do + matcher <- Limit.getMatcher + map (process matcher) <$> fs + where + process matcher f = ifM (matcher $ Annex.FileInfo f f) + ( a f , return Nothing ) + +notSymlink :: FilePath -> IO Bool +notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f + +whenNotDirect :: CommandSeek -> CommandSeek +whenNotDirect a params = ifM isDirect ( return [] , a params ) + +whenDirect :: CommandSeek -> CommandSeek +whenDirect a params = ifM isDirect ( a params, return [] ) diff --git a/Setup.hs b/Setup.hs new file mode 100644 index 0000000000..7a1f6cc265 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,63 @@ +{-# LANGUAGE NamedFieldPuns #-} + +{- cabal setup file -} + +import Distribution.Simple +import Distribution.Simple.LocalBuildInfo +import Distribution.Simple.Setup +import Distribution.Simple.Utils (installOrdinaryFiles, rawSystemExit) +import Distribution.PackageDescription (PackageDescription(..)) +import Distribution.Verbosity (Verbosity) +import System.FilePath +import Control.Applicative +import Control.Monad +import System.Directory + +import qualified Build.InstallDesktopFile as InstallDesktopFile +import qualified Build.Configure as Configure + +main = defaultMainWithHooks simpleUserHooks + { preConf = configure + , postInst = myPostInst + } + +configure _ _ = do + Configure.run Configure.tests + return (Nothing, []) + +myPostInst :: Args -> InstallFlags -> PackageDescription -> LocalBuildInfo -> IO () +myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do + installGitAnnexShell dest verbosity pkg lbi + installManpages dest verbosity pkg lbi + installDesktopFile dest verbosity pkg lbi + where + dest = NoCopyDest + verbosity = fromFlag installVerbosity + +installGitAnnexShell :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () +installGitAnnexShell copyDest verbosity pkg lbi = + rawSystemExit verbosity "ln" + ["-sf", "git-annex", dstBinDir "git-annex-shell"] + where + dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest + +{- See http://www.haskell.org/haskellwiki/Cabal/Developer-FAQ#Installing_manpages + - + - Man pages are provided prebuilt in the tarball in cabal, + - but may not be available otherwise, in which case, skip installing them. + -} +installManpages :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () +installManpages copyDest verbosity pkg lbi = + installOrdinaryFiles verbosity dstManDir =<< srcManpages + where + dstManDir = mandir (absoluteInstallDirs pkg lbi copyDest) "man1" + srcManpages = zip (repeat srcManDir) + <$> filterM doesFileExist manpages + srcManDir = "" + manpages = ["git-annex.1", "git-annex-shell.1"] + +installDesktopFile :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO () +installDesktopFile copyDest verbosity pkg lbi = + InstallDesktopFile.install $ dstBinDir "git-annex" + where + dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest diff --git a/Test.hs b/Test.hs new file mode 100644 index 0000000000..308769b26d --- /dev/null +++ b/Test.hs @@ -0,0 +1,1019 @@ +{- git-annex test suite + - + - Copyright 2010-2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Test where + +import Test.HUnit +import Test.QuickCheck +import Test.QuickCheck.Test + +import System.Posix.Directory (changeWorkingDirectory) +import System.Posix.Files +import System.Posix.Env +import Control.Exception.Extensible +import qualified Data.Map as M +import System.IO.HVFS (SystemFS(..)) +import qualified Text.JSON + +import Common + +import qualified Utility.SafeCommand +import qualified Annex +import qualified Annex.UUID +import qualified Backend +import qualified Git.CurrentRepo +import qualified Git.Filename +import qualified Locations +import qualified Types.KeySource +import qualified Types.Backend +import qualified Types.TrustLevel +import qualified Types +import qualified GitAnnex +import qualified Logs.UUIDBased +import qualified Logs.Trust +import qualified Logs.Remote +import qualified Logs.Unused +import qualified Logs.Transfer +import qualified Logs.Presence +import qualified Remote +import qualified Types.Key +import qualified Types.Messages +import qualified Config +import qualified Crypto +import qualified Utility.Path +import qualified Utility.FileMode +import qualified Utility.Gpg +import qualified Build.SysConfig +import qualified Utility.Format +import qualified Utility.Verifiable +import qualified Utility.Process +import qualified Utility.Misc +import qualified Utility.InodeCache + +main :: IO () +main = do + divider + putStrLn "First, some automated quick checks of properties ..." + divider + qcok <- all isSuccess <$> sequence quickcheck + divider + putStrLn "Now, some broader checks ..." + putStrLn " (Do not be alarmed by odd output here; it's normal." + putStrLn " wait for the last line to see how it went.)" + prepare + rs <- forM hunit $ \t -> do + divider + t + cleanup tmpdir + divider + propigate rs qcok + where + divider = putStrLn $ take 70 $ repeat '-' + +propigate :: [Counts] -> Bool -> IO () +propigate cs qcok + | countsok && qcok = putStrLn "All tests ok." + | otherwise = do + unless qcok $ + putStrLn "Quick check tests failed! This is a bug in git-annex." + unless countsok $ do + putStrLn "Some tests failed!" + putStrLn " (This could be due to a bug in git-annex, or an incompatability" + putStrLn " with utilities, such as git, installed on this system.)" + exitFailure + where + noerrors (Counts { errors = e , failures = f }) = e + f == 0 + countsok = all noerrors cs + +quickcheck :: [IO Result] +quickcheck = + [ check "prop_idempotent_deencode_git" Git.Filename.prop_idempotent_deencode + , check "prop_idempotent_deencode" Utility.Format.prop_idempotent_deencode + , check "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey + , check "prop_idempotent_key_encode" Types.Key.prop_idempotent_key_encode + , check "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape + , check "prop_idempotent_shellEscape_multiword" Utility.SafeCommand.prop_idempotent_shellEscape_multiword + , check "prop_idempotent_configEscape" Logs.Remote.prop_idempotent_configEscape + , check "prop_parse_show_Config" Logs.Remote.prop_parse_show_Config + , check "prop_parentDir_basics" Utility.Path.prop_parentDir_basics + , check "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics + , check "prop_relPathDirToFile_regressionTest" Utility.Path.prop_relPathDirToFile_regressionTest + , check "prop_cost_sane" Config.prop_cost_sane + , check "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane + , check "prop_TimeStamp_sane" Logs.UUIDBased.prop_TimeStamp_sane + , check "prop_addLog_sane" Logs.UUIDBased.prop_addLog_sane + , check "prop_verifiable_sane" Utility.Verifiable.prop_verifiable_sane + , check "prop_segment_regressionTest" Utility.Misc.prop_segment_regressionTest + , check "prop_read_write_transferinfo" Logs.Transfer.prop_read_write_transferinfo + , check "prop_read_show_inodecache" Utility.InodeCache.prop_read_show_inodecache + , check "prop_parse_show_log" Logs.Presence.prop_parse_show_log + , check "prop_read_show_TrustLevel" Types.TrustLevel.prop_read_show_TrustLevel + , check "prop_parse_show_TrustLog" Logs.Trust.prop_parse_show_TrustLog + ] + where + check desc prop = do + putStrLn desc + quickCheckResult prop + +hunit :: [IO Counts] +hunit = + -- test order matters, later tests may rely on state from earlier + [ check "init" test_init + , check "add" test_add + , check "reinject" test_reinject + , check "unannex" test_unannex + , check "drop" test_drop + , check "get" test_get + , check "move" test_move + , check "copy" test_copy + , check "lock" test_lock + , check "edit" test_edit + , check "fix" test_fix + , check "trust" test_trust + , check "fsck" test_fsck + , check "migrate" test_migrate + , check" unused" test_unused + , check "describe" test_describe + , check "find" test_find + , check "merge" test_merge + , check "status" test_status + , check "version" test_version + , check "sync" test_sync + , check "sync regression" test_sync_regression + , check "map" test_map + , check "uninit" test_uninit + , check "upgrade" test_upgrade + , check "whereis" test_whereis + , check "hook remote" test_hook_remote + , check "directory remote" test_directory_remote + , check "rsync remote" test_rsync_remote + , check "bup remote" test_bup_remote + , check "crypto" test_crypto + ] + where + check desc t = do + putStrLn desc + runTestTT t + +test_init :: Test +test_init = "git-annex init" ~: TestCase $ innewrepo $ do + git_annex "init" [reponame] @? "init failed" + where + reponame = "test repo" + +test_add :: Test +test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs] + where + -- this test case runs in the main repo, to set up a basic + -- annexed file that later tests will use + basic = TestCase $ inmainrepo $ do + writeFile annexedfile $ content annexedfile + git_annex "add" [annexedfile] @? "add failed" + annexed_present annexedfile + writeFile sha1annexedfile $ content sha1annexedfile + git_annex "add" [sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed" + annexed_present sha1annexedfile + checkbackend sha1annexedfile backendSHA1 + writeFile wormannexedfile $ content wormannexedfile + git_annex "add" [wormannexedfile, "--backend=WORM"] @? "add with WORM failed" + annexed_present wormannexedfile + checkbackend wormannexedfile backendWORM + boolSystem "git" [Params "rm --force -q", File wormannexedfile] @? "git rm failed" + writeFile ingitfile $ content ingitfile + boolSystem "git" [Param "add", File ingitfile] @? "git add failed" + boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed" + git_annex "add" [ingitfile] @? "add ingitfile should be no-op" + unannexed ingitfile + sha1dup = TestCase $ intmpclonerepo $ do + writeFile sha1annexedfiledup $ content sha1annexedfiledup + git_annex "add" [sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed" + annexed_present sha1annexedfiledup + annexed_present sha1annexedfile + subdirs = TestCase $ intmpclonerepo $ do + createDirectory "dir" + writeFile "dir/foo" $ content annexedfile + git_annex "add" ["dir"] @? "add of subdir failed" + createDirectory "dir2" + writeFile "dir2/foo" $ content annexedfile + changeWorkingDirectory "dir" + git_annex "add" ["../dir2"] @? "add of ../subdir failed" + +test_reinject :: Test +test_reinject = "git-annex reinject/fromkey" ~: TestCase $ intmpclonerepo $ do + git_annex "drop" ["--force", sha1annexedfile] @? "drop failed" + writeFile tmp $ content sha1annexedfile + r <- annexeval $ Types.Backend.getKey backendSHA1 $ + Types.KeySource.KeySource { Types.KeySource.keyFilename = tmp, Types.KeySource.contentLocation = tmp, Types.KeySource.inodeCache = Nothing } + let key = Types.Key.key2file $ fromJust r + git_annex "reinject" [tmp, sha1annexedfile] @? "reinject failed" + git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed" + annexed_present sha1annexedfiledup + where + tmp = "tmpfile" + +test_unannex :: Test +test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] + where + nocopy = "no content" ~: intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex "unannex" [annexedfile] @? "unannex failed with no copy" + annexed_notpresent annexedfile + withcopy = "with content" ~: intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + git_annex "unannex" [annexedfile, sha1annexedfile] @? "unannex failed" + unannexed annexedfile + git_annex "unannex" [annexedfile] @? "unannex failed on non-annexed file" + unannexed annexedfile + git_annex "unannex" [ingitfile] @? "unannex ingitfile should be no-op" + unannexed ingitfile + +test_drop :: Test +test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote] + where + noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get failed" + boolSystem "git" [Params "remote rm origin"] + @? "git remote rm origin failed" + not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of file" + annexed_present annexedfile + git_annex "drop" ["--force", annexedfile] @? "drop --force failed" + annexed_notpresent annexedfile + git_annex "drop" [annexedfile] @? "drop of dropped file failed" + git_annex "drop" [ingitfile] @? "drop ingitfile should be no-op" + unannexed ingitfile + withremote = "with remote" ~: TestCase $ intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + git_annex "drop" [annexedfile] @? "drop failed though origin has copy" + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + untrustedremote = "untrusted remote" ~: TestCase $ intmpclonerepo $ do + git_annex "untrust" ["origin"] @? "untrust of origin failed" + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile] @? "drop wrongly suceeded with only an untrusted copy of the file" + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile + +test_get :: Test +test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile + git_annex "get" [annexedfile] @? "get of file failed" + inmainrepo $ annexed_present annexedfile + annexed_present annexedfile + git_annex "get" [annexedfile] @? "get of file already here failed" + inmainrepo $ annexed_present annexedfile + annexed_present annexedfile + inmainrepo $ unannexed ingitfile + unannexed ingitfile + git_annex "get" [ingitfile] @? "get ingitfile should be no-op" + inmainrepo $ unannexed ingitfile + unannexed ingitfile + +test_move :: Test +test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "move" ["--from", "origin", annexedfile] @? "move --from of file failed" + annexed_present annexedfile + inmainrepo $ annexed_notpresent annexedfile + git_annex "move" ["--from", "origin", annexedfile] @? "move --from of file already here failed" + annexed_present annexedfile + inmainrepo $ annexed_notpresent annexedfile + git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file failed" + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile + git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file already there failed" + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile + git_annex "move" ["--to", "origin", ingitfile] @? "move of ingitfile should be no-op" + unannexed ingitfile + inmainrepo $ unannexed ingitfile + git_annex "move" ["--from", "origin", ingitfile] @? "move of ingitfile should be no-op" + unannexed ingitfile + inmainrepo $ unannexed ingitfile + +test_copy :: Test +test_copy = "git-annex copy" ~: TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "copy" ["--from", "origin", annexedfile] @? "copy --from of file failed" + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "copy" ["--from", "origin", annexedfile] @? "copy --from of file already here failed" + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "copy" ["--to", "origin", annexedfile] @? "copy --to of file already there failed" + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile + git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file already there failed" + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile + git_annex "copy" ["--to", "origin", ingitfile] @? "copy of ingitfile should be no-op" + unannexed ingitfile + inmainrepo $ unannexed ingitfile + git_annex "copy" ["--from", "origin", ingitfile] @? "copy of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile + +test_lock :: Test +test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do + -- regression test: unlock of not present file should skip it + annexed_notpresent annexedfile + not <$> git_annex "unlock" [annexedfile] @? "unlock failed to fail with not present file" + annexed_notpresent annexedfile + + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "unlock" [annexedfile] @? "unlock failed" + unannexed annexedfile + -- write different content, to verify that lock + -- throws it away + changecontent annexedfile + writeFile annexedfile $ content annexedfile ++ "foo" + git_annex "lock" [annexedfile] @? "lock failed" + annexed_present annexedfile + git_annex "unlock" [annexedfile] @? "unlock failed" + unannexed annexedfile + changecontent annexedfile + git_annex "add" [annexedfile] @? "add of modified file failed" + runchecks [checklink, checkunwritable] annexedfile + c <- readFile annexedfile + assertEqual "content of modified file" c (changedcontent annexedfile) + r' <- git_annex "drop" [annexedfile] + not r' @? "drop wrongly succeeded with no known copy of modified file" + +test_edit :: Test +test_edit = "git-annex edit/commit" ~: TestList [t False, t True] + where t precommit = TestCase $ intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "edit" [annexedfile] @? "edit failed" + unannexed annexedfile + changecontent annexedfile + if precommit + then do + -- pre-commit depends on the file being + -- staged, normally git commit does this + boolSystem "git" [Param "add", File annexedfile] + @? "git add of edited file failed" + git_annex "pre-commit" [] + @? "pre-commit failed" + else do + boolSystem "git" [Params "commit -q -a -m contentchanged"] + @? "git commit of edited file failed" + runchecks [checklink, checkunwritable] annexedfile + c <- readFile annexedfile + assertEqual "content of modified file" c (changedcontent annexedfile) + not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of modified file" + +test_fix :: Test +test_fix = "git-annex fix" ~: intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex "fix" [annexedfile] @? "fix of not present failed" + annexed_notpresent annexedfile + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "fix" [annexedfile] @? "fix of present file failed" + annexed_present annexedfile + createDirectory subdir + boolSystem "git" [Param "mv", File annexedfile, File subdir] + @? "git mv failed" + git_annex "fix" [newfile] @? "fix of moved file failed" + runchecks [checklink, checkunwritable] newfile + c <- readFile newfile + assertEqual "content of moved file" c (content annexedfile) + where + subdir = "s" + newfile = subdir ++ "/" ++ annexedfile + +test_trust :: Test +test_trust = "git-annex trust/untrust/semitrust/dead" ~: intmpclonerepo $ do + git_annex "trust" [repo] @? "trust failed" + trustcheck Logs.Trust.Trusted "trusted 1" + git_annex "trust" [repo] @? "trust of trusted failed" + trustcheck Logs.Trust.Trusted "trusted 2" + git_annex "untrust" [repo] @? "untrust failed" + trustcheck Logs.Trust.UnTrusted "untrusted 1" + git_annex "untrust" [repo] @? "untrust of untrusted failed" + trustcheck Logs.Trust.UnTrusted "untrusted 2" + git_annex "dead" [repo] @? "dead failed" + trustcheck Logs.Trust.DeadTrusted "deadtrusted 1" + git_annex "dead" [repo] @? "dead of dead failed" + trustcheck Logs.Trust.DeadTrusted "deadtrusted 2" + git_annex "semitrust" [repo] @? "semitrust failed" + trustcheck Logs.Trust.SemiTrusted "semitrusted 1" + git_annex "semitrust" [repo] @? "semitrust of semitrusted failed" + trustcheck Logs.Trust.SemiTrusted "semitrusted 2" + where + repo = "origin" + trustcheck expected msg = do + present <- annexeval $ do + l <- Logs.Trust.trustGet expected + u <- Remote.nameToUUID repo + return $ u `elem` l + assertBool msg present + +test_fsck :: Test +test_fsck = "git-annex fsck" ~: TestList [basicfsck, barefsck, withlocaluntrusted, withremoteuntrusted] + where + basicfsck = TestCase $ intmpclonerepo $ do + git_annex "fsck" [] @? "fsck failed" + boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed" + fsck_should_fail "numcopies unsatisfied" + boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed" + corrupt annexedfile + corrupt sha1annexedfile + barefsck = TestCase $ intmpbareclonerepo $ do + git_annex "fsck" [] @? "fsck failed" + withlocaluntrusted = TestCase $ intmpclonerepo $ do + git_annex "get" [annexedfile] @? "get failed" + git_annex "untrust" ["origin"] @? "untrust of origin repo failed" + git_annex "untrust" ["."] @? "untrust of current repo failed" + fsck_should_fail "content only available in untrusted (current) repository" + git_annex "trust" ["."] @? "trust of current repo failed" + git_annex "fsck" [annexedfile] @? "fsck failed on file present in trusted repo" + withremoteuntrusted = TestCase $ intmpclonerepo $ do + boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed" + git_annex "get" [annexedfile] @? "get failed" + git_annex "get" [sha1annexedfile] @? "get failed" + git_annex "fsck" [] @? "fsck failed with numcopies=2 and 2 copies" + git_annex "untrust" ["origin"] @? "untrust of origin failed" + fsck_should_fail "content not replicated to enough non-untrusted repositories" + + corrupt f = do + git_annex "get" [f] @? "get of file failed" + Utility.FileMode.allowWrite f + writeFile f (changedcontent f) + not <$> git_annex "fsck" [] @? "fsck failed to fail with corrupted file content" + git_annex "fsck" [] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f + fsck_should_fail m = do + not <$> git_annex "fsck" [] @? "fsck failed to fail with " ++ m + +test_migrate :: Test +test_migrate = "git-annex migrate" ~: TestList [t False, t True] + where t usegitattributes = TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + annexed_notpresent sha1annexedfile + git_annex "migrate" [annexedfile] @? "migrate of not present failed" + git_annex "migrate" [sha1annexedfile] @? "migrate of not present failed" + git_annex "get" [annexedfile] @? "get of file failed" + git_annex "get" [sha1annexedfile] @? "get of file failed" + annexed_present annexedfile + annexed_present sha1annexedfile + if usegitattributes + then do + writeFile ".gitattributes" $ "* annex.backend=SHA1" + git_annex "migrate" [sha1annexedfile] + @? "migrate sha1annexedfile failed" + git_annex "migrate" [annexedfile] + @? "migrate annexedfile failed" + else do + git_annex "migrate" [sha1annexedfile, "--backend", "SHA1"] + @? "migrate sha1annexedfile failed" + git_annex "migrate" [annexedfile, "--backend", "SHA1"] + @? "migrate annexedfile failed" + annexed_present annexedfile + annexed_present sha1annexedfile + checkbackend annexedfile backendSHA1 + checkbackend sha1annexedfile backendSHA1 + + -- check that reversing a migration works + writeFile ".gitattributes" $ "* annex.backend=SHA256" + git_annex "migrate" [sha1annexedfile] + @? "migrate sha1annexedfile failed" + git_annex "migrate" [annexedfile] + @? "migrate annexedfile failed" + annexed_present annexedfile + annexed_present sha1annexedfile + checkbackend annexedfile backendSHA256 + checkbackend sha1annexedfile backendSHA256 + +test_unused :: Test +test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do + -- keys have to be looked up before files are removed + annexedfilekey <- annexeval $ findkey annexedfile + sha1annexedfilekey <- annexeval $ findkey sha1annexedfile + git_annex "get" [annexedfile] @? "get of file failed" + git_annex "get" [sha1annexedfile] @? "get of file failed" + checkunused [] "after get" + boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed" + checkunused [] "after rm" + boolSystem "git" [Params "commit -q -m foo"] @? "git commit failed" + checkunused [] "after commit" + -- unused checks origin/master; once it's gone it is really unused + boolSystem "git" [Params "remote rm origin"] @? "git remote rm origin failed" + checkunused [annexedfilekey] "after origin branches are gone" + boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed" + boolSystem "git" [Params "commit -q -m foo"] @? "git commit failed" + checkunused [annexedfilekey, sha1annexedfilekey] "after rm sha1annexedfile" + + -- good opportunity to test dropkey also + git_annex "dropkey" ["--force", Types.Key.key2file annexedfilekey] + @? "dropkey failed" + checkunused [sha1annexedfilekey] ("after dropkey --force " ++ Types.Key.key2file annexedfilekey) + + git_annex "dropunused" ["1", "2"] @? "dropunused failed" + checkunused [] "after dropunused" + git_annex "dropunused" ["10", "501"] @? "dropunused failed on bogus numbers" + + where + checkunused expectedkeys desc = do + git_annex "unused" [] @? "unused failed" + unusedmap <- annexeval $ Logs.Unused.readUnusedLog "" + let unusedkeys = M.elems unusedmap + assertEqual ("unused keys differ " ++ desc) + (sort expectedkeys) (sort unusedkeys) + findkey f = do + r <- Backend.lookupFile f + return $ fst $ fromJust r + +test_describe :: Test +test_describe = "git-annex describe" ~: intmpclonerepo $ do + git_annex "describe" [".", "this repo"] @? "describe 1 failed" + git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed" + +test_find :: Test +test_find = "git-annex find" ~: intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex_expectoutput "find" [] [] + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + annexed_notpresent sha1annexedfile + git_annex_expectoutput "find" [] [annexedfile] + git_annex_expectoutput "find" ["--exclude", annexedfile, "--and", "--exclude", sha1annexedfile] [] + git_annex_expectoutput "find" ["--include", annexedfile] [annexedfile] + git_annex_expectoutput "find" ["--not", "--in", "origin"] [] + git_annex_expectoutput "find" ["--copies", "1", "--and", "--not", "--copies", "2"] [sha1annexedfile] + git_annex_expectoutput "find" ["--inbackend", "SHA1"] [sha1annexedfile] + git_annex_expectoutput "find" ["--inbackend", "WORM"] [] + + {- --include=* should match files in subdirectories too, + - and --exclude=* should exclude them. -} + createDirectory "dir" + writeFile "dir/subfile" "subfile" + git_annex "add" ["dir"] @? "add of subdir failed" + git_annex_expectoutput "find" ["--include", "*", "--exclude", annexedfile, "--exclude", sha1annexedfile] ["dir/subfile"] + git_annex_expectoutput "find" ["--exclude", "*"] [] + +test_merge :: Test +test_merge = "git-annex merge" ~: intmpclonerepo $ do + git_annex "merge" [] @? "merge failed" + +test_status :: Test +test_status = "git-annex status" ~: intmpclonerepo $ do + json <- git_annex_output "status" ["--json"] + case Text.JSON.decodeStrict json :: Text.JSON.Result (Text.JSON.JSObject Text.JSON.JSValue) of + Text.JSON.Ok _ -> return () + Text.JSON.Error e -> assertFailure e + +test_version :: Test +test_version = "git-annex version" ~: intmpclonerepo $ do + git_annex "version" [] @? "version failed" + +test_sync :: Test +test_sync = "git-annex sync" ~: intmpclonerepo $ do + git_annex "sync" [] @? "sync failed" + +{- Regression test for sync merge bug fixed in + - 0214e0fb175a608a49b812d81b4632c081f63027 -} +test_sync_regression :: Test +test_sync_regression = "git-annex sync_regression" ~: + {- We need 3 repos to see this bug. -} + withtmpclonerepo False $ \r1 -> do + withtmpclonerepo False $ \r2 -> do + withtmpclonerepo False $ \r3 -> do + forM_ [r1, r2, r3] $ \r -> indir r $ do + when (r /= r1) $ + boolSystem "git" [Params "remote add r1", File ("../../" ++ r1)] @? "remote add" + when (r /= r2) $ + boolSystem "git" [Params "remote add r2", File ("../../" ++ r2)] @? "remote add" + when (r /= r3) $ + boolSystem "git" [Params "remote add r3", File ("../../" ++ r3)] @? "remote add" + git_annex "get" [annexedfile] @? "get failed" + boolSystem "git" [Params "remote rm origin"] @? "remote rm" + forM_ [r3, r2, r1] $ \r -> indir r $ + git_annex "sync" [] @? "sync failed" + forM_ [r3, r2] $ \r -> indir r $ + git_annex "drop" ["--force", annexedfile] @? "drop failed" + indir r1 $ do + git_annex "sync" [] @? "sync failed in r1" + git_annex_expectoutput "find" ["--in", "r3"] [] + {- This was the bug. The sync + - mangled location log data and it + - thought the file was still in r2 -} + git_annex_expectoutput "find" ["--in", "r2"] [] + +test_map :: Test +test_map = "git-annex map" ~: intmpclonerepo $ do + -- set descriptions, that will be looked for in the map + git_annex "describe" [".", "this repo"] @? "describe 1 failed" + git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed" + -- --fast avoids it running graphviz, not a build dependency + git_annex "map" ["--fast"] @? "map failed" + +test_uninit :: Test +test_uninit = "git-annex uninit" ~: intmpclonerepo $ do + git_annex "get" [] @? "get failed" + annexed_present annexedfile + boolSystem "git" [Params "checkout git-annex"] @? "git checkout git-annex" + not <$> git_annex "uninit" [] @? "uninit failed to fail when git-annex branch was checked out" + boolSystem "git" [Params "checkout master"] @? "git checkout master" + _ <- git_annex "uninit" [] -- exit status not checked; does abnormal exit + checkregularfile annexedfile + doesDirectoryExist ".git" @? ".git vanished in uninit" + not <$> doesDirectoryExist ".git/annex" @? ".git/annex still present after uninit" + +test_upgrade :: Test +test_upgrade = "git-annex upgrade" ~: intmpclonerepo $ do + git_annex "upgrade" [] @? "upgrade from same version failed" + +test_whereis :: Test +test_whereis = "git-annex whereis" ~: intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex "whereis" [annexedfile] @? "whereis on non-present file failed" + git_annex "untrust" ["origin"] @? "untrust failed" + not <$> git_annex "whereis" [annexedfile] @? "whereis on non-present file only present in untrusted repo failed to fail" + git_annex "get" [annexedfile] @? "get failed" + annexed_present annexedfile + git_annex "whereis" [annexedfile] @? "whereis on present file failed" + +test_hook_remote :: Test +test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do + git_annex "initremote" (words "foo type=hook encryption=none hooktype=foo") @? "initremote failed" + createDirectory dir + git_config "annex.foo-store-hook" $ + "cp $ANNEX_FILE " ++ loc + git_config "annex.foo-retrieve-hook" $ + "cp " ++ loc ++ " $ANNEX_FILE" + git_config "annex.foo-remove-hook" $ + "rm -f " ++ loc + git_config "annex.foo-checkpresent-hook" $ + "if [ -e " ++ loc ++ " ]; then echo $ANNEX_KEY; fi" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to hook remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from hook remote failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" + annexed_present annexedfile + where + dir = "dir" + loc = dir ++ "/$ANNEX_KEY" + git_config k v = boolSystem "git" [Param "config", Param k, Param v] + @? "git config failed" + +test_directory_remote :: Test +test_directory_remote = "git-annex directory remote" ~: intmpclonerepo $ do + createDirectory "dir" + git_annex "initremote" (words $ "foo type=directory encryption=none directory=dir") @? "initremote failed" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to directory remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from directory remote failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" + annexed_present annexedfile + +test_rsync_remote :: Test +test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do + createDirectory "dir" + git_annex "initremote" (words $ "foo type=rsync encryption=none rsyncurl=dir") @? "initremote failed" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to rsync remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from rsync remote failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" + annexed_present annexedfile + +test_bup_remote :: Test +test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ when Build.SysConfig.bup $ do + dir <- absPath "dir" -- bup special remote needs an absolute path + createDirectory dir + git_annex "initremote" (words $ "foo type=bup encryption=none buprepo="++dir) @? "initremote failed" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to bup remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "copy" [annexedfile, "--from", "foo"] @? "copy --from bup remote failed" + annexed_present annexedfile + not <$> git_annex "move" [annexedfile, "--from", "foo"] @? "move --from bup remote failed to fail" + annexed_present annexedfile + +-- gpg is not a build dependency, so only test when it's available +test_crypto :: Test +test_crypto = "git-annex crypto" ~: intmpclonerepo $ when Build.SysConfig.gpg $ do + -- force gpg into batch mode for the tests + setEnv "GPG_BATCH" "1" True + Utility.Gpg.testTestHarness @? "test harness self-test failed" + Utility.Gpg.testHarness $ do + createDirectory "dir" + let initremote = git_annex "initremote" + [ "foo" + , "type=directory" + , "encryption=" ++ Utility.Gpg.testKeyId + , "directory=dir" + ] + initremote @? "initremote failed" + initremote @? "initremote failed when run twice in a row" + git_annex "get" [annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed" + annexed_present annexedfile + git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed" + annexed_notpresent annexedfile + git_annex "move" [annexedfile, "--from", "foo"] @? "move --from encrypted remote failed" + annexed_present annexedfile + not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail" + annexed_present annexedfile + +-- This is equivilant to running git-annex, but it's all run in-process +-- so test coverage collection works. +git_annex :: String -> [String] -> IO Bool +git_annex command params = do + -- catch all errors, including normally fatal errors + r <- try (run)::IO (Either SomeException ()) + case r of + Right _ -> return True + Left _ -> return False + where + run = GitAnnex.run (command:"-q":params) + +{- Runs git-annex and returns its output. -} +git_annex_output :: String -> [String] -> IO String +git_annex_output command params = do + got <- Utility.Process.readProcess "git-annex" (command:params) + -- XXX since the above is a separate process, code coverage stats are + -- not gathered for things run in it. + -- Run same command again, to get code coverage. + _ <- git_annex command params + return got + +git_annex_expectoutput :: String -> [String] -> [String] -> IO () +git_annex_expectoutput command params expected = do + got <- lines <$> git_annex_output command params + got == expected @? ("unexpected value running " ++ command ++ " " ++ show params ++ " -- got: " ++ show got ++ " expected: " ++ show expected) + +-- Runs an action in the current annex. Note that shutdown actions +-- are not run; this should only be used for actions that query state. +annexeval :: Types.Annex a -> IO a +annexeval a = do + s <- Annex.new =<< Git.CurrentRepo.get + Annex.eval s $ do + Annex.setOutput Types.Messages.QuietOutput + a + +innewrepo :: Assertion -> Assertion +innewrepo a = withgitrepo $ \r -> indir r a + +inmainrepo :: Assertion -> Assertion +inmainrepo a = indir mainrepodir a + +intmpclonerepo :: Assertion -> Assertion +intmpclonerepo a = withtmpclonerepo False $ \r -> indir r a + +intmpbareclonerepo :: Assertion -> Assertion +intmpbareclonerepo a = withtmpclonerepo True $ \r -> indir r a + +withtmpclonerepo :: Bool -> (FilePath -> Assertion) -> Assertion +withtmpclonerepo bare a = do + dir <- tmprepodir + bracket (clonerepo mainrepodir dir bare) cleanup a + +withgitrepo :: (FilePath -> Assertion) -> Assertion +withgitrepo = bracket (setuprepo mainrepodir) return + +indir :: FilePath -> Assertion -> Assertion +indir dir a = do + cwd <- getCurrentDirectory + -- Assertion failures throw non-IO errors; catch + -- any type of error and change back to cwd before + -- rethrowing. + r <- bracket_ (changeToTmpDir dir) (changeWorkingDirectory cwd) + (try (a)::IO (Either SomeException ())) + case r of + Right () -> return () + Left e -> throw e + +setuprepo :: FilePath -> IO FilePath +setuprepo dir = do + cleanup dir + ensuretmpdir + boolSystem "git" [Params "init -q", File dir] @? "git init failed" + indir dir $ do + boolSystem "git" [Params "config user.name", Param "Test User"] @? "git config failed" + boolSystem "git" [Params "config user.email test@example.com"] @? "git config failed" + return dir + +-- clones are always done as local clones; we cannot test ssh clones +clonerepo :: FilePath -> FilePath -> Bool -> IO FilePath +clonerepo old new bare = do + cleanup new + ensuretmpdir + let b = if bare then " --bare" else "" + boolSystem "git" [Params ("clone -q" ++ b), File old, File new] @? "git clone failed" + indir new $ git_annex "init" ["-q", new] @? "git annex init failed" + return new + +ensuretmpdir :: IO () +ensuretmpdir = do + e <- doesDirectoryExist tmpdir + unless e $ + createDirectory tmpdir + +cleanup :: FilePath -> IO () +cleanup dir = do + e <- doesDirectoryExist dir + when e $ do + -- git-annex prevents annexed file content from being + -- removed via directory permissions; undo + recurseDir SystemFS dir >>= + filterM doesDirectoryExist >>= + mapM_ Utility.FileMode.allowWrite + removeDirectoryRecursive dir + +checklink :: FilePath -> Assertion +checklink f = do + s <- getSymbolicLinkStatus f + isSymbolicLink s @? f ++ " is not a symlink" + +checkregularfile :: FilePath -> Assertion +checkregularfile f = do + s <- getSymbolicLinkStatus f + isRegularFile s @? f ++ " is not a normal file" + return () + +checkcontent :: FilePath -> Assertion +checkcontent f = do + c <- readFile f + assertEqual ("checkcontent " ++ f) c (content f) + +checkunwritable :: FilePath -> Assertion +checkunwritable f = do + -- Look at permissions bits rather than trying to write or using + -- fileAccess because if run as root, any file can be modified + -- despite permissions. + s <- getFileStatus f + let mode = fileMode s + if (mode == mode `unionFileModes` ownerWriteMode) + then assertFailure $ "able to modify annexed file's " ++ f ++ " content" + else return () + +checkwritable :: FilePath -> Assertion +checkwritable f = do + r <- tryIO $ writeFile f $ content f + case r of + Left _ -> assertFailure $ "unable to modify " ++ f + Right _ -> return () + +checkdangling :: FilePath -> Assertion +checkdangling f = do + r <- tryIO $ readFile f + case r of + Left _ -> return () -- expected; dangling link + Right _ -> assertFailure $ f ++ " was not a dangling link as expected" + +checklocationlog :: FilePath -> Bool -> Assertion +checklocationlog f expected = do + thisuuid <- annexeval Annex.UUID.getUUID + r <- annexeval $ Backend.lookupFile f + case r of + Just (k, _) -> do + uuids <- annexeval $ Remote.keyLocations k + assertEqual ("bad content in location log for " ++ f ++ " key " ++ (Types.Key.key2file k) ++ " uuid " ++ show thisuuid) + expected (thisuuid `elem` uuids) + _ -> assertFailure $ f ++ " failed to look up key" + +checkbackend :: FilePath -> Types.Backend -> Assertion +checkbackend file expected = do + r <- annexeval $ Backend.lookupFile file + let b = snd $ fromJust r + assertEqual ("backend for " ++ file) expected b + +inlocationlog :: FilePath -> Assertion +inlocationlog f = checklocationlog f True + +notinlocationlog :: FilePath -> Assertion +notinlocationlog f = checklocationlog f False + +runchecks :: [FilePath -> Assertion] -> FilePath -> Assertion +runchecks [] _ = return () +runchecks (a:as) f = do + a f + runchecks as f + +annexed_notpresent :: FilePath -> Assertion +annexed_notpresent = runchecks + [checklink, checkdangling, notinlocationlog] + +annexed_present :: FilePath -> Assertion +annexed_present = runchecks + [checklink, checkcontent, checkunwritable, inlocationlog] + +unannexed :: FilePath -> Assertion +unannexed = runchecks [checkregularfile, checkcontent, checkwritable] + +prepare :: IO () +prepare = do + whenM (doesDirectoryExist tmpdir) $ + error $ "The temporary directory " ++ tmpdir ++ " already exists; cannot run test suite." + + -- While PATH is mostly avoided, the commit hook does run it, + -- and so does git_annex_output. Make sure that the just-built + -- git annex is used. + cwd <- getCurrentDirectory + p <- getEnvDefault "PATH" "" + setEnv "PATH" (cwd ++ ":" ++ p) True + setEnv "TOPDIR" cwd True + -- Avoid git complaining if it cannot determine the user's email + -- address, or exploding if it doesn't know the user's name. + setEnv "GIT_AUTHOR_EMAIL" "test@example.com" True + setEnv "GIT_AUTHOR_NAME" "git-annex test" True + setEnv "GIT_COMMITTER_EMAIL" "test@example.com" True + setEnv "GIT_COMMITTER_NAME" "git-annex test" True + +changeToTmpDir :: FilePath -> IO () +changeToTmpDir t = do + -- Hack alert. Threading state to here was too much bother. + topdir <- getEnvDefault "TOPDIR" "" + changeWorkingDirectory $ topdir ++ "/" ++ t + +tmpdir :: String +tmpdir = ".t" + +mainrepodir :: FilePath +mainrepodir = tmpdir ++ "/repo" + +tmprepodir :: IO FilePath +tmprepodir = go (0 :: Int) + where + go n = do + let d = tmpdir ++ "/tmprepo" ++ show n + ifM (doesDirectoryExist d) + ( go $ n + 1 + , return d + ) + +annexedfile :: String +annexedfile = "foo" + +wormannexedfile :: String +wormannexedfile = "apple" + +sha1annexedfile :: String +sha1annexedfile = "sha1foo" + +sha1annexedfiledup :: String +sha1annexedfiledup = "sha1foodup" + +ingitfile :: String +ingitfile = "bar" + +content :: FilePath -> String +content f + | f == annexedfile = "annexed file content" + | f == ingitfile = "normal file content" + | f == sha1annexedfile ="sha1 annexed file content" + | f == sha1annexedfiledup = content sha1annexedfile + | f == wormannexedfile = "worm annexed file content" + | otherwise = "unknown file " ++ f + +changecontent :: FilePath -> IO () +changecontent f = writeFile f $ changedcontent f + +changedcontent :: FilePath -> String +changedcontent f = (content f) ++ " (modified)" + +backendSHA1 :: Types.Backend +backendSHA1 = backend_ "SHA1" + +backendSHA256 :: Types.Backend +backendSHA256 = backend_ "SHA256" + +backendWORM :: Types.Backend +backendWORM = backend_ "WORM" + +backend_ :: String -> Types.Backend +backend_ name = Backend.lookupBackendName name diff --git a/Types.hs b/Types.hs new file mode 100644 index 0000000000..5a4fed3be0 --- /dev/null +++ b/Types.hs @@ -0,0 +1,32 @@ +{- git-annex abstract data types + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types ( + Annex, + Backend, + Key, + UUID, + GitConfig(..), + RemoteGitConfig(..), + Remote, + RemoteType, + Option, + MeterUpdate +) where + +import Annex +import Types.Backend +import Types.GitConfig +import Types.Key +import Types.UUID +import Types.Remote +import Types.Option +import Types.Meters + +type Backend = BackendA Annex +type Remote = RemoteA Annex +type RemoteType = RemoteTypeA Annex diff --git a/Types/Backend.hs b/Types/Backend.hs new file mode 100644 index 0000000000..c7d962db06 --- /dev/null +++ b/Types/Backend.hs @@ -0,0 +1,26 @@ +{- git-annex key/value backend data type + - + - Most things should not need this, using Types instead + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Backend where + +import Types.Key +import Types.KeySource + +data BackendA a = Backend + { name :: String + , getKey :: KeySource -> a (Maybe Key) + , fsckKey :: Maybe (Key -> FilePath -> a Bool) + , canUpgradeKey :: Maybe (Key -> Bool) + } + +instance Show (BackendA a) where + show backend = "Backend { name =\"" ++ name backend ++ "\" }" + +instance Eq (BackendA a) where + a == b = name a == name b diff --git a/Types/BranchState.hs b/Types/BranchState.hs new file mode 100644 index 0000000000..2f7948ebbf --- /dev/null +++ b/Types/BranchState.hs @@ -0,0 +1,16 @@ +{- git-annex BranchState data type + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.BranchState where + +data BranchState = BranchState + { branchUpdated :: Bool -- has the branch been updated this run? + , indexChecked :: Bool -- has the index file been checked to exist? + } + +startBranchState :: BranchState +startBranchState = BranchState False False diff --git a/Types/Command.hs b/Types/Command.hs new file mode 100644 index 0000000000..b652bdad59 --- /dev/null +++ b/Types/Command.hs @@ -0,0 +1,57 @@ +{- git-annex command data types + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Command where + +import Data.Ord + +import Types + +{- A command runs in these stages. + - + - a. The check stage runs checks, that error out if + - anything prevents the command from running. -} +data CommandCheck = CommandCheck { idCheck :: Int, runCheck :: Annex () } +{- b. The seek stage takes the parameters passed to the command, + - looks through the repo to find the ones that are relevant + - to that command (ie, new files to add), and generates + - a list of start stage actions. -} +type CommandSeek = [String] -> Annex [CommandStart] +{- c. The start stage is run before anything is printed about the + - command, is passed some input, and can early abort it + - if the input does not make sense. It should run quickly and + - should not modify Annex state. -} +type CommandStart = Annex (Maybe CommandPerform) +{- d. The perform stage is run after a message is printed about the command + - being run, and it should be where the bulk of the work happens. -} +type CommandPerform = Annex (Maybe CommandCleanup) +{- e. The cleanup stage is run only if the perform stage succeeds, and it + - returns the overall success/fail of the command. -} +type CommandCleanup = Annex Bool + +{- A command is defined by specifying these things. -} +data Command = Command + { cmdoptions :: [Option] -- command-specific options + , cmdnorepo :: Maybe (IO ()) -- an action to run when not in a repo + , cmdcheck :: [CommandCheck] -- check stage + , cmdnocommit :: Bool -- don't commit journalled state changes + , cmdname :: String + , cmdparamdesc :: String -- description of params for usage + , cmdseek :: [CommandSeek] -- seek stage + , cmddesc :: String -- description of command for usage + } + +{- CommandCheck functions can be compared using their unique id. -} +instance Eq CommandCheck where + a == b = idCheck a == idCheck b + +instance Eq Command where + a == b = cmdname a == cmdname b + +{- Order commands by name -} +instance Ord Command where + compare = comparing cmdname diff --git a/Types/Crypto.hs b/Types/Crypto.hs new file mode 100644 index 0000000000..135522ba11 --- /dev/null +++ b/Types/Crypto.hs @@ -0,0 +1,20 @@ +{- git-annex crypto types + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Crypto ( + Cipher(..), + StorableCipher(..), + KeyIds(..), +) where + +import Utility.Gpg (KeyIds(..)) + +-- XXX ideally, this would be a locked memory region +newtype Cipher = Cipher String + +data StorableCipher = EncryptedCipher String KeyIds | SharedCipher String + deriving (Ord, Eq) diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs new file mode 100644 index 0000000000..2430a73a70 --- /dev/null +++ b/Types/GitConfig.hs @@ -0,0 +1,131 @@ +{- git-annex configuration + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.GitConfig ( + GitConfig(..), + extractGitConfig, + RemoteGitConfig(..), + extractRemoteGitConfig, +) where + +import Common +import qualified Git +import qualified Git.Config +import Utility.DataUnits + +{- Main git-annex settings. Each setting corresponds to a git-config key + - such as annex.foo -} +data GitConfig = GitConfig + { annexVersion :: Maybe String + , annexNumCopies :: Int + , annexDiskReserve :: Integer + , annexDirect :: Bool + , annexBackends :: [String] + , annexQueueSize :: Maybe Int + , annexBloomCapacity :: Maybe Int + , annexBloomAccuracy :: Maybe Int + , annexSshCaching :: Maybe Bool + , annexAlwaysCommit :: Bool + , annexDelayAdd :: Maybe Int + , annexHttpHeaders :: [String] + , annexHttpHeadersCommand :: Maybe String + , annexAutoCommit :: Bool + , annexWebOptions :: [String] + , annexCrippledFileSystem :: Bool + , coreSymlinks :: Bool + } + +extractGitConfig :: Git.Repo -> GitConfig +extractGitConfig r = GitConfig + { annexVersion = notempty $ getmaybe (annex "version") + , annexNumCopies = get (annex "numcopies") 1 + , annexDiskReserve = fromMaybe onemegabyte $ + readSize dataUnits =<< getmaybe (annex "diskreserve") + , annexDirect = getbool (annex "direct") False + , annexBackends = getwords (annex "backends") + , annexQueueSize = getmayberead (annex "queuesize") + , annexBloomCapacity = getmayberead (annex "bloomcapacity") + , annexBloomAccuracy = getmayberead (annex "bloomaccuracy") + , annexSshCaching = getmaybebool (annex "sshcaching") + , annexAlwaysCommit = getbool (annex "alwayscommit") True + , annexDelayAdd = getmayberead (annex "delayadd") + , annexHttpHeaders = getlist (annex "http-headers") + , annexHttpHeadersCommand = getmaybe (annex "http-headers-command") + , annexAutoCommit = getbool (annex "autocommit") True + , annexWebOptions = getwords (annex "web-options") + , annexCrippledFileSystem = getbool (annex "crippledfilesystem") False + , coreSymlinks = getbool "core.symlinks" True + } + where + get k def = fromMaybe def $ getmayberead k + getbool k def = fromMaybe def $ getmaybebool k + getmaybebool k = Git.Config.isTrue =<< getmaybe k + getmayberead k = readish =<< getmaybe k + getmaybe k = Git.Config.getMaybe k r + getlist k = Git.Config.getList k r + getwords k = fromMaybe [] $ words <$> getmaybe k + + annex k = "annex." ++ k + + onemegabyte = 1000000 + +{- Per-remote git-annex settings. Each setting corresponds to a git-config + - key such as .annex-foo, or if that is not set, a default from + - annex.foo -} +data RemoteGitConfig = RemoteGitConfig + { remoteAnnexCost :: Maybe Int + , remoteAnnexCostCommand :: Maybe String + , remoteAnnexIgnore :: Bool + , remoteAnnexSync :: Bool + , remoteAnnexTrustLevel :: Maybe String + , remoteAnnexStartCommand :: Maybe String + , remoteAnnexStopCommand :: Maybe String + + -- these settings are specific to particular types of remotes + , remoteAnnexSshOptions :: [String] + , remoteAnnexRsyncOptions :: [String] + , remoteAnnexRsyncUrl :: Maybe String + , remoteAnnexBupRepo :: Maybe String + , remoteAnnexBupSplitOptions :: [String] + , remoteAnnexDirectory :: Maybe FilePath + , remoteAnnexHookType :: Maybe String + } + +extractRemoteGitConfig :: Git.Repo -> String -> RemoteGitConfig +extractRemoteGitConfig r remotename = RemoteGitConfig + { remoteAnnexCost = getmayberead "cost" + , remoteAnnexCostCommand = notempty $ getmaybe "cost-command" + , remoteAnnexIgnore = getbool "ignore" False + , remoteAnnexSync = getbool "sync" True + , remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel" + , remoteAnnexStartCommand = notempty $ getmaybe "start-command" + , remoteAnnexStopCommand = notempty $ getmaybe "stop-command" + + , remoteAnnexSshOptions = getoptions "ssh-options" + , remoteAnnexRsyncOptions = getoptions "rsync-options" + , remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl" + , remoteAnnexBupRepo = getmaybe "buprepo" + , remoteAnnexBupSplitOptions = getoptions "bup-split-options" + , remoteAnnexDirectory = notempty $ getmaybe "directory" + , remoteAnnexHookType = notempty $ getmaybe "hooktype" + } + where + getbool k def = fromMaybe def $ getmaybebool k + getmaybebool k = Git.Config.isTrue =<< getmaybe k + getmayberead k = readish =<< getmaybe k + getmaybe k = maybe (Git.Config.getMaybe (key k) r) Just $ + Git.Config.getMaybe (remotekey k) r + getoptions k = fromMaybe [] $ words <$> getmaybe k + + key k = "annex." ++ k + remotekey k = "remote." ++ remotename ++ ".annex-" ++ k + +notempty :: Maybe String -> Maybe String +notempty Nothing = Nothing +notempty (Just "") = Nothing +notempty (Just s) = Just s + diff --git a/Types/Group.hs b/Types/Group.hs new file mode 100644 index 0000000000..88bc352077 --- /dev/null +++ b/Types/Group.hs @@ -0,0 +1,27 @@ +{- git-annex repo groups + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Group ( + Group, + GroupMap(..), + emptyGroupMap +) where + +import Types.UUID + +import qualified Data.Map as M +import qualified Data.Set as S + +type Group = String + +data GroupMap = GroupMap + { groupsByUUID :: M.Map UUID (S.Set Group) + , uuidsByGroup :: M.Map Group (S.Set UUID) + } + +emptyGroupMap :: GroupMap +emptyGroupMap = GroupMap M.empty M.empty diff --git a/Types/Key.hs b/Types/Key.hs new file mode 100644 index 0000000000..51449ca33e --- /dev/null +++ b/Types/Key.hs @@ -0,0 +1,86 @@ +{- git-annex Key data type + - + - Most things should not need this, using Types instead + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Key ( + Key(..), + stubKey, + key2file, + file2key, + + prop_idempotent_key_encode +) where + +import System.Posix.Types + +import Common +import Utility.QuickCheck + +{- A Key has a unique name, is associated with a key/value backend, + - and may contain other optional metadata. -} +data Key = Key { + keyName :: String, + keyBackendName :: String, + keySize :: Maybe Integer, + keyMtime :: Maybe EpochTime +} deriving (Eq, Ord, Read, Show) + +stubKey :: Key +stubKey = Key { + keyName = "", + keyBackendName = "", + keySize = Nothing, + keyMtime = Nothing +} + +fieldSep :: Char +fieldSep = '-' + +{- Converts a key to a string that is suitable for use as a filename. + - The name field is always shown last, separated by doubled fieldSeps, + - and is the only field allowed to contain the fieldSep. -} +key2file :: Key -> FilePath +key2file Key { keyBackendName = b, keySize = s, keyMtime = m, keyName = n } = + b +++ ('s' ?: s) +++ ('m' ?: m) +++ (fieldSep : n) + where + "" +++ y = y + x +++ "" = x + x +++ y = x ++ fieldSep:y + c ?: (Just v) = c : show v + _ ?: _ = "" + +file2key :: FilePath -> Maybe Key +file2key s = if key == Just stubKey then Nothing else key + where + key = startbackend stubKey s + + startbackend k v = sepfield k v addbackend + + sepfield k v a = case span (/= fieldSep) v of + (v', _:r) -> findfields r $ a k v' + _ -> Nothing + + findfields (c:v) (Just k) + | c == fieldSep = Just $ k { keyName = v } + | otherwise = sepfield k v $ addfield c + findfields _ v = v + + addbackend k v = Just k { keyBackendName = v } + addfield 's' k v = Just k { keySize = readish v } + addfield 'm' k v = Just k { keyMtime = readish v } + addfield _ _ _ = Nothing + +instance Arbitrary Key where + arbitrary = Key + <$> arbitrary + <*> (listOf1 $ elements ['A'..'Z']) -- BACKEND + <*> ((abs <$>) <$> arbitrary) -- size cannot be negative + <*> arbitrary + +prop_idempotent_key_encode :: Key -> Bool +prop_idempotent_key_encode k = Just k == (file2key . key2file) k diff --git a/Types/KeySource.hs b/Types/KeySource.hs new file mode 100644 index 0000000000..fd4af07a68 --- /dev/null +++ b/Types/KeySource.hs @@ -0,0 +1,29 @@ +{- KeySource data type + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.KeySource where + +import Utility.InodeCache + +{- When content is in the process of being added to the annex, + - and a Key generated from it, this data type is used. + - + - The contentLocation may be different from the filename + - associated with the key. For example, the add command + - may temporarily hard link the content into a lockdown directory + - for checking. The migrate command uses the content + - of a different Key. + - + - The inodeCache can be used to detect some types of modifications to + - files that may be made while they're in the process of being added. + -} +data KeySource = KeySource + { keyFilename :: FilePath + , contentLocation :: FilePath + , inodeCache :: Maybe InodeCache + } + deriving (Show) diff --git a/Types/Messages.hs b/Types/Messages.hs new file mode 100644 index 0000000000..4fcce79f80 --- /dev/null +++ b/Types/Messages.hs @@ -0,0 +1,24 @@ +{- git-annex Messages data types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Messages where + +import qualified Data.Set as S + +data OutputType = NormalOutput | QuietOutput | JSONOutput + +data SideActionBlock = NoBlock | StartBlock | InBlock + deriving (Eq) + +data MessageState = MessageState + { outputType :: OutputType + , sideActionBlock :: SideActionBlock + , fileNotFoundShown :: S.Set FilePath + } + +defaultMessageState :: MessageState +defaultMessageState = MessageState NormalOutput NoBlock S.empty diff --git a/Types/Meters.hs b/Types/Meters.hs new file mode 100644 index 0000000000..ef304d1aef --- /dev/null +++ b/Types/Meters.hs @@ -0,0 +1,12 @@ +{- git-annex meter types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Meters where + +{- An action that can be run repeatedly, feeding it the number of + - bytes sent or retrieved so far. -} +type MeterUpdate = (Integer -> IO ()) diff --git a/Types/Option.hs b/Types/Option.hs new file mode 100644 index 0000000000..0362578388 --- /dev/null +++ b/Types/Option.hs @@ -0,0 +1,17 @@ +{- git-annex command options + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Option where + +import System.Console.GetOpt + +import Annex + +{- Each dashed command-line option results in generation of an action + - in the Annex monad that performs the necessary setting. + -} +type Option = OptDescr (Annex ()) diff --git a/Types/Remote.hs b/Types/Remote.hs new file mode 100644 index 0000000000..e1fd029b04 --- /dev/null +++ b/Types/Remote.hs @@ -0,0 +1,90 @@ +{- git-annex remotes types + - + - Most things should not need this, using Types instead + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.Remote where + +import Data.Map as M +import Data.Ord + +import qualified Git +import Types.Key +import Types.UUID +import Types.Meters +import Types.GitConfig + +type RemoteConfigKey = String +type RemoteConfig = M.Map RemoteConfigKey String + +{- There are different types of remotes. -} +data RemoteTypeA a = RemoteType { + -- human visible type name + typename :: String, + -- enumerates remotes of this type + enumerate :: a [Git.Repo], + -- generates a remote of this type + generate :: Git.Repo -> (Maybe UUID) -> RemoteConfig -> RemoteGitConfig -> a (RemoteA a), + -- initializes or changes a remote + setup :: UUID -> RemoteConfig -> a RemoteConfig +} + +instance Eq (RemoteTypeA a) where + x == y = typename x == typename y + +{- A filename associated with a Key, for display to user. -} +type AssociatedFile = Maybe FilePath + +{- An individual remote. -} +data RemoteA a = Remote { + -- each available Remote has a unique uuid + -- remotes may be unavailable or not fully set up and have Nothing + uuid :: Maybe UUID, + -- each Remote has a human visible name + name :: String, + -- Remotes have a use cost; higher is more expensive + cost :: Int, + -- Transfers a key to the remote. + storeKey :: Key -> AssociatedFile -> MeterUpdate -> a Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> a Bool, + -- retrieves a key's contents to a tmp file, if it can be done cheaply + retrieveKeyFileCheap :: Key -> FilePath -> a Bool, + -- removes a key's contents + removeKey :: Key -> a Bool, + -- Checks if a key is present in the remote; if the remote + -- cannot be accessed returns a Left error message. + hasKey :: Key -> a (Either String Bool), + -- Some remotes can check hasKey without an expensive network + -- operation. + hasKeyCheap :: Bool, + -- Some remotes can provide additional details for whereis. + whereisKey :: Maybe (Key -> a [String]), + -- a Remote has a persistent configuration store + config :: RemoteConfig, + -- git repo for the Remote + repo :: Git.Repo, + -- a Remote's configuration from git + gitconfig :: RemoteGitConfig, + -- a Remote can be assocated with a specific local filesystem path + localpath :: Maybe FilePath, + -- a Remote can be known to be readonly + readonly :: Bool, + -- the type of the remote + remotetype :: RemoteTypeA a +} + +instance Show (RemoteA a) where + show remote = "Remote { name =\"" ++ name remote ++ "\" }" + +-- two remotes are the same if they have the same uuid +instance Eq (RemoteA a) where + x == y = uuid x == uuid y + +-- order remotes by cost +instance Ord (RemoteA a) where + compare = comparing cost diff --git a/Types/StandardGroups.hs b/Types/StandardGroups.hs new file mode 100644 index 0000000000..a8ac89b4ec --- /dev/null +++ b/Types/StandardGroups.hs @@ -0,0 +1,56 @@ +{- git-annex standard repository groups + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.StandardGroups where + +data StandardGroup + = ClientGroup + | TransferGroup + | BackupGroup + | SmallArchiveGroup + | FullArchiveGroup + | SourceGroup + | ManualGroup + deriving (Eq, Ord, Enum, Bounded, Show) + +fromStandardGroup :: StandardGroup -> String +fromStandardGroup ClientGroup = "client" +fromStandardGroup TransferGroup = "transfer" +fromStandardGroup BackupGroup = "backup" +fromStandardGroup SmallArchiveGroup = "smallarchive" +fromStandardGroup FullArchiveGroup = "archive" +fromStandardGroup SourceGroup = "source" +fromStandardGroup ManualGroup = "manual" + +toStandardGroup :: String -> Maybe StandardGroup +toStandardGroup "client" = Just ClientGroup +toStandardGroup "transfer" = Just TransferGroup +toStandardGroup "backup" = Just BackupGroup +toStandardGroup "smallarchive" = Just SmallArchiveGroup +toStandardGroup "archive" = Just FullArchiveGroup +toStandardGroup "source" = Just SourceGroup +toStandardGroup "manual" = Just ManualGroup +toStandardGroup _ = Nothing + +descStandardGroup :: StandardGroup -> String +descStandardGroup ClientGroup = "client: a repository on your computer" +descStandardGroup TransferGroup = "transfer: distributes files to clients" +descStandardGroup BackupGroup = "backup: backs up all files" +descStandardGroup SmallArchiveGroup = "small archive: archives files located in \"archive\" directories" +descStandardGroup FullArchiveGroup = "full archive: archives all files not archived elsewhere" +descStandardGroup SourceGroup = "file source: moves files on to other repositories" +descStandardGroup ManualGroup = "manual mode: only stores files you manually choose" + +{- See doc/preferred_content.mdwn for explanations of these expressions. -} +preferredContent :: StandardGroup -> String +preferredContent ClientGroup = "exclude=*/archive/* and exclude=archive/*" +preferredContent TransferGroup = "not (inallgroup=client and copies=client:2) and " ++ preferredContent ClientGroup +preferredContent BackupGroup = "include=*" +preferredContent SmallArchiveGroup = "(include=*/archive/* or include=archive/*) and " ++ preferredContent FullArchiveGroup +preferredContent FullArchiveGroup = "not (copies=archive:1 or copies=smallarchive:1)" +preferredContent SourceGroup = "not (copies=1)" +preferredContent ManualGroup = "present and exclude=*/archive/* and exclude=archive/*" diff --git a/Types/TrustLevel.hs b/Types/TrustLevel.hs new file mode 100644 index 0000000000..27325cd2bf --- /dev/null +++ b/Types/TrustLevel.hs @@ -0,0 +1,41 @@ +{- git-annex trust levels + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.TrustLevel ( + TrustLevel(..), + TrustMap, + readTrustLevel, + showTrustLevel, + prop_read_show_TrustLevel +) where + +import qualified Data.Map as M + +import Types.UUID + +data TrustLevel = Trusted | SemiTrusted | UnTrusted | DeadTrusted + deriving (Eq, Enum, Ord, Bounded) + +type TrustMap = M.Map UUID TrustLevel + +readTrustLevel :: String -> Maybe TrustLevel +readTrustLevel "trusted" = Just Trusted +readTrustLevel "untrusted" = Just UnTrusted +readTrustLevel "semitrusted" = Just SemiTrusted +readTrustLevel "dead" = Just DeadTrusted +readTrustLevel _ = Nothing + +showTrustLevel :: TrustLevel -> String +showTrustLevel Trusted = "trusted" +showTrustLevel UnTrusted = "untrusted" +showTrustLevel SemiTrusted = "semitrusted" +showTrustLevel DeadTrusted = "dead" + +prop_read_show_TrustLevel :: Bool +prop_read_show_TrustLevel = all check [minBound .. maxBound] + where + check l = readTrustLevel (showTrustLevel l) == Just l diff --git a/Types/UUID.hs b/Types/UUID.hs new file mode 100644 index 0000000000..1a272fddcf --- /dev/null +++ b/Types/UUID.hs @@ -0,0 +1,36 @@ +{- git-annex UUID type + - + - Copyright 2011,2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.UUID ( + UUID, + UUIDMap, + fromUUID, + toUUID, + genUUID +) where + +import qualified Data.Map as M +import qualified Data.UUID as U +import System.Random +import Control.Applicative + +-- A UUID an arbitrary opaque string, which cannot be empty. +data UUID = UUID String + deriving (Eq, Ord, Show, Read) + +type UUIDMap = M.Map UUID String + +fromUUID :: UUID -> String +fromUUID (UUID u) = u + +toUUID :: String -> Maybe UUID +toUUID [] = Nothing +toUUID s = Just (UUID s) + +{- Generates a random UUID, that does not include the MAC address. -} +genUUID :: IO UUID +genUUID = UUID . show <$> (randomIO :: IO U.UUID) diff --git a/Upgrade.hs b/Upgrade.hs new file mode 100644 index 0000000000..705b190d85 --- /dev/null +++ b/Upgrade.hs @@ -0,0 +1,22 @@ +{- git-annex upgrade support + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade where + +import Common.Annex +import Annex.Version +import qualified Upgrade.V0 +import qualified Upgrade.V1 +import qualified Upgrade.V2 + +upgrade :: Annex Bool +upgrade = go =<< getVersion + where + go (Just "0") = Upgrade.V0.upgrade + go (Just "1") = Upgrade.V1.upgrade + go (Just "2") = Upgrade.V2.upgrade + go _ = return True diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs new file mode 100644 index 0000000000..00a08cb458 --- /dev/null +++ b/Upgrade/V0.hs @@ -0,0 +1,49 @@ +{- git-annex v0 -> v1 upgrade support + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V0 where + +import Common.Annex +import Annex.Content +import qualified Upgrade.V1 + +upgrade :: Annex Bool +upgrade = do + showAction "v0 to v1" + + -- do the reorganisation of the key files + olddir <- fromRepo gitAnnexDir + keys <- getKeysPresent0 olddir + forM_ keys $ \k -> moveAnnex k $ olddir keyFile0 k + + -- update the symlinks to the key files + -- No longer needed here; V1.upgrade does the same thing + + -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 + Upgrade.V1.upgrade + +-- these stayed unchanged between v0 and v1 +keyFile0 :: Key -> FilePath +keyFile0 = Upgrade.V1.keyFile1 +fileKey0 :: FilePath -> Key +fileKey0 = Upgrade.V1.fileKey1 +lookupFile0 :: FilePath -> Annex (Maybe (Key, Backend)) +lookupFile0 = Upgrade.V1.lookupFile1 + +getKeysPresent0 :: FilePath -> Annex [Key] +getKeysPresent0 dir = ifM (liftIO $ doesDirectoryExist dir) + ( liftIO $ map fileKey0 + <$> (filterM present =<< getDirectoryContents dir) + , return [] + ) + where + present d = do + result <- tryIO $ + getFileStatus $ dir ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs new file mode 100644 index 0000000000..72e105d167 --- /dev/null +++ b/Upgrade/V1.hs @@ -0,0 +1,241 @@ +{- git-annex v1 -> v2 upgrade support + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V1 where + +import System.Posix.Types +import Data.Char + +import Common.Annex +import Types.Key +import Annex.Content +import Logs.Presence +import qualified Annex.Queue +import qualified Git +import qualified Git.LsFiles as LsFiles +import Backend +import Annex.Version +import Utility.FileMode +import Utility.TempFile +import qualified Upgrade.V2 + +-- v2 adds hashing of filenames of content and location log files. +-- Key information is encoded in filenames differently, so +-- both content and location log files move around, and symlinks +-- to content need to be changed. +-- +-- When upgrading a v1 key to v2, file size metadata ought to be +-- added to the key (unless it is a WORM key, which encoded +-- mtime:size in v1). This can only be done when the file content +-- is present. Since upgrades need to happen consistently, +-- (so that two repos get changed the same way by the upgrade, and +-- will merge), that metadata cannot be added on upgrade. +-- +-- Note that file size metadata +-- will only be used for detecting situations where git-annex +-- would run out of disk space, so if some keys don't have it, +-- the impact is minor. At least initially. It could be used in the +-- future by smart auto-repo balancing code, etc. +-- +-- Anyway, since v2 plans ahead for other metadata being included +-- in keys, there should probably be a way to update a key. +-- Something similar to the migrate subcommand could be used, +-- and users could then run that at their leisure. + +upgrade :: Annex Bool +upgrade = do + showAction "v1 to v2" + + ifM (fromRepo Git.repoIsLocalBare) + ( do + moveContent + setVersion defaultVersion + , do + moveContent + updateSymlinks + moveLocationLogs + + Annex.Queue.flush + setVersion defaultVersion + ) + + Upgrade.V2.upgrade + +moveContent :: Annex () +moveContent = do + showAction "moving content" + files <- getKeyFilesPresent1 + forM_ files move + where + move f = do + let k = fileKey1 (takeFileName f) + let d = parentDir f + liftIO $ allowWrite d + liftIO $ allowWrite f + moveAnnex k f + liftIO $ removeDirectory d + +updateSymlinks :: Annex () +updateSymlinks = do + showAction "updating symlinks" + top <- fromRepo Git.repoPath + (files, cleanup) <- inRepo $ LsFiles.inRepo [top] + forM_ files fixlink + void $ liftIO cleanup + where + fixlink f = do + r <- lookupFile1 f + case r of + Nothing -> noop + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.Queue.addCommand "add" [Param "--"] [f] + +moveLocationLogs :: Annex () +moveLocationLogs = do + showAction "moving location logs" + logkeys <- oldlocationlogs + forM_ logkeys move + where + oldlocationlogs = do + dir <- fromRepo Upgrade.V2.gitStateDir + ifM (liftIO $ doesDirectoryExist dir) + ( mapMaybe oldlog2key + <$> (liftIO $ getDirectoryContents dir) + , return [] + ) + move (l, k) = do + dest <- fromRepo $ logFile2 k + dir <- fromRepo Upgrade.V2.gitStateDir + let f = dir l + liftIO $ createDirectoryIfMissing True (parentDir dest) + -- could just git mv, but this way deals with + -- log files that are not checked into git, + -- as well as merging with already upgraded + -- logs that have been pulled from elsewhere + old <- liftIO $ readLog1 f + new <- liftIO $ readLog1 dest + liftIO $ writeLog1 dest (old++new) + Annex.Queue.addCommand "add" [Param "--"] [dest] + Annex.Queue.addCommand "add" [Param "--"] [f] + Annex.Queue.addCommand "rm" [Param "--quiet", Param "-f", Param "--"] [f] + +oldlog2key :: FilePath -> Maybe (FilePath, Key) +oldlog2key l + | drop len l == ".log" && sane = Just (l, k) + | otherwise = Nothing + where + len = length l - 4 + k = readKey1 (take len l) + sane = (not . null $ keyName k) && (not . null $ keyBackendName k) + +-- WORM backend keys: "WORM:mtime:size:filename" +-- all the rest: "backend:key" +-- +-- If the file looks like "WORM:XXX-...", then it was created by mixing +-- v2 and v1; that infelicity is worked around by treating the value +-- as the v2 key that it is. +readKey1 :: String -> Key +readKey1 v + | mixup = fromJust $ file2key $ join ":" $ Prelude.tail bits + | otherwise = Key + { keyName = n + , keyBackendName = b + , keySize = s + , keyMtime = t + } + where + bits = split ":" v + b = Prelude.head bits + n = join ":" $ drop (if wormy then 3 else 1) bits + t = if wormy + then Just (Prelude.read (bits !! 1) :: EpochTime) + else Nothing + s = if wormy + then Just (Prelude.read (bits !! 2) :: Integer) + else Nothing + wormy = Prelude.head bits == "WORM" + mixup = wormy && isUpper (Prelude.head $ bits !! 1) + +showKey1 :: Key -> String +showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } = + join ":" $ filter (not . null) [b, showifhere t, showifhere s, n] + where + showifhere Nothing = "" + showifhere (Just v) = show v + +keyFile1 :: Key -> FilePath +keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ showKey1 key + +fileKey1 :: FilePath -> Key +fileKey1 file = readKey1 $ + replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + +writeLog1 :: FilePath -> [LogLine] -> IO () +writeLog1 file ls = viaTmp writeFile file (showLog ls) + +readLog1 :: FilePath -> IO [LogLine] +readLog1 file = catchDefaultIO [] $ + parseLog <$> readFileStrict file + +lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend)) +lookupFile1 file = do + tl <- liftIO $ tryIO getsymlink + case tl of + Left _ -> return Nothing + Right l -> makekey l + where + getsymlink = takeFileName <$> readSymbolicLink file + makekey l = case maybeLookupBackendName bname of + Nothing -> do + unless (null kname || null bname || + not (isLinkToAnnex l)) $ + warning skip + return Nothing + Just backend -> return $ Just (k, backend) + where + k = fileKey1 l + bname = keyBackendName k + kname = keyName k + skip = "skipping " ++ file ++ + " (unknown backend " ++ bname ++ ")" + +getKeyFilesPresent1 :: Annex [FilePath] +getKeyFilesPresent1 = getKeyFilesPresent1' =<< fromRepo gitAnnexObjectDir +getKeyFilesPresent1' :: FilePath -> Annex [FilePath] +getKeyFilesPresent1' dir = + ifM (liftIO $ doesDirectoryExist dir) + ( do + dirs <- liftIO $ getDirectoryContents dir + let files = map (\d -> dir ++ "/" ++ d ++ "/" ++ takeFileName d) dirs + liftIO $ filterM present files + , return [] + ) + where + present f = do + result <- tryIO $ getFileStatus f + case result of + Right s -> return $ isRegularFile s + Left _ -> return False + +logFile1 :: Git.Repo -> Key -> String +logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log" + +logFile2 :: Key -> Git.Repo -> String +logFile2 = logFile' hashDirLower + +logFile' :: (Key -> FilePath) -> Key -> Git.Repo -> String +logFile' hasher key repo = + gitStateDir repo ++ hasher key ++ keyFile key ++ ".log" + +stateDir :: FilePath +stateDir = addTrailingPathSeparator ".git-annex" + +gitStateDir :: Git.Repo -> FilePath +gitStateDir repo = addTrailingPathSeparator $ Git.repoPath repo stateDir diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs new file mode 100644 index 0000000000..935fc48258 --- /dev/null +++ b/Upgrade/V2.hs @@ -0,0 +1,137 @@ +{- git-annex v2 -> v3 upgrade support + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V2 where + +import Common.Annex +import qualified Git +import qualified Git.Command +import qualified Git.Ref +import qualified Annex.Branch +import Logs.Location +import Annex.Content +import Utility.TempFile + +olddir :: Git.Repo -> FilePath +olddir g + | Git.repoIsLocalBare g = "" + | otherwise = ".git-annex" + +{- .git-annex/ moved to a git-annex branch. + - + - Strategy: + - + - * Create the git-annex branch. + - * Find each location log file in .git-annex/, and inject its content + - into the git-annex branch, unioning with any content already in + - there. (in passing, this deals with the semi transition that left + - some location logs hashed two different ways; both are found and + - merged). + - * Also inject remote.log, trust.log, and uuid.log. + - * git rm -rf .git-annex + - * Remove stuff that used to be needed in .gitattributes. + - * Commit changes. + -} +upgrade :: Annex Bool +upgrade = do + showAction "v2 to v3" + bare <- fromRepo Git.repoIsLocalBare + old <- fromRepo olddir + + Annex.Branch.create + showProgress + + e <- liftIO $ doesDirectoryExist old + when e $ do + mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs + mapM_ (\f -> inject f f) =<< logFiles old + + saveState False + showProgress + + when e $ do + inRepo $ Git.Command.run [Param "rm", Param "-r", Param "-f", Param "-q", File old] + unless bare $ inRepo gitAttributesUnWrite + showProgress + + unless bare push + + return True + +locationLogs :: Annex [(Key, FilePath)] +locationLogs = do + dir <- fromRepo gitStateDir + liftIO $ do + levela <- dirContents dir + levelb <- mapM tryDirContents levela + files <- mapM tryDirContents (concat levelb) + return $ mapMaybe islogfile (concat files) + where + tryDirContents d = catchDefaultIO [] $ dirContents d + islogfile f = maybe Nothing (\k -> Just (k, f)) $ + logFileKey $ takeFileName f + +inject :: FilePath -> FilePath -> Annex () +inject source dest = do + old <- fromRepo olddir + new <- liftIO (readFile $ old source) + Annex.Branch.change dest $ \prev -> + unlines $ nub $ lines prev ++ lines new + +logFiles :: FilePath -> Annex [FilePath] +logFiles dir = return . filter (".log" `isSuffixOf`) + <=< liftIO $ getDirectoryContents dir + +push :: Annex () +push = do + origin_master <- inRepo $ Git.Ref.exists $ Git.Ref "origin/master" + origin_gitannex <- Annex.Branch.hasOrigin + case (origin_master, origin_gitannex) of + (_, True) -> do + -- Merge in the origin's git-annex branch, + -- so that pushing the git-annex branch + -- will immediately work. Not pushed here, + -- because it's less obnoxious to let the user + -- push. + Annex.Branch.update + (True, False) -> do + -- push git-annex to origin, so that + -- "git push" will from then on + -- automatically push it + Annex.Branch.update -- just in case + showAction "pushing new git-annex branch to origin" + showOutput + inRepo $ Git.Command.run + [Param "push", Param "origin", Param $ show Annex.Branch.name] + _ -> do + -- no origin exists, so just let the user + -- know about the new branch + Annex.Branch.update + showLongNote $ + "git-annex branch created\n" ++ + "Be sure to push this branch when pushing to remotes.\n" + +{- Old .gitattributes contents, not needed anymore. -} +attrLines :: [String] +attrLines = + [ stateDir "*.log merge=union" + , stateDir "*/*/*.log merge=union" + ] + +gitAttributesUnWrite :: Git.Repo -> IO () +gitAttributesUnWrite repo = do + let attributes = Git.attributes repo + whenM (doesFileExist attributes) $ do + c <- readFileStrict attributes + liftIO $ viaTmp writeFile attributes $ unlines $ + filter (`notElem` attrLines) $ lines c + Git.Command.run [Param "add", File attributes] repo + +stateDir :: FilePath +stateDir = addTrailingPathSeparator ".git-annex" +gitStateDir :: Git.Repo -> FilePath +gitStateDir repo = addTrailingPathSeparator $ Git.repoPath repo stateDir diff --git a/Usage.hs b/Usage.hs new file mode 100644 index 0000000000..fc62bf5d2d --- /dev/null +++ b/Usage.hs @@ -0,0 +1,99 @@ +{- git-annex usage messages + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Usage where + +import Common.Annex +import System.Console.GetOpt + +import Types.Command + +{- Usage message with lists of commands and options. -} +usage :: String -> [Command] -> [Option] -> String +usage header cmds commonoptions = unlines $ + [ header + , "" + , "Options:" + ] ++ optlines ++ + [ "" + , "Commands:" + , "" + ] ++ cmdlines + where + -- To get consistent indentation of options, generate the + -- usage for all options at once. A command's options will + -- be displayed after the command. + alloptlines = filter (not . null) $ + lines $ usageInfo "" $ + concatMap cmdoptions scmds ++ commonoptions + (cmdlines, optlines) = go scmds alloptlines [] + go [] os ls = (ls, os) + go (c:cs) os ls = go cs os' (ls++(l:o)) + where + (o, os') = splitAt (length $ cmdoptions c) os + l = concat + [ cmdname c + , namepad (cmdname c) + , cmdparamdesc c + , descpad (cmdparamdesc c) + , cmddesc c + ] + pad n s = replicate (n - length s) ' ' + namepad = pad $ longest cmdname + 1 + descpad = pad $ longest cmdparamdesc + 2 + longest f = foldl max 0 $ map (length . f) cmds + scmds = sort cmds + +{- Descriptions of params used in usage messages. -} +paramPaths :: String +paramPaths = paramOptional $ paramRepeating paramPath -- most often used +paramPath :: String +paramPath = "PATH" +paramKey :: String +paramKey = "KEY" +paramDesc :: String +paramDesc = "DESC" +paramUrl :: String +paramUrl = "URL" +paramNumber :: String +paramNumber = "NUMBER" +paramNumRange :: String +paramNumRange = "NUM|RANGE" +paramRemote :: String +paramRemote = "REMOTE" +paramGlob :: String +paramGlob = "GLOB" +paramName :: String +paramName = "NAME" +paramValue :: String +paramValue = "VALUE" +paramUUID :: String +paramUUID = "UUID" +paramType :: String +paramType = "TYPE" +paramDate :: String +paramDate = "DATE" +paramTime :: String +paramTime = "TIME" +paramFormat :: String +paramFormat = "FORMAT" +paramFile :: String +paramFile = "FILE" +paramGroup :: String +paramGroup = "GROUP" +paramSize :: String +paramSize = "SIZE" +paramKeyValue :: String +paramKeyValue = "K=V" +paramNothing :: String +paramNothing = "" +paramRepeating :: String -> String +paramRepeating s = s ++ " ..." +paramOptional :: String -> String +paramOptional s = "[" ++ s ++ "]" +paramPair :: String -> String -> String +paramPair a b = a ++ " " ++ b diff --git a/Utility/Applicative.hs b/Utility/Applicative.hs new file mode 100644 index 0000000000..64400c8012 --- /dev/null +++ b/Utility/Applicative.hs @@ -0,0 +1,16 @@ +{- applicative stuff + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Applicative where + +{- Like <$> , but supports one level of currying. + - + - foo v = bar <$> action v == foo = bar <$$> action + -} +(<$$>) :: Functor f => (a -> b) -> (c -> f a) -> c -> f b +f <$$> v = fmap f . v +infixr 4 <$$> diff --git a/Utility/Base64.hs b/Utility/Base64.hs new file mode 100644 index 0000000000..ed803a00a5 --- /dev/null +++ b/Utility/Base64.hs @@ -0,0 +1,18 @@ +{- Simple Base64 access + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Base64 (toB64, fromB64) where + +import Codec.Binary.Base64 +import Data.Bits.Utils + +toB64 :: String -> String +toB64 = encode . s2w8 + +fromB64 :: String -> String +fromB64 s = maybe bad w82s $ decode s + where bad = error "bad base64 encoded data" diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs new file mode 100644 index 0000000000..7a2a5fe8e0 --- /dev/null +++ b/Utility/CoProcess.hs @@ -0,0 +1,35 @@ +{- Interface for running a shell command as a coprocess, + - sending it queries and getting back results. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.CoProcess ( + CoProcessHandle, + start, + stop, + query +) where + +import Common + +type CoProcessHandle = (ProcessHandle, Handle, Handle, CreateProcess) + +start :: FilePath -> [String] -> Maybe [(String, String)] -> IO CoProcessHandle +start command params env = do + (from, to, _err, pid) <- runInteractiveProcess command params Nothing env + return (pid, to, from, proc command params) + +stop :: CoProcessHandle -> IO () +stop (pid, from, to, p) = do + hClose to + hClose from + forceSuccessProcess p pid + +query :: CoProcessHandle -> (Handle -> IO a) -> (Handle -> IO b) -> IO b +query (_, from, to, _) send receive = do + _ <- send to + hFlush to + receive from diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs new file mode 100644 index 0000000000..18290669d9 --- /dev/null +++ b/Utility/CopyFile.hs @@ -0,0 +1,25 @@ +{- git-annex file copying + - + - Copyright 2010,2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.CopyFile (copyFileExternal) where + +import Common +import qualified Build.SysConfig as SysConfig + +{- The cp command is used, because I hate reinventing the wheel, + - and because this allows easy access to features like cp --reflink. -} +copyFileExternal :: FilePath -> FilePath -> IO Bool +copyFileExternal src dest = do + whenM (doesFileExist dest) $ + removeFile dest + boolSystem "cp" $ params ++ [File src, File dest] + where + params = map snd $ filter fst + [ (SysConfig.cp_reflink_auto, Param "--reflink=auto") + , (SysConfig.cp_a, Param "-a") + , (SysConfig.cp_p && not SysConfig.cp_a, Param "-p") + ] diff --git a/Utility/DBus.hs b/Utility/DBus.hs new file mode 100644 index 0000000000..3523a3aa35 --- /dev/null +++ b/Utility/DBus.hs @@ -0,0 +1,84 @@ +{- DBus utilities + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-} + +module Utility.DBus where + +import Utility.Exception + +import DBus.Client +import DBus +import Data.Maybe +import Control.Concurrent +import Control.Exception as E + +type ServiceName = String + +listServiceNames :: Client -> IO [ServiceName] +listServiceNames client = do + reply <- callDBus client "ListNames" [] + return $ fromMaybe [] $ fromVariant (methodReturnBody reply !! 0) + +callDBus :: Client -> MemberName -> [Variant] -> IO MethodReturn +callDBus client name params = call_ client $ + (methodCall "/org/freedesktop/DBus" "org.freedesktop.DBus" name) + { methodCallDestination = Just "org.freedesktop.DBus" + , methodCallBody = params + } + +{- Connects to the bus, and runs the client action. + - + - Throws a ClientError, and closes the connection if it fails to + - process an incoming message, or if the connection is lost. + - Unlike DBus's usual interface, this error is thrown at the top level, + - rather than inside the clientThreadRunner, so it can be caught, and + - runClient re-run as needed. -} +runClient :: IO (Maybe Address) -> (Client -> IO ()) -> IO () +runClient getaddr clientaction = do + env <- getaddr + case env of + Nothing -> throwIO (clientError "runClient: unable to determine DBUS address") + Just addr -> do + {- The clientaction will set up listeners, which + - run in a different thread. We block while + - they're running, until our threadrunner catches + - a ClientError, which it will put into the MVar + - to be rethrown here. -} + mv <- newEmptyMVar + let tr = threadrunner (putMVar mv) + let opts = defaultClientOptions { clientThreadRunner = tr } + client <- connectWith opts addr + clientaction client + e <- takeMVar mv + disconnect client + throw e + where + threadrunner storeerr io = loop + where + loop = catchClientError (io >> loop) storeerr + +{- Connects to the bus, and runs the client action. + - + - If the connection is lost, runs onretry, which can do something like + - a delay, or printing a warning, and has a state value (useful for + - exponential backoff). Once onretry returns, the connection is retried. + -} +persistentClient :: IO (Maybe Address) -> v -> (SomeException -> v -> IO v) -> (Client -> IO ()) -> IO () +persistentClient getaddr v onretry clientaction = + {- runClient can fail with not just ClientError, but also other + - things, if dbus is not running. Let async exceptions through. -} + runClient getaddr clientaction `catchNonAsync` retry + where + retry e = do + v' <- onretry e v + persistentClient getaddr v' onretry clientaction + +{- Catches only ClientError -} +catchClientError :: IO () -> (ClientError -> IO ()) -> IO () +catchClientError io handler = + either handler return =<< (E.try io :: IO (Either ClientError ())) diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs new file mode 100644 index 0000000000..ff13a3b8ad --- /dev/null +++ b/Utility/Daemon.hs @@ -0,0 +1,97 @@ +{- daemon support + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Daemon where + +import Common +import Utility.LogFile + +import System.Posix + +{- Run an action as a daemon, with all output sent to a file descriptor. + - + - Can write its pid to a file, to guard against multiple instances + - running and allow easy termination. + - + - When successful, does not return. -} +daemonize :: Fd -> Maybe FilePath -> Bool -> IO () -> IO () +daemonize logfd pidfile changedirectory a = do + maybe noop checkalreadyrunning pidfile + _ <- forkProcess child1 + out + where + checkalreadyrunning f = maybe noop (const $ alreadyRunning) + =<< checkDaemon f + child1 = do + _ <- createSession + _ <- forkProcess child2 + out + child2 = do + maybe noop lockPidFile pidfile + when changedirectory $ + setCurrentDirectory "/" + nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags + redir nullfd stdInput + redirLog logfd + a + out + out = exitImmediately ExitSuccess + +{- Locks the pid file, with an exclusive, non-blocking lock. + - Writes the pid to the file, fully atomically. + - Fails if the pid file is already locked by another process. -} +lockPidFile :: FilePath -> IO () +lockPidFile file = do + createDirectoryIfMissing True (parentDir file) + fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags + locked <- catchMaybeIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0) + fd' <- openFd newfile ReadWrite (Just stdFileMode) defaultFileFlags + { trunc = True } + locked' <- catchMaybeIO $ setLock fd' (WriteLock, AbsoluteSeek, 0, 0) + case (locked, locked') of + (Nothing, _) -> alreadyRunning + (_, Nothing) -> alreadyRunning + _ -> do + _ <- fdWrite fd' =<< show <$> getProcessID + renameFile newfile file + closeFd fd + where + newfile = file ++ ".new" + +alreadyRunning :: IO () +alreadyRunning = error "Daemon is already running." + +{- Checks if the daemon is running, by checking that the pid file + - is locked by the same process that is listed in the pid file. + - + - If it's running, returns its pid. -} +checkDaemon :: FilePath -> IO (Maybe ProcessID) +checkDaemon pidfile = do + v <- catchMaybeIO $ + openFd pidfile ReadOnly (Just stdFileMode) defaultFileFlags + case v of + Just fd -> do + locked <- getLock fd (ReadLock, AbsoluteSeek, 0, 0) + p <- readish <$> readFile pidfile + return $ check locked p + Nothing -> return Nothing + where + check Nothing _ = Nothing + check _ Nothing = Nothing + check (Just (pid, _)) (Just pid') + | pid == pid' = Just pid + | otherwise = error $ + "stale pid in " ++ pidfile ++ + " (got " ++ show pid' ++ + "; expected " ++ show pid ++ " )" + +{- Stops the daemon, safely. -} +stopDaemon :: FilePath -> IO () +stopDaemon pidfile = go =<< checkDaemon pidfile + where + go Nothing = noop + go (Just pid) = signalProcess sigTERM pid diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs new file mode 100644 index 0000000000..c6990fdfb0 --- /dev/null +++ b/Utility/DataUnits.hs @@ -0,0 +1,160 @@ +{- data size display and parsing + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + - + - + - And now a rant: + - + - In the beginning, we had powers of two, and they were good. + - + - Disk drive manufacturers noticed that some powers of two were + - sorta close to some powers of ten, and that rounding down to the nearest + - power of ten allowed them to advertise their drives were bigger. This + - was sorta annoying. + - + - Then drives got big. Really, really big. This was good. + - + - Except that the small rounding error perpretrated by the drive + - manufacturers suffered the fate of a small error, and became a large + - error. This was bad. + - + - So, a committee was formed. And it arrived at a committee-like decision, + - which satisfied noone, confused everyone, and made the world an uglier + - place. As with all committees, this was meh. + - + - And the drive manufacturers happily continued selling drives that are + - increasingly smaller than you'd expect, if you don't count on your + - fingers. But that are increasingly too big for anyone to much notice. + - This caused me to need git-annex. + - + - Thus, I use units here that I loathe. Because if I didn't, people would + - be confused that their drives seem the wrong size, and other people would + - complain at me for not being standards compliant. And we call this + - progress? + -} + +module Utility.DataUnits ( + dataUnits, + storageUnits, + memoryUnits, + bandwidthUnits, + oldSchoolUnits, + + roughSize, + compareSizes, + readSize +) where + +import Data.List +import Data.Char + +type ByteSize = Integer +type Name = String +type Abbrev = String +data Unit = Unit ByteSize Abbrev Name + deriving (Ord, Show, Eq) + +dataUnits :: [Unit] +dataUnits = storageUnits ++ memoryUnits + +{- Storage units are (stupidly) powers of ten. -} +storageUnits :: [Unit] +storageUnits = + [ Unit (p 8) "YB" "yottabyte" + , Unit (p 7) "ZB" "zettabyte" + , Unit (p 6) "EB" "exabyte" + , Unit (p 5) "PB" "petabyte" + , Unit (p 4) "TB" "terabyte" + , Unit (p 3) "GB" "gigabyte" + , Unit (p 2) "MB" "megabyte" + , Unit (p 1) "kB" "kilobyte" -- weird capitalization thanks to committe + , Unit (p 0) "B" "byte" + ] + where + p :: Integer -> Integer + p n = 1000^n + +{- Memory units are (stupidly named) powers of 2. -} +memoryUnits :: [Unit] +memoryUnits = + [ Unit (p 8) "YiB" "yobibyte" + , Unit (p 7) "ZiB" "zebibyte" + , Unit (p 6) "EiB" "exbibyte" + , Unit (p 5) "PiB" "pebibyte" + , Unit (p 4) "TiB" "tebibyte" + , Unit (p 3) "GiB" "gibibyte" + , Unit (p 2) "MiB" "mebibyte" + , Unit (p 1) "KiB" "kibibyte" + , Unit (p 0) "B" "byte" + ] + where + p :: Integer -> Integer + p n = 2^(n*10) + +{- Bandwidth units are only measured in bits if you're some crazy telco. -} +bandwidthUnits :: [Unit] +bandwidthUnits = error "stop trying to rip people off" + +{- Do you yearn for the days when men were men and megabytes were megabytes? -} +oldSchoolUnits :: [Unit] +oldSchoolUnits = zipWith (curry mingle) storageUnits memoryUnits + where + mingle (Unit _ a n, Unit s' _ _) = Unit s' a n + +{- approximate display of a particular number of bytes -} +roughSize :: [Unit] -> Bool -> ByteSize -> String +roughSize units abbrev i + | i < 0 = '-' : findUnit units' (negate i) + | otherwise = findUnit units' i + where + units' = reverse $ sort units -- largest first + + findUnit (u@(Unit s _ _):us) i' + | i' >= s = showUnit i' u + | otherwise = findUnit us i' + findUnit [] i' = showUnit i' (last units') -- bytes + + showUnit i' (Unit s a n) = let num = chop i' s in + show num ++ " " ++ + (if abbrev then a else plural num n) + + chop :: Integer -> Integer -> Integer + chop i' d = round $ (fromInteger i' :: Double) / fromInteger d + + plural n u + | n == 1 = u + | otherwise = u ++ "s" + +{- displays comparison of two sizes -} +compareSizes :: [Unit] -> Bool -> ByteSize -> ByteSize -> String +compareSizes units abbrev old new + | old > new = roughSize units abbrev (old - new) ++ " smaller" + | old < new = roughSize units abbrev (new - old) ++ " larger" + | otherwise = "same" + +{- Parses strings like "10 kilobytes" or "0.5tb". -} +readSize :: [Unit] -> String -> Maybe ByteSize +readSize units input + | null parsednum || null parsedunit = Nothing + | otherwise = Just $ round $ number * fromIntegral multiplier + where + (number, rest) = head parsednum + multiplier = head parsedunit + unitname = takeWhile isAlpha $ dropWhile isSpace rest + + parsednum = reads input :: [(Double, String)] + parsedunit = lookupUnit units unitname + + lookupUnit _ [] = [1] -- no unit given, assume bytes + lookupUnit [] _ = [] + lookupUnit (Unit s a n:us) v + | a ~~ v || n ~~ v = [s] + | plural n ~~ v || a ~~ byteabbrev v = [s] + | otherwise = lookupUnit us v + + a ~~ b = map toLower a == map toLower b + + plural n = n ++ "s" + byteabbrev a = a ++ "b" diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs new file mode 100644 index 0000000000..ba420401eb --- /dev/null +++ b/Utility/DirWatcher.hs @@ -0,0 +1,145 @@ +{- generic directory watching interface + - + - Uses inotify, or kqueue, or fsevents to watch a directory + - (and subdirectories) for changes, and runs hooks for different + - sorts of events as they occur. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Utility.DirWatcher where + +import Utility.Types.DirWatcher + +#if WITH_INOTIFY +import qualified Utility.INotify as INotify +import qualified System.INotify as INotify +#endif +#if WITH_KQUEUE +import qualified Utility.Kqueue as Kqueue +import Control.Concurrent +#endif +#if WITH_FSEVENTS +import qualified Utility.FSEvents as FSEvents +import qualified System.OSX.FSEvents as FSEvents +#endif + +type Pruner = FilePath -> Bool + +canWatch :: Bool +#if (WITH_INOTIFY || WITH_KQUEUE || WITH_FSEVENTS) +canWatch = True +#else +#if defined linux_HOST_OS +#warning "Building without inotify support" +#endif +canWatch = False +#endif + +{- With inotify, discrete events will be received when making multiple changes + - to the same filename. For example, adding it, deleting it, and adding it + - again will be three events. + - + - OTOH, with kqueue, often only one event is received, indicating the most + - recent state of the file. -} +eventsCoalesce :: Bool +#if (WITH_INOTIFY || WITH_FSEVENTS) +eventsCoalesce = False +#else +#if WITH_KQUEUE +eventsCoalesce = True +#else +eventsCoalesce = undefined +#endif +#endif + +{- With inotify, file closing is tracked to some extent, so an add event + - will always be received for a file once its writer closes it, and + - (typically) not before. This may mean multiple add events for the same file. + - + - fsevents behaves similarly, although different event types are used for + - creating and modification of the file. + - + - OTOH, with kqueue, add events will often be received while a file is + - still being written to, and then no add event will be received once the + - writer closes it. -} +closingTracked :: Bool +#if (WITH_INOTIFY || WITH_FSEVENTS) +closingTracked = True +#else +#if WITH_KQUEUE +closingTracked = False +#else +closingTracked = undefined +#endif +#endif + +{- With inotify, modifications to existing files can be tracked. + - Kqueue does not support this. + - Fsevents generates events when an existing file is reopened and rewritten, + - but not necessarily when it's opened once and modified repeatedly. -} +modifyTracked :: Bool +#if (WITH_INOTIFY || WITH_FSEVENTS) +modifyTracked = True +#else +#if WITH_KQUEUE +modifyTracked = False +#else +modifyTracked = undefined +#endif +#endif + +{- Starts a watcher thread. The runstartup action is passed a scanner action + - to run, that will return once the initial directory scan is complete. + - Once runstartup returns, the watcher thread continues running, + - and processing events. Returns a DirWatcherHandle that can be used + - to shutdown later. -} +#if WITH_INOTIFY +type DirWatcherHandle = INotify.INotify +watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle +watchDir dir prune hooks runstartup = do + i <- INotify.initINotify + runstartup $ INotify.watchDir i dir prune hooks + return i +#else +#if WITH_KQUEUE +type DirWatcherHandle = ThreadId +watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle +watchDir dir prune hooks runstartup = do + kq <- runstartup $ Kqueue.initKqueue dir prune + forkIO $ Kqueue.runHooks kq hooks +#else +#if WITH_FSEVENTS +type DirWatcherHandle = FSEvents.EventStream +watchDir :: FilePath -> Pruner -> WatchHooks -> (IO FSEvents.EventStream -> IO FSEvents.EventStream) -> IO DirWatcherHandle +watchDir dir prune hooks runstartup = + runstartup $ FSEvents.watchDir dir prune hooks +#else +type DirWatcherHandle = () +watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle +watchDir = undefined +#endif +#endif +#endif + +#if WITH_INOTIFY +stopWatchDir :: DirWatcherHandle -> IO () +stopWatchDir = INotify.killINotify +#else +#if WITH_KQUEUE +stopWatchDir :: DirWatcherHandle -> IO () +stopWatchDir = killThread +#else +#if WITH_FSEVENTS +stopWatchDir :: DirWatcherHandle -> IO () +stopWatchDir = FSEvents.eventStreamDestroy +#else +stopWatchDir :: DirWatcherHandle -> IO () +stopWatchDir = undefined +#endif +#endif +#endif diff --git a/Utility/Directory.hs b/Utility/Directory.hs new file mode 100644 index 0000000000..7cce4a68f8 --- /dev/null +++ b/Utility/Directory.hs @@ -0,0 +1,93 @@ +{- directory manipulation + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Directory where + +import System.IO.Error +import System.Posix.Files +import System.Directory +import Control.Exception (throw) +import Control.Monad +import Control.Monad.IfElse +import System.FilePath +import Control.Applicative +import System.IO.Unsafe (unsafeInterleaveIO) + +import Utility.SafeCommand +import Utility.TempFile +import Utility.Exception +import Utility.Monad + +dirCruft :: FilePath -> Bool +dirCruft "." = True +dirCruft ".." = True +dirCruft _ = False + +{- Lists the contents of a directory. + - Unlike getDirectoryContents, paths are not relative to the directory. -} +dirContents :: FilePath -> IO [FilePath] +dirContents d = map (d ) . filter (not . dirCruft) <$> getDirectoryContents d + +{- Gets files in a directory, and then its subdirectories, recursively, + - and lazily. If the directory does not exist, no exception is thrown, + - instead, [] is returned. -} +dirContentsRecursive :: FilePath -> IO [FilePath] +dirContentsRecursive topdir = dirContentsRecursive' [topdir] + +dirContentsRecursive' :: [FilePath] -> IO [FilePath] +dirContentsRecursive' [] = return [] +dirContentsRecursive' (dir:dirs) = unsafeInterleaveIO $ do + (files, dirs') <- collect [] [] =<< catchDefaultIO [] (dirContents dir) + files' <- dirContentsRecursive' (dirs' ++ dirs) + return (files ++ files') + where + collect files dirs' [] = return (reverse files, reverse dirs') + collect files dirs' (entry:entries) + | dirCruft entry = collect files dirs' entries + | otherwise = do + ifM (doesDirectoryExist entry) + ( collect files (entry:dirs') entries + , collect (entry:files) dirs' entries + ) + +{- Moves one filename to another. + - First tries a rename, but falls back to moving across devices if needed. -} +moveFile :: FilePath -> FilePath -> IO () +moveFile src dest = tryIO (rename src dest) >>= onrename + where + onrename (Right _) = noop + onrename (Left e) + | isPermissionError e = rethrow + | isDoesNotExistError e = rethrow + | otherwise = do + -- copyFile is likely not as optimised as + -- the mv command, so we'll use the latter. + -- But, mv will move into a directory if + -- dest is one, which is not desired. + whenM (isdir dest) rethrow + viaTmp mv dest undefined + where + rethrow = throw e + mv tmp _ = do + ok <- boolSystem "mv" [Param "-f", Param src, Param tmp] + unless ok $ do + -- delete any partial + _ <- tryIO $ removeFile tmp + rethrow + + isdir f = do + r <- tryIO $ getFileStatus f + case r of + (Left _) -> return False + (Right s) -> return $ isDirectory s + +{- Removes a file, which may or may not exist. + - + - Note that an exception is thrown if the file exists but + - cannot be removed. -} +nukeFile :: FilePath -> IO () +nukeFile file = whenM (doesFileExist file) $ removeFile file diff --git a/Utility/DiskFree.hs b/Utility/DiskFree.hs new file mode 100644 index 0000000000..4532441752 --- /dev/null +++ b/Utility/DiskFree.hs @@ -0,0 +1,29 @@ +{- disk free space checking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE ForeignFunctionInterface #-} + +module Utility.DiskFree ( getDiskFree ) where + +import Common + +import Foreign.C.Types +import Foreign.C.String +import Foreign.C.Error + +foreign import ccall safe "libdiskfree.h diskfree" c_diskfree + :: CString -> IO CULLong + +getDiskFree :: FilePath -> IO (Maybe Integer) +getDiskFree path = withFilePath path $ \c_path -> do + free <- c_diskfree c_path + ifM (safeErrno <$> getErrno) + ( return $ Just $ toInteger free + , return Nothing + ) + where + safeErrno (Errno v) = v == 0 diff --git a/Utility/Dot.hs b/Utility/Dot.hs new file mode 100644 index 0000000000..e57bf009f6 --- /dev/null +++ b/Utility/Dot.hs @@ -0,0 +1,63 @@ +{- a simple graphviz / dot(1) digraph description generator library + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Dot where -- import qualified + +{- generates a graph description from a list of lines -} +graph :: [String] -> String +graph s = unlines $ [header] ++ map indent s ++ [footer] + where + header = "digraph map {" + footer= "}" + +{- a node in the graph -} +graphNode :: String -> String -> String +graphNode nodeid desc = label desc $ quote nodeid + +{- an edge between two nodes -} +graphEdge :: String -> String -> Maybe String -> String +graphEdge fromid toid desc = indent $ maybe edge (`label` edge) desc + where + edge = quote fromid ++ " -> " ++ quote toid + +{- adds a label to a node or edge -} +label :: String -> String -> String +label = attr "label" + +{- adds an attribute to a node or edge + - (can be called multiple times for multiple attributes) -} +attr :: String -> String -> String -> String +attr a v s = s ++ " [ " ++ a ++ "=" ++ quote v ++ " ]" + +{- fills a node with a color -} +fillColor :: String -> String -> String +fillColor color s = attr "fillcolor" color $ attr "style" "filled" s + +{- apply to graphNode to put the node in a labeled box -} +subGraph :: String -> String -> String -> String -> String +subGraph subid l color s = + "subgraph " ++ name ++ " {\n" ++ + ii setlabel ++ + ii setfilled ++ + ii setcolor ++ + ii s ++ + indent "}" + where + -- the "cluster_" makes dot draw a box + name = quote ("cluster_" ++ subid) + setlabel = "label=" ++ quote l + setfilled = "style=" ++ quote "filled" + setcolor = "fillcolor=" ++ quote color + ii x = indent (indent x) ++ "\n" + +indent ::String -> String +indent s = '\t' : s + +quote :: String -> String +quote s = "\"" ++ s' ++ "\"" + where + s' = filter (/= '"') s diff --git a/Utility/Exception.hs b/Utility/Exception.hs new file mode 100644 index 0000000000..45f2aecec1 --- /dev/null +++ b/Utility/Exception.hs @@ -0,0 +1,51 @@ +{- Simple IO exception handling (and some more) + - + - Copyright 2011-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE ScopedTypeVariables #-} + +module Utility.Exception where + +import Prelude hiding (catch) +import Control.Exception +import Control.Applicative + +{- Catches IO errors and returns a Bool -} +catchBoolIO :: IO Bool -> IO Bool +catchBoolIO a = catchDefaultIO False a + +{- Catches IO errors and returns a Maybe -} +catchMaybeIO :: IO a -> IO (Maybe a) +catchMaybeIO a = catchDefaultIO Nothing $ Just <$> a + +{- Catches IO errors and returns a default value. -} +catchDefaultIO :: a -> IO a -> IO a +catchDefaultIO def a = catchIO a (const $ return def) + +{- Catches IO errors and returns the error message. -} +catchMsgIO :: IO a -> IO (Either String a) +catchMsgIO a = either (Left . show) Right <$> tryIO a + +{- catch specialized for IO errors only -} +catchIO :: IO a -> (IOException -> IO a) -> IO a +catchIO = catch + +{- try specialized for IO errors only -} +tryIO :: IO a -> IO (Either IOException a) +tryIO = try + +{- Catches all exceptions except for async exceptions. + - This is often better to use than catching them all, so that + - ThreadKilled and UserInterrupt get through. + -} +catchNonAsync :: IO a -> (SomeException -> IO a) -> IO a +catchNonAsync a onerr = a `catches` + [ Handler (\ (e :: AsyncException) -> throw e) + , Handler (\ (e :: SomeException) -> onerr e) + ] + +tryNonAsync :: IO a -> IO (Either SomeException a) +tryNonAsync a = (Right <$> a) `catchNonAsync` (return . Left) diff --git a/Utility/FSEvents.hs b/Utility/FSEvents.hs new file mode 100644 index 0000000000..af8c2fc2ff --- /dev/null +++ b/Utility/FSEvents.hs @@ -0,0 +1,92 @@ +{- FSEvents interface + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.FSEvents where + +import Common hiding (isDirectory) +import Utility.Types.DirWatcher + +import System.OSX.FSEvents +import qualified System.Posix.Files as Files +import Data.Bits ((.&.)) + +watchDir :: FilePath -> (FilePath -> Bool) -> WatchHooks -> IO EventStream +watchDir dir ignored hooks = do + unlessM fileLevelEventsSupported $ + error "Need at least OSX 10.7.0 for file-level FSEvents" + scan dir + eventStreamCreate [dir] 1.0 True True True handle + where + handle evt + | ignoredPath ignored (eventPath evt) = noop + | otherwise = do + {- More than one flag may be set, if events occurred + - close together. + - + - Order is important.. + - If a file is added and then deleted, we'll see it's + - not present, and addHook won't run. + - OTOH, if a file is deleted and then re-added, + - the delHook will run first, followed by the addHook. + -} + + when (hasflag eventFlagItemRemoved) $ + if hasflag eventFlagItemIsDir + then runhook delDirHook Nothing + else runhook delHook Nothing + when (hasflag eventFlagItemCreated) $ + maybe noop handleadd =<< getstatus (eventPath evt) + {- When a file or dir is renamed, a rename event is + - received for both its old and its new name. -} + when (hasflag eventFlagItemRenamed) $ + if hasflag eventFlagItemIsDir + then ifM (doesDirectoryExist $ eventPath evt) + ( scan $ eventPath evt + , runhook delDirHook Nothing + ) + else maybe (runhook delHook Nothing) handleadd + =<< getstatus (eventPath evt) + {- Add hooks are run when a file is modified for + - compatability with INotify, which calls the add + - hook when a file is closed, and so tends to call + - both add and modify for file modifications. -} + when (hasflag eventFlagItemModified && not (hasflag eventFlagItemIsDir)) $ do + ms <- getstatus $ eventPath evt + maybe noop handleadd ms + runhook modifyHook ms + where + hasflag f = eventFlags evt .&. f /= 0 + runhook h s = maybe noop (\a -> a (eventPath evt) s) (h hooks) + handleadd s + | Files.isSymbolicLink s = runhook addSymlinkHook $ Just s + | Files.isRegularFile s = runhook addHook $ Just s + | otherwise = noop + + scan d = unless (ignoredPath ignored d) $ + mapM_ go =<< dirContentsRecursive d + where + go f + | ignoredPath ignored f = noop + | otherwise = do + ms <- getstatus f + case ms of + Nothing -> noop + Just s + | Files.isSymbolicLink s -> + runhook addSymlinkHook ms + | Files.isRegularFile s -> + runhook addHook ms + | otherwise -> + noop + where + runhook h s = maybe noop (\a -> a f s) (h hooks) + + getstatus = catchMaybeIO . getSymbolicLinkStatus + +{- Check each component of the path to see if it's ignored. -} +ignoredPath :: (FilePath -> Bool) -> FilePath -> Bool +ignoredPath ignored = any ignored . map dropTrailingPathSeparator . splitPath diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs new file mode 100644 index 0000000000..0f70463339 --- /dev/null +++ b/Utility/FileMode.hs @@ -0,0 +1,112 @@ +{- File mode utilities. + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.FileMode where + +import Common + +import Control.Exception (bracket) +import System.Posix.Types +import Foreign (complement) + +{- Applies a conversion function to a file's mode. -} +modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO () +modifyFileMode f convert = void $ modifyFileMode' f convert +modifyFileMode' :: FilePath -> (FileMode -> FileMode) -> IO FileMode +modifyFileMode' f convert = do + s <- getFileStatus f + let old = fileMode s + let new = convert old + when (new /= old) $ + setFileMode f new + return old + +{- Adds the specified FileModes to the input mode, leaving the rest + - unchanged. -} +addModes :: [FileMode] -> FileMode -> FileMode +addModes ms m = combineModes (m:ms) + +{- Removes the specified FileModes from the input mode. -} +removeModes :: [FileMode] -> FileMode -> FileMode +removeModes ms m = m `intersectFileModes` complement (combineModes ms) + +{- Runs an action after changing a file's mode, then restores the old mode. -} +withModifiedFileMode :: FilePath -> (FileMode -> FileMode) -> IO a -> IO a +withModifiedFileMode file convert a = bracket setup cleanup go + where + setup = modifyFileMode' file convert + cleanup oldmode = modifyFileMode file (const oldmode) + go _ = a + +writeModes :: [FileMode] +writeModes = [ownerWriteMode, groupWriteMode, otherWriteMode] + +readModes :: [FileMode] +readModes = [ownerReadMode, groupReadMode, otherReadMode] + +executeModes :: [FileMode] +executeModes = [ownerExecuteMode, groupExecuteMode, otherExecuteMode] + +{- Removes the write bits from a file. -} +preventWrite :: FilePath -> IO () +preventWrite f = modifyFileMode f $ removeModes writeModes + +{- Turns a file's owner write bit back on. -} +allowWrite :: FilePath -> IO () +allowWrite f = modifyFileMode f $ addModes [ownerWriteMode] + +{- Allows owner and group to read and write to a file. -} +groupWriteRead :: FilePath -> IO () +groupWriteRead f = modifyFileMode f $ addModes + [ ownerWriteMode, groupWriteMode + , ownerReadMode, groupReadMode + ] + +checkMode :: FileMode -> FileMode -> Bool +checkMode checkfor mode = checkfor `intersectFileModes` mode == checkfor + +{- Checks if a file mode indicates it's a symlink. -} +isSymLink :: FileMode -> Bool +isSymLink = checkMode symbolicLinkMode + +{- Checks if a file has any executable bits set. -} +isExecutable :: FileMode -> Bool +isExecutable mode = combineModes executeModes `intersectFileModes` mode /= 0 + +{- Runs an action without that pesky umask influencing it, unless the + - passed FileMode is the standard one. -} +noUmask :: FileMode -> IO a -> IO a +noUmask mode a + | mode == stdFileMode = a + | otherwise = bracket setup cleanup go + where + setup = setFileCreationMask nullFileMode + cleanup = setFileCreationMask + go _ = a + +combineModes :: [FileMode] -> FileMode +combineModes [] = undefined +combineModes [m] = m +combineModes (m:ms) = foldl unionFileModes m ms + +stickyMode :: FileMode +stickyMode = 512 + +isSticky :: FileMode -> Bool +isSticky = checkMode stickyMode + +setSticky :: FilePath -> IO () +setSticky f = modifyFileMode f $ addModes [stickyMode] + +{- Writes a file, ensuring that its modes do not allow it to be read + - by anyone other than the current user, before any content is written. -} +writeFileProtected :: FilePath -> String -> IO () +writeFileProtected file content = do + h <- openFile file WriteMode + modifyFileMode file $ removeModes [groupReadMode, otherReadMode] + hPutStr h content + hClose h diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs new file mode 100644 index 0000000000..98f2f7755c --- /dev/null +++ b/Utility/FileSystemEncoding.hs @@ -0,0 +1,77 @@ +{- GHC File system encoding handling. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.FileSystemEncoding ( + fileEncoding, + withFilePath, + md5FilePath, + decodeW8, + encodeW8 +) where + +import qualified GHC.Foreign as GHC +import qualified GHC.IO.Encoding as Encoding +import Foreign.C +import System.IO +import System.IO.Unsafe +import qualified Data.Hash.MD5 as MD5 +import Data.Word +import Data.Bits.Utils + +{- Sets a Handle to use the filesystem encoding. This causes data + - written or read from it to be encoded/decoded the same + - as ghc 7.4 does to filenames etc. This special encoding + - allows "arbitrary undecodable bytes to be round-tripped through it". -} +fileEncoding :: Handle -> IO () +fileEncoding h = hSetEncoding h =<< Encoding.getFileSystemEncoding + +{- Marshal a Haskell FilePath into a NUL terminated C string using temporary + - storage. The FilePath is encoded using the filesystem encoding, + - reversing the decoding that should have been done when the FilePath + - was obtained. -} +withFilePath :: FilePath -> (CString -> IO a) -> IO a +withFilePath fp f = Encoding.getFileSystemEncoding + >>= \enc -> GHC.withCString enc fp f + +{- Encodes a FilePath into a String, applying the filesystem encoding. + - + - There are very few things it makes sense to do with such an encoded + - string. It's not a legal filename; it should not be displayed. + - So this function is not exported, but instead used by the few functions + - that can usefully consume it. + - + - This use of unsafePerformIO is belived to be safe; GHC's interface + - only allows doing this conversion with CStrings, and the CString buffer + - is allocated, used, and deallocated within the call, with no side + - effects. + -} +{-# NOINLINE _encodeFilePath #-} +_encodeFilePath :: FilePath -> String +_encodeFilePath fp = unsafePerformIO $ do + enc <- Encoding.getFileSystemEncoding + GHC.withCString enc fp $ GHC.peekCString Encoding.char8 + +{- Encodes a FilePath into a Md5.Str, applying the filesystem encoding. -} +md5FilePath :: FilePath -> MD5.Str +md5FilePath = MD5.Str . _encodeFilePath + +{- Converts a [Word8] to a FilePath, encoding using the filesystem encoding. + - + - w82c produces a String, which may contain Chars that are invalid + - unicode. From there, this is really a simple matter of applying the + - file system encoding, only complicated by GHC's interface to doing so. + -} +{-# NOINLINE encodeW8 #-} +encodeW8 :: [Word8] -> FilePath +encodeW8 w8 = unsafePerformIO $ do + enc <- Encoding.getFileSystemEncoding + GHC.withCString Encoding.char8 (w82s w8) $ GHC.peekCString enc + +{- Useful when you want the actual number of bytes that will be used to + - represent the FilePath on disk. -} +decodeW8 :: FilePath -> [Word8] +decodeW8 = s2w8 . _encodeFilePath diff --git a/Utility/Format.hs b/Utility/Format.hs new file mode 100644 index 0000000000..97a966ac1a --- /dev/null +++ b/Utility/Format.hs @@ -0,0 +1,173 @@ +{- Formatted string handling. + - + - Copyright 2010, 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Format ( + Format, + gen, + format, + decode_c, + encode_c, + prop_idempotent_deencode +) where + +import Text.Printf (printf) +import Data.Char (isAlphaNum, isOctDigit, isSpace, chr, ord) +import Data.Maybe (fromMaybe) +import Data.Word (Word8) +import Data.List (isPrefixOf) +import qualified Codec.Binary.UTF8.String +import qualified Data.Map as M + +import Utility.PartialPrelude + +type FormatString = String + +{- A format consists of a list of fragments. -} +type Format = [Frag] + +{- A fragment is either a constant string, + - or a variable, with a justification. -} +data Frag = Const String | Var String Justify + deriving (Show) + +data Justify = LeftJustified Int | RightJustified Int | UnJustified + deriving (Show) + +type Variables = M.Map String String + +{- Expands a Format using some variables, generating a formatted string. + - This can be repeatedly called, efficiently. -} +format :: Format -> Variables -> String +format f vars = concatMap expand f + where + expand (Const s) = s + expand (Var name j) + | "escaped_" `isPrefixOf` name = + justify j $ encode_c_strict $ + getvar $ drop (length "escaped_") name + | otherwise = justify j $ getvar name + getvar name = fromMaybe "" $ M.lookup name vars + justify UnJustified s = s + justify (LeftJustified i) s = s ++ pad i s + justify (RightJustified i) s = pad i s ++ s + pad i s = take (i - length s) spaces + spaces = repeat ' ' + +{- Generates a Format that can be used to expand variables in a + - format string, such as "${foo} ${bar;10} ${baz;-10}\n" + - + - (This is the same type of format string used by dpkg-query.) + -} +gen :: FormatString -> Format +gen = filter (not . empty) . fuse [] . scan [] . decode_c + where + -- The Format is built up in reverse, for efficiency, + -- and can have many adjacent Consts. Fusing it fixes both + -- problems. + fuse f [] = f + fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs + fuse f (v:vs) = fuse (v:f) vs + + scan f (a:b:cs) + | a == '$' && b == '{' = invar f [] cs + | otherwise = scan (Const [a] : f ) (b:cs) + scan f v = Const v : f + + invar f var [] = Const (novar var) : f + invar f var (c:cs) + | c == '}' = foundvar f var UnJustified cs + | isAlphaNum c || c == '_' = invar f (c:var) cs + | c == ';' = inpad "" f var cs + | otherwise = scan ((Const $ novar $ c:var):f) cs + + inpad p f var (c:cs) + | c == '}' = foundvar f var (readjustify $ reverse p) cs + | otherwise = inpad (c:p) f var cs + inpad p f var [] = Const (novar $ p++";"++var) : f + readjustify = getjustify . fromMaybe 0 . readish + getjustify i + | i == 0 = UnJustified + | i < 0 = LeftJustified (-1 * i) + | otherwise = RightJustified i + novar v = "${" ++ reverse v + foundvar f v p = scan (Var (reverse v) p : f) + +empty :: Frag -> Bool +empty (Const "") = True +empty _ = False + +{- Decodes a C-style encoding, where \n is a newline, \NNN is an octal + - encoded character, etc. + -} +decode_c :: FormatString -> FormatString +decode_c [] = [] +decode_c s = unescape ("", s) + where + e = '\\' + unescape (b, []) = b + -- look for escapes starting with '\' + unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair) + where + pair = span (/= e) v + isescape x = x == e + -- \NNN is an octal encoded character + handle (x:n1:n2:n3:rest) + | isescape x && alloctal = (fromoctal, rest) + where + alloctal = isOctDigit n1 && isOctDigit n2 && isOctDigit n3 + fromoctal = [chr $ readoctal [n1, n2, n3]] + readoctal o = Prelude.read $ "0o" ++ o :: Int + -- \C is used for a few special characters + handle (x:nc:rest) + | isescape x = ([echar nc], rest) + where + echar 'a' = '\a' + echar 'b' = '\b' + echar 'f' = '\f' + echar 'n' = '\n' + echar 'r' = '\r' + echar 't' = '\t' + echar 'v' = '\v' + echar a = a + handle n = ("", n) + +{- Inverse of decode_c. -} +encode_c :: FormatString -> FormatString +encode_c = encode_c' (const False) + +{- Encodes more strictly, including whitespace. -} +encode_c_strict :: FormatString -> FormatString +encode_c_strict = encode_c' isSpace + +encode_c' :: (Char -> Bool) -> FormatString -> FormatString +encode_c' p = concatMap echar + where + e c = '\\' : [c] + echar '\a' = e 'a' + echar '\b' = e 'b' + echar '\f' = e 'f' + echar '\n' = e 'n' + echar '\r' = e 'r' + echar '\t' = e 't' + echar '\v' = e 'v' + echar '\\' = e '\\' + echar '"' = e '"' + echar c + | ord c < 0x20 = e_asc c -- low ascii + | ord c >= 256 = e_utf c -- unicode + | ord c > 0x7E = e_asc c -- high ascii + | p c = e_asc c -- unprintable ascii + | otherwise = [c] -- printable ascii + -- unicode character is decomposed to individual Word8s, + -- and each is shown in octal + e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8]) + e_asc c = showoctal $ ord c + showoctal i = '\\' : printf "%03o" i + +{- for quickcheck -} +prop_idempotent_deencode :: String -> Bool +prop_idempotent_deencode s = s == decode_c (encode_c s) diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs new file mode 100644 index 0000000000..e3ced6d748 --- /dev/null +++ b/Utility/FreeDesktop.hs @@ -0,0 +1,127 @@ +{- Freedesktop.org specifications + - + - http://standards.freedesktop.org/basedir-spec/latest/ + - http://standards.freedesktop.org/desktop-entry-spec/latest/ + - http://standards.freedesktop.org/menu-spec/latest/ + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.FreeDesktop ( + DesktopEntry, + genDesktopEntry, + buildDesktopMenuFile, + writeDesktopMenuFile, + desktopMenuFilePath, + autoStartPath, + systemDataDir, + systemConfigDir, + userDataDir, + userConfigDir, + userDesktopDir +) where + +import Utility.Exception +import Utility.Path +import Utility.UserInfo +import Utility.Process +import Utility.PartialPrelude + +import System.Environment +import System.Directory +import System.FilePath +import Data.List +import Data.String.Utils +import Control.Applicative + +type DesktopEntry = [(Key, Value)] + +type Key = String + +data Value = StringV String | BoolV Bool | NumericV Float | ListV [Value] + +toString :: Value -> String +toString (StringV s) = s +toString (BoolV b) + | b = "true" + | otherwise = "false" +toString(NumericV f) = show f +toString (ListV l) + | null l = "" + | otherwise = (intercalate ";" $ map (escapesemi . toString) l) ++ ";" + where + escapesemi = join "\\;" . split ";" + +genDesktopEntry :: String -> String -> Bool -> FilePath -> [String] -> DesktopEntry +genDesktopEntry name comment terminal program categories = + [ item "Type" StringV "Application" + , item "Version" NumericV 1.0 + , item "Name" StringV name + , item "Comment" StringV comment + , item "Terminal" BoolV terminal + , item "Exec" StringV program + , item "Categories" ListV (map StringV categories) + ] + where + item x c y = (x, c y) + +buildDesktopMenuFile :: DesktopEntry -> String +buildDesktopMenuFile d = unlines ("[Desktop Entry]" : map keyvalue d) ++ "\n" + where + keyvalue (k, v) = k ++ "=" ++ toString v + +writeDesktopMenuFile :: DesktopEntry -> String -> IO () +writeDesktopMenuFile d file = do + createDirectoryIfMissing True (parentDir file) + writeFile file $ buildDesktopMenuFile d + +{- Path to use for a desktop menu file, in either the systemDataDir or + - the userDataDir -} +desktopMenuFilePath :: String -> FilePath -> FilePath +desktopMenuFilePath basename datadir = + datadir "applications" desktopfile basename + +{- Path to use for a desktop autostart file, in either the systemDataDir + - or the userDataDir -} +autoStartPath :: String -> FilePath -> FilePath +autoStartPath basename configdir = + configdir "autostart" desktopfile basename + +desktopfile :: FilePath -> FilePath +desktopfile f = f ++ ".desktop" + +{- Directory used for installation of system wide data files.. -} +systemDataDir :: FilePath +systemDataDir = "/usr/share" + +{- Directory used for installation of system wide config files. -} +systemConfigDir :: FilePath +systemConfigDir = "/etc/xdg" + +{- Directory for user data files. -} +userDataDir :: IO FilePath +userDataDir = xdgEnvHome "DATA_HOME" ".local/share" + +{- Directory for user config files. -} +userConfigDir :: IO FilePath +userConfigDir = xdgEnvHome "CONFIG_HOME" ".config" + +{- Directory for the user's Desktop, may be localized. + - + - This is not looked up very fast; the config file is in a shell format + - that is best parsed by shell, so xdg-user-dir is used, with a fallback + - to ~/Desktop. -} +userDesktopDir :: IO FilePath +userDesktopDir = maybe fallback return =<< (parse <$> xdg_user_dir) + where + parse = maybe Nothing (headMaybe . lines) + xdg_user_dir = catchMaybeIO $ readProcess "xdg-user-dir" ["DESKTOP"] + fallback = xdgEnvHome "DESKTOP_DIR" "Desktop" + +xdgEnvHome :: String -> String -> IO String +xdgEnvHome envbase homedef = do + home <- myHomeDir + catchDefaultIO (home homedef) $ + getEnv $ "XDG_" ++ envbase diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs new file mode 100644 index 0000000000..c31755d623 --- /dev/null +++ b/Utility/Gpg.hs @@ -0,0 +1,226 @@ +{- gpg interface + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Gpg where + +import System.Posix.Types +import Control.Applicative +import Control.Concurrent +import Control.Exception (bracket) +import System.Posix.Env (setEnv, unsetEnv, getEnv) + +import Common + +newtype KeyIds = KeyIds [String] + deriving (Ord, Eq) + +stdParams :: [CommandParam] -> IO [String] +stdParams params = do + -- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous + -- gpg output about password prompts. GPG_BATCH is set by the test + -- suite for a similar reason. + e <- getEnv "GPG_AGENT_INFO" + b <- getEnv "GPG_BATCH" + let batch = if isNothing e && isNothing b + then [] + else ["--batch", "--no-tty", "--use-agent"] + return $ batch ++ defaults ++ toCommand params + where + -- be quiet, even about checking the trustdb + defaults = ["--quiet", "--trust-model", "always"] + +{- Runs gpg with some params and returns its stdout, strictly. -} +readStrict :: [CommandParam] -> IO String +readStrict params = do + params' <- stdParams params + withHandle StdoutHandle createProcessSuccess (proc "gpg" params') $ \h -> do + hSetBinaryMode h True + hGetContentsStrict h + +{- Runs gpg, piping an input value to it, and returning its stdout, + - strictly. -} +pipeStrict :: [CommandParam] -> String -> IO String +pipeStrict params input = do + params' <- stdParams params + withBothHandles createProcessSuccess (proc "gpg" params') $ \(to, from) -> do + hSetBinaryMode to True + hSetBinaryMode from True + hPutStr to input + hClose to + hGetContentsStrict from + +{- Runs gpg with some parameters. First sends it a passphrase via + - --passphrase-fd. Then runs a feeder action that is passed a handle and + - should write to it all the data to input to gpg. Finally, runs + - a reader action that is passed a handle to gpg's output. + - + - Runs gpg in batch mode; this is necessary to avoid gpg 2.x prompting for + - the passphrase. + - + - Note that to avoid deadlock with the cleanup stage, + - the reader must fully consume gpg's input before returning. -} +feedRead :: [CommandParam] -> String -> (Handle -> IO ()) -> (Handle -> IO a) -> IO a +feedRead params passphrase feeder reader = do + -- pipe the passphrase into gpg on a fd + (frompipe, topipe) <- createPipe + void $ forkIO $ do + toh <- fdToHandle topipe + hPutStrLn toh passphrase + hClose toh + let Fd pfd = frompipe + let passphrasefd = [Param "--passphrase-fd", Param $ show pfd] + + params' <- stdParams $ [Param "--batch"] ++ passphrasefd ++ params + closeFd frompipe `after` + withBothHandles createProcessSuccess (proc "gpg" params') go + where + go (to, from) = do + void $ forkIO $ do + feeder to + hClose to + reader from + +{- Finds gpg public keys matching some string. (Could be an email address, + - a key id, or a name. -} +findPubKeys :: String -> IO KeyIds +findPubKeys for = KeyIds . parse <$> readStrict params + where + params = [Params "--with-colons --list-public-keys", Param for] + parse = catMaybes . map (keyIdField . split ":") . lines + keyIdField ("pub":_:_:_:f:_) = Just f + keyIdField _ = Nothing + +{- Creates a block of high-quality random data suitable to use as a cipher. + - It is armored, to avoid newlines, since gpg only reads ciphers up to the + - first newline. -} +genRandom :: Int -> IO String +genRandom size = checksize <$> readStrict + [ Params params + , Param $ show randomquality + , Param $ show size + ] + where + params = "--gen-random --armor" + + -- 1 is /dev/urandom; 2 is /dev/random + randomquality = 1 :: Int + + {- The size is the number of bytes of entropy desired; the data is + - base64 encoded, so needs 8 bits to represent every 6 bytes of + - entropy. -} + expectedlength = size * 8 `div` 6 + + checksize s = let len = length s in + if len >= expectedlength + then s + else shortread len + + shortread got = error $ unwords + [ "Not enough bytes returned from gpg", params + , "(got", show got, "; expected", show expectedlength, ")" + ] + +{- A test key. This is provided pre-generated since generating a new gpg + - key is too much work (requires too much entropy) for a test suite to + - do. + - + - This key was generated with no exipiration date, and a small keysize. + - It has an empty passphrase. -} +testKeyId :: String +testKeyId = "129D6E0AC537B9C7" +testKey :: String +testKey = keyBlock True + [ "mI0ETvFAZgEEAKnqwWgZqznMhi1RQExem2H8t3OyKDxaNN3rBN8T6LWGGqAYV4wT" + , "r8In5tfsnz64bKpE1Qi68JURFwYmthgUL9N48tbODU8t3xzijdjLOSaTyqkH1ik6" + , "EyulfKN63xLne9i4F9XqNwpiZzukXYbNfHkDA2yb0M6g4UFKLY/fNzGXABEBAAG0" + , "W2luc2VjdXJlIHRlc3Qga2V5ICh0aGlzIGlzIGEgdGVzdCBrZXksIGRvIG5vdCB1" + , "c2UgZm9yIGFjdHVhbCBlbmNyeXB0aW9uKSA8dGVzdEBleGFtcGxlLmNvbT6IuAQT" + , "AQgAIgUCTvFAZgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQEp1uCsU3" + , "uceQ9wP/YMd1f0+/eLLcwGXNBvGqyVhUOfAKknO1bMzGbqTsq9g60qegy/cldqee" + , "xVxNfy0VN//JeMfgdcb8+RgJYLoaMrTy9CcsUcFPxtwN9tcLmsM0V2/fNmmFBO9t" + , "v75iH+zeFbNg0/FbPkHiN6Mjw7P2gXYKQXgTvQZBWaphk8oQlBm4jQRO8UBmAQQA" + , "vdi50M/WRCkOLt2RsUve8V8brMWYTJBJTTWoHUeRr82v4NCdX7OE1BsoVK8cy/1Q" + , "Y+gLOH9PqinuGGNWRmPV2Ju/RYn5H7sdewXA8E80xWhc4phHRMJ8Jjhg/GVPamkJ" + , "8B5zeKF0jcLFl7cuVdOyQakhoeDWJd0CyfW837nmPtMAEQEAAYifBBgBCAAJBQJO" + , "8UBmAhsMAAoJEBKdbgrFN7nHclAEAKBShuP/toH03atDUQTbGE34CA4yEC9BVghi" + , "7kviOZlOz2s8xAfp/8AYsrECx1kgbXcA7JD902eNyp7NzXsdJX0zJwHqiuZW0XlD" + , "T8ZJu4qrYRYgl/790WPESZ+ValvHD/fqkR38RF4tfxvyoMhhp0roGmJY33GASIG/" + , "+gQkDF9/" + , "=1k11" + ] +testSecretKey :: String +testSecretKey = keyBlock False + [ "lQHYBE7xQGYBBACp6sFoGas5zIYtUUBMXpth/Ldzsig8WjTd6wTfE+i1hhqgGFeM" + , "E6/CJ+bX7J8+uGyqRNUIuvCVERcGJrYYFC/TePLWzg1PLd8c4o3Yyzkmk8qpB9Yp" + , "OhMrpXyjet8S53vYuBfV6jcKYmc7pF2GzXx5AwNsm9DOoOFBSi2P3zcxlwARAQAB" + , "AAP+PlRboxy7Z0XjuG70N6+CrzSddQbW5KCwgPFrxYsPk7sAPFcBkmRMVlv9vZpS" + , "phbP4bvDK+MrSntM51g+9uE802yhPhSWdmEbImiWfV2ucEhlLjD8gw7JDex9XZ0a" + , "EbTOV56wOsILuedX/jF/6i6IQzy5YmuMeo+ip1XQIsIN+80CAMyXepOBJgHw/gBD" + , "VdXh/l//vUkQQlhInQYwgkKbr0POCTdr8DM1qdKLcUD9Q1khgNRp0vZGGz+5xsrc" + , "KaODUlMCANSczLJcYWa8yPqB3S14yTe7qmtDiOS362+SeVUwQA7eQ06PcHLPsN+p" + , "NtWoHRfYazxrs+g0JvmoQOYdj4xSQy0CAMq7H/l6aeG1n8tpyMxqE7OvBOsvzdu5" + , "XS7I1AnwllVFgvTadVvqgf7b+hdYd91doeHDUGqSYO78UG1GgaBHJdylqrRbaW5z" + , "ZWN1cmUgdGVzdCBrZXkgKHRoaXMgaXMgYSB0ZXN0IGtleSwgZG8gbm90IHVzZSBm" + , "b3IgYWN0dWFsIGVuY3J5cHRpb24pIDx0ZXN0QGV4YW1wbGUuY29tPoi4BBMBCAAi" + , "BQJO8UBmAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRASnW4KxTe5x5D3" + , "A/9gx3V/T794stzAZc0G8arJWFQ58AqSc7VszMZupOyr2DrSp6DL9yV2p57FXE1/" + , "LRU3/8l4x+B1xvz5GAlguhoytPL0JyxRwU/G3A321wuawzRXb982aYUE722/vmIf" + , "7N4Vs2DT8Vs+QeI3oyPDs/aBdgpBeBO9BkFZqmGTyhCUGZ0B2ARO8UBmAQQAvdi5" + , "0M/WRCkOLt2RsUve8V8brMWYTJBJTTWoHUeRr82v4NCdX7OE1BsoVK8cy/1QY+gL" + , "OH9PqinuGGNWRmPV2Ju/RYn5H7sdewXA8E80xWhc4phHRMJ8Jjhg/GVPamkJ8B5z" + , "eKF0jcLFl7cuVdOyQakhoeDWJd0CyfW837nmPtMAEQEAAQAD/RaVtFFTkF1udun7" + , "YOwzJvQXCO9OWHZvSdEeG4BUNdAwy4YWu0oZzKkBDBS6+lWILqqb/c28U4leUJ1l" + , "H+viz5svN9BWWyj/UpI00uwUo9JaIqalemwfLx6vsh69b54L1B4exLZHYGLvy/B3" + , "5T6bT0gpOE+53BRtKcJaOh/McQeJAgDTOCBU5weWOf6Bhqnw3Vr/gRfxntAz2okN" + , "gqz/h79mWbCc/lHKoYQSsrCdMiwziHSjXwvehUrdWE/AcomtW0vbAgDmGJqJ2fNr" + , "HvdsGx4Ld/BxyiZbCURJLUQ5CwzfHGIvBu9PMT8zM26NOSncaXRjxDna2Ggh8Uum" + , "ANEwbnhxFwZpAf9L9RLYIMTtAqwBjfXJg/lHcc2R+VP0hL5c8zFz+S+w7bRqINwL" + , "ff1JstKuHT2nJnu0ustK66by8YI3T0hDFFahnNCInwQYAQgACQUCTvFAZgIbDAAK" + , "CRASnW4KxTe5x3JQBACgUobj/7aB9N2rQ1EE2xhN+AgOMhAvQVYIYu5L4jmZTs9r" + , "PMQH6f/AGLKxAsdZIG13AOyQ/dNnjcqezc17HSV9MycB6ormVtF5Q0/GSbuKq2EW" + , "IJf+/dFjxEmflWpbxw/36pEd/EReLX8b8qDIYadK6BpiWN9xgEiBv/oEJAxffw==" + , "=LDsg" + ] +keyBlock :: Bool -> [String] -> String +keyBlock public ls = unlines + [ "-----BEGIN PGP "++t++" KEY BLOCK-----" + , "Version: GnuPG v1.4.11 (GNU/Linux)" + , "" + , unlines ls + , "-----END PGP "++t++" KEY BLOCK-----" + ] + where + t + | public = "PUBLIC" + | otherwise = "PRIVATE" + +{- Runs an action using gpg in a test harness, in which gpg does + - not use ~/.gpg/, but a directory with the test key set up to be used. -} +testHarness :: IO a -> IO a +testHarness a = do + orig <- getEnv var + bracket setup (cleanup orig) (const a) + where + var = "GNUPGHOME" + + setup = do + base <- getTemporaryDirectory + dir <- mktmpdir $ base "gpgtmpXXXXXX" + setEnv var dir True + _ <- pipeStrict [Params "--import -q"] $ unlines + [testSecretKey, testKey] + return dir + + cleanup orig tmpdir = removeDirectoryRecursive tmpdir >> reset orig + reset (Just v) = setEnv var v True + reset _ = unsetEnv var + +{- Tests the test harness. -} +testTestHarness :: IO Bool +testTestHarness = do + keys <- testHarness $ findPubKeys testKeyId + return $ KeyIds [testKeyId] == keys diff --git a/Utility/HumanTime.hs b/Utility/HumanTime.hs new file mode 100644 index 0000000000..038d1228eb --- /dev/null +++ b/Utility/HumanTime.hs @@ -0,0 +1,26 @@ +{- Time for humans. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.HumanTime where + +import Utility.PartialPrelude + +import Data.Time.Clock.POSIX (POSIXTime) + +{- Parses a human-input time duration, of the form "5h" or "1m". -} +parseDuration :: String -> Maybe POSIXTime +parseDuration s = do + num <- readish s :: Maybe Integer + units <- findUnits =<< lastMaybe s + return $ fromIntegral num * units + where + findUnits 's' = Just 1 + findUnits 'm' = Just 60 + findUnits 'h' = Just $ 60 * 60 + findUnits 'd' = Just $ 60 * 60 * 24 + findUnits 'y' = Just $ 60 * 60 * 24 * 365 + findUnits _ = Nothing diff --git a/Utility/INotify.hs b/Utility/INotify.hs new file mode 100644 index 0000000000..2b5789479a --- /dev/null +++ b/Utility/INotify.hs @@ -0,0 +1,174 @@ +{- higher-level inotify interface + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.INotify where + +import Common hiding (isDirectory) +import Utility.ThreadLock +import Utility.Types.DirWatcher + +import System.INotify +import qualified System.Posix.Files as Files +import System.IO.Error +import Control.Exception (throw) + +{- Watches for changes to files in a directory, and all its subdirectories + - that are not ignored, using inotify. This function returns after + - its initial scan is complete, leaving a thread running. Callbacks are + - made for different events. + - + - Inotify is weak at recursive directory watching; the whole directory + - tree must be scanned and watches set explicitly for each subdirectory. + - + - To notice newly created subdirectories, inotify is used, and + - watches are registered for those directories. There is a race there; + - things can be added to a directory before the watch gets registered. + - + - To close the inotify race, each time a new directory is found, it also + - recursively scans it, assuming all files in it were just added, + - and registering each subdirectory. + - + - Note: Due to the race amelioration, multiple add events may occur + - for the same file. + - + - Note: Moving a file will cause events deleting it from its old location + - and adding it to the new location. + - + - Note: It's assumed that when a file that was open for write is closed, + - it's finished being written to, and can be added. + - + - Note: inotify has a limit to the number of watches allowed, + - /proc/sys/fs/inotify/max_user_watches (default 8192). + - So this will fail if there are too many subdirectories. The + - errHook is called when this happens. + -} +watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> WatchHooks -> IO () +watchDir i dir ignored hooks + | ignored dir = noop + | otherwise = do + -- Use a lock to make sure events generated during initial + -- scan come before real inotify events. + lock <- newLock + let handler event = withLock lock (void $ go event) + void (addWatch i watchevents dir handler) + `catchIO` failedaddwatch + withLock lock $ + mapM_ scan =<< filter (not . dirCruft) <$> + getDirectoryContents dir + where + recurse d = watchDir i d ignored hooks + + -- Select only inotify events required by the enabled + -- hooks, but always include Create so new directories can + -- be scanned. + watchevents = Create : addevents ++ delevents ++ modifyevents + addevents + | hashook addHook || hashook addSymlinkHook = [MoveIn, CloseWrite] + | otherwise = [] + delevents + | hashook delHook || hashook delDirHook = [MoveOut, Delete] + | otherwise = [] + modifyevents + | hashook modifyHook = [Modify] + | otherwise = [] + + scan f = unless (ignored f) $ do + ms <- getstatus f + case ms of + Nothing -> return () + Just s + | Files.isDirectory s -> + recurse $ indir f + | Files.isSymbolicLink s -> + runhook addSymlinkHook f ms + | Files.isRegularFile s -> + runhook addHook f ms + | otherwise -> + noop + + -- Ignore creation events for regular files, which won't be + -- done being written when initially created, but handle for + -- directories and symlinks. + go (Created { isDirectory = isd, filePath = f }) + | isd = recurse $ indir f + | hashook addSymlinkHook = + checkfiletype Files.isSymbolicLink addSymlinkHook f + | otherwise = noop + -- Closing a file is assumed to mean it's done being written. + go (Closed { isDirectory = False, maybeFilePath = Just f }) = + checkfiletype Files.isRegularFile addHook f + -- When a file or directory is moved in, scan it to add new + -- stuff. + go (MovedIn { filePath = f }) = scan f + go (MovedOut { isDirectory = isd, filePath = f }) + | isd = runhook delDirHook f Nothing + | otherwise = runhook delHook f Nothing + -- Verify that the deleted item really doesn't exist, + -- since there can be spurious deletion events for items + -- in a directory that has been moved out, but is still + -- being watched. + go (Deleted { isDirectory = isd, filePath = f }) + | isd = guarded $ runhook delDirHook f Nothing + | otherwise = guarded $ runhook delHook f Nothing + where + guarded = unlessM (filetype (const True) f) + go (Modified { isDirectory = isd, maybeFilePath = Just f }) + | isd = noop + | otherwise = runhook modifyHook f Nothing + go _ = noop + + hashook h = isJust $ h hooks + + runhook h f s + | ignored f = noop + | otherwise = maybe noop (\a -> a (indir f) s) (h hooks) + + indir f = dir f + + getstatus f = catchMaybeIO $ getSymbolicLinkStatus $ indir f + checkfiletype check h f = do + ms <- getstatus f + case ms of + Just s + | check s -> runhook h f ms + _ -> noop + filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f) + + -- Inotify fails when there are too many watches with a + -- disk full error. + failedaddwatch e + | isFullError e = + case errHook hooks of + Nothing -> throw e + Just hook -> tooManyWatches hook dir + | otherwise = throw e + +tooManyWatches :: (String -> Maybe FileStatus -> IO ()) -> FilePath -> IO () +tooManyWatches hook dir = do + sysctlval <- querySysctl [Param maxwatches] :: IO (Maybe Integer) + hook (unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval) Nothing + where + maxwatches = "fs.inotify.max_user_watches" + basewarning = "Too many directories to watch! (Not watching " ++ dir ++")" + withoutsysctl = ["Increase the value in /proc/sys/fs/inotify/max_user_watches"] + withsysctl n = let new = n * 10 in + [ "Increase the limit permanently by running:" + , " echo " ++ maxwatches ++ "=" ++ show new ++ + " | sudo tee -a /etc/sysctl.conf; sudo sysctl -p" + , "Or temporarily by running:" + , " sudo sysctl -w " ++ maxwatches ++ "=" ++ show new + ] + +querySysctl :: Read a => [CommandParam] -> IO (Maybe a) +querySysctl ps = getM go ["sysctl", "/sbin/sysctl", "/usr/sbin/sysctl"] + where + go p = do + v <- catchMaybeIO $ readProcess p (toCommand ps) + case v of + Nothing -> return Nothing + Just s -> return $ parsesysctl s + parsesysctl s = readish =<< lastMaybe (words s) diff --git a/Utility/InodeCache.hs b/Utility/InodeCache.hs new file mode 100644 index 0000000000..5b9bdebcb8 --- /dev/null +++ b/Utility/InodeCache.hs @@ -0,0 +1,57 @@ +{- Caching a file's inode, size, and modification time to see when it's changed. + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.InodeCache where + +import Common +import System.Posix.Types +import Utility.QuickCheck + +data InodeCache = InodeCache FileID FileOffset EpochTime + deriving (Eq, Show) + +{- Weak comparison of the inode caches, comparing the size and mtime, but + - not the actual inode. Useful when inodes have changed, perhaps + - due to some filesystems being remounted. -} +compareWeak :: InodeCache -> InodeCache -> Bool +compareWeak (InodeCache _ size1 mtime1) (InodeCache _ size2 mtime2) = + size1 == size2 && mtime1 == mtime2 + +showInodeCache :: InodeCache -> String +showInodeCache (InodeCache inode size mtime) = unwords + [ show inode + , show size + , show mtime + ] + +readInodeCache :: String -> Maybe InodeCache +readInodeCache s = case words s of + (inode:size:mtime:_) -> InodeCache + <$> readish inode + <*> readish size + <*> readish mtime + _ -> Nothing + +genInodeCache :: FilePath -> IO (Maybe InodeCache) +genInodeCache f = catchDefaultIO Nothing $ toInodeCache <$> getFileStatus f + +toInodeCache :: FileStatus -> Maybe InodeCache +toInodeCache s + | isRegularFile s = Just $ InodeCache + (fileID s) + (fileSize s) + (modificationTime s) + | otherwise = Nothing + +instance Arbitrary InodeCache where + arbitrary = InodeCache + <$> arbitrary + <*> arbitrary + <*> arbitrary + +prop_read_show_inodecache :: InodeCache -> Bool +prop_read_show_inodecache c = readInodeCache (showInodeCache c) == Just c diff --git a/Utility/JSONStream.hs b/Utility/JSONStream.hs new file mode 100644 index 0000000000..f3e93c3dac --- /dev/null +++ b/Utility/JSONStream.hs @@ -0,0 +1,44 @@ +{- Streaming JSON output. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.JSONStream ( + start, + add, + end +) where + +import Text.JSON + +{- Text.JSON does not support building up a larger JSON document piece by + - piece as a stream. To support streaming, a hack. The JSObject is converted + - to a string with its final "}" is left off, allowing it to be added to + - later. -} +start :: JSON a => [(String, a)] -> String +start l + | last s == endchar = init s + | otherwise = bad s + where + s = encodeStrict $ toJSObject l + +add :: JSON a => [(String, a)] -> String +add l + | head s == startchar = ',' : drop 1 s + | otherwise = bad s + where + s = start l + +end :: String +end = [endchar, '\n'] + +startchar :: Char +startchar = '{' + +endchar :: Char +endchar = '}' + +bad :: String -> a +bad s = error $ "Text.JSON returned unexpected string: " ++ s diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs new file mode 100644 index 0000000000..f0559e531b --- /dev/null +++ b/Utility/Kqueue.hs @@ -0,0 +1,267 @@ +{- BSD kqueue file modification notification interface + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE ForeignFunctionInterface #-} + +module Utility.Kqueue ( + Kqueue, + initKqueue, + stopKqueue, + waitChange, + Change(..), + changedFile, + runHooks, +) where + +import Common +import Utility.Types.DirWatcher + +import System.Posix.Types +import Foreign.C.Types +import Foreign.C.Error +import Foreign.Ptr +import Foreign.Marshal +import qualified Data.Map as M +import qualified Data.Set as S +import qualified System.Posix.Files as Files +import Control.Concurrent + +data Change + = Deleted FilePath + | DeletedDir FilePath + | Added FilePath + deriving (Show) + +isAdd :: Change -> Bool +isAdd (Added _) = True +isAdd (Deleted _) = False +isAdd (DeletedDir _) = False + +changedFile :: Change -> FilePath +changedFile (Added f) = f +changedFile (Deleted f) = f +changedFile (DeletedDir f) = f + +data Kqueue = Kqueue + { kqueueFd :: Fd + , kqueueTop :: FilePath + , kqueueMap :: DirMap + , _kqueuePruner :: Pruner + } + +type Pruner = FilePath -> Bool + +type DirMap = M.Map Fd DirInfo + +{- Enough information to uniquely identify a file in a directory, + - but not too much. -} +data DirEnt = DirEnt + { dirEnt :: FilePath -- relative to the parent directory + , _dirInode :: FileID -- included to notice file replacements + , isSubDir :: Bool + } + deriving (Eq, Ord, Show) + +{- A directory, and its last known contents. -} +data DirInfo = DirInfo + { dirName :: FilePath + , dirCache :: S.Set DirEnt + } + deriving (Show) + +getDirInfo :: FilePath -> IO DirInfo +getDirInfo dir = do + l <- filter (not . dirCruft) <$> getDirectoryContents dir + contents <- S.fromList . catMaybes <$> mapM getDirEnt l + return $ DirInfo dir contents + where + getDirEnt f = catchMaybeIO $ do + s <- getSymbolicLinkStatus (dir f) + return $ DirEnt f (fileID s) (isDirectory s) + +{- Difference between the dirCaches of two DirInfos. -} +(//) :: DirInfo -> DirInfo -> [Change] +oldc // newc = deleted ++ added + where + deleted = calc gendel oldc newc + added = calc genadd newc oldc + gendel x = (if isSubDir x then DeletedDir else Deleted) $ + dirName oldc dirEnt x + genadd x = Added $ dirName newc dirEnt x + calc a x y = map a $ S.toList $ + S.difference (dirCache x) (dirCache y) + +{- Builds a map of directories in a tree, possibly pruning some. + - Opens each directory in the tree, and records its current contents. -} +scanRecursive :: FilePath -> Pruner -> IO DirMap +scanRecursive topdir prune = M.fromList <$> walk [] [topdir] + where + walk c [] = return c + walk c (dir:rest) + | prune dir = walk c rest + | otherwise = do + minfo <- catchMaybeIO $ getDirInfo dir + case minfo of + Nothing -> walk c rest + Just info -> do + mfd <- catchMaybeIO $ + openFd dir ReadOnly Nothing defaultFileFlags + case mfd of + Nothing -> walk c rest + Just fd -> do + let subdirs = map (dir ) . map dirEnt $ + S.toList $ dirCache info + walk ((fd, info):c) (subdirs ++ rest) + +{- Adds a list of subdirectories (and all their children), unless pruned to a + - directory map. Adding a subdirectory that's already in the map will + - cause its contents to be refreshed. -} +addSubDirs :: DirMap -> Pruner -> [FilePath] -> IO DirMap +addSubDirs dirmap prune dirs = do + newmap <- foldr M.union M.empty <$> + mapM (\d -> scanRecursive d prune) dirs + return $ M.union newmap dirmap -- prefer newmap + +{- Removes a subdirectory (and all its children) from a directory map. -} +removeSubDir :: DirMap -> FilePath -> IO DirMap +removeSubDir dirmap dir = do + mapM_ closeFd $ M.keys toremove + return rest + where + (toremove, rest) = M.partition (dirContains dir . dirName) dirmap + +findDirContents :: DirMap -> FilePath -> [FilePath] +findDirContents dirmap dir = concatMap absolutecontents $ search + where + absolutecontents i = map (dirName i ) + (map dirEnt $ S.toList $ dirCache i) + search = map snd $ M.toList $ + M.filter (\i -> dirName i == dir) dirmap + +foreign import ccall safe "libkqueue.h init_kqueue" c_init_kqueue + :: IO Fd +foreign import ccall safe "libkqueue.h addfds_kqueue" c_addfds_kqueue + :: Fd -> CInt -> Ptr Fd -> IO () +foreign import ccall safe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue + :: Fd -> IO Fd + +{- Initializes a Kqueue to watch a directory, and all its subdirectories. -} +initKqueue :: FilePath -> Pruner -> IO Kqueue +initKqueue dir pruned = do + dirmap <- scanRecursive dir pruned + h <- c_init_kqueue + let kq = Kqueue h dir dirmap pruned + updateKqueue kq + return kq + +{- Updates a Kqueue, adding watches for its map. -} +updateKqueue :: Kqueue -> IO () +updateKqueue (Kqueue h _ dirmap _) = + withArrayLen (M.keys dirmap) $ \fdcnt c_fds -> do + c_addfds_kqueue h (fromIntegral fdcnt) c_fds + +{- Stops a Kqueue. Note: Does not directly close the Fds in the dirmap, + - so it can be reused. -} +stopKqueue :: Kqueue -> IO () +stopKqueue = closeFd . kqueueFd + +{- Waits for a change on a Kqueue. + - May update the Kqueue. + -} +waitChange :: Kqueue -> IO (Kqueue, [Change]) +waitChange kq@(Kqueue h _ dirmap _) = do + changedfd <- c_waitchange_kqueue h + if changedfd == -1 + then ifM ((==) eINTR <$> getErrno) + (yield >> waitChange kq, nochange) + else case M.lookup changedfd dirmap of + Nothing -> nochange + Just info -> handleChange kq changedfd info + where + nochange = return (kq, []) + +{- The kqueue interface does not tell what type of change took place in + - the directory; it could be an added file, a deleted file, a renamed + - file, a new subdirectory, or a deleted subdirectory, or a moved + - subdirectory. + - + - So to determine this, the contents of the directory are compared + - with its last cached contents. The Kqueue is updated to watch new + - directories as necessary. + -} +handleChange :: Kqueue -> Fd -> DirInfo -> IO (Kqueue, [Change]) +handleChange kq@(Kqueue _ _ dirmap pruner) fd olddirinfo = + go =<< catchMaybeIO (getDirInfo $ dirName olddirinfo) + where + go (Just newdirinfo) = do + let changes = filter (not . pruner . changedFile) $ + olddirinfo // newdirinfo + let (added, deleted) = partition isAdd changes + + -- Scan newly added directories to add to the map. + -- (Newly added files will fail getDirInfo.) + newdirinfos <- catMaybes <$> + mapM (catchMaybeIO . getDirInfo . changedFile) added + newmap <- addSubDirs dirmap pruner $ map dirName newdirinfos + + -- Remove deleted directories from the map. + newmap' <- foldM removeSubDir newmap (map changedFile deleted) + + -- Update the cached dirinfo just looked up. + let newmap'' = M.insertWith' const fd newdirinfo newmap' + + -- When new directories were added, need to update + -- the kqueue to watch them. + let kq' = kq { kqueueMap = newmap'' } + unless (null newdirinfos) $ + updateKqueue kq' + + return (kq', changes) + go Nothing = do + -- The directory has been moved or deleted, so + -- remove it from our map. + newmap <- removeSubDir dirmap (dirName olddirinfo) + return (kq { kqueueMap = newmap }, []) + +{- Processes changes on the Kqueue, calling the hooks as appropriate. + - Never returns. -} +runHooks :: Kqueue -> WatchHooks -> IO () +runHooks kq hooks = do + -- First, synthetic add events for the whole directory tree contents, + -- to catch any files created beforehand. + recursiveadd (kqueueMap kq) (Added $ kqueueTop kq) + loop kq + where + loop q = do + (q', changes) <- waitChange q + forM_ changes $ dispatch (kqueueMap q') + loop q' + + dispatch _ change@(Deleted _) = + callhook delHook Nothing change + dispatch _ change@(DeletedDir _) = + callhook delDirHook Nothing change + dispatch dirmap change@(Added _) = + withstatus change $ dispatchadd dirmap + + dispatchadd dirmap change s + | Files.isSymbolicLink s = callhook addSymlinkHook (Just s) change + | Files.isDirectory s = recursiveadd dirmap change + | Files.isRegularFile s = callhook addHook (Just s) change + | otherwise = noop + + recursiveadd dirmap change = do + let contents = findDirContents dirmap $ changedFile change + forM_ contents $ \f -> + withstatus (Added f) $ dispatchadd dirmap + + callhook h s change = case h hooks of + Nothing -> noop + Just a -> a (changedFile change) s + + withstatus change a = maybe noop (a change) =<< + (catchMaybeIO (getSymbolicLinkStatus (changedFile change))) diff --git a/Utility/LogFile.hs b/Utility/LogFile.hs new file mode 100644 index 0000000000..1ff3006fe9 --- /dev/null +++ b/Utility/LogFile.hs @@ -0,0 +1,54 @@ +{- log files + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.LogFile where + +import Common + +import System.Posix + +openLog :: FilePath -> IO Fd +openLog logfile = do + rotateLog logfile + openFd logfile WriteOnly (Just stdFileMode) + defaultFileFlags { append = True } + +rotateLog :: FilePath -> IO () +rotateLog logfile = go 0 + where + go num + | num > maxLogs = return () + | otherwise = whenM (doesFileExist currfile) $ do + go (num + 1) + renameFile currfile nextfile + where + currfile = filename num + nextfile = filename (num + 1) + filename n + | n == 0 = logfile + | otherwise = rotatedLog logfile n + +rotatedLog :: FilePath -> Int -> FilePath +rotatedLog logfile n = logfile ++ "." ++ show n + +{- Lists most recent logs last. -} +listLogs :: FilePath -> IO [FilePath] +listLogs logfile = filterM doesFileExist $ reverse $ + logfile : map (rotatedLog logfile) [1..maxLogs] + +maxLogs :: Int +maxLogs = 9 + +redirLog :: Fd -> IO () +redirLog logfd = do + mapM_ (redir logfd) [stdOutput, stdError] + closeFd logfd + +redir :: Fd -> Fd -> IO () +redir newh h = do + closeFd h + void $ dupTo newh h diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs new file mode 100644 index 0000000000..8db514d797 --- /dev/null +++ b/Utility/Lsof.hs @@ -0,0 +1,120 @@ +{- lsof interface + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE BangPatterns, CPP #-} + +module Utility.Lsof where + +import Common +import Build.SysConfig as SysConfig + +import System.Posix.Types +import System.Posix.Env + +data LsofOpenMode = OpenReadWrite | OpenReadOnly | OpenWriteOnly | OpenUnknown + deriving (Show, Eq) + +type CmdLine = String + +data ProcessInfo = ProcessInfo ProcessID CmdLine + deriving (Show) + +{- lsof is not in PATH on all systems, so SysConfig may have the absolute + - path where the program was found. Make sure at runtime that lsof is + - available, and if it's not in PATH, adjust PATH to contain it. -} +setupLsof :: IO () +setupLsof = do + let cmd = fromMaybe "lsof" SysConfig.lsof + when (isAbsolute cmd) $ do + path <- getSearchPath + let path' = takeDirectory cmd : path + setEnv "PATH" (join [searchPathSeparator] path') True + +{- Checks each of the files in a directory to find open files. + - Note that this will find hard links to files elsewhere that are open. -} +queryDir :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)] +queryDir path = query ["+d", path] + +{- Runs lsof with some parameters. + - + - Ignores nonzero exit code; lsof returns that when no files are open. + - + - Note: If lsof is not available, this always returns [] ! + -} +query :: [String] -> IO [(FilePath, LsofOpenMode, ProcessInfo)] +query opts = + withHandle StdoutHandle (createProcessChecked checkSuccessProcess) p $ \h -> do + fileEncoding h + parse <$> hGetContentsStrict h + where + p = proc "lsof" ("-F0can" : opts) + +type LsofParser = String -> [(FilePath, LsofOpenMode, ProcessInfo)] + +parse :: LsofParser +#ifdef __ANDROID__ +parse = parseDefault +#else +parse = parseFormatted +#endif + +{- Parsing null-delimited output like: + - + - pPID\0cCMDLINE\0 + - aMODE\0nFILE\0 + - aMODE\0nFILE\0 + - pPID\0[...] + - + - Where each new process block is started by a pid, and a process can + - have multiple files open. + -} +parseFormatted :: LsofParser +parseFormatted s = bundle $ go [] $ lines s + where + bundle = concatMap (\(fs, p) -> map (\(f, m) -> (f, m, p)) fs) + + go c [] = c + go c ((t:r):ls) + | t == 'p' = + let (fs, ls') = parsefiles [] ls + in go ((fs, parseprocess r):c) ls' + | otherwise = parsefail + go _ _ = parsefail + + parseprocess l = case splitnull l of + [pid, 'c':cmdline, ""] -> + case readish pid of + (Just n) -> ProcessInfo n cmdline + Nothing -> parsefail + _ -> parsefail + + parsefiles c [] = (c, []) + parsefiles c (l:ls) = case splitnull l of + ['a':mode, 'n':file, ""] -> + parsefiles ((file, parsemode mode):c) ls + (('p':_):_) -> (c, l:ls) + _ -> parsefail + + parsemode ('r':_) = OpenReadOnly + parsemode ('w':_) = OpenWriteOnly + parsemode ('u':_) = OpenReadWrite + parsemode _ = OpenUnknown + + splitnull = split "\0" + + parsefail = error $ "failed to parse lsof output: " ++ show s + +{- Parses lsof's default output format. -} +parseDefault :: LsofParser +parseDefault = catMaybes . map parseline . drop 1 . lines + where + parseline l = case words l of + (command : spid : _user : _fd : _type : _device : _size : _node : rest) -> + case readish spid of + Nothing -> Nothing + Just pid -> Just (unwords rest, OpenUnknown, ProcessInfo pid command) + _ -> Nothing diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs new file mode 100644 index 0000000000..89a4e7d0c4 --- /dev/null +++ b/Utility/Matcher.hs @@ -0,0 +1,111 @@ +{- A generic matcher. + - + - Can be used to check if a user-supplied condition, + - like "foo and ( bar or not baz )" matches. The condition must already + - be tokenized, and can contain arbitrary operations. + - + - If operations are not separated by and/or, they are defaulted to being + - anded together, so "foo bar baz" all must match. + - + - Is forgiving about misplaced closing parens, so "foo and (bar or baz" + - will be handled, as will "foo and ( bar or baz ) )" + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE Rank2Types, KindSignatures #-} + +module Utility.Matcher ( + Token(..), + Matcher, + token, + tokens, + generate, + match, + matchM, + matchMrun, + isEmpty +) where + +import Common + +{- A Token can be an Operation of an arbitrary type, or one of a few + - predefined peices of syntax. -} +data Token op = Operation op | And | Or | Not | Open | Close + deriving (Show, Eq) + +data Matcher op = MAny + | MAnd (Matcher op) (Matcher op) + | MOr (Matcher op) (Matcher op) + | MNot (Matcher op) + | MOp op + deriving (Show, Eq) + +{- Converts a word of syntax into a token. Doesn't handle operations. -} +token :: String -> Token op +token "and" = And +token "or" = Or +token "not" = Not +token "(" = Open +token ")" = Close +token t = error $ "unknown token " ++ t + +tokens :: [String] +tokens = words "and or not ( )" + +{- Converts a list of Tokens into a Matcher. -} +generate :: [Token op] -> Matcher op +generate = go MAny + where + go m [] = m + go m ts = uncurry go $ consume m ts + +{- Consumes one or more Tokens, constructs a new Matcher, + - and returns unconsumed Tokens. -} +consume :: Matcher op -> [Token op] -> (Matcher op, [Token op]) +consume m [] = (m, []) +consume m (t:ts) = go t + where + go And = cont $ m `MAnd` next + go Or = cont $ m `MOr` next + go Not = cont $ m `MAnd` MNot next + go Open = let (n, r) = consume next rest in (m `MAnd` n, r) + go Close = (m, ts) + go (Operation o) = (m `MAnd` MOp o, ts) + + (next, rest) = consume MAny ts + cont v = (v, rest) + +{- Checks if a Matcher matches, using a supplied function to check + - the value of Operations. -} +match :: (op -> v -> Bool) -> Matcher op -> v -> Bool +match a m v = go m + where + go MAny = True + go (MAnd m1 m2) = go m1 && go m2 + go (MOr m1 m2) = go m1 || go m2 + go (MNot m1) = not $ go m1 + go (MOp o) = a o v + +{- Runs a monadic Matcher, where Operations are actions in the monad. -} +matchM :: Monad m => Matcher (v -> m Bool) -> v -> m Bool +matchM m v = matchMrun m $ \o -> o v + +{- More generic running of a monadic Matcher, with full control over running + - of Operations. Mostly useful in order to match on more than one + - parameter. -} +matchMrun :: forall o (m :: * -> *). Monad m => Matcher o -> (o -> m Bool) -> m Bool +matchMrun m run = go m + where + go MAny = return True + go (MAnd m1 m2) = go m1 <&&> go m2 + go (MOr m1 m2) = go m1 <||> go m2 + go (MNot m1) = liftM not (go m1) + go (MOp o) = run o + +{- Checks if a matcher contains no limits. -} +isEmpty :: Matcher a -> Bool +isEmpty MAny = True +isEmpty _ = False diff --git a/Utility/Misc.hs b/Utility/Misc.hs new file mode 100644 index 0000000000..c04409563b --- /dev/null +++ b/Utility/Misc.hs @@ -0,0 +1,112 @@ +{- misc utility functions + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Misc where + +import System.IO +import Control.Monad +import Foreign +import Data.Char +import Control.Applicative +import System.Posix.Process (getAnyProcessStatus) + +import Utility.Exception + +{- A version of hgetContents that is not lazy. Ensures file is + - all read before it gets closed. -} +hGetContentsStrict :: Handle -> IO String +hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s + +{- A version of readFile that is not lazy. -} +readFileStrict :: FilePath -> IO String +readFileStrict = readFile >=> \s -> length s `seq` return s + +{- Like break, but the character matching the condition is not included + - in the second result list. + - + - separate (== ':') "foo:bar" = ("foo", "bar") + - separate (== ':') "foobar" = ("foobar", "") + -} +separate :: (a -> Bool) -> [a] -> ([a], [a]) +separate c l = unbreak $ break c l + where + unbreak r@(a, b) + | null b = r + | otherwise = (a, tail b) + +{- Breaks out the first line. -} +firstLine :: String -> String +firstLine = takeWhile (/= '\n') + +{- Splits a list into segments that are delimited by items matching + - a predicate. (The delimiters are not included in the segments.) + - Segments may be empty. -} +segment :: (a -> Bool) -> [a] -> [[a]] +segment p l = map reverse $ go [] [] l + where + go c r [] = reverse $ c:r + go c r (i:is) + | p i = go [] (c:r) is + | otherwise = go (i:c) r is + +prop_segment_regressionTest :: Bool +prop_segment_regressionTest = all id + -- Even an empty list is a segment. + [ segment (== "--") [] == [[]] + -- There are two segements in this list, even though the first is empty. + , segment (== "--") ["--", "foo", "bar"] == [[],["foo","bar"]] + ] + +{- Includes the delimiters as segments of their own. -} +segmentDelim :: (a -> Bool) -> [a] -> [[a]] +segmentDelim p l = map reverse $ go [] [] l + where + go c r [] = reverse $ c:r + go c r (i:is) + | p i = go [] ([i]:c:r) is + | otherwise = go (i:c) r is + +{- Given two orderings, returns the second if the first is EQ and returns + - the first otherwise. + - + - Example use: + - + - compare lname1 lname2 `thenOrd` compare fname1 fname2 + -} +thenOrd :: Ordering -> Ordering -> Ordering +thenOrd EQ x = x +thenOrd x _ = x +{-# INLINE thenOrd #-} + +{- Wrapper around hGetBufSome that returns a String. + - + - The null string is returned on eof, otherwise returns whatever + - data is currently available to read from the handle, or waits for + - data to be written to it if none is currently available. + - + - Note on encodings: The normal encoding of the Handle is ignored; + - each byte is converted to a Char. Not unicode clean! + -} +hGetSomeString :: Handle -> Int -> IO String +hGetSomeString h sz = do + fp <- mallocForeignPtrBytes sz + len <- withForeignPtr fp $ \buf -> hGetBufSome h buf sz + map (chr . fromIntegral) <$> withForeignPtr fp (peekbytes len) + where + peekbytes :: Int -> Ptr Word8 -> IO [Word8] + peekbytes len buf = mapM (peekElemOff buf) [0..pred len] + +{- Reaps any zombie git processes. + - + - Warning: Not thread safe. Anything that was expecting to wait + - on a process and get back an exit status is going to be confused + - if this reap gets there first. -} +reapZombies :: IO () +reapZombies = do + -- throws an exception when there are no child processes + catchDefaultIO Nothing (getAnyProcessStatus False True) + >>= maybe (return ()) (const reapZombies) diff --git a/Utility/Monad.hs b/Utility/Monad.hs new file mode 100644 index 0000000000..4c3c30473b --- /dev/null +++ b/Utility/Monad.hs @@ -0,0 +1,65 @@ +{- monadic stuff + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Monad where + +import Data.Maybe +import Control.Monad (liftM) + +{- Return the first value from a list, if any, satisfying the given + - predicate -} +firstM :: Monad m => (a -> m Bool) -> [a] -> m (Maybe a) +firstM _ [] = return Nothing +firstM p (x:xs) = ifM (p x) (return $ Just x , firstM p xs) + +{- Runs the action on values from the list until it succeeds, returning + - its result. -} +getM :: Monad m => (a -> m (Maybe b)) -> [a] -> m (Maybe b) +getM _ [] = return Nothing +getM p (x:xs) = maybe (getM p xs) (return . Just) =<< p x + +{- Returns true if any value in the list satisfies the predicate, + - stopping once one is found. -} +anyM :: Monad m => (a -> m Bool) -> [a] -> m Bool +anyM p = liftM isJust . firstM p + +{- Runs an action on values from a list until it succeeds. -} +untilTrue :: Monad m => [a] -> (a -> m Bool) -> m Bool +untilTrue = flip anyM + +{- if with a monadic conditional. -} +ifM :: Monad m => m Bool -> (m a, m a) -> m a +ifM cond (thenclause, elseclause) = do + c <- cond + if c then thenclause else elseclause + +{- short-circuiting monadic || -} +(<||>) :: Monad m => m Bool -> m Bool -> m Bool +ma <||> mb = ifM ma ( return True , mb ) + +{- short-circuiting monadic && -} +(<&&>) :: Monad m => m Bool -> m Bool -> m Bool +ma <&&> mb = ifM ma ( mb , return False ) + +{- Same fixity as && and || -} +infixr 3 <&&> +infixr 2 <||> + +{- Runs an action, passing its value to an observer before returning it. -} +observe :: Monad m => (a -> m b) -> m a -> m a +observe observer a = do + r <- a + _ <- observer r + return r + +{- b `after` a runs first a, then b, and returns the value of a -} +after :: Monad m => m b -> m a -> m a +after = observe . const + +{- do nothing -} +noop :: Monad m => m () +noop = return () diff --git a/Utility/Mounts.hsc b/Utility/Mounts.hsc new file mode 100644 index 0000000000..c21a68032e --- /dev/null +++ b/Utility/Mounts.hsc @@ -0,0 +1,69 @@ +{- Interface to mtab (and fstab) + - + - Derived from hsshellscript, originally written by + - Volker Wysk + - + - Modified to support BSD and Mac OS X by + - Joey Hess + - + - Licensed under the GNU LGPL version 2.1 or higher. + -} + +{-# LANGUAGE ForeignFunctionInterface #-} + +module Utility.Mounts ( + Mntent(..), + getMounts +) where + +import Control.Monad +import Foreign +import Foreign.C +import GHC.IO hiding (finally, bracket) +import Prelude hiding (catch) + +#include "libmounts.h" + +{- This is a stripped down mntent, containing only + - fields available everywhere. -} +data Mntent = Mntent + { mnt_fsname :: String + , mnt_dir :: FilePath + , mnt_type :: String + } deriving (Read, Show, Eq, Ord) + +getMounts :: IO [Mntent] +getMounts = do + h <- c_mounts_start + when (h == nullPtr) $ + throwErrno "getMounts" + mntent <- getmntent h [] + _ <- c_mounts_end h + return mntent + + where + getmntent h c = do + ptr <- c_mounts_next h + if (ptr == nullPtr) + then return $ reverse c + else do + mnt_fsname_str <- #{peek struct mntent, mnt_fsname} ptr >>= peekCString + mnt_dir_str <- #{peek struct mntent, mnt_dir} ptr >>= peekCString + mnt_type_str <- #{peek struct mntent, mnt_type} ptr >>= peekCString + let ent = Mntent + { mnt_fsname = mnt_fsname_str + , mnt_dir = mnt_dir_str + , mnt_type = mnt_type_str + } + getmntent h (ent:c) + +{- Using unsafe imports because the C functions are belived to never block. + - Note that getmntinfo is called with MNT_NOWAIT to avoid possibly blocking; + - while getmntent only accesses a file in /etc (or /proc) that should not + - block. -} +foreign import ccall unsafe "libmounts.h mounts_start" c_mounts_start + :: IO (Ptr ()) +foreign import ccall unsafe "libmounts.h mounts_next" c_mounts_next + :: Ptr () -> IO (Ptr ()) +foreign import ccall unsafe "libmounts.h mounts_end" c_mounts_end + :: Ptr () -> IO CInt diff --git a/Utility/Network.hs b/Utility/Network.hs new file mode 100644 index 0000000000..62523c9e98 --- /dev/null +++ b/Utility/Network.hs @@ -0,0 +1,21 @@ +{- network functions + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Network where + +import Utility.Process +import Utility.Exception + +import Control.Applicative + +{- Haskell lacks uname(2) bindings, except in the + - Bindings.Uname addon. Rather than depend on that, + - use uname -n when available. -} +getHostname :: IO (Maybe String) +getHostname = catchMaybeIO uname_node + where + uname_node = takeWhile (/= '\n') <$> readProcess "uname" ["-n"] diff --git a/Utility/NotificationBroadcaster.hs b/Utility/NotificationBroadcaster.hs new file mode 100644 index 0000000000..413ec2d755 --- /dev/null +++ b/Utility/NotificationBroadcaster.hs @@ -0,0 +1,77 @@ +{- notification broadcaster + - + - This is used to allow clients to block until there is a new notification + - that some thing occurred. It does not communicate what the change is, + - it only provides blocking reads to wait on notifications. + - + - Multiple clients are supported. Each has a unique id. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.NotificationBroadcaster ( + NotificationBroadcaster, + NotificationHandle, + NotificationId, + newNotificationBroadcaster, + newNotificationHandle, + notificationHandleToId, + notificationHandleFromId, + sendNotification, + waitNotification, +) where + +import Common + +import Control.Concurrent.STM +import Control.Concurrent.MSampleVar + +{- One MSampleVar per client. The TMVar is never empty, so never blocks. -} +type NotificationBroadcaster = TMVar [MSampleVar ()] + +newtype NotificationId = NotificationId Int + deriving (Read, Show, Eq, Ord) + +{- Handle given out to an individual client. -} +data NotificationHandle = NotificationHandle NotificationBroadcaster NotificationId + +newNotificationBroadcaster :: IO NotificationBroadcaster +newNotificationBroadcaster = atomically $ newTMVar [] + +{- Allocates a notification handle for a client to use. -} +newNotificationHandle :: NotificationBroadcaster -> IO NotificationHandle +newNotificationHandle b = NotificationHandle + <$> pure b + <*> addclient + where + addclient = do + s <- newEmptySV + atomically $ do + l <- takeTMVar b + putTMVar b $ l ++ [s] + return $ NotificationId $ length l + +{- Extracts the identifier from a notification handle. + - This can be used to eg, pass the identifier through to a WebApp. -} +notificationHandleToId :: NotificationHandle -> NotificationId +notificationHandleToId (NotificationHandle _ i) = i + +notificationHandleFromId :: NotificationBroadcaster -> NotificationId -> NotificationHandle +notificationHandleFromId = NotificationHandle + +{- Sends a notification to all clients. -} +sendNotification :: NotificationBroadcaster -> IO () +sendNotification b = do + l <- atomically $ readTMVar b + mapM_ notify l + where + notify s = writeSV s () + +{- Used by a client to block until a new notification is available since + - the last time it tried. -} +waitNotification :: NotificationHandle -> IO () +waitNotification (NotificationHandle b (NotificationId i)) = do + l <- atomically $ readTMVar b + readSV (l !! i) diff --git a/Utility/OSX.hs b/Utility/OSX.hs new file mode 100644 index 0000000000..f9d992575a --- /dev/null +++ b/Utility/OSX.hs @@ -0,0 +1,44 @@ +{- OSX stuff + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.OSX where + +import Utility.UserInfo + +import System.FilePath + +autoStartBase :: String -> FilePath +autoStartBase label = "Library" "LaunchAgents" label ++ ".plist" + +systemAutoStart :: String -> FilePath +systemAutoStart label = "/" autoStartBase label + +userAutoStart :: String -> IO FilePath +userAutoStart label = do + home <- myHomeDir + return $ home autoStartBase label + +{- Generates an OSX autostart plist file with a given label, command, and + - params to run at boot or login. -} +genOSXAutoStartFile :: String -> String -> [String] -> String +genOSXAutoStartFile label command params = unlines + [ "" + , "" + , "" + , "" + , "Label" + , "" ++ label ++ "" + , "ProgramArguments" + , "" + , unlines $ map (\v -> "" ++ v ++ "") (command:params) + , "" + , "RunAtLoad" + , "" + , "" + , "" + ] + diff --git a/Utility/Observed.hs b/Utility/Observed.hs new file mode 100644 index 0000000000..3ee9734298 --- /dev/null +++ b/Utility/Observed.hs @@ -0,0 +1,43 @@ +module Utility.Observed where + +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S +import System.IO +import System.IO.Unsafe +import Foreign.Storable (Storable(sizeOf)) + +{- This is like L.hGetContents, but after each chunk is read, an action + - is run to observe the size of the chunk. + - + - Note that the observer is run in unsafeInterleaveIO, which means that + - it can be run at any time. It's even possible for observers to run out + - of order, as different parts of the ByteString are consumed. + - + - All the usual caveats about using unsafeInterleaveIO apply to the observers, + - so use caution. + -} +hGetContentsObserved :: Handle -> (Int -> IO ()) -> IO L.ByteString +hGetContentsObserved h observe = lazyRead + where + lazyRead = unsafeInterleaveIO loop + + loop = do + c <- S.hGetSome h defaultChunkSize + if S.null c + then do + hClose h + return $ L.empty + else do + observe $ S.length c + {- unsafeInterleaveIO causes this to be + - deferred until the data is read from the + - ByteString. -} + cs <- lazyRead + return $ L.append (L.fromChunks [c]) cs + +{- Same default chunk size Lazy ByteStrings use. -} +defaultChunkSize :: Int +defaultChunkSize = 32 * k - chunkOverhead + where + k = 1024 + chunkOverhead = 2 * sizeOf (undefined :: Int) -- GHC specific diff --git a/Utility/Parallel.hs b/Utility/Parallel.hs new file mode 100644 index 0000000000..b398803557 --- /dev/null +++ b/Utility/Parallel.hs @@ -0,0 +1,35 @@ +{- parallel processing via threads + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Parallel where + +import Common + +import Control.Concurrent +import Control.Exception + +{- Runs an action in parallel with a set of values, in a set of threads. + - In order for the actions to truely run in parallel, requires GHC's + - threaded runtime, + - + - Returns the values partitioned into ones with which the action succeeded, + - and ones with which it failed. -} +inParallel :: (v -> IO Bool) -> [v] -> IO ([v], [v]) +inParallel a l = do + mvars <- mapM thread l + statuses <- mapM takeMVar mvars + return $ reduce $ partition snd $ zip l statuses + where + reduce (x,y) = (map fst x, map fst y) + thread v = do + mvar <- newEmptyMVar + _ <- forkIO $ do + r <- try (a v) :: IO (Either SomeException Bool) + case r of + Left _ -> putMVar mvar False + Right b -> putMVar mvar b + return mvar diff --git a/Utility/PartialPrelude.hs b/Utility/PartialPrelude.hs new file mode 100644 index 0000000000..6efa093fd3 --- /dev/null +++ b/Utility/PartialPrelude.hs @@ -0,0 +1,68 @@ +{- Parts of the Prelude are partial functions, which are a common source of + - bugs. + - + - This exports functions that conflict with the prelude, which avoids + - them being accidentially used. + -} + +module Utility.PartialPrelude where + +import qualified Data.Maybe + +{- read should be avoided, as it throws an error + - Instead, use: readish -} +read :: Read a => String -> a +read = Prelude.read + +{- head is a partial function; head [] is an error + - Instead, use: take 1 or headMaybe -} +head :: [a] -> a +head = Prelude.head + +{- tail is also partial + - Instead, use: drop 1 -} +tail :: [a] -> [a] +tail = Prelude.tail + +{- init too + - Instead, use: beginning -} +init :: [a] -> [a] +init = Prelude.init + +{- last too + - Instead, use: end or lastMaybe -} +last :: [a] -> a +last = Prelude.last + +{- Attempts to read a value from a String. + - + - Ignores leading/trailing whitespace, and throws away any trailing + - text after the part that can be read. + - + - readMaybe is available in Text.Read in new versions of GHC, + - but that one requires the entire string to be consumed. + -} +readish :: Read a => String -> Maybe a +readish s = case reads s of + ((x,_):_) -> Just x + _ -> Nothing + +{- Like head but Nothing on empty list. -} +headMaybe :: [a] -> Maybe a +headMaybe = Data.Maybe.listToMaybe + +{- Like last but Nothing on empty list. -} +lastMaybe :: [a] -> Maybe a +lastMaybe [] = Nothing +lastMaybe v = Just $ Prelude.last v + +{- All but the last element of a list. + - (Like init, but no error on an empty list.) -} +beginning :: [a] -> [a] +beginning [] = [] +beginning l = Prelude.init l + +{- Like last, but no error on an empty list. -} +end :: [a] -> [a] +end [] = [] +end l = [Prelude.last l] diff --git a/Utility/Path.hs b/Utility/Path.hs new file mode 100644 index 0000000000..d058082df5 --- /dev/null +++ b/Utility/Path.hs @@ -0,0 +1,165 @@ +{-# LANGUAGE PackageImports #-} +{- path manipulation + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Path where + +import Data.String.Utils +import "MissingH" System.Path +import System.FilePath +import System.Directory +import Data.List +import Data.Maybe +import Control.Applicative + +import Utility.Monad +import Utility.UserInfo + +{- Returns the parent directory of a path. Parent of / is "" -} +parentDir :: FilePath -> FilePath +parentDir dir + | not $ null dirs = slash ++ join s (init dirs) + | otherwise = "" + where + dirs = filter (not . null) $ split s dir + slash = if isAbsolute dir then s else "" + s = [pathSeparator] + +prop_parentDir_basics :: FilePath -> Bool +prop_parentDir_basics dir + | null dir = True + | dir == "/" = parentDir dir == "" + | otherwise = p /= dir + where + p = parentDir dir + +{- Checks if the first FilePath is, or could be said to contain the second. + - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc + - are all equivilant. + -} +dirContains :: FilePath -> FilePath -> Bool +dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b' + where + norm p = fromMaybe "" $ absNormPath p "." + a' = norm a + b' = norm b + +{- Converts a filename into a normalized, absolute path. + - + - Unlike Directory.canonicalizePath, this does not require the path + - already exists. -} +absPath :: FilePath -> IO FilePath +absPath file = do + cwd <- getCurrentDirectory + return $ absPathFrom cwd file + +{- Converts a filename into a normalized, absolute path + - from the specified cwd. -} +absPathFrom :: FilePath -> FilePath -> FilePath +absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file + where + bad = error $ "unable to normalize " ++ file + +{- Constructs a relative path from the CWD to a file. + - + - For example, assuming CWD is /tmp/foo/bar: + - relPathCwdToFile "/tmp/foo" == ".." + - relPathCwdToFile "/tmp/foo/bar" == "" + -} +relPathCwdToFile :: FilePath -> IO FilePath +relPathCwdToFile f = relPathDirToFile <$> getCurrentDirectory <*> absPath f + +{- Constructs a relative path from a directory to a file. + - + - Both must be absolute, and normalized (eg with absNormpath). + -} +relPathDirToFile :: FilePath -> FilePath -> FilePath +relPathDirToFile from to = join s $ dotdots ++ uncommon + where + s = [pathSeparator] + pfrom = split s from + pto = split s to + common = map fst $ takeWhile same $ zip pfrom pto + same (c,d) = c == d + uncommon = drop numcommon pto + dotdots = replicate (length pfrom - numcommon) ".." + numcommon = length common + +prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool +prop_relPathDirToFile_basics from to + | from == to = null r + | otherwise = not (null r) + where + r = relPathDirToFile from to + +prop_relPathDirToFile_regressionTest :: Bool +prop_relPathDirToFile_regressionTest = same_dir_shortcurcuits_at_difference + where + {- Two paths have the same directory component at the same + - location, but it's not really the same directory. + - Code used to get this wrong. -} + same_dir_shortcurcuits_at_difference = + relPathDirToFile "/tmp/r/lll/xxx/yyy/18" "/tmp/r/.git/annex/objects/18/gk/SHA256-foo/SHA256-foo" == "../../../../.git/annex/objects/18/gk/SHA256-foo/SHA256-foo" + +{- Given an original list of paths, and an expanded list derived from it, + - generates a list of lists, where each sublist corresponds to one of the + - original paths. When the original path is a direcotry, any items + - in the expanded list that are contained in that directory will appear in + - its segment. + -} +segmentPaths :: [FilePath] -> [FilePath] -> [[FilePath]] +segmentPaths [] new = [new] +segmentPaths [_] new = [new] -- optimisation +segmentPaths (l:ls) new = [found] ++ segmentPaths ls rest + where + (found, rest)=partition (l `dirContains`) new + +{- This assumes that it's cheaper to call segmentPaths on the result, + - than it would be to run the action separately with each path. In + - the case of git file list commands, that assumption tends to hold. + -} +runSegmentPaths :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [[FilePath]] +runSegmentPaths a paths = segmentPaths paths <$> a paths + +{- Converts paths in the home directory to use ~/ -} +relHome :: FilePath -> IO String +relHome path = do + home <- myHomeDir + return $ if dirContains home path + then "~/" ++ relPathDirToFile home path + else path + +{- Checks if a command is available in PATH. + - + - The command may be fully-qualified, in which case, this succeeds as + - long as it exists. -} +inPath :: String -> IO Bool +inPath command = isJust <$> searchPath command + +{- Finds a command in PATH and returns the full path to it. + - + - The command may be fully qualified already, in which case it will + - be returned if it exists. + -} +searchPath :: String -> IO (Maybe FilePath) +searchPath command + | isAbsolute command = check command + | otherwise = getSearchPath >>= getM indir + where + indir d = check $ d command + check f = ifM (doesFileExist f) ( return (Just f), return Nothing ) + +{- Checks if a filename is a unix dotfile. All files inside dotdirs + - count as dotfiles. -} +dotfile :: FilePath -> Bool +dotfile file + | f == "." = False + | f == ".." = False + | f == "" = False + | otherwise = "." `isPrefixOf` f || dotfile (takeDirectory file) + where + f = takeFileName file diff --git a/Utility/Percentage.hs b/Utility/Percentage.hs new file mode 100644 index 0000000000..1c6b500627 --- /dev/null +++ b/Utility/Percentage.hs @@ -0,0 +1,38 @@ +{- percentages + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Percentage ( + Percentage, + percentage, + showPercentage +) where + +import Data.Ratio + +newtype Percentage = Percentage (Ratio Integer) + +instance Show Percentage where + show = showPercentage 0 + +{- Normally the big number comes first. But 110% is allowed if desired. :) -} +percentage :: Integer -> Integer -> Percentage +percentage 0 _ = Percentage 0 +percentage full have = Percentage $ have * 100 % full + +{- Pretty-print a Percentage, with a specified level of precision. -} +showPercentage :: Int -> Percentage -> String +showPercentage precision (Percentage p) + | precision == 0 || remainder == 0 = go $ show int + | otherwise = go $ show int ++ "." ++ strip0s (show remainder) + where + go v = v ++ "%" + int :: Integer + (int, frac) = properFraction (fromRational p) + remainder = floor (frac * multiplier) :: Integer + strip0s = reverse . dropWhile (== '0') . reverse + multiplier :: Float + multiplier = 10 ** (fromIntegral precision) diff --git a/Utility/Process.hs b/Utility/Process.hs new file mode 100644 index 0000000000..b2bac99a12 --- /dev/null +++ b/Utility/Process.hs @@ -0,0 +1,310 @@ +{- System.Process enhancements, including additional ways of running + - processes, and logging. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE Rank2Types #-} + +module Utility.Process ( + module X, + CreateProcess, + StdHandle(..), + readProcess, + readProcessEnv, + writeReadProcessEnv, + forceSuccessProcess, + checkSuccessProcess, + ignoreFailureProcess, + createProcessSuccess, + createProcessChecked, + createBackgroundProcess, + processTranscript, + withHandle, + withBothHandles, + withQuietOutput, + createProcess, + runInteractiveProcess, + stdinHandle, + stdoutHandle, + stderrHandle, +) where + +import qualified System.Process +import System.Process as X hiding (CreateProcess(..), createProcess, runInteractiveProcess, readProcess, readProcessWithExitCode, system, rawSystem, runInteractiveCommand, runProcess) +import System.Process hiding (createProcess, runInteractiveProcess, readProcess) +import System.Exit +import System.IO +import System.Log.Logger +import Control.Concurrent +import qualified Control.Exception as E +import Control.Monad +import Data.Maybe +import System.Posix.IO + +import Utility.Misc + +type CreateProcessRunner = forall a. CreateProcess -> ((Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> IO a) -> IO a + +data StdHandle = StdinHandle | StdoutHandle | StderrHandle + deriving (Eq) + +{- Normally, when reading from a process, it does not need to be fed any + - standard input. -} +readProcess :: FilePath -> [String] -> IO String +readProcess cmd args = readProcessEnv cmd args Nothing + +readProcessEnv :: FilePath -> [String] -> Maybe [(String, String)] -> IO String +readProcessEnv cmd args environ = + withHandle StdoutHandle createProcessSuccess p $ \h -> do + output <- hGetContentsStrict h + hClose h + return output + where + p = (proc cmd args) + { std_out = CreatePipe + , env = environ + } + +{- Writes a string to a process on its stdin, + - returns its output, and also allows specifying the environment. + -} +writeReadProcessEnv + :: FilePath + -> [String] + -> Maybe [(String, String)] + -> String + -> (Maybe (Handle -> IO ())) + -> IO String +writeReadProcessEnv cmd args environ input adjusthandle = do + (Just inh, Just outh, _, pid) <- createProcess p + + maybe (return ()) (\a -> a inh) adjusthandle + maybe (return ()) (\a -> a outh) adjusthandle + + -- fork off a thread to start consuming the output + output <- hGetContents outh + outMVar <- newEmptyMVar + _ <- forkIO $ E.evaluate (length output) >> putMVar outMVar () + + -- now write and flush any input + when (not (null input)) $ do hPutStr inh input; hFlush inh + hClose inh -- done with stdin + + -- wait on the output + takeMVar outMVar + hClose outh + + -- wait on the process + forceSuccessProcess p pid + + return output + + where + p = (proc cmd args) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = Inherit + , env = environ + } + +{- Waits for a ProcessHandle, and throws an IOError if the process + - did not exit successfully. -} +forceSuccessProcess :: CreateProcess -> ProcessHandle -> IO () +forceSuccessProcess p pid = do + code <- waitForProcess pid + case code of + ExitSuccess -> return () + ExitFailure n -> fail $ showCmd p ++ " exited " ++ show n + +{- Waits for a ProcessHandle and returns True if it exited successfully. + - Note that using this with createProcessChecked will throw away + - the Bool, and is only useful to ignore the exit code of a process, + - while still waiting for it. -} +checkSuccessProcess :: ProcessHandle -> IO Bool +checkSuccessProcess pid = do + code <- waitForProcess pid + return $ code == ExitSuccess + +ignoreFailureProcess :: ProcessHandle -> IO Bool +ignoreFailureProcess pid = do + void $ waitForProcess pid + return True + +{- Runs createProcess, then an action on its handles, and then + - forceSuccessProcess. -} +createProcessSuccess :: CreateProcessRunner +createProcessSuccess p a = createProcessChecked (forceSuccessProcess p) p a + +{- Runs createProcess, then an action on its handles, and then + - an action on its exit code. -} +createProcessChecked :: (ProcessHandle -> IO b) -> CreateProcessRunner +createProcessChecked checker p a = do + t@(_, _, _, pid) <- createProcess p + r <- a t + _ <- checker pid + return r + +{- Leaves the process running, suitable for lazy streaming. + - Note: Zombies will result, and must be waited on. -} +createBackgroundProcess :: CreateProcessRunner +createBackgroundProcess p a = a =<< createProcess p + +{- Runs a process, optionally feeding it some input, and + - returns a transcript combining its stdout and stderr, and + - whether it succeeded or failed. -} +processTranscript :: String -> [String] -> (Maybe String) -> IO (String, Bool) +processTranscript cmd opts input = do + (readf, writef) <- createPipe + readh <- fdToHandle readf + writeh <- fdToHandle writef + p@(_, _, _, pid) <- createProcess $ + (proc cmd opts) + { std_in = if isJust input then CreatePipe else Inherit + , std_out = UseHandle writeh + , std_err = UseHandle writeh + } + hClose writeh + + -- fork off a thread to start consuming the output + transcript <- hGetContents readh + outMVar <- newEmptyMVar + _ <- forkIO $ E.evaluate (length transcript) >> putMVar outMVar () + + -- now write and flush any input + case input of + Just s -> do + let inh = stdinHandle p + unless (null s) $ do + hPutStr inh s + hFlush inh + hClose inh + Nothing -> return () + + -- wait on the output + takeMVar outMVar + hClose readh + + ok <- checkSuccessProcess pid + return (transcript, ok) + + +{- Runs a CreateProcessRunner, on a CreateProcess structure, that + - is adjusted to pipe only from/to a single StdHandle, and passes + - the resulting Handle to an action. -} +withHandle + :: StdHandle + -> CreateProcessRunner + -> CreateProcess + -> (Handle -> IO a) + -> IO a +withHandle h creator p a = creator p' $ a . select + where + base = p + { std_in = Inherit + , std_out = Inherit + , std_err = Inherit + } + (select, p') + | h == StdinHandle = + (stdinHandle, base { std_in = CreatePipe }) + | h == StdoutHandle = + (stdoutHandle, base { std_out = CreatePipe }) + | h == StderrHandle = + (stderrHandle, base { std_err = CreatePipe }) + +{- Like withHandle, but passes (stdin, stdout) handles to the action. -} +withBothHandles + :: CreateProcessRunner + -> CreateProcess + -> ((Handle, Handle) -> IO a) + -> IO a +withBothHandles creator p a = creator p' $ a . bothHandles + where + p' = p + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = Inherit + } + +{- Forces the CreateProcessRunner to run quietly; + - both stdout and stderr are discarded. -} +withQuietOutput + :: CreateProcessRunner + -> CreateProcess + -> IO () +withQuietOutput creator p = withFile "/dev/null" WriteMode $ \devnull -> do + let p' = p + { std_out = UseHandle devnull + , std_err = UseHandle devnull + } + creator p' $ const $ return () + +{- Extract a desired handle from createProcess's tuple. + - These partial functions are safe as long as createProcess is run + - with appropriate parameters to set up the desired handle. + - Get it wrong and the runtime crash will always happen, so should be + - easily noticed. -} +type HandleExtractor = (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> Handle +stdinHandle :: HandleExtractor +stdinHandle (Just h, _, _, _) = h +stdinHandle _ = error "expected stdinHandle" +stdoutHandle :: HandleExtractor +stdoutHandle (_, Just h, _, _) = h +stdoutHandle _ = error "expected stdoutHandle" +stderrHandle :: HandleExtractor +stderrHandle (_, _, Just h, _) = h +stderrHandle _ = error "expected stderrHandle" +bothHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle) +bothHandles (Just hin, Just hout, _, _) = (hin, hout) +bothHandles _ = error "expected bothHandles" + +{- Debugging trace for a CreateProcess. -} +debugProcess :: CreateProcess -> IO () +debugProcess p = do + debugM "Utility.Process" $ unwords + [ action ++ ":" + , showCmd p + ] + where + action + | piped (std_in p) && piped (std_out p) = "chat" + | piped (std_in p) = "feed" + | piped (std_out p) = "read" + | otherwise = "call" + piped Inherit = False + piped _ = True + +{- Shows the command that a CreateProcess will run. -} +showCmd :: CreateProcess -> String +showCmd = go . cmdspec + where + go (ShellCommand s) = s + go (RawCommand c ps) = c ++ " " ++ show ps + +{- Wrappers for System.Process functions that do debug logging. + - + - More could be added, but these are the only ones I usually need. + -} + +createProcess :: CreateProcess -> IO (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) +createProcess p = do + debugProcess p + System.Process.createProcess p + +runInteractiveProcess + :: FilePath + -> [String] + -> Maybe FilePath + -> Maybe [(String, String)] + -> IO (Handle, Handle, Handle, ProcessHandle) +runInteractiveProcess f args c e = do + debugProcess $ (proc f args) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = CreatePipe + , env = e + } + System.Process.runInteractiveProcess f args c e diff --git a/Utility/QuickCheck.hs b/Utility/QuickCheck.hs new file mode 100644 index 0000000000..078b10c8bc --- /dev/null +++ b/Utility/QuickCheck.hs @@ -0,0 +1,45 @@ +{- QuickCheck with additional instances + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE TypeSynonymInstances #-} + +module Utility.QuickCheck + ( module X + , module Utility.QuickCheck + ) where + +import Test.QuickCheck as X +import Data.Time.Clock.POSIX +import System.Posix.Types +import qualified Data.Map as M +import Control.Applicative + +instance (Arbitrary k, Arbitrary v, Eq k, Ord k) => Arbitrary (M.Map k v) where + arbitrary = M.fromList <$> arbitrary + +{- Times before the epoch are excluded. -} +instance Arbitrary POSIXTime where + arbitrary = nonNegative arbitrarySizedIntegral + +instance Arbitrary EpochTime where + arbitrary = nonNegative arbitrarySizedIntegral + +{- Pids are never negative, or 0. -} +instance Arbitrary ProcessID where + arbitrary = arbitrarySizedBoundedIntegral `suchThat` (> 0) + +{- Inodes are never negative. -} +instance Arbitrary FileID where + arbitrary = nonNegative arbitrarySizedIntegral + +{- File sizes are never negative. -} +instance Arbitrary FileOffset where + arbitrary = nonNegative arbitrarySizedIntegral + +nonNegative :: (Num a, Ord a) => Gen a -> Gen a +nonNegative g = g `suchThat` (>= 0) diff --git a/Utility/Rsync.hs b/Utility/Rsync.hs new file mode 100644 index 0000000000..e038242392 --- /dev/null +++ b/Utility/Rsync.hs @@ -0,0 +1,129 @@ +{- various rsync stuff + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Rsync where + +import Common + +import Data.Char + +{- Generates parameters to make rsync use a specified command as its remote + - shell. -} +rsyncShell :: [CommandParam] -> [CommandParam] +rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand command)] + where + {- rsync requires some weird, non-shell like quoting in + - here. A doubled single quote inside the single quoted + - string is a single quote. -} + escape s = "'" ++ join "''" (split "'" s) ++ "'" + +{- Runs rsync in server mode to send a file. -} +rsyncServerSend :: FilePath -> IO Bool +rsyncServerSend file = rsync $ + rsyncServerParams ++ [Param "--sender", File file] + +{- Runs rsync in server mode to receive a file. -} +rsyncServerReceive :: FilePath -> IO Bool +rsyncServerReceive file = rsync $ rsyncServerParams ++ [File file] + +rsyncServerParams :: [CommandParam] +rsyncServerParams = + [ Param "--server" + -- preserve timestamps + , Param "-t" + -- allow resuming of transfers of big files + , Param "--inplace" + -- other options rsync normally uses in server mode + , Params "-e.Lsf ." + ] + +rsync :: [CommandParam] -> IO Bool +rsync = boolSystem "rsync" + +{- Runs rsync, but intercepts its progress output and feeds bytes + - complete values into the callback. The progress output is also output + - to stdout. + - + - The params must enable rsync's --progress mode for this to work. + -} +rsyncProgress :: (Integer -> IO ()) -> [CommandParam] -> IO Bool +rsyncProgress callback params = do + r <- withHandle StdoutHandle createProcessSuccess p (feedprogress 0 []) + {- For an unknown reason, piping rsync's output like this does + - causes it to run a second ssh process, which it neglects to wait + - on. Reap the resulting zombie. -} + reapZombies + return r + where + p = proc "rsync" (toCommand params) + feedprogress prev buf h = do + s <- hGetSomeString h 80 + if null s + then return True + else do + putStr s + hFlush stdout + let (mbytes, buf') = parseRsyncProgress (buf++s) + case mbytes of + Nothing -> feedprogress prev buf' h + (Just bytes) -> do + when (bytes /= prev) $ + callback bytes + feedprogress bytes buf' h + +{- Checks if an rsync url involves the remote shell (ssh or rsh). + - Use of such urls with rsync requires additional shell + - escaping. -} +rsyncUrlIsShell :: String -> Bool +rsyncUrlIsShell s + | "rsync://" `isPrefixOf` s = False + | otherwise = go s + where + -- host::dir is rsync protocol, while host:dir is ssh/rsh + go [] = False + go (c:cs) + | c == '/' = False -- got to directory with no colon + | c == ':' = not $ ":" `isPrefixOf` cs + | otherwise = go cs + +{- Checks if a rsync url is really just a local path. -} +rsyncUrlIsPath :: String -> Bool +rsyncUrlIsPath s + | rsyncUrlIsShell s = False + | otherwise = ':' `notElem` s + +{- Parses the String looking for rsync progress output, and returns + - Maybe the number of bytes rsynced so far, and any any remainder of the + - string that could be an incomplete progress output. That remainder + - should be prepended to future output, and fed back in. This interface + - allows the output to be read in any desired size chunk, or even one + - character at a time. + - + - Strategy: Look for chunks prefixed with \r (rsync writes a \r before + - the first progress output, and each thereafter). The first number + - after the \r is the number of bytes processed. After the number, + - there must appear some whitespace, or we didn't get the whole number, + - and return the \r and part we did get, for later processing. + -} +parseRsyncProgress :: String -> (Maybe Integer, String) +parseRsyncProgress = go [] . reverse . progresschunks + where + go remainder [] = (Nothing, remainder) + go remainder (x:xs) = case parsebytes (findbytesstart x) of + Nothing -> go (delim:x++remainder) xs + Just b -> (Just b, remainder) + + delim = '\r' + {- Find chunks that each start with delim. + - The first chunk doesn't start with it + - (it's empty when delim is at the start of the string). -} + progresschunks = drop 1 . split [delim] + findbytesstart s = dropWhile isSpace s + parsebytes s = case break isSpace s of + ([], _) -> Nothing + (_, []) -> Nothing + (b, _) -> readish b diff --git a/Utility/SRV.hs b/Utility/SRV.hs new file mode 100644 index 0000000000..b39bf71b26 --- /dev/null +++ b/Utility/SRV.hs @@ -0,0 +1,107 @@ +{- SRV record lookup + - + - Uses either the ADNS Haskell library, or the standalone Haskell DNS + - package, or the host command. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Utility.SRV ( + mkSRVTcp, + mkSRV, + lookupSRV, + lookupSRVHost, +) where + +import Utility.Process +import Utility.Exception +import Utility.PartialPrelude + +import Network +import Data.Function +import Data.List +import Control.Applicative +import Data.Maybe + +#ifdef WITH_ADNS +import ADNS.Resolver +import Data.Either +#else +#ifdef WITH_DNS +import qualified Network.DNS.Lookup as DNS +import Network.DNS.Resolver +import qualified Data.ByteString.UTF8 as B8 +#endif +#endif + +newtype SRV = SRV String + deriving (Show, Eq) + +type HostPort = (HostName, PortID) + +type PriorityWeight = (Int, Int) -- sort by priority first, then weight + +mkSRV :: String -> String -> HostName -> SRV +mkSRV transport protocol host = SRV $ concat + ["_", protocol, "._", transport, ".", host] + +mkSRVTcp :: String -> HostName -> SRV +mkSRVTcp = mkSRV "tcp" + +{- Returns an ordered list, with highest priority hosts first. + - + - On error, returns an empty list. -} +lookupSRV :: SRV -> IO [HostPort] +#ifdef WITH_ADNS +lookupSRV (SRV srv) = initResolver [] $ \resolver -> do + r <- catchDefaultIO (Right []) $ + resolveSRV resolver srv + return $ either (\_ -> []) id r +#else +#ifdef WITH_DNS +lookupSRV (SRV srv) = do + seed <- makeResolvSeed defaultResolvConf + print srv + r <- withResolver seed $ flip DNS.lookupSRV $ B8.fromString srv + print r + return $ maybe [] (orderHosts . map tohosts) r + where + tohosts (priority, weight, port, hostname) = + ( (priority, weight) + , (B8.toString hostname, PortNumber $ fromIntegral port) + ) +#else +lookupSRV = lookupSRVHost +#endif +#endif + +lookupSRVHost :: SRV -> IO [HostPort] +lookupSRVHost (SRV srv) = catchDefaultIO [] $ + parseSrvHost <$> readProcessEnv "host" ["-t", "SRV", "--", srv] + -- clear environment, to avoid LANG affecting output + (Just []) + +parseSrvHost :: String -> [HostPort] +parseSrvHost = orderHosts . catMaybes . map parse . lines + where + parse l = case words l of + [_, _, _, _, spriority, sweight, sport, hostname] -> do + let v = + ( readish sport :: Maybe Int + , readish spriority :: Maybe Int + , readish sweight :: Maybe Int + ) + case v of + (Just port, Just priority, Just weight) -> Just + ( (priority, weight) + , (hostname, PortNumber $ fromIntegral port) + ) + _ -> Nothing + _ -> Nothing + +orderHosts :: [(PriorityWeight, HostPort)] -> [HostPort] +orderHosts = map snd . sortBy (compare `on` fst) diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs new file mode 100644 index 0000000000..e6075e888d --- /dev/null +++ b/Utility/SafeCommand.hs @@ -0,0 +1,104 @@ +{- safely running shell commands + - + - Copyright 2010-2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.SafeCommand where + +import System.Exit +import Utility.Process +import System.Process (env) +import Data.String.Utils +import Control.Applicative + +{- A type for parameters passed to a shell command. A command can + - be passed either some Params (multiple parameters can be included, + - whitespace-separated, or a single Param (for when parameters contain + - whitespace), or a File. + -} +data CommandParam = Params String | Param String | File FilePath + deriving (Eq, Show, Ord) + +{- Used to pass a list of CommandParams to a function that runs + - a command and expects Strings. -} +toCommand :: [CommandParam] -> [String] +toCommand = (>>= unwrap) + where + unwrap (Param s) = [s] + unwrap (Params s) = filter (not . null) (split " " s) + -- Files that start with a dash are modified to avoid + -- the command interpreting them as options. + unwrap (File s@('-':_)) = ["./" ++ s] + unwrap (File s) = [s] + +{- Run a system command, and returns True or False + - if it succeeded or failed. + -} +boolSystem :: FilePath -> [CommandParam] -> IO Bool +boolSystem command params = boolSystemEnv command params Nothing + +boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool +boolSystemEnv command params environ = dispatch <$> safeSystemEnv command params environ + where + dispatch ExitSuccess = True + dispatch _ = False + +{- Runs a system command, returning the exit status. -} +safeSystem :: FilePath -> [CommandParam] -> IO ExitCode +safeSystem command params = safeSystemEnv command params Nothing + +safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode +safeSystemEnv command params environ = do + (_, _, _, pid) <- createProcess (proc command $ toCommand params) + { env = environ } + waitForProcess pid + +{- Escapes a filename or other parameter to be safely able to be exposed to + - the shell. -} +shellEscape :: String -> String +shellEscape f = "'" ++ escaped ++ "'" + where + -- replace ' with '"'"' + escaped = join "'\"'\"'" $ split "'" f + +{- Unescapes a set of shellEscaped words or filenames. -} +shellUnEscape :: String -> [String] +shellUnEscape [] = [] +shellUnEscape s = word : shellUnEscape rest + where + (word, rest) = findword "" s + findword w [] = (w, "") + findword w (c:cs) + | c == ' ' = (w, cs) + | c == '\'' = inquote c w cs + | c == '"' = inquote c w cs + | otherwise = findword (w++[c]) cs + inquote _ w [] = (w, "") + inquote q w (c:cs) + | c == q = findword w cs + | otherwise = inquote q (w++[c]) cs + +{- For quickcheck. -} +prop_idempotent_shellEscape :: String -> Bool +prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s +prop_idempotent_shellEscape_multiword :: [String] -> Bool +prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s + +{- Segements a list of filenames into groups that are all below the manximum + - command-line length limit. Does not preserve order. -} +segmentXargs :: [FilePath] -> [[FilePath]] +segmentXargs l = go l [] 0 [] + where + go [] c _ r = c:r + go (f:fs) c accumlen r + | len < maxlen && newlen > maxlen = go (f:fs) [] 0 (c:r) + | otherwise = go fs (f:c) newlen r + where + len = length f + newlen = accumlen + len + + {- 10k of filenames per command, well under Linux's 20k limit; + - allows room for other parameters etc. -} + maxlen = 10240 diff --git a/Utility/Shell.hs b/Utility/Shell.hs new file mode 100644 index 0000000000..f3858af7f5 --- /dev/null +++ b/Utility/Shell.hs @@ -0,0 +1,20 @@ +{- /bin/sh handling + - + - Copyright 2013 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Utility.Shell where + +shellPath :: FilePath +#ifndef __ANDROID__ +shellPath = "/bin/sh" +#else +shellPath = "/system/bin/sh" +#endif + +shebang :: String +shebang = "#!" ++ shellPath diff --git a/Utility/State.hs b/Utility/State.hs new file mode 100644 index 0000000000..7f89190823 --- /dev/null +++ b/Utility/State.hs @@ -0,0 +1,28 @@ +{- state monad support + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE PackageImports #-} + +module Utility.State where + +import "mtl" Control.Monad.State.Strict + +{- Modifies Control.Monad.State's state, forcing a strict update. + - This avoids building thunks in the state and leaking. + - Why it's not the default, I don't know. + - + - Example: changeState $ \s -> s { foo = bar } + -} +changeState :: MonadState s m => (s -> s) -> m () +changeState f = do + x <- get + put $! f x + +{- Gets a value from the internal state, selected by the passed value + - constructor. -} +getState :: MonadState s m => (s -> a) -> m a +getState = gets diff --git a/Utility/TSet.hs b/Utility/TSet.hs new file mode 100644 index 0000000000..bb711a4fba --- /dev/null +++ b/Utility/TSet.hs @@ -0,0 +1,39 @@ +{- Transactional sets + - + - Copyright 2012 Joey Hess + -} + +module Utility.TSet where + +import Common + +import Control.Concurrent.STM + +type TSet = TChan + +runTSet :: STM a -> IO a +runTSet = atomically + +newTSet :: IO (TSet a) +newTSet = atomically newTChan + +{- Gets the contents of the TSet. Blocks until at least one item is + - present. -} +getTSet :: TSet a -> IO [a] +getTSet tset = runTSet $ do + c <- readTChan tset + go [c] + where + go l = do + v <- tryReadTChan tset + case v of + Nothing -> return l + Just c -> go (c:l) + +{- Puts items into a TSet. -} +putTSet :: TSet a -> [a] -> IO () +putTSet tset vs = runTSet $ mapM_ (writeTChan tset) vs + +{- Put a single item into a TSet. -} +putTSet1 :: TSet a -> a -> IO () +putTSet1 tset v = void $ runTSet $ writeTChan tset v diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs new file mode 100644 index 0000000000..6dbea693ab --- /dev/null +++ b/Utility/TempFile.hs @@ -0,0 +1,58 @@ +{- temp file functions + - + - Copyright 2010-2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.TempFile where + +import Control.Exception (bracket) +import System.IO +import System.Posix.Process +import System.Directory + +import Utility.Exception +import Utility.Path +import System.FilePath + +{- Runs an action like writeFile, writing to a temp file first and + - then moving it into place. The temp file is stored in the same + - directory as the final file to avoid cross-device renames. -} +viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO () +viaTmp a file content = do + pid <- getProcessID + let tmpfile = file ++ ".tmp" ++ show pid + createDirectoryIfMissing True (parentDir file) + a tmpfile content + renameFile tmpfile file + +type Template = String + +{- Runs an action with a temp file, then removes the file. -} +withTempFile :: Template -> (FilePath -> Handle -> IO a) -> IO a +withTempFile template a = bracket create remove use + where + create = do + tmpdir <- catchDefaultIO "." getTemporaryDirectory + openTempFile tmpdir template + remove (name, handle) = do + hClose handle + catchBoolIO (removeFile name >> return True) + use (name, handle) = a name handle + +{- Runs an action with a temp directory, then removes the directory and + - all its contents. -} +withTempDir :: Template -> (FilePath -> IO a) -> IO a +withTempDir template = bracket create remove + where + remove = removeDirectoryRecursive + create = do + tmpdir <- catchDefaultIO "." getTemporaryDirectory + createDirectoryIfMissing True tmpdir + pid <- getProcessID + makedir tmpdir (template ++ show pid) (0 :: Int) + makedir tmpdir t n = do + let dir = tmpdir t ++ "." ++ show n + r <- tryIO $ createDirectory dir + either (const $ makedir tmpdir t $ n + 1) (const $ return dir) r diff --git a/Utility/Tense.hs b/Utility/Tense.hs new file mode 100644 index 0000000000..60b3fa513d --- /dev/null +++ b/Utility/Tense.hs @@ -0,0 +1,57 @@ +{- Past and present tense text. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings #-} + +module Utility.Tense where + +import qualified Data.Text as T +import Data.Text (Text) +import GHC.Exts( IsString(..) ) + +data Tense = Present | Past + deriving (Eq) + +data TenseChunk = Tensed Text Text | UnTensed Text + deriving (Eq, Ord, Show) + +newtype TenseText = TenseText [TenseChunk] + deriving (Eq, Ord) + +{- Allows OverloadedStrings to be used, to build UnTensed chunks. -} +instance IsString TenseChunk where + fromString = UnTensed . T.pack + +{- Allows OverloadedStrings to be used, to provide UnTensed TenseText. -} +instance IsString TenseText where + fromString s = TenseText [fromString s] + +renderTense :: Tense -> TenseText -> Text +renderTense tense (TenseText chunks) = T.concat $ map render chunks + where + render (Tensed present past) + | tense == Present = present + | otherwise = past + render (UnTensed s) = s + +{- Builds up a TenseText, separating chunks with spaces. + - + - However, rather than just intersperse new chunks for the spaces, + - the spaces are appended to the end of the chunks. + -} +tenseWords :: [TenseChunk] -> TenseText +tenseWords = TenseText . go [] + where + go c [] = reverse c + go c (w:[]) = reverse (w:c) + go c ((UnTensed w):ws) = go (UnTensed (addspace w) : c) ws + go c ((Tensed w1 w2):ws) = + go (Tensed (addspace w1) (addspace w2) : c) ws + addspace w = T.append w " " + +unTensed :: Text -> TenseText +unTensed t = TenseText [UnTensed t] diff --git a/Utility/ThreadLock.hs b/Utility/ThreadLock.hs new file mode 100644 index 0000000000..c029a2b0c8 --- /dev/null +++ b/Utility/ThreadLock.hs @@ -0,0 +1,19 @@ +{- locking between threads + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.ThreadLock where + +import Control.Concurrent.MVar + +type Lock = MVar () + +newLock :: IO Lock +newLock = newMVar () + +{- Runs an action with a lock held, so only one thread at a time can run it. -} +withLock :: Lock -> IO a -> IO a +withLock lock = withMVar lock . const diff --git a/Utility/ThreadScheduler.hs b/Utility/ThreadScheduler.hs new file mode 100644 index 0000000000..a32606cfdd --- /dev/null +++ b/Utility/ThreadScheduler.hs @@ -0,0 +1,63 @@ +{- thread scheduling + - + - Copyright 2012 Joey Hess + - Copyright 2011 Bas van Dijk & Roel van Dijk + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Utility.ThreadScheduler where + +import Common + +import Control.Concurrent +import System.Posix.Signals +#ifndef __ANDROID__ +import System.Posix.Terminal +#endif + +newtype Seconds = Seconds { fromSeconds :: Int } + deriving (Eq, Ord, Show) + +{- Runs an action repeatedly forever, sleeping at least the specified number + - of seconds in between. -} +runEvery :: Seconds -> IO a -> IO a +runEvery n a = forever $ do + threadDelaySeconds n + a + +threadDelaySeconds :: Seconds -> IO () +threadDelaySeconds (Seconds n) = unboundDelay (fromIntegral n * oneSecond) + where + oneSecond = 1000000 -- microseconds + +{- Like threadDelay, but not bounded by an Int. + - + - There is no guarantee that the thread will be rescheduled promptly when the + - delay has expired, but the thread will never continue to run earlier than + - specified. + - + - Taken from the unbounded-delay package to avoid a dependency for 4 lines + - of code. + -} +unboundDelay :: Integer -> IO () +unboundDelay time = do + let maxWait = min time $ toInteger (maxBound :: Int) + threadDelay $ fromInteger maxWait + when (maxWait /= time) $ unboundDelay (time - maxWait) + +{- Pauses the main thread, letting children run until program termination. -} +waitForTermination :: IO () +waitForTermination = do + lock <- newEmptyMVar + check softwareTermination lock +#ifndef __ANDROID__ + whenM (queryTerminal stdInput) $ + check keyboardSignal lock +#endif + takeMVar lock + where + check sig lock = void $ + installHandler sig (CatchOnce $ putMVar lock ()) Nothing diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc new file mode 100644 index 0000000000..53dd719fb4 --- /dev/null +++ b/Utility/Touch.hsc @@ -0,0 +1,120 @@ +{- More control over touching a file. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE ForeignFunctionInterface #-} + +module Utility.Touch ( + TimeSpec(..), + touchBoth, + touch +) where + +import Utility.FileSystemEncoding + +import Foreign +import Foreign.C +import Control.Monad (when) + +newtype TimeSpec = TimeSpec CTime + +{- Changes the access and modification times of an existing file. + Can follow symlinks, or not. Throws IO error on failure. -} +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () + +touch :: FilePath -> TimeSpec -> Bool -> IO () +touch file mtime = touchBoth file mtime mtime + +#include +#include +#include +#include + +#ifndef _BSD_SOURCE +#define _BSD_SOURCE +#endif + +#if (defined UTIME_OMIT && defined UTIME_NOW && defined AT_FDCWD && defined AT_SYMLINK_NOFOLLOW) + +at_fdcwd :: CInt +at_fdcwd = #const AT_FDCWD + +at_symlink_nofollow :: CInt +at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW + +instance Storable TimeSpec where + -- use the larger alignment of the two types in the struct + alignment _ = max sec_alignment nsec_alignment + where + sec_alignment = alignment (undefined::CTime) + nsec_alignment = alignment (undefined::CLong) + sizeOf _ = #{size struct timespec} + peek ptr = do + sec <- #{peek struct timespec, tv_sec} ptr + return $ TimeSpec sec + poke ptr (TimeSpec sec) = do + #{poke struct timespec, tv_sec} ptr sec + #{poke struct timespec, tv_nsec} ptr (0 :: CLong) + +{- While its interface is beastly, utimensat is in recent + POSIX standards, unlike lutimes. -} +foreign import ccall "utimensat" + c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt + +touchBoth file atime mtime follow = + allocaArray 2 $ \ptr -> + withFilePath file $ \f -> do + pokeArray ptr [atime, mtime] + r <- c_utimensat at_fdcwd f ptr flags + when (r /= 0) $ throwErrno "touchBoth" + where + flags + | follow = 0 + | otherwise = at_symlink_nofollow + +#else +#if 0 +{- Using lutimes is needed for BSD. + - + - TODO: test if lutimes is available. May have to do it in configure. + - TODO: TimeSpec uses a CTime, while tv_sec is a CLong. It is implementation + - dependent whether these are the same; need to find a cast that works. + - (Without the cast it works on linux i386, but + - maybe not elsewhere.) + -} + +instance Storable TimeSpec where + alignment _ = alignment (undefined::CLong) + sizeOf _ = #{size struct timeval} + peek ptr = do + sec <- #{peek struct timeval, tv_sec} ptr + return $ TimeSpec sec + poke ptr (TimeSpec sec) = do + #{poke struct timeval, tv_sec} ptr sec + #{poke struct timeval, tv_usec} ptr (0 :: CLong) + +foreign import ccall "utimes" + c_utimes :: CString -> Ptr TimeSpec -> IO CInt +foreign import ccall "lutimes" + c_lutimes :: CString -> Ptr TimeSpec -> IO CInt + +touchBoth file atime mtime follow = + allocaArray 2 $ \ptr -> + withFilePath file $ \f -> do + pokeArray ptr [atime, mtime] + r <- syscall f ptr + when (r /= 0) $ + throwErrno "touchBoth" + where + syscall + | follow = c_lutimes + | otherwise = c_utimes + +#else +#warning "utimensat and lutimes not available; building without symlink timestamp preservation support" +touchBoth _ _ _ _ = return () +#endif +#endif diff --git a/Utility/Types/DirWatcher.hs b/Utility/Types/DirWatcher.hs new file mode 100644 index 0000000000..30ada9c68c --- /dev/null +++ b/Utility/Types/DirWatcher.hs @@ -0,0 +1,26 @@ +{- generic directory watching types + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Utility.Types.DirWatcher where + +import Common + +type Hook a = Maybe (a -> Maybe FileStatus -> IO ()) + +data WatchHooks = WatchHooks + { addHook :: Hook FilePath + , addSymlinkHook :: Hook FilePath + , delHook :: Hook FilePath + , delDirHook :: Hook FilePath + , errHook :: Hook String -- error message + , modifyHook :: Hook FilePath + } + +mkWatchHooks :: WatchHooks +mkWatchHooks = WatchHooks Nothing Nothing Nothing Nothing Nothing Nothing diff --git a/Utility/Url.hs b/Utility/Url.hs new file mode 100644 index 0000000000..da7863091d --- /dev/null +++ b/Utility/Url.hs @@ -0,0 +1,95 @@ +{- Url downloading. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Utility.Url ( + URLString, + check, + exists, + download, + get +) where + +import Common +import Network.URI + +type URLString = String + +type Headers = [String] + +{- Checks that an url exists and could be successfully downloaded, + - also checking that its size, if available, matches a specified size. -} +check :: URLString -> Headers -> Maybe Integer -> IO Bool +check url headers expected_size = handle <$> exists url headers + where + handle (False, _) = False + handle (True, Nothing) = True + handle (True, s) = expected_size == s + +{- Checks that an url exists and could be successfully downloaded, + - also returning its size if available. -} +exists :: URLString -> Headers -> IO (Bool, Maybe Integer) +exists url headers = case parseURI url of + Just u + | uriScheme u == "file:" -> do + s <- catchMaybeIO $ getFileStatus (uriPath u) + case s of + Just stat -> return (True, Just $ fromIntegral $ fileSize stat) + Nothing -> dne + | otherwise -> do + output <- readProcess "curl" curlparams + case lastMaybe (lines output) of + Just ('2':_:_) -> return (True, extractsize output) + _ -> dne + Nothing -> dne + where + dne = return (False, Nothing) + + curlparams = + [ "-s" + , "--head" + , "-L" + , url + , "-w", "%{http_code}" + ] ++ concatMap (\h -> ["-H", h]) headers + + extractsize s = case lastMaybe $ filter ("Content-Length:" `isPrefixOf`) (lines s) of + Just l -> case lastMaybe $ words l of + Just sz -> readish sz + _ -> Nothing + _ -> Nothing + +{- Used to download large files, such as the contents of keys. + - + - Uses wget or curl program for its progress bar. (Wget has a better one, + - so is preferred.) Which program to use is determined at run time; it + - would not be appropriate to test at configure time and build support + - for only one in. + - + - Curl is always used for file:// urls, as wget does not support them. + -} +download :: URLString -> Headers -> [CommandParam] -> FilePath -> IO Bool +download url headers options file + | "file://" `isPrefixOf` url = curl + | otherwise = ifM (inPath "wget") (wget , curl) + where + headerparams = map (\h -> Param $ "--header=" ++ h) headers + wget = go "wget" $ headerparams ++ [Params "-c -O"] + {- Uses the -# progress display, because the normal + - one is very confusing when resuming, showing + - the remainder to download as the whole file, + - and not indicating how much percent was + - downloaded before the resume. -} + curl = go "curl" $ headerparams ++ [Params "-L -C - -# -o"] + go cmd opts = boolSystem cmd $ + options++opts++[File file, File url] + +{- Downloads a small file. -} +get :: URLString -> Headers -> IO String +get url headers = readProcess "curl" $ + ["-s", "-L", url] ++ concatMap (\h -> ["-H", h]) headers diff --git a/Utility/UserInfo.hs b/Utility/UserInfo.hs new file mode 100644 index 0000000000..916ebb1918 --- /dev/null +++ b/Utility/UserInfo.hs @@ -0,0 +1,42 @@ +{- user info + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Utility.UserInfo ( + myHomeDir, + myUserName, + myUserGecos, +) where + +import Control.Applicative +import System.Posix.User +import System.Posix.Env + +{- Current user's home directory. + - + - getpwent will fail on LDAP or NIS, so use HOME if set. -} +myHomeDir :: IO FilePath +myHomeDir = myVal ["HOME"] homeDirectory + +{- Current user's user name. -} +myUserName :: IO String +myUserName = myVal ["USER", "LOGNAME"] userName + +myUserGecos :: IO String +#ifdef __ANDROID__ +myUserGecos = return "" -- userGecos crashes on Android +#else +myUserGecos = myVal [] userGecos +#endif + +myVal :: [String] -> (UserEntry -> String) -> IO String +myVal envvars extract = maybe (extract <$> getpwent) return =<< check envvars + where + check [] = return Nothing + check (v:vs) = maybe (check vs) (return . Just) =<< getEnv v + getpwent = getUserEntryForID =<< getEffectiveUserID diff --git a/Utility/Verifiable.hs b/Utility/Verifiable.hs new file mode 100644 index 0000000000..4f88cb9f29 --- /dev/null +++ b/Utility/Verifiable.hs @@ -0,0 +1,37 @@ +{- values verified using a shared secret + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Verifiable where + +import Data.Digest.Pure.SHA +import Data.ByteString.Lazy.UTF8 (fromString) +import qualified Data.ByteString.Lazy as L + +type Secret = L.ByteString +type HMACDigest = String + +{- A value, verifiable using a HMAC digest and a secret. -} +data Verifiable a = Verifiable + { verifiableVal :: a + , verifiableDigest :: HMACDigest + } + deriving (Eq, Read, Show) + +mkVerifiable :: Show a => a -> Secret -> Verifiable a +mkVerifiable a secret = Verifiable a (calcDigest (show a) secret) + +verify :: (Eq a, Show a) => Verifiable a -> Secret -> Bool +verify v secret = v == mkVerifiable (verifiableVal v) secret + +calcDigest :: String -> Secret -> HMACDigest +calcDigest v secret = showDigest $ hmacSha1 secret $ fromString v + +{- for quickcheck -} +prop_verifiable_sane :: String -> String -> Bool +prop_verifiable_sane a s = verify (mkVerifiable a secret) secret + where + secret = fromString s diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs new file mode 100644 index 0000000000..c6aae9db5d --- /dev/null +++ b/Utility/WebApp.hs @@ -0,0 +1,208 @@ +{- Yesod webapp + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE OverloadedStrings, CPP, RankNTypes #-} + +module Utility.WebApp where + +import Common +import Utility.TempFile +import Utility.FileMode + +import qualified Yesod +import qualified Network.Wai as Wai +import Network.Wai.Handler.Warp +import Network.Wai.Logger +import Control.Monad.IO.Class +import Network.HTTP.Types +import System.Log.Logger +import Data.ByteString.Lazy.UTF8 +import qualified Data.CaseInsensitive as CI +import Network.Socket +import Control.Exception +import Crypto.Random +import Data.Digest.Pure.SHA +import qualified Web.ClientSession as CS +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as B +import qualified Data.Text as T +import qualified Data.Text.Encoding as TE +import Blaze.ByteString.Builder.Char.Utf8 (fromText) +import Blaze.ByteString.Builder (Builder) +import Data.Monoid +import Control.Arrow ((***)) +import Control.Concurrent + +localhost :: String +localhost = "localhost" + +{- Command to use to run a web browser. -} +browserCommand :: FilePath +#ifdef darwin_HOST_OS +browserCommand = "open" +#else +browserCommand = "xdg-open" +#endif + +{- Binds to a socket on localhost, and runs a webapp on it. + - + - An IO action can also be run, to do something with the address, + - such as start a web browser to view the webapp. + -} +runWebApp :: Wai.Application -> (SockAddr -> IO ()) -> IO () +runWebApp app observer = do + sock <- localSocket + void $ forkIO $ runSettingsSocket defaultSettings sock app + observer =<< getSocketName sock + +{- Binds to a local socket, selecting any free port. + - + - Prefers to bind to the ipv4 address rather than the ipv6 address + - of localhost, if it's available. + - + - As a (very weak) form of security, only connections from + - localhost are accepted. -} +localSocket :: IO Socket +localSocket = do + addrs <- getAddrInfo (Just hints) (Just localhost) Nothing + case (partition (\a -> addrFamily a == AF_INET) addrs) of + (v4addr:_, _) -> go v4addr + (_, v6addr:_) -> go v6addr + _ -> error "unable to bind to a local socket" + where + hints = defaultHints + { addrFlags = [AI_ADDRCONFIG] + , addrSocketType = Stream + } + {- Repeated attempts because bind sometimes fails for an + - unknown reason on OSX. -} + go addr = go' 100 addr + go' :: Int -> AddrInfo -> IO Socket + go' 0 _ = error "unable to bind to local socket" + go' n addr = do + r <- tryIO $ bracketOnError (open addr) sClose (use addr) + either (const $ go' (pred n) addr) return r + open addr = socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr) + use addr sock = do + setSocketOption sock ReuseAddr 1 + bindSocket sock (addrAddress addr) + listen sock maxListenQueue + return sock + +{- Checks if debugging is actually enabled. -} +debugEnabled :: IO Bool +debugEnabled = do + l <- getRootLogger + return $ getLevel l <= Just DEBUG + +{- WAI middleware that logs using System.Log.Logger at debug level. + - + - Recommend only inserting this middleware when debugging is actually + - enabled, as it's not optimised at all. + -} +httpDebugLogger :: Wai.Middleware +httpDebugLogger waiApp req = do + logRequest req + waiApp req + +logRequest :: MonadIO m => Wai.Request -> m () +logRequest req = do + liftIO $ debugM "WebApp" $ unwords + [ showSockAddr $ Wai.remoteHost req + , frombs $ Wai.requestMethod req + , frombs $ Wai.rawPathInfo req + --, show $ Wai.httpVersion req + --, frombs $ lookupRequestField "referer" req + , frombs $ lookupRequestField "user-agent" req + ] + where + frombs v = toString $ L.fromChunks [v] + +lookupRequestField :: CI.CI B.ByteString -> Wai.Request -> B.ByteString +lookupRequestField k req = fromMaybe "" . lookup k $ Wai.requestHeaders req + +{- Rather than storing a session key on disk, use a random key + - that will only be valid for this run of the webapp. -} +webAppSessionBackend :: Yesod.Yesod y => y -> IO (Maybe (Yesod.SessionBackend y)) +webAppSessionBackend _ = do + g <- newGenIO :: IO SystemRandom + case genBytes 96 g of + Left e -> error $ "failed to generate random key: " ++ show e + Right (s, _) -> case CS.initKey s of + Left e -> error $ "failed to initialize key: " ++ show e + Right key -> return $ Just $ + Yesod.clientSessionBackend key 120 + +{- Generates a random sha512 string, suitable to be used for an + - authentication secret. -} +genRandomToken :: IO String +genRandomToken = do + g <- newGenIO :: IO SystemRandom + return $ + case genBytes 512 g of + Left e -> error $ "failed to generate secret token: " ++ show e + Right (s, _) -> showDigest $ sha512 $ L.fromChunks [s] + +{- A Yesod isAuthorized method, which checks the auth cgi parameter + - against a token extracted from the Yesod application. + - + - Note that the usual Yesod error page is bypassed on error, to avoid + - possibly leaking the auth token in urls on that page! + -} +checkAuthToken :: forall t sub. (t -> T.Text) -> Yesod.GHandler sub t Yesod.AuthResult +checkAuthToken extractToken = do + webapp <- Yesod.getYesod + req <- Yesod.getRequest + let params = Yesod.reqGetParams req + if lookup "auth" params == Just (extractToken webapp) + then return Yesod.Authorized + else Yesod.sendResponseStatus unauthorized401 () + +{- A Yesod joinPath method, which adds an auth cgi parameter to every + - url matching a predicate, containing a token extracted from the + - Yesod application. + - + - A typical predicate would exclude files under /static. + -} +insertAuthToken :: forall y. (y -> T.Text) + -> ([T.Text] -> Bool) + -> y + -> T.Text + -> [T.Text] + -> [(T.Text, T.Text)] + -> Builder +insertAuthToken extractToken predicate webapp root pathbits params = + fromText root `mappend` encodePath pathbits' encodedparams + where + pathbits' = if null pathbits then [T.empty] else pathbits + encodedparams = map (TE.encodeUtf8 *** go) params' + go "" = Nothing + go x = Just $ TE.encodeUtf8 x + authparam = (T.pack "auth", extractToken webapp) + params' + | predicate pathbits = authparam:params + | otherwise = params + +{- Creates a html shim file that's used to redirect into the webapp, + - to avoid exposing the secret token when launching the web browser. -} +writeHtmlShim :: String -> String -> FilePath -> IO () +writeHtmlShim title url file = viaTmp writeFileProtected file $ genHtmlShim title url + +{- TODO: generate this static file using Yesod. -} +genHtmlShim :: String -> String -> String +genHtmlShim title url = unlines + [ "" + , "" + , ""++ title ++ "" + , "" + , "" + , "

" + , "" ++ title ++ "" + , "

" + , "" + , "" + ] diff --git a/Utility/Yesod.hs b/Utility/Yesod.hs new file mode 100644 index 0000000000..93000587cc --- /dev/null +++ b/Utility/Yesod.hs @@ -0,0 +1,37 @@ +{- Yesod stuff, that's typically found in the scaffolded site. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +#if defined VERSION_yesod_default +#if ! MIN_VERSION_yesod_default(1,1,0) +#define WITH_OLD_YESOD +#endif +#endif + +module Utility.Yesod where + +import Yesod.Default.Util +import Language.Haskell.TH.Syntax +#ifndef WITH_OLD_YESOD +import Data.Default (def) +import Text.Hamlet +#endif + +widgetFile :: String -> Q Exp +#ifdef WITH_OLD_YESOD +widgetFile = widgetFileNoReload +#else +widgetFile = widgetFileNoReload $ def + { wfsHamletSettings = defaultHamletSettings + { hamletNewlines = AlwaysNewlines + } + } +#endif + +hamletTemplate :: FilePath -> FilePath +hamletTemplate f = globFile "hamlet" f diff --git a/Utility/libdiskfree.c b/Utility/libdiskfree.c new file mode 100644 index 0000000000..d2843ed203 --- /dev/null +++ b/Utility/libdiskfree.c @@ -0,0 +1,73 @@ +/* disk free space checking, C mini-library + * + * Copyright 2012 Joey Hess + * + * Licensed under the GNU GPL version 3 or higher. + */ + +/* Include appropriate headers for the OS, and define what will be used to + * check the free space. */ +#if defined(__APPLE__) +# include +# include +/* In newer OSX versions, statfs64 is deprecated, in favor of statfs, + * which is 64 bit only with a build option -- but statfs64 still works, + * and this keeps older OSX also supported. */ +# define STATCALL statfs64 +# define STATSTRUCT statfs64 +#else +#if defined (__FreeBSD__) +# include +# include +# define STATCALL statfs /* statfs64 not yet tested on a real FreeBSD machine */ +# define STATSTRUCT statfs +#else +#if defined __ANDROID__ +# warning free space checking code not available for Android +# define UNKNOWN +#else +#if defined (__linux__) || defined (__FreeBSD_kernel__) +/* Linux or Debian kFreeBSD */ +/* This is a POSIX standard, so might also work elsewhere too. */ +# include +# define STATCALL statvfs +# define STATSTRUCT statvfs +#else +# warning free space checking code not available for this OS +# define UNKNOWN +#endif +#endif +#endif +#endif + +#include +#include + +/* Checks the amount of disk that is available to regular (non-root) users. + * (If there's an error, or this is not supported, + * returns 0 and sets errno to nonzero.) + */ +unsigned long long int diskfree(const char *path) { +#ifdef UNKNOWN + errno = 1; + return 0; +#else + unsigned long long int available, blocksize; + struct STATSTRUCT buf; + + if (STATCALL(path, &buf) != 0) + return 0; /* errno is set */ + else + errno = 0; + + available = buf.f_bavail; + blocksize = buf.f_bsize; + return available * blocksize; +#endif +} + +/* +main () { + printf("%lli\n", diskfree(".")); +} +*/ diff --git a/Utility/libdiskfree.h b/Utility/libdiskfree.h new file mode 100644 index 0000000000..e5b84754fe --- /dev/null +++ b/Utility/libdiskfree.h @@ -0,0 +1 @@ +unsigned long long int diskfree(const char *path); diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c new file mode 100644 index 0000000000..b5a19a1350 --- /dev/null +++ b/Utility/libkqueue.c @@ -0,0 +1,73 @@ +/* kqueue interface, C mini-library + * + * Copyright 2012 Joey Hess + * + * Licensed under the GNU GPL version 3 or higher. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* The specified fds are added to the set of fds being watched for changes. + * Fds passed to prior calls still take effect, so it's most efficient to + * not pass the same fds repeatedly. + * + * Returns the fd that changed, or -1 on error. + */ +signed int helper(const int kq, const int fdcnt, const int *fdlist, int nodelay) { + int i, nev; + struct kevent evlist[1]; + struct kevent chlist[fdcnt]; + struct timespec avoiddelay = {0, 0}; + struct timespec *timeout = nodelay ? &avoiddelay : NULL; + + for (i = 0; i < fdcnt; i++) { + EV_SET(&chlist[i], fdlist[i], EVFILT_VNODE, + EV_ADD | EV_ENABLE | EV_CLEAR, + NOTE_WRITE, + 0, 0); + } + + nev = kevent(kq, chlist, fdcnt, evlist, 1, timeout); + if (nev == 1) + return evlist[0].ident; + else + return -1; +} + +/* Initializes a new, empty kqueue. */ +int init_kqueue() { + int kq; + if ((kq = kqueue()) == -1) { + perror("kqueue"); + exit(1); + } + return kq; +} + +/* Adds fds to the set that should be watched. */ +void addfds_kqueue(const int kq, const int fdcnt, const int *fdlist) { + helper(kq, fdcnt, fdlist, 1); +} + +/* Waits for a change event on a kqueue. */ +signed int waitchange_kqueue(const int kq) { + return helper(kq, 0, NULL, 0); +} + +/* +main () { + int list[1]; + int kq; + list[0]=open(".", O_RDONLY); + kq = init_kqueue(); + addfds_kqueue(kq, 1, list) + printf("change: %i\n", waitchange_kqueue(kq)); +} +*/ diff --git a/Utility/libkqueue.h b/Utility/libkqueue.h new file mode 100644 index 0000000000..692b47f14e --- /dev/null +++ b/Utility/libkqueue.h @@ -0,0 +1,3 @@ +int init_kqueue(); +void addfds_kqueue(const int kq, const int fdcnt, const int *fdlist); +signed int waitchange_kqueue(const int kq); diff --git a/Utility/libmounts.c b/Utility/libmounts.c new file mode 100644 index 0000000000..8669f33ea9 --- /dev/null +++ b/Utility/libmounts.c @@ -0,0 +1,103 @@ +/* mounted filesystems, C mini-library + * + * Copyright (c) 1980, 1989, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * Copyright (c) 2001 + * David Rufino + * Copyright 2012 + * Joey Hess + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "libmounts.h" + +#ifdef GETMNTENT +/* direct passthrough the getmntent */ +FILE *mounts_start (void) { + return setmntent("/etc/mtab", "r"); +} +int mounts_end (FILE *fp) { + return endmntent(fp); +} +struct mntent *mounts_next (FILE *fp) { + return getmntent(fp); +} +#endif + +#ifdef GETMNTINFO +/* getmntent emulation using getmntinfo */ +FILE *mounts_start (void) { + return ((FILE *)0x1); /* dummy non-NULL FILE pointer, not used */ +} +int mounts_end (FILE *fp) { + return 1; +} + +static struct mntent _mntent; + +static struct mntent *statfs_to_mntent (struct statfs *mntbuf) { + _mntent.mnt_fsname = mntbuf->f_mntfromname; + _mntent.mnt_dir = mntbuf->f_mntonname; + _mntent.mnt_type = mntbuf->f_fstypename; + + _mntent.mnt_opts = '\0'; + _mntent.mnt_freq = 0; + _mntent.mnt_passno = 0; + + return (&_mntent); +} + +static int pos = -1; +static int mntsize = -1; +struct statfs *mntbuf = NULL; + +struct mntent *mounts_next (FILE *fp) { + + if (pos == -1 || mntsize == -1) + mntsize = getmntinfo(&mntbuf, MNT_NOWAIT); + ++pos; + if (pos == mntsize) { + pos = mntsize = -1; + mntbuf = NULL; + return NULL; + } + + return (statfs_to_mntent(&mntbuf[pos])); +} +#endif + +#ifdef UNKNOWN +/* dummy, do-nothing version */ +FILE *mounts_start (void) { + return ((FILE *)0x1); +} +int mounts_end (FILE *fp) { + return 1; +} +struct mntent *mounts_next (FILE *fp) { + return NULL; +} +#endif diff --git a/Utility/libmounts.h b/Utility/libmounts.h new file mode 100644 index 0000000000..460fcc7abd --- /dev/null +++ b/Utility/libmounts.h @@ -0,0 +1,38 @@ +/* Include appropriate headers for the OS, and define what will be used. */ +#if defined (__FreeBSD__) || defined (__APPLE__) +# include +# include +# include +# define GETMNTINFO +#else +#if defined __ANDROID__ +# warning mounts listing code not available for Android +# define UNKNOWN +#else +#if defined (__linux__) || defined (__FreeBSD_kernel__) +/* Linux or Debian kFreeBSD */ +#include +# define GETMNTENT +#else +# warning mounts listing code not available for this OS +# define UNKNOWN +#endif +#endif +#endif + +#include + +#ifndef GETMNTENT +struct mntent { + char *mnt_fsname; + char *mnt_dir; + char *mnt_type; + char *mnt_opts; /* not filled in */ + int mnt_freq; /* not filled in */ + int mnt_passno; /* not filled in */ +}; +#endif + +FILE *mounts_start (void); +int mounts_end (FILE *fp); +struct mntent *mounts_next (FILE *fp); diff --git a/configure.hs b/configure.hs new file mode 100644 index 0000000000..15833e62a7 --- /dev/null +++ b/configure.hs @@ -0,0 +1,6 @@ +{- configure program -} + +import Build.Configure + +main :: IO () +main = run tests diff --git a/debian/NEWS b/debian/NEWS new file mode 100644 index 0000000000..1c95146912 --- /dev/null +++ b/debian/NEWS @@ -0,0 +1,36 @@ +git-annex (3.20120123) unstable; urgency=low + + There was a bug in the handling of directory special remotes that + could cause partial file contents to be stored in them. If you use + a directory special remote, you should fsck it, to avoid potential + data loss. + + Example: git annex fsck --from mydirectory + + -- Joey Hess Thu, 19 Jan 2012 15:24:23 -0400 + +git-annex (3.20110624) experimental; urgency=low + + There has been another change to the git-annex data store. + Use `git annex upgrade` to migrate your repositories to the new + layout. See or + /usr/share/doc/git-annex/html/upgrades.html + + The significant change this time is that the .git-annex/ directory + is gone; instead there is a git-annex branch that is automatically + maintained by git-annex, and encapsulates all its state nicely out + of your way. + + You should make sure you include the git-annex branch when + git pushing and pulling. + + -- Joey Hess Tue, 21 Jun 2011 20:18:00 -0400 + +git-annex (0.20110316) experimental; urgency=low + + This version reorganises the layout of git-annex's files in your repository. + There is an upgrade process to convert a repository from the old git-annex + to this version. See or + /usr/share/doc/git-annex/html/upgrades.html + + -- Joey Hess Wed, 16 Mar 2011 15:49:15 -0400 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000000..8af9120b4c --- /dev/null +++ b/debian/changelog @@ -0,0 +1,1701 @@ +git-annex (4.20130228) UNRELEASED; urgency=low + + * Stop depending on testpack. + * Android: Enable test suite. + * webapp: Only show up to 10 queued transfers. + * Several improvements to Makefile and cabal file. Thanks, Peter Simmons + * assistant: Logs are rotated to avoid them using too much disk space. + * assistant: Avoid noise in logs from git commit about typechanged + files in direct mode repositories. + * assistant: Fix dropping content when a file is moved to an archive + directory. + * assistant: Set gc.auto=0 when creating repositories to prevent + automatic commits from causing git-gc runs. + * assistant: If gc.auto=0, run git-gc once a day, packing loose objects + very non-aggressively. + * webapp: New preferences page allows enabling/disabling debug logging + at runtime, as well as configuring numcopies and diskreserve. + * assistant: Fix bug in direct mode that could occur when a symlink is + moved out of an archive directory, and resulted in the file not being + set to direct mode when it was transferred. + * webapp: Proceed automatically on from "Configure jabber account" + to pairing. + * Bugfix: If the UUID of a remote is not known, prevent --from, --to, + and other ways of specifying remotes by name from selecting it, + since it is not possible to sanely use it. + * Run ssh with -T to avoid tty allocation and any login scripts that + may do undesired things with it. + + -- Joey Hess Wed, 27 Feb 2013 23:20:40 -0400 + +git-annex (4.20130227) unstable; urgency=low + + * annex.version is now set to 4 for direct mode repositories. + * Should now fully support git repositories with core.symlinks=false; + always using git's pseudosymlink files in such repositories. + * webapp: Allow creating repositories on filesystems that lack support for + symlinks. + * webapp: Can now add a new local repository, and make it sync with + the main local repository. + * Android: Bundle now includes openssh. + * Android: Support ssh connection caching. + * Android: Assistant is fully working. (But no webapp yet.) + * Direct mode: Support filesystems like FAT which can change their inodes + each time they are mounted. + * Direct mode: Fix support for adding a modified file. + * Avoid passing -p to rsync, to interoperate with crippled filesystems. + Closes: #700282 + * Additional GIT_DIR support bugfixes. May actually work now. + * webapp: Display any error message from git init if it fails to create + a repository. + * Fix a reversion in matching globs introduced in the last release, + where "*" did not match files inside subdirectories. No longer uses + the Glob library. + * copy: Update location log when no copy was performed, if the location + log was out of date. + * Makefile now builds using cabal, taking advantage of cabal's automatic + detection of appropriate build flags. + * test: The test suite is now built into the git-annex binary, and can + be run at any time. + + -- Joey Hess Wed, 27 Feb 2013 14:07:24 -0400 + +git-annex (3.20130216) unstable; urgency=low + + * Now uses the Haskell uuid library, rather than needing a uuid program. + * Now uses the Haskell Glob library, rather than pcre-light, avoiding + the need to install libpcre. Currently done only for Cabal or when + the Makefile is made to use -DWITH_GLOB + * Android port now available (command-line only). + * New annex.crippledfilesystem setting, allows use of git-annex + repositories on FAT and even worse filesystems; avoiding use of + hard links and locked down permissions settings. (Support is incomplete.) + * init: Detect when the repository is on a filesystem that does not + support hard links, or symlinks, or unix permissions, and set + annex.crippledfilesystem, as well as annex.direct. + * add: Improved detection of files that are modified while being added. + * Fix a bug in direct mode, introduced in the previous release, where + if a file was dropped and then got back, it would be stored in indirect + mode. + + -- Joey Hess Sat, 16 Feb 2013 10:03:26 -0400 + +git-annex (3.20130207) unstable; urgency=low + + * webapp: Now allows restarting any threads that crash. + * Adjust debian package to only build-depend on DAV on architectures + where it is available. + * addurl --fast: Use curl, rather than haskell HTTP library, to support https. + * annex.autocommit: New setting, can be used to disable autocommit + of changed files by the assistant, while it still does data syncing + and other tasks. + * assistant: Ignore .DS_Store on OSX. + * assistant: Fix location log when adding new file in direct mode. + * Deal with stale mappings for deleted file in direct mode. + * pre-commit: Update direct mode mappings. + * uninit, unannex --fast: If hard link creation fails, fall back to slow + mode. + * Clean up direct mode cache and mapping info when dropping keys. + * dropunused: Clean up stale direct mode cache and mapping info not + removed before. + + -- Joey Hess Thu, 07 Feb 2013 12:45:25 -0400 + +git-annex (3.20130124) unstable; urgency=low + + * Added source repository group, that only retains files until they've + been transferred to another repository. Useful for things like + repositories on cameras. + * Added manual repository group. Use to prevent the assistant from + downloading any file contents to keep things in sync. Instead + `git annex get`, `git annex drop` etc can be used manually as desired. + * webapp: More adjustments to longpoll code to deal with changes in + variable quoting in different versions of shakespeare-js. + * webapp: Avoid an error if a transfer is stopped just as it finishes. + Closes: #698184 + * webapp: Now always logs to .git/annex/daemon.log + * webapp: Has a page to view the log, accessed from the control menu. + * webapp: Fix crash adding removable drive that has an annex directory + in it that is not a git repository. + * Deal with incompatability in gpg2, which caused prompts for encryption + passphrases rather than using the supplied --passphrase-fd. + * bugfix: Union merges involving two or more repositories could sometimes + result in data from one repository getting lost. This could result + in the location log data becoming wrong, and fsck being needed to fix it. + * sync: Automatic merge conflict resolution now stages deleted files. + * Depend on git 1.7.7.6 for --no-edit. Closes: #698399 + * Fix direct mode mapping code to always store direct mode filenames + relative to the top of the repository, even when operating inside a + subdirectory. + * fsck: Detect and fix consistency errors in direct mode mapping files. + * Avoid filename encoding errors when writing direct mode mappings. + + -- Joey Hess Tue, 22 Jan 2013 07:11:59 +1100 + +git-annex (3.20130114) unstable; urgency=low + + * Now handles the case where a file that's being transferred to a remote + is modified in place, which direct mode allows. When this + happens, the transfer now fails, rather than allow possibly corrupt + data into the remote. + * fsck: Better checking of file content in direct mode. + * drop: Suggest using git annex move when numcopies prevents dropping a file. + * webapp: Repo switcher filters out repos that do not exist any more + (or are on a drive that's not mounted). + * webapp: Use IP address, rather than localhost, since some systems may + have configuration problems or other issues that prevent web browsers + from connecting to the right localhost IP for the webapp. + * webapp: Adjust longpoll code to work with recent versions of + shakespeare-js. + * assistant: Support new gvfs dbus names used in Gnome 3.6. + * In direct mode, files with the same key are no longer hardlinked, as + that would cause a surprising behavior if modifying one, where the other + would also change. + * webapp: Avoid illegal characters in hostname when creating S3 or + Glacier remote. + * assistant: Avoid committer crashing if a file is deleted at the wrong + instant. + + -- Joey Hess Mon, 14 Jan 2013 15:25:18 -0400 + +git-annex (3.20130107) unstable; urgency=low + + * webapp: Add UI to stop and restart assistant. + * committer: Fix a file handle leak. + * assistant: Make expensive transfer scan work fully in direct mode. + * More commands work in direct mode repositories: find, whereis, move, copy, + drop, log, fsck, add, addurl. + * sync: No longer automatically adds files in direct mode. + * assistant: Detect when system is not configured with a user name, + and set environment to prevent git from failing. + * direct: Avoid hardlinking symlinks that point to the same content + when the content is not present. + * Fix transferring files to special remotes in direct mode. + + -- Joey Hess Mon, 07 Jan 2013 01:01:41 -0400 + +git-annex (3.20130102) unstable; urgency=low + + * direct, indirect: New commands, that switch a repository to and from + direct mode. In direct mode, files are accessed directly, rather than + via symlinks. Note that direct mode is currently experimental. Many + git-annex commands do not work in direct mode. Some git commands can + cause data loss when used in direct mode repositories. + * assistant: Now uses direct mode by default when setting up a new + local repository. + * OSX assistant: Uses the FSEvents API to detect file changes. + This avoids issues with running out of file descriptors on large trees, + as well as allowing detection of modification of files in direct mode. + Other BSD systems still use kqueue. + * kqueue: Fix bug that made broken symlinks not be noticed. + * vicfg: Quote filename. Closes: #696193 + * Bugfix: Fixed bug parsing transfer info files, where the newline after + the filename was included in it. This was generally benign, but in + the assistant, it caused unexpected dropping of preferred content. + * Bugfix: Remove leading \ from checksums output by sha*sum commands, + when the filename contains \ or a newline. Closes: #696384 + * fsck: Still accept checksums with a leading \ as valid, now that + above bug is fixed. + * SHA*E backends: Exclude non-alphanumeric characters from extensions. + * migrate: Remove leading \ in SHA* checksums, and non-alphanumerics + from extensions of SHA*E keys. + + -- Joey Hess Wed, 02 Jan 2013 13:21:34 -0400 + +git-annex (3.20121211) unstable; urgency=low + + * webapp: Defaults to sharing box.com account info with friends, allowing + one-click enabling of the repository. + * Fix broken .config/git-annex/program installed by standalone tarball. + * assistant: Retrival from glacier now handled. + * Include ssh in standalone tarball and OSX app. + * watch: Avoid leaving hard links to files behind in .git/annex/tmp + if a file is deleted or moved while it's being quarantined in preparation + to being added to the annex. + * Allow `git annex drop --from web`; of course this does not remove + any file from the web, but it does make git-annex remove all urls + associated with a file. + * webapp: S3 and Glacier forms now have a select list of all + currently-supported AWS regions. + * webdav: Avoid trying to set props, avoiding incompatability with + livedrive.com. Needs DAV version 0.3. + * webapp: Prettify error display. + * webapp: Fix bad interaction between required fields and modals. + * webapp: Added help buttons and links next to fields that require + explanations. + * webapp: Encryption can be disabled when setting up remotes. + * assistant: Avoid trying to drop content from remotes that don't have it. + * assistant: Allow periods in ssh key comments. + * get/copy --auto: Transfer data even if it would exceed numcopies, + when preferred content settings want it. + * drop --auto: Fix dropping content when there are no preferred content + settings. + * webapp: Allow user to specify the port when setting up a ssh or rsync + remote. + * assistant: Fix syncing to just created ssh remotes. + * Enable WebDAV support in Debian package. Closes: #695532 + + -- Joey Hess Tue, 11 Dec 2012 11:25:03 -0400 + +git-annex (3.20121127) unstable; urgency=low + + * Fix dirContentsRecursive, which had missed some files in deeply nested + subdirectories. Could affect various parts of git-annex. + * rsync: Fix bug introduced in last release that broke encrypted rsync + special remotes. + * The standalone builds now unset their special path and library path + variables before running the system web browser. + + -- Joey Hess Tue, 27 Nov 2012 17:07:32 -0400 + +git-annex (3.20121126) unstable; urgency=low + + * New webdav and Amazon glacier special remotes. + * Display a warning when a non-existing file or directory is specified. + * webapp: Added configurator for Box.com. + * webapp: Show error messages to user when testing XMPP creds. + * Fix build of assistant without yesod. + * webapp: The list of repositiories refreshes when new repositories are + added, including when new repository configurations are pushed in from + remotes. + * OSX: Fix RunAtLoad value in plist file. + * Getting a file from chunked directory special remotes no longer buffers + it all in memory. + * S3: Added progress display for uploading and downloading. + * directory special remote: Made more efficient and robust. + * Bugfix: directory special remote could loop forever storing a key + when a too small chunksize was configured. + * Allow controlling whether login credentials for S3 and webdav are + committed to the repository, by setting embedcreds=yes|no when running + initremote. + * Added smallarchive repository group, that only archives files that are + in archive directories. Used by default for glacier when set up in the + webapp. + * assistant: Fixed handling of toplevel archive directory and + client repository group. + * assistant: Apply preferred content settings when a new symlink + is created, or a symlink gets renamed. Made archive directories work. + + -- Joey Hess Mon, 26 Nov 2012 11:37:49 -0400 + +git-annex (3.20121112) unstable; urgency=low + + * assistant: Can use XMPP to notify other nodes about pushes made to other + repositories, as well as pushing to them directly over XMPP. + * wepapp: Added an XMPP configuration interface. + * webapp: Supports pairing over XMPP, with both friends, and other repos + using the same account. + * assistant: Drops non-preferred content when possible. + * assistant: Notices, and applies config changes as they are made to + the git-annex branch, including config changes pushed in from remotes. + * git-annex-shell: GIT_ANNEX_SHELL_DIRECTORY can be set to limit it + to operating on a specified directory. + * webapp: When setting up authorized_keys, use GIT_ANNEX_SHELL_DIRECTORY. + * Preferred content path matching bugfix. + * Preferred content expressions cannot use "in=". + * Preferred content expressions can use "present". + * Fix handling of GIT_DIR when it refers to a git submodule. + * Depend on and use the Haskell SafeSemaphore library, which provides + exception-safe versions of SampleVar and QSemN. + Thanks, Ben Gamari for an excellent patch set. + * file:/// URLs can now be used with the web special remote. + * webapp: Allow dashes in ssh key comments when pairing. + * uninit: Check and abort if there are symlinks to annexed content that + are not checked into git. + * webapp: Switched to using the same multicast IP address that avahi uses. + * bup: Don't pass - to bup-split to make it read stdin; bup 0.25 + does not accept that. + * bugfix: Don't fail transferring content from read-only repos. + Closes: #691341 + * configure: Check that checksum programs produce correct checksums. + * Re-enable dbus, using a new version of the library that fixes the memory + leak. + * NetWatcher: When dbus connection is lost, try to reconnect. + * Use USER and HOME environment when set, and only fall back to getpwent, + which doesn't work with LDAP or NIS. + * rsync special remote: Include annex-rsync-options when running rsync + to test a key's presence. + * The standalone tarball's runshell now takes care of installing a + ~/.ssh/git-annex-shell wrapper the first time it's run. + * webapp: Make an initial, empty commit so there is a master branch + * assistant: Fix syncing local drives. + * webapp: Fix creation of rsync.net repositories. + * webapp: Fix renaming of special remotes. + * webapp: Generate better git remote names. + * webapp: Ensure that rsync special remotes are enabled using the same + name they were originally created using. + * Bugfix: Fix hang in webapp when setting up a ssh remote with an absolute + path. + + -- Joey Hess Mon, 12 Nov 2012 10:39:47 -0400 + +git-annex (3.20121017) unstable; urgency=low + + * Fix zombie cleanup reversion introduced in 3.20121009. + * Additional fix to support git submodules. + + -- Joey Hess Tue, 16 Oct 2012 21:10:14 -0400 + +git-annex (3.20121016) unstable; urgency=low + + * vicfg: New file format, avoids ambiguity with repos that have the same + description, or no description. + * Bug fix: A recent change caused git-annex-shell to crash. + * Better preferred content expression for transfer repos. + * webapp: Repository edit form can now edit the name of a repository. + * webapp: Make bare repositories on removable drives, as there is nothing + to ensure non-bare repos get updated when syncing. + * webapp: Better behavior when pausing syncing to a remote when a transfer + scan is running and queueing new transfers for that remote. + * The standalone binaries are now built to not use ssh connection caching, + in order to work with old versions of ssh. + * A relative core.worktree is relative to the gitdir. Now that this is + handled correctly, git-annex can be used in git submodules. + * Temporarily disable use of dbus, as the haskell dbus library blows up + when losing connection, which will need to be fixed upstream. + + -- Joey Hess Tue, 16 Oct 2012 15:25:22 -0400 + +git-annex (3.20121010) unstable; urgency=low + + * Renamed --ingroup to --inallgroup. + * Standard groups changed to client, transfer, archive, and backup. + Each of these has its own standard preferred content setting. + * dead: Remove dead repository from all groups. + * Avoid unsetting HOME when running certian git commands. Closes: #690193 + * test: Fix threaded runtime hang. + * Makefile: Avoid building with -threaded if the ghc threaded runtime does + not exist. + * webapp: Improve wording of intro display. Closes: #689848 + * webapp: Repositories can now be configured, to change their description, + their group, or even to disable syncing to them. + * git config remote.name.annex-sync can be used to control whether + a remote gets synced. + * Fix a crash when merging files in the git-annex branch that contain + invalid utf8. + * Automatically detect when a ssh remote does not have git-annex-shell + installed, and set annex-ignore. + + -- Joey Hess Fri, 12 Oct 2012 13:45:21 -0400 + +git-annex (3.20121009) unstable; urgency=low + + * watch, assistant: It's now safe to git annex unlock files while + the watcher is running, as well as modify files checked into git + as normal files. Additionally, .gitignore settings are now honored. + Closes: #689979 + * group, ungroup: New commands to indicate groups of repositories. + * webapp: Adds newly created repositories to one of these groups: + clients, drives, servers + * vicfg: New command, allows editing (or simply viewing) most + of the repository configuration settings stored in the git-annex branch. + * Added preferred content expressions, configurable using vicfg. + * get --auto: If the local repository has preferred content + configured, only get that content. + * drop --auto: If the repository the content is dropped from has + preferred content configured, drop only content that is not preferred. + * copy --auto: Only transfer content that the destination repository prefers. + * assistant: Now honors preferred content settings when deciding what to + transfer. + * --copies=group:number can now be used to match files that are present + in a specified number of repositories in a group. + * Added --smallerthan, --largerthan, and --inall limits. + * Only build-depend on libghc-clientsession-dev on arches that will have + the webapp. + * uninit: Unset annex.version. Closes: #689852 + + -- Joey Hess Tue, 09 Oct 2012 15:13:23 -0400 + +git-annex (3.20121001) unstable; urgency=low + + * fsck: Now has an incremental mode. Start a new incremental fsck pass + with git annex fsck --incremental. Now the fsck can be interrupted + as desired, and resumed with git annex fsck --more. + Thanks, Justin Azoff + * New --time-limit option, makes long git-annex commands stop after + a specified amount of time. + * fsck: New --incremental-schedule option which is nice for scheduling + eg, monthly incremental fsck runs in cron jobs. + * Fix fallback to ~/Desktop when xdg-user-dir is not available. + Closes: #688833 + * S3: When using a shared cipher, S3 credentials are not stored encrypted + in the git repository, as that would allow anyone with access to + the repository access to the S3 account. Instead, they're stored + in a 600 mode file in the local git repo. + * webapp: Avoid crashing when ssh-keygen -F chokes on an invalid known_hosts + file. + * Always do a system wide installation when DESTDIR is set. Closes: #689052 + * The Makefile now builds with the new yesod by default. + Systems like Debian that have the old yesod 1.0.1 should set + GIT_ANNEX_LOCAL_FEATURES=-DWITH_OLD_YESOD + * copy: Avoid updating the location log when no copy is performed. + * configure: Test that uuid -m works, falling back to plain uuid if not. + * Avoid building the webapp on Debian architectures that do not yet + have template haskell and thus yesod. (Should be available for arm soonish + I hope). + + -- Joey Hess Mon, 01 Oct 2012 13:56:55 -0400 + +git-annex (3.20120924) unstable; urgency=low + + * assistant: New command, a daemon which does everything watch does, + as well as automatically syncing file contents between repositories. + * webapp: An interface for managing and configuring the assistant. + * The default backend used when adding files to the annex is changed + from SHA256 to SHA256E, to simplify interoperability with OSX, media + players, and various programs that needlessly look at symlink targets. + To get old behavior, add a .gitattributes containing: * annex.backend=SHA256 + * init: If no description is provided for a new repository, one will + automatically be generated, like "joey@gnu:~/foo" + * test: Set a lot of git environment variables so testing works in strange + environments that normally need git config to set names, etc. + Closes: #682351 Thanks, gregor herrmann + * Disable ssh connection caching if the path to the control socket would be + too long (and use relative path to minimise path to the control socket). + * migrate: Check content before generating the new key, to avoid generating + a key for corrupt data. + * Support repositories created with --separate-git-dir. Closes: #684405 + * reinject: When the provided file doesn't match, leave it where it is, + rather than moving to .git/annex/bad/ + * Avoid crashing on encoding errors in filenames when writing transfer info + files and reading from checksum commands. + * sync: Pushes the git-annex branch to remote/synced/git-annex, rather + than directly to remote/git-annex. + * Now supports matching files that are present on a number of remotes + with a specified trust level. Example: --copies=trusted:2 + Thanks, Nicolas Pouillard + + -- Joey Hess Mon, 24 Sep 2012 13:47:48 -0400 + +git-annex (3.20120825) unstable; urgency=low + + * S3: Add fileprefix setting. + * Pass --use-agent to gpg when in no tty mode. Thanks, Eskild Hustvedt. + * Bugfix: Fix fsck in SHA*E backends, when the key contains composite + extensions, as added in 3.20120721. + + -- Joey Hess Sat, 25 Aug 2012 10:00:10 -0400 + +git-annex (3.20120807) unstable; urgency=low + + * initremote: Avoid recording remote's description before checking + that its config is valid. + * unused, status: Avoid crashing when ran in bare repo. + * Avoid crashing when "git annex get" fails to download from one + location, and falls back to downloading from a second location. + + -- Joey Hess Tue, 07 Aug 2012 13:35:07 -0400 + +git-annex (3.20120721) unstable; urgency=low + + * get, move, copy: Now refuse to do anything when the requested file + transfer is already in progress by another process. + * status: Lists transfers that are currently in progress. + * Fix passing --uuid to git-annex-shell. + * When shaNsum commands cannot be found, use the Haskell SHA library + (already a dependency) to do the checksumming. This may be slower, + but avoids portability problems. + * Use SHA library for files less than 50 kb in size, at which point it's + faster than forking the more optimised external program. + * SHAnE backends are now smarter about composite extensions, such as + .tar.gz Closes: #680450 + * map: Write map.dot to .git/annex, which avoids watch trying to annex it. + + -- Joey Hess Sat, 21 Jul 2012 16:52:48 -0400 + +git-annex (3.20120629) unstable; urgency=low + + * cabal: Only try to use inotify on Linux. + * Version build dependency on STM, and allow building without it, + which disables the watch command. + * Avoid ugly failure mode when moving content from a local repository + that is not available. + * Got rid of the last place that did utf8 decoding. + * Accept arbitrarily encoded repository filepaths etc when reading + git config output. This fixes support for remotes with unusual characters + in their names. + * sync: Automatically resolves merge conflicts. + + -- Joey Hess Fri, 29 Jun 2012 10:17:49 -0400 + +git-annex (3.20120624) unstable; urgency=low + + * watch: New subcommand, a daemon which notices changes to + files and automatically annexes new files, etc, so you don't + need to manually run git commands when manipulating files. + Available on Linux, BSDs, and OSX! + * Enable diskfree on kfreebsd, using kqueue. + * unused: Fix crash when key names contain invalid utf8. + * sync: Avoid recent git's interactive merge. + + -- Joey Hess Sun, 24 Jun 2012 12:36:50 -0400 + +git-annex (3.20120614) unstable; urgency=medium + + * addurl: Was broken by a typo introduced 2 released ago, now fixed. + Closes: #677576 + * Install man page when run by cabal, in a location where man will + find it, even when installing under $HOME. Thanks, Nathan Collins + + -- Joey Hess Thu, 14 Jun 2012 20:21:29 -0400 + +git-annex (3.20120611) unstable; urgency=medium + + * add: Prevent (most) modifications from being made to a file while it + is being added to the annex. + * initremote: Automatically describe a remote when creating it. + * uninit: Refuse to run in a subdirectory. Closes: #677076 + + -- Joey Hess Mon, 11 Jun 2012 10:32:01 -0400 + +git-annex (3.20120605) unstable; urgency=low + + * sync: Show a nicer message if a user tries to sync to a special remote. + * lock: Reset unlocked file to index, rather than to branch head. + * import: New subcommand, pulls files from a directory outside the annex + and adds them. + * Fix display of warning message when encountering a file that uses an + unsupported backend. + * Require that the SHA256 backend can be used when building, since it's the + default. + * Preserve parent environment when running hooks of the hook special remote. + + -- Joey Hess Tue, 05 Jun 2012 14:03:39 -0400 + +git-annex (3.20120522) unstable; urgency=low + + * Pass -a to cp even when it supports --reflink=auto, to preserve + permissions. + * Clean up handling of git directory and git worktree. + * Add support for core.worktree, and fix support for GIT_WORK_TREE and + GIT_DIR. + + -- Joey Hess Tue, 22 May 2012 11:16:13 -0400 + +git-annex (3.20120511) unstable; urgency=low + + * Rsync special remotes can be configured with shellescape=no + to avoid shell quoting that is normally done when using rsync over ssh. + This is known to be needed for certian rsync hosting providers + (specificially hidrive.strato.com) that use rsync over ssh but do not + pass it through the shell. + * dropunused: Allow specifying ranges to drop. + * addunused: New command, the opposite of dropunused, it relinks unused + content into the git repository. + * Fix use of several config settings: annex.ssh-options, + annex.rsync-options, annex.bup-split-options. (And adjust types to avoid + the bugs that broke several config settings.) + + -- Joey Hess Fri, 11 May 2012 12:29:30 -0400 + +git-annex (3.20120430) unstable; urgency=low + + * Fix use of annex.diskreserve config setting. + * Directory special remotes now check annex.diskreserve. + * Support git's core.sharedRepository configuration. + * Add annex.http-headers and annex.http-headers-command config + settings, to allow custom headers to be sent with all HTTP requests. + (Requested by the Internet Archive) + * uninit: Clear annex.uuid from .git/config. Closes: #670639 + * Added shared cipher mode to encryptable special remotes. This option + avoids gpg key distribution, at the expense of flexability, and with + the requirement that all clones of the git repository be equally trusted. + + -- Joey Hess Mon, 30 Apr 2012 13:16:10 -0400 + +git-annex (3.20120418) unstable; urgency=low + + * bugfix: Adding a dotfile also caused all non-dotfiles to be added. + * bup: Properly handle key names with spaces or other things that are + not legal git refs. + * git-annex (but not git-annex-shell) supports the git help.autocorrect + configuration setting, doing fuzzy matching using the restricted + Damerau-Levenshtein edit distance, just as git does. This adds a build + dependency on the haskell edit-distance library. + * Renamed diskfree.c to avoid OSX case insensativity bug. + * cabal now installs git-annex-shell as a symlink to git-annex. + * cabal file now autodetects whether S3 support is available. + + -- Joey Hess Wed, 18 Apr 2012 12:11:32 -0400 + +git-annex (3.20120406) unstable; urgency=low + + * Disable diskfree on kfreebsd, as I have a build failure on kfreebsd-i386 + that is quite likely caused by it. + + -- Joey Hess Sat, 07 Apr 2012 15:50:36 -0400 + +git-annex (3.20120405) unstable; urgency=low + + * Rewrote free disk space checking code, moving the portability + handling into a small C library. + * status: Display amount of free disk space. + + -- Joey Hess Thu, 05 Apr 2012 16:19:10 -0400 + +git-annex (3.20120315) unstable; urgency=low + + * fsck: Fix up any broken links and misplaced content caused by the + directory hash calculation bug fixed in the last release. + * sync: Sync to lower cost remotes first. + * status: Fixed to run in constant space. + * status: More accurate display of sizes of tmp and bad keys. + * unused: Now uses a bloom filter, and runs in constant space. + Use of a bloom filter does mean it will not notice a small + number of unused keys. For repos with up to half a million keys, + it will miss one key in 1000. + * Added annex.bloomcapacity and annex.bloomaccuracy, which can be + adjusted as desired to tune the bloom filter. + * status: Display amount of memory used by bloom filter, and + detect when it's too small for the number of keys in a repository. + * git-annex-shell: Runs hooks/annex-content after content is received + or dropped. + * Work around a bug in rsync (IMHO) introduced by openSUSE's SIP patch. + * git-annex now behaves as git-annex-shell if symlinked to and run by that + name. The Makefile sets this up, saving some 8 mb of installed size. + * git-union-merge is a demo program, so it is no longer built by default. + + -- Joey Hess Thu, 15 Mar 2012 11:05:28 -0400 + +git-annex (3.20120309) unstable; urgency=low + + * Fix key directory hash calculation code to behave as it did before + version 3.20120227 when a key contains non-ascii characters (only + WORM backend is likely to have been affected). + + -- Joey Hess Fri, 09 Mar 2012 20:05:09 -0400 + +git-annex (3.20120230) unstable; urgency=low + + * "here" can be used to refer to the current repository, + which can read better than the old "." (which still works too). + * Directory special remotes now support chunking files written to them, + avoiding writing files larger than a specified size. + * Add progress bar display to the directory special remote. + * Add configurable hooks that are run when git-annex starts and stops + using a remote: remote.name.annex-start-command and + remote.name.annex-stop-command + * Fix a bug in symlink calculation code, that triggered in rare + cases where an annexed file is in a subdirectory that nearly + matched to the .git/annex/object/xx/yy subdirectories. + + -- Joey Hess Mon, 05 Mar 2012 13:38:13 -0400 + +git-annex (3.20120229) unstable; urgency=low + + * Fix test suite to not require a unicode locale. + * Fix cabal build failure. Thanks, Sergei Trofimovich + + -- Joey Hess Wed, 29 Feb 2012 02:31:31 -0400 + +git-annex (3.20120227) unstable; urgency=low + + * Modifications to support ghc 7.4's handling of filenames. + This version can only be built with ghc 7.4 or newer. See the ghc7.0 + branch for older ghcs. + * S3: Fix irrefutable pattern failure when accessing encrypted S3 + credentials. + * Use the haskell IfElse library. + * Fix teardown of stale cached ssh connections. + * Fixed to use the strict state monad, to avoid leaking all kinds of memory + due to lazy state update thunks when adding/fixing many files. + * Fixed some memory leaks that occurred when committing journal files. + * Added a annex.queuesize setting, useful when adding hundreds of thousands + of files on a system with plenty of memory. + * whereis: Prints the urls of files that the web special remote knows about. + * addurl --fast: Verifies that the url can be downloaded (only getting + its head), and records the size in the key. + * When checking that an url has a key, verify that the Content-Length, + if available, matches the size of the key. + * addurl: Added a --file option, which can be used to specify what + file the url is added to. This can be used to override the default + filename that is used when adding an url, which is based on the url. + Or, when the file already exists, the url is recorded as another + location of the file. + * addurl: Normalize badly encoded urls. + * addurl: Add --pathdepth option. + * rekey: New plumbing level command, can be used to change the keys used + for files en masse. + * Store web special remote url info in a more efficient location. + (Urls stored with this version will not be visible to older versions.) + * Deal with NFS problem that caused a failure to remove a directory + when removing content from the annex. + * Make a single location log commit after a remote has received or + dropped files. Uses a new "git-annex-shell commit" command when available. + * To avoid commits of data to the git-annex branch after each command + is run, set annex.alwayscommit=false. Its data will then be committed + less frequently, when a merge or sync is done. + * configure: Check if ssh connection caching is supported by the installed + version of ssh and default annex.sshcaching accordingly. + * move --from, copy --from: Now 10 times faster when scanning to find + files in a remote on a local disk; rather than go through the location log + to see which files are present on the remote, it simply looks at the + disk contents directly. + + -- Joey Hess Mon, 27 Feb 2012 12:58:21 -0400 + +git-annex (3.20120123) unstable; urgency=low + + * fsck --from: Fscking a remote is now supported. It's done by retrieving + the contents of the specified files from the remote, and checking them, + so can be an expensive operation. Still, if the remote is a special + remote, or a git repository that you cannot run fsck in locally, it's + nice to have the ability to fsck it. + * If you have any directory special remotes, now would be a good time to + fsck them, in case you were hit by the data loss bug fixed in the + previous release! + * fsck --from remote --fast: Avoids expensive file transfers, at the + expense of not checking file size and/or contents. + * Ssh connection caching is now enabled automatically by git-annex. + Only one ssh connection is made to each host per git-annex run, which + can speed some things up a lot, as well as avoiding repeated password + prompts. Concurrent git-annex processes also share ssh connections. + Cached ssh connections are shut down when git-annex exits. + * To disable the ssh caching (if for example you have your own broader + ssh caching configuration), set annex.sshcaching=false. + + -- Joey Hess Mon, 23 Jan 2012 13:48:48 -0400 + +git-annex (3.20120116) unstable; urgency=medium + + * Fix data loss bug in directory special remote, when moving a file + to the remote failed, and partially transferred content was left + behind in the directory, re-running the same move would think it + succeeded and delete the local copy. + + -- Joey Hess Mon, 16 Jan 2012 16:43:45 -0400 + +git-annex (3.20120115) unstable; urgency=low + + * Add a sanity check for bad StatFS results. On architectures + where StatFS does not currently work (s390, mips, powerpc, sparc), + this disables the diskreserve checking code, and attempting to + configure an annex.diskreserve will result in an error. + * Fix QuickCheck dependency in cabal file. + * Minor optimisations. + + -- Joey Hess Sun, 15 Jan 2012 13:54:20 -0400 + +git-annex (3.20120113) unstable; urgency=low + + * log: Add --gource mode, which generates output usable by gource. + * map: Fix display of remote repos + * Add annex-trustlevel configuration settings, which can be used to + override the trust level of a remote. + * git-annex, git-union-merge: Support GIT_DIR and GIT_WORK_TREE. + * Add libghc-testpack-dev to build depends on all arches. + + -- Joey Hess Fri, 13 Jan 2012 15:35:17 -0400 + +git-annex (3.20120106) unstable; urgency=low + + * Support unescaped repository urls, like git does. + * log: New command that displays the location log for files, + showing each repository they were added to and removed from. + * Fix overbroad gpg --no-tty fix from last release. + + -- Joey Hess Sat, 07 Jan 2012 13:16:23 -0400 + +git-annex (3.20120105) unstable; urgency=low + + * Added annex-web-options configuration settings, which can be + used to provide parameters to whichever of wget or curl git-annex uses + (depends on which is available, but most of their important options + suitable for use here are the same). + * Dotfiles, and files inside dotdirs are not added by "git annex add" + unless the dotfile or directory is explicitly listed. So "git annex add ." + will add all untracked files in the current directory except for those in + dotdirs. + * Added quickcheck to build dependencies, and fail if test suite cannot be + built. + * fsck: Do backend-specific check before checking numcopies is satisfied. + * Run gpg with --no-tty. Closes: #654721 + + -- Joey Hess Thu, 05 Jan 2012 13:44:12 -0400 + +git-annex (3.20111231) unstable; urgency=low + + * sync: Improved to work well without a central bare repository. + Thanks to Joachim Breitner. + * Rather than manually committing, pushing, pulling, merging, and git annex + merging, we encourage you to give "git annex sync" a try. + * sync --fast: Selects some of the remotes with the lowest annex.cost + and syncs those, in addition to any specified at the command line. + * Union merge now finds the least expensive way to represent the merge. + * reinject: Add a sanity check for using an annexed file as the source file. + * Properly handle multiline git config values. + * Fix the hook special remote, which bitrotted a while ago. + * map: --fast disables use of dot to display map + * Test suite improvements. Current top-level test coverage: 75% + * Improve deletion of files from rsync special remotes. Closes: #652849 + * Add --include, which is the same as --not --exclude. + * Format strings can be specified using the new --format option, to control + what is output by git annex find. + * Support git annex find --json + * Fixed behavior when multiple insteadOf configs are provided for the + same url base. + * Can now be built with older git versions (before 1.7.7); the resulting + binary should only be used with old git. + * Updated to build with monad-control 0.3. + + -- Joey Hess Sat, 31 Dec 2011 14:55:29 -0400 + +git-annex (3.20111211) unstable; urgency=medium + + * Fix bug in last version in getting contents from bare repositories. + * Ensure that git-annex branch changes are merged into git-annex's index, + which fixes a bug that could cause changes that were pushed to the + git-annex branch to get reverted. As a side effect, it's now safe + for users to check out and commit changes directly to the git-annex + branch. + * map: Fix a failure to detect a loop when both repositories are local + and refer to each other with relative paths. + * Prevent key names from containing newlines. + * add: If interrupted, add can leave files converted to symlinks but not + yet added to git. Running the add again will now clean up this situtation. + * Fix caching of decrypted ciphers, which failed when drop had to check + multiple different encrypted special remotes. + * unannex: Can be run on files that have been added to the annex, but not + yet committed. + * sync: New command that synchronises the local repository and default + remote, by running git commit, pull, and push for you. + * Version monad-control dependency in cabal file. + + -- Joey Hess Sun, 11 Dec 2011 21:24:39 -0400 + +git-annex (3.20111203) unstable; urgency=low + + * The VFAT filesystem on recent versions of Linux, when mounted with + shortname=mixed, does not get along well with git-annex's mixed case + .git/annex/objects hash directories. To avoid this problem, new content + is now stored in all-lowercase hash directories. Except for non-bare + repositories which would be a pain to transition and cannot be put on FAT. + (Old mixed-case hash directories are still tried for backwards + compatibility.) + * Flush json output, avoiding a buffering problem that could result in + doubled output. + * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright. + * Bugfix: dropunused did not drop keys with two spaces in their name. + * Support for storing .git/annex on a different device than the rest of the + git repository. + * --inbackend can be used to make git-annex only operate on files + whose content is stored using a specified key-value backend. + * dead: A command which says that a repository is gone for good + and you don't want git-annex to mention it again. + + -- Joey Hess Sat, 03 Dec 2011 21:01:45 -0400 + +git-annex (3.20111122) unstable; urgency=low + + * merge: Improve commit messages to mention what was merged. + * Avoid doing auto-merging in commands that don't need fully current + information from the git-annex branch. In particular, git annex add + no longer needs to auto-merge. + * init: When run in an already initalized repository, and without + a description specified, don't delete the old description. + * Optimised union merging; now only runs git cat-file once, and runs + in constant space. + * status: Now displays trusted, untrusted, and semitrusted repositories + separately. + * status: Include all special remotes in the list of repositories. + * status: Fix --json mode. + * status: --fast is back + * Fix support for insteadOf url remapping. Closes: #644278 + * When not run in a git repository, git-annex can still display a usage + message, and "git annex version" even works. + * migrate: Don't fall over a stale temp file. + * Avoid excessive escaping for rsync special remotes that are not accessed + over ssh. + * find: Support --print0 + + -- Joey Hess Tue, 22 Nov 2011 14:31:45 -0400 + +git-annex (3.20111111) unstable; urgency=low + + * Handle a case where an annexed file is moved into a gitignored directory, + by having fix --force add its change. + * Avoid cyclic drop problems. + * Optimized copy --from and get --from to avoid checking the location log + for files that are already present. + * Automatically fix up badly formatted uuid.log entries produced by + 3.20111105, whenever the uuid.log is changed (ie, by init or describe). + * map: Support remotes with /~/ and /~user/ + + -- Joey Hess Fri, 11 Nov 2011 13:44:18 -0400 + +git-annex (3.20111107) unstable; urgency=low + + * merge: Use fast-forward merges when possible. + Thanks Valentin Haenel for a test case showing how non-fast-forward + merges could result in an ongoing pull/merge/push cycle. + * Don't try to read config from repos with annex-ignore set. + * Bugfix: In the past two releases, git-annex init has written the uuid.log + in the wrong format, with the UUID and description flipped. + + -- Joey Hess Mon, 07 Nov 2011 12:47:44 -0400 + +git-annex (3.20111105) unstable; urgency=low + + * The default backend used when adding files to the annex is changed + from WORM to SHA256. + To get old behavior, add a .gitattributes containing: * annex.backend=WORM + * Sped up some operations on remotes that are on the same host. + * copy --to: Fixed leak when copying many files to a remote on the same + host. + * uninit: Add guard against being run with the git-annex branch checked out. + * Fail if --from or --to is passed to commands that do not support them. + * drop --from is now supported to remove file content from a remote. + * status: Now always shows the current repository, even when it does not + appear in uuid.log. + * fsck: Now works in bare repositories. Checks location log information, + and file contents. Does not check that numcopies is satisfied, as + .gitattributes information about numcopies is not available in a bare + repository. + * unused, dropunused: Now work in bare repositories. + * Removed the setkey command, and added a reinject command with a more + useful interface. + * The fromkey command now takes the key as its first parameter. The --key + option is no longer used. + * Built without any filename containing .git being excluded. Closes: #647215 + * Record uuid when auto-initializing a remote so it shows in status. + * Bugfix: Fixed git-annex init crash in a bare repository when there was + already an existing git-annex branch. + * Pass -t to rsync to preserve timestamps. + + -- Joey Hess Sat, 05 Nov 2011 15:47:52 -0400 + +git-annex (3.20111025) unstable; urgency=low + + * A remote can have a annexUrl configured, that is used by git-annex + instead of its usual url. (Similar to pushUrl.) + * migrate: Copy url logs for keys when migrating. + * git-annex-shell: GIT_ANNEX_SHELL_READONLY and GIT_ANNEX_SHELL_LIMITED + environment variables can be set to limit what commands can be run. + This is used by gitolite's new git-annex support! + + -- Joey Hess Tue, 25 Oct 2011 13:03:08 -0700 + +git-annex (3.20111011) unstable; urgency=low + + * This version of git-annex only works with git 1.7.7 and newer. + The breakage with old versions is subtle, and affects the + annex.numcopies settings in .gitattributes, so be sure to upgrade git + to 1.7.7. (Debian package now depends on that version.) + * Don't pass absolute paths to git show-attr, as it started following + symlinks when that's done in 1.7.7. Instead, use relative paths, + which show-attr only handles 100% correctly in 1.7.7. Closes: #645046 + * Fix referring to remotes by uuid. + * New or changed repository descriptions in uuid.log now have a timestamp, + which is used to ensure the newest description is used when the uuid.log + has been merged. + * Note that older versions of git-annex will display the timestamp as part + of the repository description, which is ugly but otherwise harmless. + * Add timestamps to trust.log and remote.log too. + * git-annex-shell: Added the --uuid option. + * git-annex now asks git-annex-shell to verify that it's operating in + the expected repository. + * Note that this git-annex will not interoperate with remotes using + older versions of git-annex-shell. + * Now supports git's insteadOf configuration, to modify the url + used to access a remote. Note that pushInsteadOf is not used; + that and pushurl are reserved for actual git pushes. Closes: #644278 + * status: List all known repositories. + * When displaying a list of repositories, show git remote names + in addition to their descriptions. + * Add locking to avoid races when changing the git-annex branch. + * Various speed improvements gained by using ByteStrings. + * Contain the zombie hordes. + + -- Joey Hess Tue, 11 Oct 2011 23:00:02 -0400 + +git-annex (3.20110928) unstable; urgency=low + + * --in can be used to make git-annex only operate on files + believed to be present in a given repository. + * Arbitrarily complex expressions can be built to limit the files git-annex + operates on, by combining the options --not --and --or -( and -) + Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -) + * --copies=N can be used to make git-annex only operate on files with + the specified number of copies. (And --not --copies=N for the inverse.) + * find: Rather than only showing files whose contents are present, + when used with --exclude --copies or --in, displays all files that + match the specified conditions. + * Note that this is a behavior change for git-annex find! Old behavior + can be gotten by using: git-annex find --in . + * status: Massively sped up; remove --fast mode. + * unused: File contents used by branches and tags are no longer + considered unused, even when not used by the current branch. This is + the final piece of the puzzle needed for git-annex to to play nicely + with branches. + + -- Joey Hess Wed, 28 Sep 2011 18:14:02 -0400 + +git-annex (3.20110915) unstable; urgency=low + + * whereis: Show untrusted locations separately and do not include in + location count. + * Fix build without S3. + * addurl: Always use whole url as destination filename, rather than + only its file component. + * get, drop, copy: Added --auto option, which decides whether + to get/drop content as needed to work toward the configured numcopies. + * bugfix: drop and fsck did not honor --exclude + + -- Joey Hess Thu, 15 Sep 2011 22:25:46 -0400 + +git-annex (3.20110906) unstable; urgency=low + + * Improve display of newlines around error and warning messages. + * Fix Makefile to work with cabal again. + + -- Joey Hess Tue, 06 Sep 2011 13:45:16 -0400 + +git-annex (3.20110902) unstable; urgency=low + + * Set EMAIL when running test suite so that git does not need to be + configured first. Closes: #638998 + * The wget command will now be used in preference to curl, if available. + * init: Make description an optional parameter. + * unused, status: Sped up by avoiding unnecessary stats of annexed files. + * unused --remote: Reduced memory use to 1/4th what was used before. + * Add --json switch, to produce machine-consumable output. + + -- Joey Hess Fri, 02 Sep 2011 21:20:37 -0400 + +git-annex (3.20110819) unstable; urgency=low + + * Now "git annex init" only has to be run once, when a git repository + is first being created. Clones will automatically notice that git-annex + is in use and automatically perform a basic initalization. It's + still recommended to run "git annex init" in any clones, to describe them. + * Added annex-cost-command configuration, which can be used to vary the + cost of a remote based on the output of a shell command. + * Fix broken upgrade from V1 repository. Closes: #638584 + + -- Joey Hess Fri, 19 Aug 2011 20:34:09 -0400 + +git-annex (3.20110817) unstable; urgency=low + + * Fix shell escaping in rsync special remote. + * addurl: --fast can be used to avoid immediately downloading the url. + * Added support for getting content from git remotes using http (and https). + * Added curl to Debian package dependencies. + + -- Joey Hess Wed, 17 Aug 2011 01:29:02 -0400 + +git-annex (3.20110719) unstable; urgency=low + + * add: Be even more robust to avoid ever leaving the file seemingly deleted. + Closes: #634233 + * Bugfix: Make add ../ work. + * Support the standard git -c name=value + * unannex: Clean up use of git commit -a. + + -- Joey Hess Tue, 19 Jul 2011 23:39:53 -0400 + +git-annex (3.20110707) unstable; urgency=low + + * Fix sign bug in disk free space checking. + * Bugfix: Forgot to de-escape keys when upgrading. Could result in + bad location log data for keys that contain [&:%] in their names. + (A workaround for this problem is to run git annex fsck.) + * add: Avoid a failure mode that resulted in the file seemingly being + deleted (content put in the annex but no symlink present). + + -- Joey Hess Thu, 07 Jul 2011 19:29:39 -0400 + +git-annex (3.20110705) unstable; urgency=low + + * uninit: Delete the git-annex branch and .git/annex/ + * unannex: In --fast mode, file content is left in the annex, and a + hard link made to it. + * uninit: Use unannex in --fast mode, to support unannexing multiple + files that link to the same content. + * Drop the dependency on the haskell curl bindings, use regular haskell HTTP. + * Fix a pipeline stall when upgrading (caused by #624389). + + -- Joey Hess Tue, 05 Jul 2011 14:37:39 -0400 + +git-annex (3.20110702) unstable; urgency=low + + * Now the web can be used as a special remote. + This feature replaces the old URL backend. + * addurl: New command to download an url and store it in the annex. + * Sped back up fsck, copy --from, and other commands that often + have to read a lot of information from the git-annex branch. Such + commands are now faster than they were before introduction of the + git-annex branch. + * Always ensure git-annex branch exists. + * Modify location log parser to allow future expansion. + * --force will cause add, etc, to operate on ignored files. + * Avoid mangling encoding when storing the description of repository + and other content. + * cabal can now be used to build git-annex. This is substantially + slower than using make, does not build or install documentation, + does not run the test suite, and is not particularly recommended, + but could be useful to some. + + -- Joey Hess Sat, 02 Jul 2011 15:00:18 -0400 + +git-annex (3.20110624) experimental; urgency=low + + * New repository format, annex.version=3. Use `git annex upgrade` to migrate. + * git-annex now stores its logs in a git-annex branch. + * merge: New subcommand. Auto-merges the new git-annex branch. + * Improved handling of bare git repos with annexes. Many more commands will + work in them. + * git-annex is now more robust; it will never leave state files + uncommitted when some other git process comes along and locks the index + at an inconvenient time. + * rsync is now used when copying files from repos on other filesystems. + cp is still used when copying file from repos on the same filesystem, + since --reflink=auto can make it significantly faster on filesystems + such as btrfs. + * Allow --trust etc to specify a repository by name, for temporarily + trusting repositories that are not configured remotes. + * unlock: Made atomic. + * git-union-merge: New git subcommand, that does a generic union merge + operation, and operates efficiently without touching the working tree. + + -- Joey Hess Fri, 24 Jun 2011 14:32:18 -0400 + +git-annex (0.20110610) unstable; urgency=low + + * Add --numcopies option. + * Add --trust, --untrust, and --semitrust options. + * get --from is the same as copy --from + * Bugfix: Fix fsck to not think all SHAnE keys are bad. + + -- Joey Hess Fri, 10 Jun 2011 11:48:40 -0400 + +git-annex (0.20110601) unstable; urgency=low + + * Minor bugfixes and error message improvements. + * Massively sped up `git annex lock` by avoiding use of the uber-slow + `git reset`, and only running `git checkout` once, even when many files + are being locked. + * Fix locking of files with staged changes. + * Somewhat sped up `git commit` of modifications to unlocked files. + * Build fix for older ghc. + + -- Joey Hess Wed, 01 Jun 2011 11:50:47 -0400 + +git-annex (0.20110522) unstable; urgency=low + + * Closer emulation of git's behavior when told to use "foo/.git" as a + git repository instead of just "foo". Closes: #627563 + * Fix bug in --exclude introduced in 0.20110516. + + -- Joey Hess Fri, 27 May 2011 20:20:41 -0400 + +git-annex (0.20110521) unstable; urgency=low + + * status: New subcommand to show info about an annex, including its size. + * --backend now overrides any backend configured in .gitattributes files. + * Add --debug option. Closes: #627499 + + -- Joey Hess Sat, 21 May 2011 11:52:53 -0400 + +git-annex (0.20110516) unstable; urgency=low + + * Add a few tweaks to make it easy to use the Internet Archive's variant + of S3. In particular, munge key filenames to comply with the IA's filename + limits, disable encryption, support their nonstandard way of creating + buckets, and allow x-archive-* headers to be specified in initremote to + set item metadata. + * Added filename extension preserving variant backends SHA1E, SHA256E, etc. + * migrate: Use current filename when generating new key, for backends + where the filename affects the key name. + * Work around a bug in Network.URI's handling of bracketed ipv6 addresses. + + -- Joey Hess Mon, 16 May 2011 14:16:52 -0400 + +git-annex (0.20110503) unstable; urgency=low + + * Fix hasKeyCheap setting for bup and rsync special remotes. + * Add hook special remotes. + * Avoid crashing when an existing key is readded to the annex. + * unused: Now also lists files fsck places in .git/annex/bad/ + * S3: When encryption is enabled, the Amazon S3 login credentials + are stored, encrypted, in .git-annex/remotes.log, so environment + variables need not be set after the remote is initialized. + + -- Joey Hess Tue, 03 May 2011 20:56:01 -0400 + +git-annex (0.20110427) unstable; urgency=low + + * Switch back to haskell SHA library, so git-annex remains buildable on + Debian stable. + * Added rsync special remotes. This could be used, for example, to + store annexed content on rsync.net (encrypted naturally). Or anywhere else. + * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a + lot of files. Possibly only occured with ghc 7. + + -- Joey Hess Wed, 27 Apr 2011 22:50:26 -0400 + +git-annex (0.20110425) unstable; urgency=low + + * Use haskell Crypto library instead of haskell SHA library. + * Remove testpack from build depends for non x86 architectures where it + is not available. The test suite will not be run if it cannot be compiled. + * Avoid using absolute paths when staging location log, as that can + confuse git when a remote's path contains a symlink. Closes: #621386 + + -- Joey Hess Mon, 25 Apr 2011 15:47:00 -0400 + +git-annex (0.20110420) unstable; urgency=low + + * Update Debian build dependencies for ghc 7. + * Debian package is now built with S3 support. + Thanks Joachim Breitner for making this possible. + * Somewhat improved memory usage of S3, still work to do. + Thanks Greg Heartsfield for ongoing work to improve the hS3 library + for git-annex. + + -- Joey Hess Thu, 21 Apr 2011 15:00:48 -0400 + +git-annex (0.20110419) unstable; urgency=low + + * Don't run gpg in batch mode, so it can prompt for passphrase when + there is no agent. + * Add missing build dep on dataenc. + * S3: Fix stalls when transferring encrypted data. + * bup: Avoid memory leak when transferring encrypted data. + + -- Joey Hess Tue, 19 Apr 2011 21:26:51 -0400 + +git-annex (0.20110417) unstable; urgency=low + + * bup is now supported as a special type of remote. + * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted + using GPG for privacy. + * Use lowercase hash directories for locationlog files, to avoid + some issues with git on OSX with the mixed-case directories. + No migration is needed; the old mixed case hash directories are still + read; new information is written to the new directories. + * Unused files on remotes, particulary special remotes, can now be + identified and dropped, by using "--from remote" with git annex unused + and git annex dropunused. + * Clear up short option confusion between --from and --force (-f is now + --from, and there is no short option for --force). + * Add build depend on perlmagick so docs are consistently built. + Closes: #621410 + * Add doc-base file. Closes: #621408 + * Periodically flush git command queue, to avoid boating memory usage + too much. + * Support "sha1" and "sha512" commands on FreeBSD, and allow building + if any/all SHA commands are not available. Thanks, Fraser Tweedale + + -- Joey Hess Sun, 17 Apr 2011 12:00:24 -0400 + +git-annex (0.20110401) experimental; urgency=low + + * Amazon S3 is now supported as a special type of remote. + Warning: Encrypting data before sending it to S3 is not yet supported. + * Note that Amazon S3 support is not built in by default on Debian yet, + as hS3 is not packaged. + * fsck: Ensure that files and directories in .git/annex/objects + have proper permissions. + * Added a special type of remote called a directory remote, which + simply stores files in an arbitrary local directory. + * Bugfix: copy --to --fast never really copied, fixed. + + -- Joey Hess Fri, 01 Apr 2011 21:27:22 -0400 + +git-annex (0.20110328) experimental; urgency=low + + * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes") + * Generalized remotes handling, laying groundwork for remotes that are + not regular git remotes. (Think Amazon S3.) + * Provide a less expensive version of `git annex copy --to`, enabled + via --fast. This assumes that location tracking information is correct, + rather than contacting the remote for every file. + * Bugfix: Keys could be received into v1 annexes from v2 annexes, via + v1 git-annex-shell. This results in some oddly named keys in the v1 + annex. Recognise and fix those keys when upgrading, instead of crashing. + + -- Joey Hess Mon, 28 Mar 2011 10:47:29 -0400 + +git-annex (0.20110325) experimental; urgency=low + + * Free space checking is now done, for transfers of data for keys + that have free space metadata. (Notably, not for SHA* keys generated + with git-annex 0.2x or earlier.) The code is believed to work on + Linux, FreeBSD, and OSX; check compile-time messages to see if it + is not enabled for your OS. + * Add annex.diskreserve config setting, to control how much free space + to reserve for other purposes and avoid using (defaults to 1 mb). + * Add --fast flag, that can enable less expensive, but also less thorough + versions of some commands. + * fsck: In fast mode, avoid checking checksums. + * unused: In fast mode, just show all existing temp files as unused, + and avoid expensive scan for other unused content. + * migrate: Support migrating v1 SHA keys to v2 SHA keys with + size information that can be used for free space checking. + * Fix space leak in fsck and drop commands. + * migrate: Bugfix for case when migrating a file results in a key that + is already present in .git/annex/objects. + * dropunused: Significantly sped up; only read unused log file once. + + -- Joey Hess Fri, 25 Mar 2011 00:47:37 -0400 + +git-annex (0.20110320) experimental; urgency=low + + * Fix dropping of files using the URL backend. + * Fix support for remotes with '.' in their names. + * Add version command to show git-annex version as well as repository + version information. + * No longer auto-upgrade to repository format 2, to avoid accidental + upgrades, etc. Use git-annex upgrade when you're ready to run this + version. + + -- Joey Hess Sun, 20 Mar 2011 16:36:33 -0400 + +git-annex (0.20110316) experimental; urgency=low + + * New repository format, annex.version=2. + * The first time git-annex is run in an old format repository, it + will automatically upgrade it to the new format, staging all + necessary changes to git. Also added a "git annex upgrade" command. + * Colons are now avoided in filenames, so bare clones of git repos + can be put on USB thumb drives formatted with vFAT or similar + filesystems. + * Added two levels of hashing to object directory and .git-annex logs, + to improve scalability with enormous numbers of annexed + objects. (With one hundred million annexed objects, each + directory would contain fewer than 1024 files.) + * The setkey, fromkey, and dropkey subcommands have changed how + the key is specified. --backend is no longer used with these. + + -- Joey Hess Wed, 16 Mar 2011 16:20:23 -0400 + +git-annex (0.24) unstable; urgency=low + + Branched the 0.24 series, which will be maintained for a while to + support v1 git-annex repos, while main development moves to the 0.2011 + series, with v2 git-annex repos. + + * Add Suggests on graphviz. Closes: #618039 + * When adding files to the annex, the symlinks pointing at the annexed + content are made to have the same mtime as the original file. + While git does not preserve that information, this allows a tool + like metastore to be used with annexed files. + (Currently this is only done on systems supporting POSIX 200809.) + + -- Joey Hess Wed, 16 Mar 2011 18:35:13 -0400 + +git-annex (0.23) unstable; urgency=low + + * Support ssh remotes with a port specified. + * whereis: New subcommand to show where a file's content has gotten to. + * Rethink filename encoding handling for display. Since filename encoding + may or may not match locale settings, any attempt to decode filenames + will fail for some files. So instead, do all output in binary mode. + + -- Joey Hess Sat, 12 Mar 2011 15:02:49 -0400 + +git-annex (0.22) unstable; urgency=low + + * Git annexes can now be attached to bare git repositories. + (Both the local and remote host must have this version of git-annex + installed for it to work.) + * Support filenames that start with a dash; when such a file is passed + to a utility it will be escaped to avoid it being interpreted as an + option. (I went a little overboard and got the type checker involved + in this, so such files are rather comprehensively supported now.) + * New backends: SHA512 SHA384 SHA256 SHA224 + (Supported on systems where corresponding shaNsum commands are available.) + * describe: New subcommand that can set or change the description of + a repository. + * Fix test suite to reap zombies. + (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang + for his help eliminating the infestation... for now.) + * Make test suite not rely on a working cp -pr. + (The Unix wars are still ON!) + * Look for dir.git directories the same as git does. + * Support remote urls specified as relative paths. + * Support non-ssh remote paths that contain tilde expansions. + * fsck: Check for and repair location log damage. + * Bugfix: When fsck detected and moved away corrupt file content, it did + not update the location log. + + -- Joey Hess Fri, 04 Mar 2011 15:10:57 -0400 + +git-annex (0.21) unstable; urgency=low + + * test: Don't rely on chmod -R working. + * unannex: Fix recently introduced bug when attempting to unannex more + than one file at a time. + * test: Set git user name and email in case git can't guess values. + * Fix display of unicode filenames. + + -- Joey Hess Fri, 11 Feb 2011 23:21:08 -0400 + +git-annex (0.20) unstable; urgency=low + + * Preserve specified file ordering when instructed to act on multiple + files or directories. For example, "git annex get a b" will now always + get "a" before "b". Previously it could operate in either order. + * unannex: Commit staged changes at end, to avoid some confusing behavior + with the pre-commit hook, which would see some types of commits after + an unannex as checking in of an unlocked file. + * map: New subcommand that uses graphviz to display a nice map of + the git repository network. + * Deal with the mtl/monads-fd conflict. + * configure: Check for sha1sum. + + -- Joey Hess Tue, 08 Feb 2011 18:57:24 -0400 + +git-annex (0.19) unstable; urgency=low + + * configure: Support using the uuidgen command if the uuid command is + not available. + * Allow --exclude to be specified more than once. + * There are now three levels of repository trust. + * untrust: Now marks the current repository as untrusted. + * semitrust: Now restores the default trust level. (What untrust used to do.) + * fsck, drop: Take untrusted repositories into account. + * Bugfix: Files were copied from trusted remotes first even if their + annex.cost was higher than other remotes. + * Improved temp file handling. Transfers of content can now be resumed + from temp files later; the resume does not have to be the immediate + next git-annex run. + * unused: Include partially transferred content in the list. + * Bugfix: Running a second git-annex while a first has a transfer in + progress no longer deletes the first processes's temp file. + + -- Joey Hess Fri, 28 Jan 2011 14:31:37 -0400 + +git-annex (0.18) unstable; urgency=low + + * Bugfix: `copy --to` and `move --to` forgot to stage location log changes + after transferring the file to the remote repository. + (Did not affect ssh remotes.) + * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ + * migrate: Fix support for --backend option. + * unlock: Fix behavior when file content is not present. + * Test suite improvements. Current top-level test coverage: 80% + + -- Joey Hess Fri, 14 Jan 2011 14:17:44 -0400 + +git-annex (0.17) unstable; urgency=low + + * unannex: Now skips files whose content is not present, rather than + it being an error. + * New migrate subcommand can be used to switch files to using a different + backend, safely and with no duplication of content. + * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.) + + -- Joey Hess Sun, 09 Jan 2011 10:04:11 -0400 + +git-annex (0.16) unstable; urgency=low + + * git-annex-shell: Avoid exposing any git repo config except for the + annex.uuid when doing configlist. + * bugfix: Running `move --to` with a remote whose UUID was not yet known + could result in git-annex not recording on the local side where the + file was moved to. This could not result in data loss, or even a + significant problem, since the remote *did* record that it had the file. + * Also, add a general guard to detect attempts to record information + about repositories with missing UUIDs. + * bugfix: Running `move --to` with a non-ssh remote failed. + * bugfix: Running `copy --to` with a non-ssh remote actually did a move. + * Many test suite improvements. Current top-level test coverage: 65% + + -- Joey Hess Fri, 07 Jan 2011 14:33:13 -0400 + +git-annex (0.15) unstable; urgency=low + + * Support scp-style urls for remotes (host:path). + * Support ssh urls containing "~". + * Add trust and untrust subcommands, to allow configuring repositories + that are trusted to retain files without explicit checking. + * Fix bug in numcopies handling when multiple remotes pointed to the + same repository. + * Introduce the git-annex-shell command. It's now possible to make + a user have it as a restricted login shell, similar to git-shell. + * Note that git-annex will always use git-annex-shell when accessing + a ssh remote, so all of your remotes need to be upgraded to this + version of git-annex at the same time. + * Now rsync is exclusively used for copying files to and from remotes. + scp is not longer supported. + + -- Joey Hess Fri, 31 Dec 2010 22:00:52 -0400 + +git-annex (0.14) unstable; urgency=low + + * Bugfix to git annex unused in a repository with nothing yet annexed. + * Support upgrading from a v0 annex with nothing in it. + * Avoid multiple calls to git ls-files when passed eg, "*". + + -- Joey Hess Fri, 24 Dec 2010 17:38:48 -0400 + +git-annex (0.13) unstable; urgency=low + + * Makefile: Install man page and html (when built). + * Makefile: Add GHCFLAGS variable. + * Fix upgrade from 0.03. + * Support remotes using git+ssh and ssh+git as protocol. + Closes: #607056 + + -- Joey Hess Tue, 14 Dec 2010 13:05:10 -0400 + +git-annex (0.12) unstable; urgency=low + + * Add --exclude option to exclude files from processing. + * mwdn2man: Fix a bug in newline supression. Closes: #606578 + * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 + * Makefile: Add PREFIX variable. + + -- Joey Hess Sat, 11 Dec 2010 17:32:00 -0400 + +git-annex (0.11) unstable; urgency=low + + * If available, rsync will be used for file transfers from remote + repositories. This allows resuming interrupted transfers. + * Added remote.annex-rsync-options. + * Avoid deleting temp files when rsync fails. + * Improve detection of version 0 repos. + * Add uninit subcommand. Closes: #605749 + + -- Joey Hess Sat, 04 Dec 2010 17:27:42 -0400 + +git-annex (0.10) unstable; urgency=low + + * In .gitattributes, the annex.numcopies attribute can be used + to control the number of copies to retain of different types of files. + * Bugfix: Always correctly handle gitattributes when in a subdirectory of + the repository. (Had worked ok for ones like "*.mp3", but failed for + ones like "dir/*".) + * fsck: Fix warning about not enough copies of a file, when locations + are known, but are not available in currently configured remotes. + * precommit: Optimise to avoid calling git-check-attr more than once. + * The git-annex-backend attribute has been renamed to annex.backend. + + -- Joey Hess Sun, 28 Nov 2010 19:28:05 -0400 + +git-annex (0.09) unstable; urgency=low + + * Add copy subcommand. + * Fix bug in setkey subcommand triggered by move --to. + + -- Joey Hess Sat, 27 Nov 2010 17:14:59 -0400 + +git-annex (0.08) unstable; urgency=low + + * Fix `git annex add ../foo` (when ran in a subdir of the repo). + * Add configure step to build process. + * Only use cp -a if it is supported, falling back to cp -p or plain cp + as needed for portability. + * cp --reflink=auto is used if supported, and will make git annex unlock + much faster on filesystems like btrfs that support copy on write. + + -- Joey Hess Sun, 21 Nov 2010 13:45:44 -0400 + +git-annex (0.07) unstable; urgency=low + + * find: New subcommand. + * unused: New subcommand, finds unused data. (Split out from fsck.) + * dropunused: New subcommand, provides for easy dropping of unused keys + by number, as listed by the unused subcommand. + * fsck: Print warnings to stderr; --quiet can now be used to only see + problems. + + -- Joey Hess Mon, 15 Nov 2010 18:41:50 -0400 + +git-annex (0.06) unstable; urgency=low + + * fsck: Check if annex.numcopies is satisfied. + * fsck: Verify the sha1 of files when the SHA1 backend is used. + * fsck: Verify the size of files when the WORM backend is used. + * fsck: Allow specifying individual files if fscking everything + is not desired. + * fsck: Fix bug, introduced in 0.04, in detection of unused data. + + -- Joey Hess Sat, 13 Nov 2010 16:24:29 -0400 + +git-annex (0.05) unstable; urgency=low + + * Optimize both pre-commit and lock subcommands to not call git diff + on every file being committed/locked. + (This actually also works around a bug in ghc, that caused + git-annex 0.04 pre-commit to sometimes corrupt filename being read + from git ls-files and fail. + See + The excessive number of calls made by pre-commit exposed the ghc bug. + Thanks Josh Triplett for the debugging.) + * Build with -O2. + + -- Joey Hess Thu, 11 Nov 2010 18:31:09 -0400 + +git-annex (0.04) unstable; urgency=low + + * Add unlock subcommand, which replaces the symlink with a copy of + the file's content in preparation of changing it. The "edit" subcommand + is an alias for unlock. + * Add lock subcommand. + * Unlocked files will now automatically be added back into the annex when + committed (and the updated symlink committed), by some magic in the + pre-commit hook. + * The SHA1 backend is now fully usable. + * Add annex.version, which will be used to automate upgrades + between incompatible versions. + * Reorganised the layout of .git/annex/ + * The new layout will be automatically upgraded to the first time + git-annex is used in a repository with the old layout. + * Note that git-annex 0.04 cannot transfer content from old repositories + that have not yet been upgraded. + * Annexed file contents are now made unwritable and put in unwriteable + directories, to avoid them accidentially being removed or modified. + (Thanks Josh Triplett for the idea.) + * Add build dep on libghc6-testpack-dev. Closes: #603016 + * Avoid using runghc to run test suite as it is not available on all + architectures. Closes: #603006 + + -- Joey Hess Wed, 10 Nov 2010 14:23:23 -0400 + +git-annex (0.03) unstable; urgency=low + + * Fix support for file:// remotes. + * Add --verbose + * Fix SIGINT handling. + * Fix handling of files with unusual characters in their name. + * Fixed memory leak; git-annex no longer reads the whole file list + from git before starting, and will be much faster with large repos. + * Fix crash on unknown symlinks. + * Added remote.annex-scp-options and remote.annex-ssh-options. + * The backends to use when adding different sets of files can be configured + via gitattributes. + * In .gitattributes, the git-annex-backend attribute can be set to the + names of backends to use when adding different types of files. + * Add fsck subcommand. (For now it only finds unused key contents in the + annex.) + + -- Joey Hess Sun, 07 Nov 2010 18:26:04 -0400 + +git-annex (0.02) unstable; urgency=low + + * Can scp annexed files from remote hosts, and check remote hosts for + file content when dropping files. + * New move subcommand, that makes it easy to move file contents from + or to a remote. + * New fromkey subcommand, for registering urls, etc. + * git-annex init will now set up a pre-commit hook that fixes up symlinks + before they are committed, to ensure that moving symlinks around does not + break them. + * More intelligent and fast staging of modified files; git add coalescing. + * Add remote.annex-ignore git config setting to allow completly disabling + a given remote. + * --from/--to can be used to control the remote repository that git-annex + uses. + * --quiet can be used to avoid verbose output + * New plumbing-level dropkey and addkey subcommands. + * Lots of bug fixes. + + -- Joey Hess Wed, 27 Oct 2010 16:39:29 -0400 + +git-annex (0.01) unstable; urgency=low + + * First prerelease. + + -- Joey Hess Wed, 20 Oct 2010 12:54:24 -0400 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000000..ec635144f6 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000000..3cc796c573 --- /dev/null +++ b/debian/control @@ -0,0 +1,82 @@ +Source: git-annex +Section: utils +Priority: optional +Build-Depends: + debhelper (>= 9), + ghc (>= 7.4), + libghc-mtl-dev (>= 2.1.1), + libghc-missingh-dev, + libghc-hslogger-dev, + libghc-pcre-light-dev, + libghc-sha-dev, + libghc-dataenc-dev, + libghc-utf8-string-dev, + libghc-hs3-dev (>= 0.5.6), + libghc-dav-dev (>= 0.3) [amd64 i386 kfreebsd-amd64 kfreebsd-i386 sparc], + libghc-quickcheck2-dev, + libghc-monad-control-dev (>= 0.3), + libghc-lifted-base-dev, + libghc-uuid-dev, + libghc-json-dev, + libghc-ifelse-dev, + libghc-bloomfilter-dev, + libghc-edit-distance-dev, + libghc-hinotify-dev [linux-any], + libghc-stm-dev (>= 2.3), + libghc-dbus-dev (>= 0.10.3) [linux-any], + libghc-yesod-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-yesod-static-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-yesod-default-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-hamlet-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-clientsession-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-warp-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-wai-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-wai-logger-dev [i386 amd64 kfreebsd-i386 kfreebsd-amd64], + libghc-case-insensitive-dev, + libghc-http-types-dev, + libghc-transformers-dev, + libghc-blaze-builder-dev, + libghc-crypto-api-dev, + libghc-network-multicast-dev, + libghc-network-info-dev, + libghc-safesemaphore-dev, + libghc-network-protocol-xmpp-dev (>= 0.4.3-1+b1), + libghc-gnutls-dev (>= 0.1.4), + libghc-xml-types-dev, + libghc-async-dev, + ikiwiki, + perlmagick, + git, + uuid, + rsync, + openssh-client, +Maintainer: Joey Hess +Standards-Version: 3.9.3 +Vcs-Git: git://git.kitenet.net/git-annex +Homepage: http://git-annex.branchable.com/ + +Package: git-annex +Architecture: any +Section: utils +Depends: ${misc:Depends}, ${shlibs:Depends}, + git (>= 1:1.7.7.6), + uuid, + rsync, + wget, + curl, + openssh-client (>= 1:5.6p1) +Recommends: lsof, gnupg, bind9-host +Suggests: graphviz, bup, libnss-mdns +Description: manage files with git, without checking their contents into git + git-annex allows managing files with git, without checking the file + contents into git. While that may seem paradoxical, it is useful when + dealing with files larger than git can currently easily handle, whether due + to limitations in memory, time, or disk space. + . + Even without file content tracking, being able to manage files with git, + move files around and delete files with versioned directory trees, and use + branches and distributed clones, are all very handy reasons to use git. And + annexed files can co-exist in the same git repository with regularly + versioned files, which is convenient for maintaining documents, Makefiles, + etc that are associated with annexed files but that benefit from full + revision control. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000000..17c928f813 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,774 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Source: native package + +Files: * +Copyright: © 2010-2013 Joey Hess +License: GPL-3+ + +Files: Assistant/WebApp.hs Assistant/WebApp/* templates/* static/* +Copyright: © 2012 Joey Hess +License: AGPL-3+ + +Files: doc/logo* */favicon.ico standalone/osx/git-annex.app/Contents/Resources/git-annex.icns +Copyright: 2007 Henrik Nyh + 2010 Joey Hess +License: other + Free to modify and redistribute with due credit, and obviously free to use. + +Files: Utility/Mounts.hsc +Copyright: Volker Wysk +License: LGPL-2.1+ + +Files: Utility/libmounts.c +Copyright: 1980, 1989, 1993, 1994 The Regents of the University of California + 2001 David Rufino + 2012 Joey Hess +License: BSD-3-clause + * Copyright (c) 1980, 1989, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * Copyright (c) 2001 + * David Rufino + * Copyright 2012 + * Joey Hess + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + +Files: static/jquery* +Copyright: © 2005-2011 by John Resig, Branden Aaron & Jörn Zaefferer + © 2011 The Dojo Foundation +License: MIT or GPL-2 + The full text of version 2 of the GPL is distributed in + /usr/share/common-licenses/GPL-2 on Debian systems. The text of the MIT + license follows: + . + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + . + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Files: static/*/bootstrap* static/img/glyphicons-halflings* +Copyright: 2012 Twitter, Inc. +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + The complete text of the Apache License is distributed in + /usr/share/common-licenses/Apache-2.0 on Debian systems. + +License: GPL-3+ + The full text of version 3 of the GPL is distributed as doc/license/GPL in + this package's source, or in /usr/share/common-licenses/GPL-3 on + Debian systems. + +License: LGPL-2.1+ + The full text of version 2.1 of the LGPL is distributed as doc/license/LGPL + in this package's source, or in /usr/share/common-licenses/LGPL-2.1 + on Debian systems. + +License: AGPL-3+ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + . + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + . + Preamble + . + The GNU Affero General Public License is a free, copyleft license for + software and other kinds of works, specifically designed to ensure + cooperation with the community in the case of network server software. + . + The licenses for most software and other practical works are designed + to take away your freedom to share and change the works. By contrast, + our General Public Licenses are intended to guarantee your freedom to + share and change all versions of a program--to make sure it remains free + software for all its users. + . + When we speak of free software, we are referring to freedom, not + price. Our General Public Licenses are designed to make sure that you + have the freedom to distribute copies of free software (and charge for + them if you wish), that you receive source code or can get it if you + want it, that you can change the software or use pieces of it in new + free programs, and that you know you can do these things. + . + Developers that use our General Public Licenses protect your rights + with two steps: (1) assert copyright on the software, and (2) offer + you this License which gives you legal permission to copy, distribute + and/or modify the software. + . + A secondary benefit of defending all users' freedom is that + improvements made in alternate versions of the program, if they + receive widespread use, become available for other developers to + incorporate. Many developers of free software are heartened and + encouraged by the resulting cooperation. However, in the case of + software used on network servers, this result may fail to come about. + The GNU General Public License permits making a modified version and + letting the public access it on a server without ever releasing its + source code to the public. + . + The GNU Affero General Public License is designed specifically to + ensure that, in such cases, the modified source code becomes available + to the community. It requires the operator of a network server to + provide the source code of the modified version running there to the + users of that server. Therefore, public use of a modified version, on + a publicly accessible server, gives the public access to the source + code of the modified version. + . + An older license, called the Affero General Public License and + published by Affero, was designed to accomplish similar goals. This is + a different license, not a version of the Affero GPL, but Affero has + released a new version of the Affero GPL which permits relicensing under + this license. + . + The precise terms and conditions for copying, distribution and + modification follow. + . + TERMS AND CONDITIONS + . + 0. Definitions. + . + "This License" refers to version 3 of the GNU Affero General Public License. + . + "Copyright" also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + . + "The Program" refers to any copyrightable work licensed under this + License. Each licensee is addressed as "you". "Licensees" and + "recipients" may be individuals or organizations. + . + To "modify" a work means to copy from or adapt all or part of the work + in a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a "modified version" of the + earlier work or a work "based on" the earlier work. + . + A "covered work" means either the unmodified Program or a work based + on the Program. + . + To "propagate" a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + . + To "convey" a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through + a computer network, with no transfer of a copy, is not conveying. + . + An interactive user interface displays "Appropriate Legal Notices" + to the extent that it includes a convenient and prominently visible + feature that (1) displays an appropriate copyright notice, and (2) + tells the user that there is no warranty for the work (except to the + extent that warranties are provided), that licensees may convey the + work under this License, and how to view a copy of this License. If + the interface presents a list of user commands or options, such as a + menu, a prominent item in the list meets this criterion. + . + 1. Source Code. + . + The "source code" for a work means the preferred form of the work + for making modifications to it. "Object code" means any non-source + form of a work. + . + A "Standard Interface" means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that + is widely used among developers working in that language. + . + The "System Libraries" of an executable work include anything, other + than the work as a whole, that (a) is included in the normal form of + packaging a Major Component, but which is not part of that Major + Component, and (b) serves only to enable use of the work with that + Major Component, or to implement a Standard Interface for which an + implementation is available to the public in source code form. A + "Major Component", in this context, means a major essential component + (kernel, window system, and so on) of the specific operating system + (if any) on which the executable work runs, or a compiler used to + produce the work, or an object code interpreter used to run it. + . + The "Corresponding Source" for a work in object code form means all + the source code needed to generate, install, and (for an executable + work) run the object code and to modify the work, including scripts to + control those activities. However, it does not include the work's + System Libraries, or general-purpose tools or generally available free + programs which are used unmodified in performing those activities but + which are not part of the work. For example, Corresponding Source + includes interface definition files associated with source files for + the work, and the source code for shared libraries and dynamically + linked subprograms that the work is specifically designed to require, + such as by intimate data communication or control flow between those + subprograms and other parts of the work. + . + The Corresponding Source need not include anything that users + can regenerate automatically from other parts of the Corresponding + Source. + . + The Corresponding Source for a work in source code form is that + same work. + . + 2. Basic Permissions. + . + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program. The output from running a + covered work is covered by this License only if the output, given its + content, constitutes a covered work. This License acknowledges your + rights of fair use or other equivalent, as provided by copyright law. + . + You may make, run and propagate covered works that you do not + convey, without conditions so long as your license otherwise remains + in force. You may convey covered works to others for the sole purpose + of having them make modifications exclusively for you, or provide you + with facilities for running those works, provided that you comply with + the terms of this License in conveying all material for which you do + not control copyright. Those thus making or running the covered works + for you must do so exclusively on your behalf, under your direction + and control, on terms that prohibit them from making any copies of + your copyrighted material outside their relationship with you. + . + Conveying under any other circumstances is permitted solely under + the conditions stated below. Sublicensing is not allowed; section 10 + makes it unnecessary. + . + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + . + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article + 11 of the WIPO copyright treaty adopted on 20 December 1996, or + similar laws prohibiting or restricting circumvention of such + measures. + . + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention + is effected by exercising rights under this License with respect to + the covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's + users, your or third parties' legal rights to forbid circumvention of + technological measures. + . + 4. Conveying Verbatim Copies. + . + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; + keep intact all notices stating that this License and any + non-permissive terms added in accord with section 7 apply to the code; + keep intact all notices of the absence of any warranty; and give all + recipients a copy of this License along with the Program. + . + You may charge any price or no price for each copy that you convey, + and you may offer support or warranty protection for a fee. + . + 5. Conveying Modified Source Versions. + . + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the + terms of section 4, provided that you also meet all of these conditions: + . + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + . + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + . + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + . + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + . + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, + and which are not combined with it such as to form a larger program, + in or on a volume of a storage or distribution medium, is called an + "aggregate" if the compilation and its resulting copyright are not + used to limit the access or legal rights of the compilation's users + beyond what the individual works permit. Inclusion of a covered work + in an aggregate does not cause this License to apply to the other + parts of the aggregate. + . + 6. Conveying Non-Source Forms. + . + You may convey a covered work in object code form under the terms + of sections 4 and 5, provided that you also convey the + machine-readable Corresponding Source under the terms of this License, + in one of these ways: + . + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + . + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + . + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + . + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + . + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + . + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be + included in conveying the object code work. + . + A "User Product" is either (1) a "consumer product", which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, "normally used" refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + . + "Installation Information" for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + . + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as + part of a transaction in which the right of possession and use of the + User Product is transferred to the recipient in perpetuity or for a + fixed term (regardless of how the transaction is characterized), the + Corresponding Source conveyed under this section must be accompanied + by the Installation Information. But this requirement does not apply + if neither you nor any third party retains the ability to install + modified object code on the User Product (for example, the work has + been installed in ROM). + . + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access to a + network may be denied when the modification itself materially and + adversely affects the operation of the network or violates the rules and + protocols for communication across the network. + . + Corresponding Source conveyed, and Installation Information provided, + in accord with this section must be in a format that is publicly + documented (and with an implementation available to the public in + source code form), and must require no special password or key for + unpacking, reading or copying. + . + 7. Additional Terms. + . + "Additional permissions" are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall + be treated as though they were included in this License, to the extent + that they are valid under applicable law. If additional permissions + apply only to part of the Program, that part may be used separately + under those permissions, but the entire Program remains governed by + this License without regard to the additional permissions. + . + When you convey a copy of a covered work, you may at your option + remove any additional permissions from that copy, or from any part of + it. (Additional permissions may be written to require their own + removal in certain cases when you modify the work.) You may place + additional permissions on material, added by you to a covered work, + for which you have or can give appropriate copyright permission. + . + Notwithstanding any other provision of this License, for material you + add to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + . + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + . + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + . + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + . + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + . + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + . + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + . + All other non-permissive additional terms are considered "further + restrictions" within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further + restriction, you may remove that term. If a license document contains + a further restriction but permits relicensing or conveying under this + License, you may add to a covered work material governed by the terms + of that license document, provided that the further restriction does + not survive such relicensing or conveying. + . + If you add terms to a covered work in accord with this section, you + must place, in the relevant source files, a statement of the + additional terms that apply to those files, or a notice indicating + where to find the applicable terms. + . + Additional terms, permissive or non-permissive, may be stated in the + form of a separately written license, or stated as exceptions; + the above requirements apply either way. + . + 8. Termination. + . + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or + modify it is void, and will automatically terminate your rights under + this License (including any patent licenses granted under the third + paragraph of section 11). + . + However, if you cease all violation of this License, then your + license from a particular copyright holder is reinstated (a) + provisionally, unless and until the copyright holder explicitly and + finally terminates your license, and (b) permanently, if the copyright + holder fails to notify you of the violation by some reasonable means + prior to 60 days after the cessation. + . + Moreover, your license from a particular copyright holder is + reinstated permanently if the copyright holder notifies you of the + violation by some reasonable means, this is the first time you have + received notice of violation of this License (for any work) from that + copyright holder, and you cure the violation prior to 30 days after + your receipt of the notice. + . + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + . + 9. Acceptance Not Required for Having Copies. + . + You are not required to accept this License in order to receive or + run a copy of the Program. Ancillary propagation of a covered work + occurring solely as a consequence of using peer-to-peer transmission + to receive a copy likewise does not require acceptance. However, + nothing other than this License grants you permission to propagate or + modify any covered work. These actions infringe copyright if you do + not accept this License. Therefore, by modifying or propagating a + covered work, you indicate your acceptance of this License to do so. + . + 10. Automatic Licensing of Downstream Recipients. + . + Each time you convey a covered work, the recipient automatically + receives a license from the original licensors, to run, modify and + propagate that work, subject to this License. You are not responsible + for enforcing compliance by third parties with this License. + . + An "entity transaction" is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered + work results from an entity transaction, each party to that + transaction who receives a copy of the work also receives whatever + licenses to the work the party's predecessor in interest had or could + give under the previous paragraph, plus a right to possession of the + Corresponding Source of the work from the predecessor in interest, if + the predecessor has it or can get it with reasonable efforts. + . + You may not impose any further restrictions on the exercise of the + rights granted or affirmed under this License. For example, you may + not impose a license fee, royalty, or other charge for exercise of + rights granted under this License, and you may not initiate litigation + (including a cross-claim or counterclaim in a lawsuit) alleging that + any patent claim is infringed by making, using, selling, offering for + sale, or importing the Program or any portion of it. + . + 11. Patents. + . + A "contributor" is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The + work thus licensed is called the contributor's "contributor version". + . + A contributor's "essential patent claims" are all patent claims + owned or controlled by the contributor, whether already acquired or + hereafter acquired, that would be infringed by some manner, permitted + by this License, of making, using, or selling its contributor version, + but do not include claims that would be infringed only as a + consequence of further modification of the contributor version. For + purposes of this definition, "control" includes the right to grant + patent sublicenses in a manner consistent with the requirements of + this License. + . + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to + make, use, sell, offer for sale, import and otherwise run, modify and + propagate the contents of its contributor version. + . + In the following three paragraphs, a "patent license" is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To "grant" such a patent license to a + party means to make such an agreement or commitment not to enforce a + patent against the party. + . + If you convey a covered work, knowingly relying on a patent license, + and the Corresponding Source of the work is not available for anyone + to copy, free of charge and under the terms of this License, through a + publicly available network server or other readily accessible means, + then you must either (1) cause the Corresponding Source to be so + available, or (2) arrange to deprive yourself of the benefit of the + patent license for this particular work, or (3) arrange, in a manner + consistent with the requirements of this License, to extend the patent + license to downstream recipients. "Knowingly relying" means you have + actual knowledge that, but for the patent license, your conveying the + covered work in a country, or your recipient's use of the covered work + in a country, would infringe one or more identifiable patents in that + country that you have reason to believe are valid. + . + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties + receiving the covered work authorizing them to use, propagate, modify + or convey a specific copy of the covered work, then the patent license + you grant is automatically extended to all recipients of the covered + work and works based on it. + . + A patent license is "discriminatory" if it does not include within + the scope of its coverage, prohibits the exercise of, or is + conditioned on the non-exercise of one or more of the rights that are + specifically granted under this License. You may not convey a covered + work if you are a party to an arrangement with a third party that is + in the business of distributing software, under which you make payment + to the third party based on the extent of your activity of conveying + the work, and under which the third party grants, to any of the + parties who would receive the covered work from you, a discriminatory + patent license (a) in connection with copies of the covered work + conveyed by you (or copies made from those copies), or (b) primarily + for and in connection with specific products or compilations that + contain the covered work, unless you entered into that arrangement, + or that patent license was granted, prior to 28 March 2007. + . + Nothing in this License shall be construed as excluding or limiting + any implied license or other defenses to infringement that may + otherwise be available to you under applicable patent law. + . + 12. No Surrender of Others' Freedom. + . + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot convey a + covered work so as to satisfy simultaneously your obligations under this + License and any other pertinent obligations, then as a consequence you may + not convey it at all. For example, if you agree to terms that obligate you + to collect a royalty for further conveying from those to whom you convey + the Program, the only way you could satisfy both those terms and this + License would be to refrain entirely from conveying the Program. + . + 13. Remote Network Interaction; Use with the GNU General Public License. + . + Notwithstanding any other provision of this License, if you modify the + Program, your modified version must prominently offer all users + interacting with it remotely through a computer network (if your version + supports such interaction) an opportunity to receive the Corresponding + Source of your version by providing access to the Corresponding Source + from a network server at no charge, through some standard or customary + means of facilitating copying of software. This Corresponding Source + shall include the Corresponding Source for any work covered by version 3 + of the GNU General Public License that is incorporated pursuant to the + following paragraph. + . + Notwithstanding any other provision of this License, you have + permission to link or combine any covered work with a work licensed + under version 3 of the GNU General Public License into a single + combined work, and to convey the resulting work. The terms of this + License will continue to apply to the part which is the covered work, + but the work with which it is combined will remain governed by version + 3 of the GNU General Public License. + . + 14. Revised Versions of this License. + . + The Free Software Foundation may publish revised and/or new versions of + the GNU Affero General Public License from time to time. Such new versions + will be similar in spirit to the present version, but may differ in detail to + address new problems or concerns. + . + Each version is given a distinguishing version number. If the + Program specifies that a certain numbered version of the GNU Affero General + Public License "or any later version" applies to it, you have the + option of following the terms and conditions either of that numbered + version or of any later version published by the Free Software + Foundation. If the Program does not specify a version number of the + GNU Affero General Public License, you may choose any version ever published + by the Free Software Foundation. + . + If the Program specifies that a proxy can decide which future + versions of the GNU Affero General Public License can be used, that proxy's + public statement of acceptance of a version permanently authorizes you + to choose that version for the Program. + . + Later license versions may give you additional or different + permissions. However, no additional obligations are imposed on any + author or copyright holder as a result of your choosing to follow a + later version. + . + 15. Disclaimer of Warranty. + . + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + . + 16. Limitation of Liability. + . + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY + GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE + USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF + DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD + PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), + EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF + SUCH DAMAGES. + . + 17. Interpretation of Sections 15 and 16. + . + If the disclaimer of warranty and limitation of liability provided + above cannot be given local legal effect according to their terms, + reviewing courts shall apply local law that most closely approximates + an absolute waiver of all civil liability in connection with the + Program, unless a warranty or assumption of liability accompanies a + copy of the Program in return for a fee. + . + END OF TERMS AND CONDITIONS + . + How to Apply These Terms to Your New Programs + . + If you develop a new program, and you want it to be of the greatest + possible use to the public, the best way to achieve this is to make it + free software which everyone can redistribute and change under these terms. + . + To do so, attach the following notices to the program. It is safest + to attach them to the start of each source file to most effectively + state the exclusion of warranty; and each file should have at least + the "copyright" line and a pointer to where the full notice is found. + . + + Copyright (C) + . + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + . + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + . + Also add information on how to contact you by electronic and paper mail. + . + If your software can interact with users remotely through a computer + network, you should also make sure that it provides a way for users to + get its source. For example, if your program is a web application, its + interface could display a "Source" link that leads users to an archive + of the code. There are many ways you could offer source, and different + solutions will be better for different programs; see section 13 for the + specific requirements. + . + You should also get your employer (if you work as a programmer) or school, + if any, to sign a "copyright disclaimer" for the program, if necessary. + For more information on this, and how to apply and follow the GNU AGPL, see + . diff --git a/debian/doc-base b/debian/doc-base new file mode 100644 index 0000000000..f71a233333 --- /dev/null +++ b/debian/doc-base @@ -0,0 +1,9 @@ +Document: git-annex +Title: git-annex documentation +Author: Joey Hess +Abstract: All the documentation from git-annex's website. +Section: File Management + +Format: HTML +Index: /usr/share/doc/git-annex/html/index.html +Files: /usr/share/doc/git-annex/html/*.html diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000000..7155baded1 --- /dev/null +++ b/debian/rules @@ -0,0 +1,8 @@ +#!/usr/bin/make -f + +%: + dh $@ + +# Not intended for use by anyone except the author. +announcedir: + @echo ${HOME}/src/git-annex/doc/news diff --git a/doc/assistant.mdwn b/doc/assistant.mdwn new file mode 100644 index 0000000000..0032698176 --- /dev/null +++ b/doc/assistant.mdwn @@ -0,0 +1,64 @@ +The git-annex assistant creates a folder on each of your computers, +removable drives, and cloud services, which +it keeps synchronised, so its contents are the same everywhere. +It's very easy to use, and has all the power of git and git-annex. + +## installation + +The git-annex assistant comes as part of git-annex, starting with version +3.20120924. See [[install]] to get it installed. + +Note that the git-annex assistant is still beta quality code. See +the [[release_notes]] for known infelicities and upgrade instructions. + +## quick start + +To get started with the git-annex assistant, just pick it from +your system's list of applications. + +[[!img assistant/menu.png]] +[[!img assistant/osx-app.png]] + +It'll prompt you to set up a folder: + +[[!img assistant/makerepo.png]] + +Then any changes you make to its folder will automatically be committed to +git, and synced to repositories on other computers. You can use the +interface to add repositories and control the git-annex assistant. + +[[!img assistant/running.png]] + +## documentation + +* Want to make two nearby computers share the same synchronised folder? + Follow the [[pairing_walkthrough]]. +* Want to share a synchronised folder with a friend? + Follow the [[share_with_a_friend_walkthrough]]. +* Want to archive data to a drive or the cloud? + Follow the [[archival_walkthrough]] + +## command line startup + +The git-annex assistant will automatically be started when you log in to +desktop environments like Mac OS X, Gnome, XFCE, and KDE, and the menu item +shown above can be used to open the webapp. On other systems, you may need +to start it by hand. + +To start the webapp, run `git annex webapp` at the command line. + +To start the assistant without opening the webapp, +you can run the command "git annex assistant --autostart". This is a +good thing to configure your system to run automatically when you log in. + +## colophon + +The git-annex assistant is being +[crowd funded on +Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/). +[[/assistant/Thanks]] to all my backers. + +I blog about my work on the git-annex assistant on a daily basis +in [[this_blog|design/assistant/blog]]. Follow along! + +See also: The [[design|/design/assistant]] pages. diff --git a/doc/assistant/addsshserver.png b/doc/assistant/addsshserver.png new file mode 100644 index 0000000000..80f5d56173 Binary files /dev/null and b/doc/assistant/addsshserver.png differ diff --git a/doc/assistant/android/appinstalled.png b/doc/assistant/android/appinstalled.png new file mode 100644 index 0000000000..166120d482 Binary files /dev/null and b/doc/assistant/android/appinstalled.png differ diff --git a/doc/assistant/android/install.png b/doc/assistant/android/install.png new file mode 100644 index 0000000000..882455df16 Binary files /dev/null and b/doc/assistant/android/install.png differ diff --git a/doc/assistant/android/terminal.png b/doc/assistant/android/terminal.png new file mode 100644 index 0000000000..9afa2720fa Binary files /dev/null and b/doc/assistant/android/terminal.png differ diff --git a/doc/assistant/archival_walkthrough.mdwn b/doc/assistant/archival_walkthrough.mdwn new file mode 100644 index 0000000000..0e3ddcc6c0 --- /dev/null +++ b/doc/assistant/archival_walkthrough.mdwn @@ -0,0 +1,28 @@ +Normally, the git-annex assistant makes your files be available +wherever you use it, and so a copy of each file is stored in each repository. +That's perfect for files you're using right now, but what about files you're +not using any more? + +You could just delete those files, but it's better to archive them, so +you can access them later. All you need to get started archiving your old +files is a USB drive, or an [Amazon Glacier](http://aws.amazon.com/glacier/) +account. + +The webapp makes it easy to make a repository on either a USB drive, +or on Amazon Glacier. Once the repository is created, be sure to +put it in either the small archive, or full archive repository group. + +[[!img repogroup.png]] + +Now when you're done with a file, just move it into a directory named +"archive". The assistant will notice you put it there, and next time it +has the opportunity (when you plug in the USB drive, or when it can +talk to Amazon Glacier over the network), will move the file's +content to your archive repository. + +You'll no longer be able to open the file once it's been archived. +If you later want to access it, you can just copy or move it out +of the archive directory, and the assistant will retrieve its +content from the archive. + +Note that retrieving data from Amazon Glacier takes 4 to 5 hours. diff --git a/doc/assistant/buddylist.png b/doc/assistant/buddylist.png new file mode 100644 index 0000000000..40b5a92382 Binary files /dev/null and b/doc/assistant/buddylist.png differ diff --git a/doc/assistant/combinerepos.png b/doc/assistant/combinerepos.png new file mode 100644 index 0000000000..7beea71bc5 Binary files /dev/null and b/doc/assistant/combinerepos.png differ diff --git a/doc/assistant/controlmenu.png b/doc/assistant/controlmenu.png new file mode 100644 index 0000000000..c1dac197b9 Binary files /dev/null and b/doc/assistant/controlmenu.png differ diff --git a/doc/assistant/crashrecovery.png b/doc/assistant/crashrecovery.png new file mode 100644 index 0000000000..fce3805d0f Binary files /dev/null and b/doc/assistant/crashrecovery.png differ diff --git a/doc/assistant/logs.png b/doc/assistant/logs.png new file mode 100644 index 0000000000..8e9e5b1ec8 Binary files /dev/null and b/doc/assistant/logs.png differ diff --git a/doc/assistant/makerepo.png b/doc/assistant/makerepo.png new file mode 100644 index 0000000000..e8f1b26216 Binary files /dev/null and b/doc/assistant/makerepo.png differ diff --git a/doc/assistant/menu.png b/doc/assistant/menu.png new file mode 100644 index 0000000000..29a7c9d0f8 Binary files /dev/null and b/doc/assistant/menu.png differ diff --git a/doc/assistant/osx-app.png b/doc/assistant/osx-app.png new file mode 100644 index 0000000000..386fb832db Binary files /dev/null and b/doc/assistant/osx-app.png differ diff --git a/doc/assistant/pairing_walkthrough.mdwn b/doc/assistant/pairing_walkthrough.mdwn new file mode 100644 index 0000000000..07b6399104 --- /dev/null +++ b/doc/assistant/pairing_walkthrough.mdwn @@ -0,0 +1,60 @@ +So you have two computers in the same building, and you want them to share +the same synchronised folder, communicating directly with each other. + +This is incredibly easy to set up with the git annex assistant. + +Let's say the two computers are your computer and your friend's computer. +We'll start on your computer, where you open up your git annex dashboard. + +[[!img addrepository.png alt="Add another repository button"]] + +`*click*` + +[[!img pairing.png alt="Pair with another computer"]] + +`*click*` + +Now the hard bit. You have to think up a secret phrase, and type it in, +(and perhaps get the spelling correct). + +[[!img secret.png alt="Enter secret phrase"]] + +Now your computer is in pairing mode. When your friend looks at her git +annex dashboard, she sees something like this. + +[[!img pairrequest.png alt="Pair request"]] + +`*click*` + +[[!img secretempty.png alt="Enter same secret phrase"]] + +Now it's up to you to let her know what the secret is. As soon as she +enters it, both your computers will be paired, and will begin to sync their +git-annex folders. Just like that you can share files. + +---- + +Something to keep in mind, especially if pairing doesn't seem to be +working, is that the two computers need to be on the same network for this +pairing process to work. Sometimes a building will have more than one +network inside it, and you'll need to connect them both to the same one. +Make sure the wireless network name is the same, or that they're both +plugged into the same router. + +Also, the file sharing set up by this pairing only works when both +computers are on the same network. If you go on a trip, any files you +edit will not be visible to your friend until you get back. + +To get around this, you'll often also want to set up +[[jabber_pairing|share_with_a_friend_walkthrough]], and a server +in the cloud, which they can use to exchange files while away. + +And also, you can pair with as many other computers as you like, not just +one! + +## What does pairing actually do behind the scenes? + +It ensures that both repositories have correctly configured +[[remotes|walkthrough/adding_a_remote]] pointing to each other. +If you have already configured this manually, you do not need to +perform pairing. diff --git a/doc/assistant/pairing_walkthrough/addrepository.png b/doc/assistant/pairing_walkthrough/addrepository.png new file mode 100644 index 0000000000..b82efdbea2 Binary files /dev/null and b/doc/assistant/pairing_walkthrough/addrepository.png differ diff --git a/doc/assistant/pairing_walkthrough/comment_1_b33deed054d3aa8cfa6c9e3958643f16._comment b/doc/assistant/pairing_walkthrough/comment_1_b33deed054d3aa8cfa6c9e3958643f16._comment new file mode 100644 index 0000000000..52bdc7f4ea --- /dev/null +++ b/doc/assistant/pairing_walkthrough/comment_1_b33deed054d3aa8cfa6c9e3958643f16._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://launchpad.net/~mattharrison" + ip="69.193.72.98" + subject="Pairing" + date="2012-10-29T17:56:06Z" + content=""" +Is there a way to do this pairing with one machine being a desktop machine and one a headless machine? +"""]] diff --git a/doc/assistant/pairing_walkthrough/comment_2_39f1162b4d43b61e957e7497df4b9e2b._comment b/doc/assistant/pairing_walkthrough/comment_2_39f1162b4d43b61e957e7497df4b9e2b._comment new file mode 100644 index 0000000000..32d56c907c --- /dev/null +++ b/doc/assistant/pairing_walkthrough/comment_2_39f1162b4d43b61e957e7497df4b9e2b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.194" + subject="easy peasy" + date="2012-10-29T19:09:43Z" + content=""" +A headless machine is probably a server. On the same page that has the \"local computer\" link, there's a \"remote server\" link that'll get you set up. +"""]] diff --git a/doc/assistant/pairing_walkthrough/pairing.png b/doc/assistant/pairing_walkthrough/pairing.png new file mode 100644 index 0000000000..dfbbf0641a Binary files /dev/null and b/doc/assistant/pairing_walkthrough/pairing.png differ diff --git a/doc/assistant/pairing_walkthrough/pairrequest.png b/doc/assistant/pairing_walkthrough/pairrequest.png new file mode 100644 index 0000000000..8d3f603bf4 Binary files /dev/null and b/doc/assistant/pairing_walkthrough/pairrequest.png differ diff --git a/doc/assistant/pairing_walkthrough/secret.png b/doc/assistant/pairing_walkthrough/secret.png new file mode 100644 index 0000000000..1eb8051222 Binary files /dev/null and b/doc/assistant/pairing_walkthrough/secret.png differ diff --git a/doc/assistant/pairing_walkthrough/secretempty.png b/doc/assistant/pairing_walkthrough/secretempty.png new file mode 100644 index 0000000000..4918787845 Binary files /dev/null and b/doc/assistant/pairing_walkthrough/secretempty.png differ diff --git a/doc/assistant/preferences.png b/doc/assistant/preferences.png new file mode 100644 index 0000000000..74fb814183 Binary files /dev/null and b/doc/assistant/preferences.png differ diff --git a/doc/assistant/release_notes.mdwn b/doc/assistant/release_notes.mdwn new file mode 100644 index 0000000000..aa79cd9b74 --- /dev/null +++ b/doc/assistant/release_notes.mdwn @@ -0,0 +1,232 @@ +## version 4.20130227 + +This release fixes a bug with globbing that broke preferred content expressions. +So, it is a recommended upgrade from the previous release, which introduced +that bug. + +In this release, the assistant is fully working on Android, although +it must be set up using the command line. + +Repositories can now be placed on filesystems that lack support for symbolic +links; FAT support is complete. + +## version 3.20130216 + +This adds a port to Android. Only usable at the command line so far; +beta qualitty. + +Also a bugfix release, and improves support for FAT. + +The following are known limitations of this release of the git-annex +assistant: + +* No Android app yet. +* On BSD operating systems (but not on OS X), the assistant uses kqueue to + watch files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[this_bug|bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. +* Also on systems with kqueue, modifications to existing files in direct + mode will not be noticed. + +## version 3.20130107, 3.20130114, 3.20130124, 3.20130207 + +These are bugfix releases. + +## version 3.20130102 + +This release makes several significant improvements to the git-annex +assistant, which is still in beta. + +The main improvement is direct mode. This allows you to directly edit files +in the repository, and the assistant will automatically commit and sync +your changes. Direct mode is the default for new repositories created +by the assistant. To convert your existing repository to use direct mode, +manually run `git annex direct` inside the repository. + +## version 3.20121211 + +This release of the git-annex assistant (which is still in beta) +consists of mostly bugfixes, user interface improvements, and improvements +to existing features. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* Using Box.com's 5 gigabytes of free storage space as a cloud transfer + point between between repositories that cannot directly contact + one-another. (Many other cloud providers are also supported, from Rsync.net + to Amazon S3, to your own ssh server.) +* Archiving or backing up files to Amazon Glacier. See [[archival_walkthrough]]. +* [[Sharing repositories with friends|share_with_a_friend_walkthrough]] + contacted through a Jabber server (such as Google Talk). +* [[Pairing|pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. + +The following are known limitations of this release of the git-annex +assistant: + +* The Max OSX standalone app may not work on all versions of Max OSX. + Please test! +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. + +## version 3.20121126 + +This adds several features to the git-annex assistant, which is still in beta. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* Using Box.com's 5 gigabytes of free storage space as a cloud transfer + point between between repositories that cannot directly contact + one-another. (Many other cloud providers are also supported, from Rsync.net + to Amazon S3, to your own ssh server.) +* Archiving or backing up files to Amazon Glacier. +* [[Sharing repositories with friends|share_with_a_friend_walkthrough]] + contacted through a Jabber server (such as Google Talk). +* [[Pairing|pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. + +The following are known limitations of this release of the git-annex +assistant: + +* The Max OSX standalone app does not work on all versions of Max OSX. +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. +* Retrieval of files from Amazon Glacier is not fully automated; the + assistant does not automatically retry in the 4 to 5 hours period + when Glacier makes the files available. + +## version 3.20121112 + +This is a major upgrade of the git-annex assistant, which is still in beta. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* [[Sharing repositories with friends|share_with_a_friend_walkthrough]] + contacted through a Jabber server (such as Google Talk). +* Setting up cloud repositories, that are used as backups, archives, + or transfer points between repositories that cannot directly contact + one-another. +* [[Pairing|pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. + +The following upgrade notes apply if you're upgrading from a previous version: + +* For best results, edit the configuration of repositories you set + up with older versions, and place them in a repository group. + This lets the assistant know how you want to use the repository; for backup, + archival, as a transfer point for clients, etc. Go to Configuration -> + Manage Repositories, and click in the "configure" link to edit a repository's + configuration. +* If you set up a cloud repository with an older version, and have multiple + clients using it, you are recommended to configure an Jabber account, + so that clients can use it to communicate when sending data to the + cloud repository. Configure Jabber by opening the webapp, and going to + Configuration -> Configure jabber account +* When setting up local pairing, the assistant did not limit the paired + computer to accessing a single git repository. This new version does, + by setting GIT_ANNEX_SHELL_DIRECTORY in `~/.ssh/authorized_keys`. + +The following are known limitations of this release of the git-annex +assistant: + +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. + +## version 3.20121009 + +This is a maintenance release of the git-annex assistant, which is still in +beta. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* [[Pairing|pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. +* Cloning your repository to a remote server, running ssh, and uploading + changes made to your files to the server. There is special support + for using the rsync.net cloud provider this way, or any shell account + on a typical unix server, such as a Linode VPS can be used. + +The following are known limitations of this release of the git-annex +assistant: + +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. +* In order to ensure that all multiple repositories are kept in sync, + each computer with a repository must be running the git-annex assistant. +* The assistant does not yet always manage to keep repositories in sync + when some are hidden from others behind firewalls. + +## version 3.20120924 + +This is the first beta release of the git-annex assistant. + +In general, anything you can configure with the assistant's web app +will work. Some examples of use cases supported by this release include: + +* [[Pairing|pairing_walkthrough]] two computers that are on the same local + network (or VPN) and automatically keeping the files in the annex in + sync as changes are made to them. +* Cloning your repository to removable drives, USB keys, etc. The assistant + will notice when the drive is mounted and keep it in sync. + Such a drive can be stored as an offline backup, or transported between + computers to keep them in sync. +* Cloning your repository to a remote server, running ssh, and uploading + changes made to your files to the server. There is special support + for using the rsync.net cloud provider this way, or any shell account + on a typical unix server, such as a Linode VPS can be used. + +The following are known limitations of this release of the git-annex +assistant: + +* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch + files. Kqueue has to open every directory it watches, so too many + directories will run it out of the max number of open files (typically + 1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]] + for a workaround. +* In order to ensure that all multiple repositories are kept in sync, + each computer with a repository must be running the git-annex assistant. +* The assistant does not yet always manage to keep repositories in sync + when some are hidden from others behind firewalls. +* If a file is checked into git as a normal file and gets modified + (or merged, etc), it will be converted into an annexed file. So you + should not mix use of the assistant with normal git files in the same + repository yet. +* If you `git annex unlock` a file, it will immediately be re-locked. + See [[bugs/watcher_commits_unlocked_files]]. diff --git a/doc/assistant/release_notes/comment_1_bd8f376c9d0c1d5ed07fb013907a60ee._comment b/doc/assistant/release_notes/comment_1_bd8f376c9d0c1d5ed07fb013907a60ee._comment new file mode 100644 index 0000000000..04cdf4039d --- /dev/null +++ b/doc/assistant/release_notes/comment_1_bd8f376c9d0c1d5ed07fb013907a60ee._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://wiggy.net/" + nickname="Wichert" + subject="OSX 10.8's gatekeeper does not like git-annex" + date="2012-11-13T14:47:52Z" + content=""" +Trying to run this release on OSX results in an error message from Gatekeeper: + +> \"git-annex\" can't be opened because it is from an unidentified developer. +> +> Your security preferences allow installation of only apps from the Mac App Store and identified developers. + +It would be nice if the binary could be signed to make Gatekeeper happy. Until then a note in the installation instructions might be useful. +"""]] diff --git a/doc/assistant/release_notes/comment_2_75e0774ad042717fbd059a8a9ec2db1e._comment b/doc/assistant/release_notes/comment_2_75e0774ad042717fbd059a8a9ec2db1e._comment new file mode 100644 index 0000000000..9033b1af6d --- /dev/null +++ b/doc/assistant/release_notes/comment_2_75e0774ad042717fbd059a8a9ec2db1e._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://wiggy.net/" + nickname="Wichert" + subject="git-annex not runnable on OSX 10.8" + date="2012-11-13T14:49:35Z" + content=""" +After telling Gatekeeper that I really want to run git-annex it still fails: + +[fog;~]-131> open Applications/git-annex.app +LSOpenURLsWithRole() failed with error -10810 for the file /Users/wichert/Applications/git-annex.app. + +"""]] diff --git a/doc/assistant/release_notes/comment_3_b3bfd8e547e20c51f7c32c6c9424e936._comment b/doc/assistant/release_notes/comment_3_b3bfd8e547e20c51f7c32c6c9424e936._comment new file mode 100644 index 0000000000..73377c714f --- /dev/null +++ b/doc/assistant/release_notes/comment_3_b3bfd8e547e20c51f7c32c6c9424e936._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 3" + date="2012-11-13T17:11:51Z" + content=""" +This has been previously reported: [[bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole()]] + +No clue what that error is supposed to mean. +"""]] diff --git a/doc/assistant/release_notes/comment_4_c6caa2b521b456bb4ce594d64919cffe._comment b/doc/assistant/release_notes/comment_4_c6caa2b521b456bb4ce594d64919cffe._comment new file mode 100644 index 0000000000..1ebaf504f8 --- /dev/null +++ b/doc/assistant/release_notes/comment_4_c6caa2b521b456bb4ce594d64919cffe._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2012-11-16T13:02:40Z" + content=""" +sadly i only have a 10.7 machine to create the builds, so I have no experience with 10.8. I haven't had a 10.6 machine in a while to create the builds. Anyone else want to work together in setting up another 10.6 or 10.8 builder for others? +"""]] diff --git a/doc/assistant/repogroup.png b/doc/assistant/repogroup.png new file mode 100644 index 0000000000..ac5aabd89f Binary files /dev/null and b/doc/assistant/repogroup.png differ diff --git a/doc/assistant/repogroups.png b/doc/assistant/repogroups.png new file mode 100644 index 0000000000..441a75084c Binary files /dev/null and b/doc/assistant/repogroups.png differ diff --git a/doc/assistant/repositories.png b/doc/assistant/repositories.png new file mode 100644 index 0000000000..2853683dca Binary files /dev/null and b/doc/assistant/repositories.png differ diff --git a/doc/assistant/rsync.net.png b/doc/assistant/rsync.net.png new file mode 100644 index 0000000000..0e09407035 Binary files /dev/null and b/doc/assistant/rsync.net.png differ diff --git a/doc/assistant/running.png b/doc/assistant/running.png new file mode 100644 index 0000000000..8c3b0eaf24 Binary files /dev/null and b/doc/assistant/running.png differ diff --git a/doc/assistant/share_with_a_friend_walkthrough.mdwn b/doc/assistant/share_with_a_friend_walkthrough.mdwn new file mode 100644 index 0000000000..a748ebdce7 --- /dev/null +++ b/doc/assistant/share_with_a_friend_walkthrough.mdwn @@ -0,0 +1,58 @@ +Want to share all the files in your repository with a friend? + +Let's suppose you use Google Mail, and so does your friend, and you +sometimes also chat in Google Talk. The git-annex assistant will +use your Google account to share with your friend. (This actually +works with any Jabber account you use, not just Google Talk.) + +Start by opening up your git annex dashboard. + +[[!img pairing_walkthrough/addrepository.png alt="Add another repository button"]] + +`*click*` + +[[!img pairing.png alt="Share with a friend"]] + +`*click*` + +[[!img xmpp.png alt="Configuring Jabber account"]] + +Fill that out, and git-annex will be able to show you a list of your +friends. + +[[!img buddylist.png alt="Buddy list"]] + +This list will refresh as friends log on and off, so you can +leave it open in a tab until a friend is available to start pairing. + +(If your friend is not using git-annex yet, now's a great time to spread +the word!) + +Once you click on "Start Pairing", your friend will see this pop up +on their git annex dashboard. + +[[!img xmppalert.png alt="Pair request"]] + +Once your friend clicks on that, your repositories will be paired. + +### But, wait, there's one more step... + +Despite the repositories being paired now, you and your friend can't yet +quite share files. You'll start to see your friend's files show up in your +git-annex folder, but you won't be able to open them yet. + +What you need to do now is set up a repository out there in the cloud, +that both you and your friend can access. This will be used to transfer +files between the two of you. + +At the end of the pairing process, a number of cloud providers are +suggested, and the git-annex assistant makes it easy to configure one of +them. Once you or your friend sets it up, it'll show up in the other +one's list of repositories: + +[[!img repolist.png alt="Repository list"]] + +The final step is to share the login information for the cloud repository +with your friend, so they can enable it too. + +With that complete, you'll be able to open your friend's files! diff --git a/doc/assistant/share_with_a_friend_walkthrough/buddylist.png b/doc/assistant/share_with_a_friend_walkthrough/buddylist.png new file mode 100644 index 0000000000..ce3d61a966 Binary files /dev/null and b/doc/assistant/share_with_a_friend_walkthrough/buddylist.png differ diff --git a/doc/assistant/share_with_a_friend_walkthrough/pairing.png b/doc/assistant/share_with_a_friend_walkthrough/pairing.png new file mode 100644 index 0000000000..533f4aef5e Binary files /dev/null and b/doc/assistant/share_with_a_friend_walkthrough/pairing.png differ diff --git a/doc/assistant/share_with_a_friend_walkthrough/repolist.png b/doc/assistant/share_with_a_friend_walkthrough/repolist.png new file mode 100644 index 0000000000..409da4aa4c Binary files /dev/null and b/doc/assistant/share_with_a_friend_walkthrough/repolist.png differ diff --git a/doc/assistant/share_with_a_friend_walkthrough/xmppalert.png b/doc/assistant/share_with_a_friend_walkthrough/xmppalert.png new file mode 100644 index 0000000000..5e2d562895 Binary files /dev/null and b/doc/assistant/share_with_a_friend_walkthrough/xmppalert.png differ diff --git a/doc/assistant/thanks.mdwn b/doc/assistant/thanks.mdwn new file mode 100644 index 0000000000..cd7f80b298 --- /dev/null +++ b/doc/assistant/thanks.mdwn @@ -0,0 +1,239 @@ +The development of the git-annex assistant was made possible by the +generous donations of many people. I want to say "Thank You!" to each of +you individually, but until I meet all 951 of you, this page will have to +do. You have my most sincere thanks. --[[Joey]] + +(If I got your name wrong, or you don't want it publically posted here, +email .) + +## Major Backers + +These people are just inspiring in their enthusiasm and generosity to this +project. + +* Jason Scott +* strager + +## Beta Testers + +Whole weeks of my time were made possible thanks to each of these +people, and their testing is invaluable to the development of +the git-annex assistant. + +* Jimmy Tang +* David Pollak +* Pater +* Francois Marier +* Paul Sherwood +* Fred Epma +* Robert Ristroph +* Josh Triplett +* David Haslem +* AJ Ashton +* Svenne Krap +* Drew Hess +* Peter van Westen + +## Prioritizers + +These forward-thinking people contributed generously just to help +set my priorities in which parts of the git-annex assistant were most +important to develop. + +Paul C. Bryan, Paul Tötterman, Don Marti, Dean Thompson, Djoume, David Johnston +Asokan Pichai, Anders Østhus, Dominik Wagenknecht, Charlie Fox, Yazz D. Atlas, +fenchel, Erik Penninga, Richard Hartmann, Graham, Stan Yamane, Ben Skelton, +Ian McEwen, asc, Paul Tagliamonte, Sherif Abouseda, Igor Támara, Anne Wind, +Mesar Hameed, Brandur K. Holm Petersen, Takahiro Inoue, Kai Hendry, +Stephen Youndt, Lee Roberson, Ben Strawbridge, Andrew Greenberg, Alfred Adams +Andrew, Aaron De Vries, Monti Knazze, Jorge Canseco, Hamish, Mark Eichin, +Sherif Abouseda, Ben Strawbridge, chee rabbits, Pedro Côrte-Real + +And special thanks to Kevin McKenzie, who also gave me a login to a Mac OSX +machine, which has proven invaluable, and Jimmy Tang who has helped +with Mac OSX autobuilding and packaging. + +## Other Backers + +Most of the success of the Kickstarter is thanks to these folks. Some of +them spent significant amounts of money in the guise of getting some +swag. For others, being listed here, and being crucial to making the +git-annex assistant happen was reward enough. Large or small, these +contributions are, literally, my bread and butter this year. + +Amitai Schlair, mvime, Romain Lenglet, James Petts, Jouni Uuksulainen, +Wichert Akkerman, Robert Bellus, Kasper Souren, rob, Michiel Buddingh', +Kevin, Rob Avina, Alon Levy, Vikash, Michael Alan Dorman, Harley Pig, +Andreas Olsson, Pietpiet, Christine Spang, Liz Young, Oleg Kosorukov, +Allard Hoeve, Valentin Haenel, Joost Baaij, Nathan Yergler, Nathan Howell, +Frédéric Schütz, Matti Eskelinen, Neil McGovern, Lane Lillquist, db48x, +Stuart Prescott, Mark Matienzo, KarlTheGood, leonm, Drew Slininger, +Andreas Fuchs, Conrad Parker, Johannes Engelke, Battlegarden, Justin Kelly, +Robin Wagner, Thad Ward, crenquis, Trudy Goold, Mike Cochrane, Adam Venturella, +Russell Foo, furankupan, Giorgio Occhioni, andy, mind, Mike Linksvayer, +Stefan Strahl, Jelmer Vernooij, Markus Fix, David Hicks, Justin Azoff, +Iain Nicol, Bob Ippolito, Thomas Lundstrøm, Jason Mandel, federico2, +Edd Cochran, Jose Ortega, Emmett Agnew, Rudy Garcia, Kodi, Nick Rusnov, +Michael Rubin, Tom de Grunt, Richard Murray, Peter, Suzanne Pierce, Jared +Marcotte, folk, Eamon, Jeff Richards, Leo Sutedja, dann frazier, Mikkel +kristiansen, Matt Thomas, Kilian Evang, Gergely Risko, Kristian Rumberg, +Peter Kropf, Mark Hepburn, greymont, D. Joe Anderson, Jeremy Zunker, ctebo, +Manuel Roumain, Jason Walsh, np, Shawn, Johan Tibell, Branden Tyree, Dinyar +Rabady, Andrew Mason, damond armstead, Ethan Aubin, TomTom Tommie, Jimmy +Kaplowitz, Steven Zakulec, mike smith, Jacob Kirkwood, Mark Hymers, Nathan +Collins, Asbjørn Sloth Tønnesen, Misty De Meo, James Shubin, +Jim Paris, Adam Sjøgren, miniBill, Taneli, Kumar Appaiah, Greg Grossmeier, +Sten Turpin, Otavio Salvador, Teemu Hukkanen, Brian Stengaard, bob walker, +bibeneus, andrelo, Yaroslav Halchenko, hesfalling, Tommy L, jlargentaye, +Serafeim Zanikolas, Don Armstrong, Chris Cormack, shayne.oneill, Radu +Raduta, Josh S, Robin Sheat, Henrik Mygind, kodx, Christian, Geoff +Crompton, Brian May, Olivier Berger, Filippo Gadotti, Daniel Curto-Millet, +Eskild Hustvedt, Douglas Soares de Andrade, Tom L, Michael Nacos, Michaël +P., William Roe, Joshua Honeycutt, Brian Kelly, Nathan Rasch, jorge, Martin +Galese, alex cox, Avery Brooks, David Whittington, Dan Martinez, Forrest +Sutton, Jouni K. Seppänen, Arnold Cano, Robert Beaty, Daniel, Kevin Savetz, +Randy, Ernie Braganza, Aaron Haviland, Brian Brunswick, asmw, sean, Michael +Williams, Alexander, Dougal Campbell, Robert Bacchas, Michael Lewis, Collin +Price, Wes Frazier, Matt Wall, Brandon Barclay, Derek van Vliet, Martin +DeMello, kitt hodsden, Stephen Kitt, Leif Bergman, Simon Lilburn, Michael +Prokop, Christiaan Conover, Nick Coombe, Tim Dysinger, Brandon Robinson, +Philip Newborough, keith, Mike Fullerton, Kyle, Phil Windley, Tyler Head, +George V. Reilly, Matthew, Ali Gündüz, Vasyl Diakonov, Paolo Capriotti, +allanfranta, Martin Haeberli, msingle, Vincent Sanders, Steven King, Dmitry +Gribanov, Brandon High, Ben Hughes, Mike Dank, JohnE, Diggory Hardy, +Michael Hanke, valhalla, Samuli Tuomola, Jeff Rau, Benjamin Lebsanft, John +Drago, James, Aidan Cooper, rondie, Paul Kohler, Matthew Knights, Aaron +Junod, Patrick R McDonald, Christopher Browne, Daniel Engel, John SJ +Anderson, Peter Sarossy, Mike Prasad, Christoph Ender, Jan Dittberner, +Zohar, Alexander Jelinek, stefan, Danny O'Brien, Matthew Thode, Nicole +Aptekar, maurice gaston, Chris Adams, Mike Klemencic, Reedy, Subito, Tobias +Gruetzmacher, Ole-Morten Duesund, André Koot, mp, gdop, Cole Scott, Blaker, +Matt Sottile, W. Craig Trader, Louis-Philippe Dubrule, Brian Boucheron, +Duncan Smith, Brenton Buchbach, Kyle Stevenson, Eliot Lash, Egon Elbre, +Praveen, williamji, Thomas Schreiber, Neil Ford, Ryan Pratt, Joshua Brand, +Peter Cox, Markus Engstrom, John Sutherland, Dean Bailey, Ed Summers, +Hillel Arnold, David Fiander, Kurt Yoder, Trevor Muñoz, keri, Ivan +Sergeyenko, Shad Bolling, Tal Kelrich, Steve Presley, gerald ocker, Essex +Taylor, Josh Thomson, Trevor Bramble, Lance Higgins, Frank Motta, Dirk +Kraft, soundray, Joe Haskins, nmjk, Apurva Desai, Colin Dean, docwhat, +Joseph Costello, Jst, flamsmark, Alex Lang, Bill Traynor, Anthony David, +Marc-André Lureau, AlNapp, Giovanni Moretti, John Lawrence, João Paulo +Pizani Flor, Jim Ray, Gregory Thrush, Alistair McGann, Andrew Wied, +Koutarou Furukawa, Xiscu Ignacio, Aaron Sachs, Matt, Quirijn, Chet +Williams, Chris Gray, Bruce Segal, Tom Conder, Louis Tovar, Alex Duryee, +booltox, d8uv, Decklin Foster, Rafael Cervantes, Micah R Ledbetter, Kevin +Sjöberg, Johan Strombom, Zachary Cohen, Jason Lewis, Yves Bilgeri, Ville +Aine, Mark Hurley, Marco Bonetti, Maximilian Haack, Hynek Schlawack, +Michael Leinartas, Andreas Liebschner, Duotrig, Nat Fairbanks, David +Deutsch, Colin Hayhurst, calca, Kyle Goodrick, Marc Bobillier, Robert +Snook, James Kim, Olivier Serres, Jon Redfern, Itai Tavor, Michael +Fladischer, Rob, Jan Schmid, Thomas H., Anders Söderbäck, Abhishek +Dasgupta, Jeff Goeke-Smith, Tommy Thorn, bonuswavepilot, Philipp Edelmann, +Nick, Alejandro Navarro Fulleda, Yann Lallemant, andrew brennan, +Dave Allen Barker Jr, Fabian, Lukas Anzinger, Carl Witty, Andy Taylor, +Andre Klärner, Andrew Chilton, Adam Gibbins, Alexander Wauck, Shane O'Dea, +Paul Waite, Iain McLaren, Maggie Ellen Robin Hess, Willard Korfhage, +Nicolas, Eric Nikolaisen, Magnus Enger, Philipp Kern, Andrew Alderwick, +Raphael Wimmer, Benjamin Schötz, Ana Guerrero, Pete, Pieter van der Eems, +Aaron Suggs, Fred Benenson, Cedric Howe, Lance Ivy, Tieg Zaharia, Kevin +Cooney, Jon Williams, Anton Kudris, Roman Komarov, Brad Hilton, Rick Dakan, +Adam Whitcomb, Paul Casagrande, Evgueni Baldin, Robert Sanders, Kagan +Kayal, Dean Gardiner, micah altman, Cameron Banga, Ross Mcnairn, Oscar +Vilaplana, Robin Graham, Dan Gervais, Jon Åslund, Ragan Webber, Noble Hays, +stephen brown, Sean True, Maciek Swiech, faser, eikenberry, Kai Laborenz, +Sergey Fedoseev, Chris Fournier, Svend Sorensen, Greg K, wojtek, Johan +Ribenfors, Anton, Benjamin, Oleg Tsarev, PsychoHazard, John Cochrane, +Kasper Lauritzen, Patrick Naish, Rob, Keith Nasman, zenmaster, David Royer, +Max Woolf, Dan Gabber, martin rhoads, Martin Schmidinger, Paul +Scott-Wilson, Tom Gromak, Andy Webster, Dale Webb, Jim Watson, Stephen +Hansen, Mircea, Dan Goodenberger, Matthias Fink, Andy Gott, Daniel, Jai +Nelson, Shrayas Rajagopal, Vladimir Rutsky, Alexander, Thorben Westerhuys, +hiruki, Tao Neuendorffer Flaherty, Elline, Marco Hegenberg, robert, Balda, +Brennen Bearnes, Richard Parkins, David Gwilliam, Mark Johnson, Jeff Eaton, +Reddawn90, Heather Pusey, Chris Heinz, Colin, Phatsaphong Thadabusapa, +valunthar, Michael Martinez, redlukas, Yury V. Zaytsev, Blake, Tobias +"betabrain" A., Leon, sopyer, Steve Burnett, bessarabov, sarble, krsch.com, +Jack Self, Jeff Welch, Sam Pettiford, Jimmy Stridh, Diego Barberá, David +Steele, Oscar Ciudad, John Braman, Jacob, Nick Jenkins, Ben Sullivan, Brian +Cleary, James Brosnahan, Darryn Ten, Alex Brem, Jonathan Hitchcock, Jan +Schmidle, Wolfrzx99, Steve Pomeroy, Matthew Sitton, Finkregh, Derek Reeve, +GDR!, Cory Chapman, Marc Olivier Chouinard, Andreas Ryland, Justin, Andreas +Persenius, Games That Teach, Walter Somerville, Bill Haecker, Brandon +Phenix, Justin Shultz, Colin Scroggins, Tim Goddard, Ben Margolin, Michael +Martinez, David Hobbs, Andre Le, Jason Roberts, Bob Lopez, Gert Van Gool, +Robert Carnel, Anders Lundqvist, Aniruddha Sathaye, Marco Gillies, Basti +von Bejga, Esko Arajärvi, Dominik Deobald, Pavel Yudaev, Fionn Behrens, +Davide Favargiotti, Perttu Luukko, Silvan Jegen, Marcelo Bittencourt, +Leonard Peris, smercer, Alexandre Dupas, Solomon Matthews, Peter Hogg, +Richard E. Varian, Ian Oswald, James W. Sarvey, Ed Grether, Frederic +Savard, Sebastian Nerz, Hans-Chr. Jehg, Matija Nalis, Josh DiMauro, Jason +Harris, Adam Byrtek, Tellef, Magnus, Bart Schuurmans, Giel van Schijndel, +Ryan, kiodos, Richard 'maddog' Collins, PawZubr, Jason Gassel, Alex +Boisvert, Richard Thompson, maddi bolton, csights, Aaron Bryson, Jason Chu, +Maxime Côté, Kineteka Systems, Joe Cabezas, Mike Czepiel, Rami Nasra, +Christian Simonsen, Wouter Beugelsdijk, Adam Gibson, Gal Buki, James +Marble, Alan Chambers, Bernd Wiehle, Simon Korzun, Daniel Glassey, Eero af +Heurlin, Mikael, Timo Engelhardt, Wesley Faulkner, Jay Wo, Mike Belle, +David Fowlkes Jr., Karl-Heinz Strass, Ed Mullins, Sam Flint, +Hendrica, Mark Emer Anderson, Joshua Cole, Jan Gondol, Henrik Lindhe, +Albert Delgado, Patrick, Alexa Avellar, Chris, sebsn1349, Maxim Kachur, +Andrew Marshall, Navjot Narula, Alwin Mathew, Christian Mangelsdorf, Avi +Shevin, Kevin S., Guillermo Sanchez Estrada, Alex Krieger, Luca Meier, Will +Jessop, Nick Ruest, Lani Aung, Ulf Benjaminsson, Rudi Engelbrecht, Miles +Matton, Cpt_Threepwood, Adam Kuyper, reacocard, David Kilsheimer, Peter +Olson, Bill Fischofer, Prashant Shah, Simon Bonnick, Alexander Grudtsov, +Antoine Boegli, Richard Warren, Sebastian Rust, AlmostHuman, Timmy +Crawford, PC, Marek Belski, pontus, Douglas S Butts, Eric Wallmander, Joe +Pelar, VIjay, Trahloc, Vernon Vantucci, Matthew baya, Viktor Štujber, +Stephen Akiki, Daniil Zamoldinov, Atley, Chris Thomson, Jacob Briggs, Falko +Richter, Andy Schmitz, Sergi Sagas, Peder Refsnes, Jonatan, Ben, Bill +Niblock, Agustin Acuna, Jeff Curl, Tim Humphrey, bib, James Zarbock, +Lachlan Devantier, Michal Berg, Jeff Lucas, Sid Irish, Franklyn, Jared +Dickson, Olli Jarva, Adam Gibson, Lukas Loesche, Jukka Määttä, Alexander +Lin, Dao Tran, Kirk, briankb, Ryan Villasenor, Daniel Wong, barista, Tomas +Jungwirth, Jesper Hansen, Nivin Singh, Alessandro Tieghi, Billy Roessler, +Peter Fetterer, Pallav Laskar, jcherney, Tyler Wang, Steve, Gigahost, Beat +Wolf, Hannibal Skorepa, aktiveradio, Mark Nunnikhoven, Bret Comnes, Alan +Ruttenberg, Anthony DiSanti, Adam Warkiewicz, Brian Bowman, Jonathan, Mark +Filley, Tobias Mohr, Christian St. Cyr, j. faceless user, Karl Miller, +Thomas Taimre, Vikram, Jason Mountcastle, Jason, Paul Elliott, Alexander, +Stephen Farmer, rayslava, Peter Leurs, Sky Kruse, JP Reeves, John J Schatz, +Martin Sandmair, Will Thompson, John Hergenroeder, Thomas, Christophe +Ponsart, Wolfdog, Eagertolearn, LukasM, Federico Hernandez, Vincent Bernat, +Christian Schmidt, Cameron Colby Thomson, Josh Duff, James Brown, Theron +Trowbridge, Falke, Don Meares, tauu, Greg Cornford, Max Fenton, Kenneth +Reitz, Bruce Bensetler, Mark Booth, Herb Mann, Sindre Sorhus, Chris +Knadler, Daniel Digerås, Derek, Sin Valentine, Ben Gamari, david +lampenscherf, fardles, Richard Burdeniuk, Tobias Kienzler, Dawid Humbla, +Bruno Barbaroxa, D Malt, krivar, James Valleroy, Peter, Tim Geddings, +Matthias Holzinger, Hanen, Petr Vacek, Raymond, Griff Maloney, Andreas +Helveg Rudolph, Nelson Blaha, Colonel Fubar, Skyjacker Captain Gavin +Phoenix, shaun, Michael, Kari Salminen, Rodrigo Miranda, Alan Chan, Justin +Eugene Evans, Isaac, Ben Staffin, Matthew Loar, Magos, Roderik, Eugenio +Piasini, Nico B, Scott Walter, Lior Amsalem, Thongrop Rodsavas, Alberto de +Paola, Shawn Poulen, John Swiderski, lluks, Waelen, Mark Slosarek, Jim +Cristol, mikesol, Bilal Quadri, LuP, Allan Nicolson, Kevin Washington, +Isaac Wedin, Paul Anguiano, ldacruz, Jason Manheim, Sawyer, Jason +Woofenden, Joe Danziger, Declan Morahan, KaptainUfolog, Vladron, bart, Jeff +McNeill, Christian Schlotter, Ben McQuillan, Anthony, Julian, Martin O, +altruism, Eric Solheim, MarkS, ndrwc, Matthew, David Lehn, Matthew +Cisneros, Mike Skoglund, Kristy Carey, fmotta, Tom Lowenthal, Branden +Tyree, Aaron Whitehouse + +## Also thanks to + +* The Kickstarter team, who have unleashed much good on the world. +* The Haskell developers, who toiled for 20 years in obscurity + before most of us noticed them, and on whose giant shoulders I now stand, + in awe of the view. +* The Git developers, for obvious reasons. +* All of git-annex's early adopters, who turned it from a personal + toy project into something much more, and showed me the interest was there. +* Rsync.net, for providing me a free account so I can make sure git-annex + works well with it. +* Anna and Mark, for the loan of the video camera; as well as the rest of + my family, for your support. Even when I couldn't explain what I was + working on. +* The Hodges, for providing such a congenial place for me to live and work + on these first world problems, while you're off helping people in the + third world. diff --git a/doc/assistant/thumbnail.png b/doc/assistant/thumbnail.png new file mode 100644 index 0000000000..346c22f025 Binary files /dev/null and b/doc/assistant/thumbnail.png differ diff --git a/doc/assistant/xmpp.png b/doc/assistant/xmpp.png new file mode 100644 index 0000000000..c3cc53ebf5 Binary files /dev/null and b/doc/assistant/xmpp.png differ diff --git a/doc/assistant/xmppnudge.png b/doc/assistant/xmppnudge.png new file mode 100644 index 0000000000..b3a0658cbf Binary files /dev/null and b/doc/assistant/xmppnudge.png differ diff --git a/doc/assistant/xmpppairingend.png b/doc/assistant/xmpppairingend.png new file mode 100644 index 0000000000..f0c9e765d1 Binary files /dev/null and b/doc/assistant/xmpppairingend.png differ diff --git a/doc/backends.mdwn b/doc/backends.mdwn new file mode 100644 index 0000000000..e1a3da218e --- /dev/null +++ b/doc/backends.mdwn @@ -0,0 +1,39 @@ +When a file is annexed, a key is generated from its content and/or metadata. +The file checked into git symlinks to the key. This key can later be used +to retrieve the file's content (its value). + +Multiple pluggable key-value backends are supported, and a single repository +can use different ones for different files. + +* `SHA256E` -- The default backend for new files. This allows + verifying that the file content is right, and can avoid duplicates of + files with the same content. Its need to generate checksums + can make it slower for large files. +* `SHA256` -- Does not include the file extension in the key, which can + lead to better deduplication. +* `WORM` ("Write Once, Read Many") This assumes that any file with + the same basename, size, and modification time has the same content. + This is the the least expensive backend, recommended for really large + files or slow systems. +* `SHA512`, `SHA512E` -- Best currently available hash, for the very paranoid. +* `SHA1`, `SHA1E` -- Smaller hash than `SHA256` for those who want a checksum + but are not concerned about security. +* `SHA384`, `SHA384E`, `SHA224`, `SHA224E` -- Hashes for people who like + unusual sizes. + +The `annex.backends` git-config setting can be used to list the backends +git-annex should use. The first one listed will be used by default when +new files are added. + +For finer control of what backend is used when adding different types of +files, the `.gitattributes` file can be used. The `annex.backend` +attribute can be set to the name of the backend to use for matching files. + +For example, to use the SHA256E backend for sound files, which tend to be +smallish and might be modified or copied over time, +while using the WORM backend for everything else, you could set +in `.gitattributes`: + + * annex.backend=WORM + *.mp3 annex.backend=SHA256E + *.ogg annex.backend=SHA256E diff --git a/doc/backends/comment_1_375bb1fb5973e8fa67b763f2dd6e404b._comment b/doc/backends/comment_1_375bb1fb5973e8fa67b763f2dd6e404b._comment new file mode 100644 index 0000000000..dc178a6fed --- /dev/null +++ b/doc/backends/comment_1_375bb1fb5973e8fa67b763f2dd6e404b._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://nanotech.nanotechcorp.net/" + nickname="NanoTech" + subject="SHA performance" + date="2012-08-10T04:37:32Z" + content=""" +It turns out that (at least on x86-64 machines) `SHA512` [is faster than][1] `SHA256`. In some benchmarks I performed1 `SHA256` was 1.8–2.2x slower than `SHA1` while `SHA512` was only 1.5–1.6x slower. + +`SHA224` and `SHA384` are effectively just truncated versions of `SHA256` and `SHA512` so their performance characteristics are identical. + +[1]: https://community.emc.com/community/edn/rsashare/blog/2010/11/01/sha-2-algorithms-when-sha-512-is-more-secure-and-faster +1 `time head -c 100000000 /dev/zero | shasum -a 512` +"""]] diff --git a/doc/backends/comment_2_1f2626eca9004b31a0b7fc1a0df8027b._comment b/doc/backends/comment_2_1f2626eca9004b31a0b7fc1a0df8027b._comment new file mode 100644 index 0000000000..0308873036 --- /dev/null +++ b/doc/backends/comment_2_1f2626eca9004b31a0b7fc1a0df8027b._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm7eqCMh_B7mxE0tnchbr0JoYu11FUAFRY" + nickname="Stéphane" + subject="Tracking remote copies not even stored locally / URL backend turned into a "special remote"." + date="2013-01-03T10:59:35Z" + content=""" +In case you came here looking for the URL backend. + +## The URL backend + +Several documents on the web refer to a special \"URL backend\", e.g. [Large file management with git-annex [LWN.net]](http://lwn.net/Articles/419241/). Historical content will never be updated yet it drives people to living places. + +## Why a URL backend ? + +It is interesting because you can: + +* let `git-annex` rest on the fact that some documents are available as extra copies available at any time (but from something that is not a git repository). +* track these documents like your own with all git features, which opens up some truly marvelous combinations, which this margin is too narrow to contain (Pierre d.F. wouldn't disapprove ;-). + +## How/Where now ? + +`git-annex` used to have a URL backend. It seems that the design changed into a \"special remote\" feature, not limited to the web. You can now track files available through plain directories, rsync, webdav, some cloud storage, etc, even clay tablets. For details see [[special remotes]]. + +"""]] diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn new file mode 100644 index 0000000000..dde74c60a6 --- /dev/null +++ b/doc/bare_repositories.mdwn @@ -0,0 +1,43 @@ +Due to popular demand, git-annex can now be used with bare repositories. + +So, for example, you can stash a file away in the origin: +`git annex move mybigfile --to origin` + +Of course, for that to work, the bare repository has to be on a system with +[[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't +use git-annex to store stuff there. + +It took a while, but bare repositories are now supported exactly as well +as non-bare repositories. Except for these caveats: + +* `git annex fsck` works in a bare repository, but does not display + warnings about insufficient + [[copies]]. To get those warnings, just run it in one of the non-bare + checkouts. +* `git annex unused` in a bare repository only knows about keys used in + branches that have been pushed to the bare repository. So use it with care.. +* Commands that need a work tree, like `git annex add` won't work in a bare + repository, of course. + +*** + +Here is a quick example of how to set this up, using `origin` as the remote name, and assuming `~/annex` contains an annex: + +On the server: + + git init --bare bare-annex.git + git annex init origin + +Now configure the remote and do the initial push: + + cd ~/annex + git remote add origin example.com:bare-annex.git + git push origin master git-annex + +Now `git annex status` should show the configured bare remote. If it does not, you may have to pull from the remote first (older versions of `git-annex`) + +If you wish to configure git such that you can push/pull without arguments, set the upstream branch: + + git branch master --set-upstream origin/master + + diff --git a/doc/bare_repositories/comment_1_148e1da70d37d311634a0309a4ff8dcd._comment b/doc/bare_repositories/comment_1_148e1da70d37d311634a0309a4ff8dcd._comment new file mode 100644 index 0000000000..c1ba9f2f4a --- /dev/null +++ b/doc/bare_repositories/comment_1_148e1da70d37d311634a0309a4ff8dcd._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmraN_ldJplGunVGmnjjLN6jL9s9TrVMGE" + nickname="Ævar Arnfjörð" + subject="How to convert bare repositories to non-bare" + date="2012-11-11T20:14:44Z" + content=""" +I made a repository bare and later wanted to convert it, this would have worked with just plain git: + + cd bare-repo.git + mkdir .git + mv .??* * .git/ + git config --unset core.bare + git reset --hard + +But because git-annex uses different hashing directories under bare repositories all the files in the repo will point to files you don't have. Here's how you can fix that up assuming you're using a backend that assigns unique hashes based on file content (e.g. the SHA256 backend): + + mv .git/annex/objects from-bare-repo + git annex add from-bare-repo + git rm -f from-bare-repo + + +"""]] diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn new file mode 100644 index 0000000000..df090a06fb --- /dev/null +++ b/doc/bugs.mdwn @@ -0,0 +1,6 @@ +This is git-annex's bug list. Link bugs to [[bugs/done]] when done. + +[[!inline pages="./bugs/* and !./bugs/*/* and !./bugs/done and !link(done) +and !*/Discussion" actions=yes postform=yes show=0 archive=yes]] + +[[!edittemplate template=templates/bugtemplate match="bugs/*" silent=yes]] diff --git a/doc/bugs/3.20121112:_build_error_in_assistant.mdwn b/doc/bugs/3.20121112:_build_error_in_assistant.mdwn new file mode 100644 index 0000000000..de11dfbf7b --- /dev/null +++ b/doc/bugs/3.20121112:_build_error_in_assistant.mdwn @@ -0,0 +1,432 @@ +Git-annex stopped compiling with GHC 7.4.2 after updating Yesod and friends to the respective latest version. The complete build log is attached below. I hope this helps. Further build logs are available at , too. + + building + make flags: PREFIX=/nix/store/9az61h33v1j6fkdmwdfy7gi0rhspsb9k-git-annex-3.20121112 + building Build/SysConfig.hs + ghc -O2 -Wall -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_DNS -DWITH_INOTIFY -DWITH_DBUS -threaded --make configure + [1 of 7] Compiling Utility.Exception ( Utility/Exception.hs, tmp/Utility/Exception.o ) + [2 of 7] Compiling Utility.Misc ( Utility/Misc.hs, tmp/Utility/Misc.o ) + [3 of 7] Compiling Utility.Process ( Utility/Process.hs, tmp/Utility/Process.o ) + [4 of 7] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, tmp/Utility/SafeCommand.o ) + [5 of 7] Compiling Build.TestConfig ( Build/TestConfig.hs, tmp/Build/TestConfig.o ) + [6 of 7] Compiling Build.Configure ( Build/Configure.hs, tmp/Build/Configure.o ) + [7 of 7] Compiling Main ( configure.hs, tmp/Main.o ) + Linking configure ... + ./configure + checking version... 3.20121112 + checking git... yes + checking git version... 1.8.0 + checking cp -a... yes + checking cp -p... yes + checking cp --reflink=auto... yes + checking uuid generator... uuidgen + checking xargs -0... yes + checking rsync... yes + checking curl... yes + checking wget... no + checking bup... no + checking gpg... no + checking lsof... no + checking ssh connection caching... yes + checking sha1... sha1sum + checking sha256... sha256sum + checking sha512... sha512sum + checking sha224... sha224sum + checking sha384... sha384sum + building Utility/Touch.hs + hsc2hs Utility/Touch.hsc + building Utility/Mounts.hs + hsc2hs Utility/Mounts.hsc + building Utility/libdiskfree.o + cc -Wall -c -o Utility/libdiskfree.o Utility/libdiskfree.c + building Utility/libmounts.o + cc -Wall -c -o Utility/libmounts.o Utility/libmounts.c + building git-annex + install -d tmp + ghc -O2 -Wall -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_DNS -DWITH_INOTIFY -DWITH_DBUS -threaded --make git-annex -o tmp/git-annex Utility/libdiskfree.o Utility/libmounts.o + [ 1 of 279] Compiling Utility.Dot ( Utility/Dot.hs, tmp/Utility/Dot.o ) + [ 2 of 279] Compiling Utility.ThreadLock ( Utility/ThreadLock.hs, tmp/Utility/ThreadLock.o ) + [ 3 of 279] Compiling Utility.Mounts ( Utility/Mounts.hs, tmp/Utility/Mounts.o ) + [ 4 of 279] Compiling Utility.Yesod ( Utility/Yesod.hs, tmp/Utility/Yesod.o ) + [ 5 of 279] Compiling Utility.Tense ( Utility/Tense.hs, tmp/Utility/Tense.o ) + [ 6 of 279] Compiling Utility.Verifiable ( Utility/Verifiable.hs, tmp/Utility/Verifiable.o ) + [ 7 of 279] Compiling Assistant.Types.TransferSlots ( Assistant/Types/TransferSlots.hs, tmp/Assistant/Types/TransferSlots.o ) + [ 8 of 279] Compiling Types.StandardGroups ( Types/StandardGroups.hs, tmp/Types/StandardGroups.o ) + [ 9 of 279] Compiling Utility.Percentage ( Utility/Percentage.hs, tmp/Utility/Percentage.o ) + [ 10 of 279] Compiling Utility.Base64 ( Utility/Base64.hs, tmp/Utility/Base64.o ) + [ 11 of 279] Compiling Utility.DataUnits ( Utility/DataUnits.hs, tmp/Utility/DataUnits.o ) + [ 12 of 279] Compiling Utility.JSONStream ( Utility/JSONStream.hs, tmp/Utility/JSONStream.o ) + [ 13 of 279] Compiling Messages.JSON ( Messages/JSON.hs, tmp/Messages/JSON.o ) + [ 14 of 279] Compiling Build.SysConfig ( Build/SysConfig.hs, tmp/Build/SysConfig.o ) + [ 15 of 279] Compiling Types.KeySource ( Types/KeySource.hs, tmp/Types/KeySource.o ) + [ 16 of 279] Compiling Utility.State ( Utility/State.hs, tmp/Utility/State.o ) + [ 17 of 279] Compiling Types.UUID ( Types/UUID.hs, tmp/Types/UUID.o ) + [ 18 of 279] Compiling Types.Messages ( Types/Messages.hs, tmp/Types/Messages.o ) + [ 19 of 279] Compiling Types.Group ( Types/Group.hs, tmp/Types/Group.o ) + [ 20 of 279] Compiling Types.TrustLevel ( Types/TrustLevel.hs, tmp/Types/TrustLevel.o ) + [ 21 of 279] Compiling Types.BranchState ( Types/BranchState.hs, tmp/Types/BranchState.o ) + [ 22 of 279] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, tmp/Utility/PartialPrelude.o ) + [ 23 of 279] Compiling Utility.HumanTime ( Utility/HumanTime.hs, tmp/Utility/HumanTime.o ) + [ 24 of 279] Compiling Utility.Format ( Utility/Format.hs, tmp/Utility/Format.o ) + [ 25 of 279] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, tmp/Utility/FileSystemEncoding.o ) + [ 26 of 279] Compiling Utility.Touch ( Utility/Touch.hs, tmp/Utility/Touch.o ) + [ 27 of 279] Compiling Utility.Applicative ( Utility/Applicative.hs, tmp/Utility/Applicative.o ) + [ 28 of 279] Compiling Utility.Monad ( Utility/Monad.hs, tmp/Utility/Monad.o ) + [ 29 of 279] Compiling Utility.Exception ( Utility/Exception.hs, tmp/Utility/Exception.o ) + [ 30 of 279] Compiling Utility.DBus ( Utility/DBus.hs, tmp/Utility/DBus.o ) + [ 31 of 279] Compiling Utility.Misc ( Utility/Misc.hs, tmp/Utility/Misc.o ) + [ 32 of 279] Compiling Utility.Process ( Utility/Process.hs, tmp/Utility/Process.o ) + [ 33 of 279] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, tmp/Utility/SafeCommand.o ) + [ 34 of 279] Compiling Utility.Network ( Utility/Network.hs, tmp/Utility/Network.o ) + [ 35 of 279] Compiling Utility.SRV ( Utility/SRV.hs, tmp/Utility/SRV.o ) + + Utility/SRV.hs:88:1: Warning: Defined but not used: `lookupSRVHost' + + Utility/SRV.hs:94:1: Warning: Defined but not used: `parseSrvHost' + [ 36 of 279] Compiling Git.Types ( Git/Types.hs, tmp/Git/Types.o ) + [ 37 of 279] Compiling Utility.UserInfo ( Utility/UserInfo.hs, tmp/Utility/UserInfo.o ) + [ 38 of 279] Compiling Utility.Path ( Utility/Path.hs, tmp/Utility/Path.o ) + [ 39 of 279] Compiling Utility.TempFile ( Utility/TempFile.hs, tmp/Utility/TempFile.o ) + [ 40 of 279] Compiling Utility.Directory ( Utility/Directory.hs, tmp/Utility/Directory.o ) + [ 41 of 279] Compiling Utility.FreeDesktop ( Utility/FreeDesktop.hs, tmp/Utility/FreeDesktop.o ) + [ 42 of 279] Compiling Assistant.Install.AutoStart ( Assistant/Install/AutoStart.hs, tmp/Assistant/Install/AutoStart.o ) + [ 43 of 279] Compiling Common ( Common.hs, tmp/Common.o ) + [ 44 of 279] Compiling Utility.FileMode ( Utility/FileMode.hs, tmp/Utility/FileMode.o ) + [ 45 of 279] Compiling Git ( Git.hs, tmp/Git.o ) + [ 46 of 279] Compiling Git.FilePath ( Git/FilePath.hs, tmp/Git/FilePath.o ) + [ 47 of 279] Compiling Utility.Matcher ( Utility/Matcher.hs, tmp/Utility/Matcher.o ) + [ 48 of 279] Compiling Utility.Gpg ( Utility/Gpg.hs, tmp/Utility/Gpg.o ) + [ 49 of 279] Compiling Types.Crypto ( Types/Crypto.hs, tmp/Types/Crypto.o ) + [ 50 of 279] Compiling Types.Key ( Types/Key.hs, tmp/Types/Key.o ) + [ 51 of 279] Compiling Types.Backend ( Types/Backend.hs, tmp/Types/Backend.o ) + [ 52 of 279] Compiling Types.Remote ( Types/Remote.hs, tmp/Types/Remote.o ) + [ 53 of 279] Compiling Git.Sha ( Git/Sha.hs, tmp/Git/Sha.o ) + [ 54 of 279] Compiling Utility.CoProcess ( Utility/CoProcess.hs, tmp/Utility/CoProcess.o ) + [ 55 of 279] Compiling Git.Command ( Git/Command.hs, tmp/Git/Command.o ) + [ 56 of 279] Compiling Git.Ref ( Git/Ref.hs, tmp/Git/Ref.o ) + [ 57 of 279] Compiling Git.Branch ( Git/Branch.hs, tmp/Git/Branch.o ) + [ 58 of 279] Compiling Git.UpdateIndex ( Git/UpdateIndex.hs, tmp/Git/UpdateIndex.o ) + [ 59 of 279] Compiling Git.Queue ( Git/Queue.hs, tmp/Git/Queue.o ) + [ 60 of 279] Compiling Git.HashObject ( Git/HashObject.hs, tmp/Git/HashObject.o ) + [ 61 of 279] Compiling Git.CatFile ( Git/CatFile.hs, tmp/Git/CatFile.o ) + [ 62 of 279] Compiling Git.UnionMerge ( Git/UnionMerge.hs, tmp/Git/UnionMerge.o ) + [ 63 of 279] Compiling Git.Url ( Git/Url.hs, tmp/Git/Url.o ) + [ 64 of 279] Compiling Git.Construct ( Git/Construct.hs, tmp/Git/Construct.o ) + [ 65 of 279] Compiling Git.Config ( Git/Config.hs, tmp/Git/Config.o ) + [ 66 of 279] Compiling Git.SharedRepository ( Git/SharedRepository.hs, tmp/Git/SharedRepository.o ) + [ 67 of 279] Compiling Git.Version ( Git/Version.hs, tmp/Git/Version.o ) + [ 68 of 279] Compiling Git.CheckAttr ( Git/CheckAttr.hs, tmp/Git/CheckAttr.o ) + [ 69 of 279] Compiling Annex ( Annex.hs, tmp/Annex.o ) + [ 70 of 279] Compiling Types.Option ( Types/Option.hs, tmp/Types/Option.o ) + [ 71 of 279] Compiling Types ( Types.hs, tmp/Types.o ) + [ 72 of 279] Compiling Messages ( Messages.hs, tmp/Messages.o ) + [ 73 of 279] Compiling Types.Command ( Types/Command.hs, tmp/Types/Command.o ) + [ 74 of 279] Compiling Locations ( Locations.hs, tmp/Locations.o ) + [ 75 of 279] Compiling Common.Annex ( Common/Annex.hs, tmp/Common/Annex.o ) + [ 76 of 279] Compiling Fields ( Fields.hs, tmp/Fields.o ) + [ 77 of 279] Compiling Annex.BranchState ( Annex/BranchState.hs, tmp/Annex/BranchState.o ) + [ 78 of 279] Compiling Annex.CatFile ( Annex/CatFile.hs, tmp/Annex/CatFile.o ) + [ 79 of 279] Compiling Annex.Perms ( Annex/Perms.hs, tmp/Annex/Perms.o ) + [ 80 of 279] Compiling Crypto ( Crypto.hs, tmp/Crypto.o ) + [ 81 of 279] Compiling Annex.Exception ( Annex/Exception.hs, tmp/Annex/Exception.o ) + [ 82 of 279] Compiling Annex.Journal ( Annex/Journal.hs, tmp/Annex/Journal.o ) + [ 83 of 279] Compiling Annex.Branch ( Annex/Branch.hs, tmp/Annex/Branch.o ) + [ 84 of 279] Compiling Usage ( Usage.hs, tmp/Usage.o ) + [ 85 of 279] Compiling Annex.CheckAttr ( Annex/CheckAttr.hs, tmp/Annex/CheckAttr.o ) + [ 86 of 279] Compiling Remote.Helper.Special ( Remote/Helper/Special.hs, tmp/Remote/Helper/Special.o ) + [ 87 of 279] Compiling Logs.Presence ( Logs/Presence.hs, tmp/Logs/Presence.o ) + [ 88 of 279] Compiling Logs.Location ( Logs/Location.hs, tmp/Logs/Location.o ) + [ 89 of 279] Compiling Logs.Web ( Logs/Web.hs, tmp/Logs/Web.o ) + [ 90 of 279] Compiling Annex.LockPool ( Annex/LockPool.hs, tmp/Annex/LockPool.o ) + [ 91 of 279] Compiling Logs.Transfer ( Logs/Transfer.hs, tmp/Logs/Transfer.o ) + [ 92 of 279] Compiling Backend.SHA ( Backend/SHA.hs, tmp/Backend/SHA.o ) + [ 93 of 279] Compiling Backend.WORM ( Backend/WORM.hs, tmp/Backend/WORM.o ) + [ 94 of 279] Compiling Backend.URL ( Backend/URL.hs, tmp/Backend/URL.o ) + [ 95 of 279] Compiling Assistant.Types.ScanRemotes ( Assistant/Types/ScanRemotes.hs, tmp/Assistant/Types/ScanRemotes.o ) + [ 96 of 279] Compiling Assistant.Types.ThreadedMonad ( Assistant/Types/ThreadedMonad.hs, tmp/Assistant/Types/ThreadedMonad.o ) + [ 97 of 279] Compiling Assistant.Types.TransferQueue ( Assistant/Types/TransferQueue.hs, tmp/Assistant/Types/TransferQueue.o ) + [ 98 of 279] Compiling Assistant.Types.Pushes ( Assistant/Types/Pushes.hs, tmp/Assistant/Types/Pushes.o ) + [ 99 of 279] Compiling Assistant.Types.BranchChange ( Assistant/Types/BranchChange.hs, tmp/Assistant/Types/BranchChange.o ) + [100 of 279] Compiling Logs.UUIDBased ( Logs/UUIDBased.hs, tmp/Logs/UUIDBased.o ) + [101 of 279] Compiling Logs.Remote ( Logs/Remote.hs, tmp/Logs/Remote.o ) + [102 of 279] Compiling Logs.Group ( Logs/Group.hs, tmp/Logs/Group.o ) + [103 of 279] Compiling Utility.DiskFree ( Utility/DiskFree.hs, tmp/Utility/DiskFree.o ) + [104 of 279] Compiling Utility.Url ( Utility/Url.hs, tmp/Utility/Url.o ) + [105 of 279] Compiling Utility.CopyFile ( Utility/CopyFile.hs, tmp/Utility/CopyFile.o ) + [106 of 279] Compiling Utility.Rsync ( Utility/Rsync.hs, tmp/Utility/Rsync.o ) + [107 of 279] Compiling Git.LsFiles ( Git/LsFiles.hs, tmp/Git/LsFiles.o ) + [108 of 279] Compiling Git.AutoCorrect ( Git/AutoCorrect.hs, tmp/Git/AutoCorrect.o ) + [109 of 279] Compiling Git.CurrentRepo ( Git/CurrentRepo.hs, tmp/Git/CurrentRepo.o ) + [110 of 279] Compiling Locations.UserConfig ( Locations/UserConfig.hs, tmp/Locations/UserConfig.o ) + [111 of 279] Compiling Utility.ThreadScheduler ( Utility/ThreadScheduler.hs, tmp/Utility/ThreadScheduler.o ) + [112 of 279] Compiling Git.Merge ( Git/Merge.hs, tmp/Git/Merge.o ) + [113 of 279] Compiling Utility.Parallel ( Utility/Parallel.hs, tmp/Utility/Parallel.o ) + [114 of 279] Compiling Git.Remote ( Git/Remote.hs, tmp/Git/Remote.o ) + [115 of 279] Compiling Assistant.Ssh ( Assistant/Ssh.hs, tmp/Assistant/Ssh.o ) + [116 of 279] Compiling Assistant.Pairing ( Assistant/Pairing.hs, tmp/Assistant/Pairing.o ) + [117 of 279] Compiling Assistant.Types.NetMessager ( Assistant/Types/NetMessager.hs, tmp/Assistant/Types/NetMessager.o ) + [118 of 279] Compiling Utility.NotificationBroadcaster ( Utility/NotificationBroadcaster.hs, tmp/Utility/NotificationBroadcaster.o ) + [119 of 279] Compiling Assistant.Types.Buddies ( Assistant/Types/Buddies.hs, tmp/Assistant/Types/Buddies.o ) + [120 of 279] Compiling Utility.TSet ( Utility/TSet.hs, tmp/Utility/TSet.o ) + [121 of 279] Compiling Assistant.Types.Commits ( Assistant/Types/Commits.hs, tmp/Assistant/Types/Commits.o ) + [122 of 279] Compiling Assistant.Types.Changes ( Assistant/Types/Changes.hs, tmp/Assistant/Types/Changes.o ) + [123 of 279] Compiling Utility.WebApp ( Utility/WebApp.hs, tmp/Utility/WebApp.o ) + [124 of 279] Compiling Utility.Daemon ( Utility/Daemon.hs, tmp/Utility/Daemon.o ) + [125 of 279] Compiling Utility.LogFile ( Utility/LogFile.hs, tmp/Utility/LogFile.o ) + [126 of 279] Compiling Git.Filename ( Git/Filename.hs, tmp/Git/Filename.o ) + [127 of 279] Compiling Git.LsTree ( Git/LsTree.hs, tmp/Git/LsTree.o ) + [128 of 279] Compiling Utility.Types.DirWatcher ( Utility/Types/DirWatcher.hs, tmp/Utility/Types/DirWatcher.o ) + [129 of 279] Compiling Utility.INotify ( Utility/INotify.hs, tmp/Utility/INotify.o ) + [130 of 279] Compiling Utility.DirWatcher ( Utility/DirWatcher.hs, tmp/Utility/DirWatcher.o ) + [131 of 279] Compiling Utility.Lsof ( Utility/Lsof.hs, tmp/Utility/Lsof.o ) + [132 of 279] Compiling Config ( Config.hs, tmp/Config.o ) + [133 of 279] Compiling Annex.UUID ( Annex/UUID.hs, tmp/Annex/UUID.o ) + [134 of 279] Compiling Logs.UUID ( Logs/UUID.hs, tmp/Logs/UUID.o ) + [135 of 279] Compiling Backend ( Backend.hs, tmp/Backend.o ) + [136 of 279] Compiling Remote.Helper.Hooks ( Remote/Helper/Hooks.hs, tmp/Remote/Helper/Hooks.o ) + [137 of 279] Compiling Remote.Helper.Encryptable ( Remote/Helper/Encryptable.hs, tmp/Remote/Helper/Encryptable.o ) + [138 of 279] Compiling Annex.Queue ( Annex/Queue.hs, tmp/Annex/Queue.o ) + [139 of 279] Compiling Annex.Content ( Annex/Content.hs, tmp/Annex/Content.o ) + [140 of 279] Compiling Remote.S3 ( Remote/S3.hs, tmp/Remote/S3.o ) + [141 of 279] Compiling Remote.Directory ( Remote/Directory.hs, tmp/Remote/Directory.o ) + [142 of 279] Compiling Remote.Rsync ( Remote/Rsync.hs, tmp/Remote/Rsync.o ) + [143 of 279] Compiling Remote.Web ( Remote/Web.hs, tmp/Remote/Web.o ) + [144 of 279] Compiling Remote.Hook ( Remote/Hook.hs, tmp/Remote/Hook.o ) + [145 of 279] Compiling Upgrade.V2 ( Upgrade/V2.hs, tmp/Upgrade/V2.o ) + [146 of 279] Compiling Annex.Ssh ( Annex/Ssh.hs, tmp/Annex/Ssh.o ) + [147 of 279] Compiling Remote.Helper.Ssh ( Remote/Helper/Ssh.hs, tmp/Remote/Helper/Ssh.o ) + [148 of 279] Compiling Remote.Bup ( Remote/Bup.hs, tmp/Remote/Bup.o ) + [149 of 279] Compiling Annex.Version ( Annex/Version.hs, tmp/Annex/Version.o ) + [150 of 279] Compiling Init ( Init.hs, tmp/Init.o ) + [151 of 279] Compiling Checks ( Checks.hs, tmp/Checks.o ) + [152 of 279] Compiling Remote.Git ( Remote/Git.hs, tmp/Remote/Git.o ) + [153 of 279] Compiling Remote.List ( Remote/List.hs, tmp/Remote/List.o ) + [154 of 279] Compiling Logs.Trust ( Logs/Trust.hs, tmp/Logs/Trust.o ) + [155 of 279] Compiling Remote ( Remote.hs, tmp/Remote.o ) + [156 of 279] Compiling Assistant.Alert ( Assistant/Alert.hs, tmp/Assistant/Alert.o ) + Loading package ghc-prim ... linking ... done. + Loading package integer-gmp ... linking ... done. + Loading package base ... linking ... done. + Loading object (static) Utility/libdiskfree.o ... done + Loading object (static) Utility/libmounts.o ... done + final link ... done + Loading package pretty-1.1.1.0 ... linking ... done. + Loading package filepath-1.3.0.0 ... linking ... done. + Loading package old-locale-1.0.0.4 ... linking ... done. + Loading package old-time-1.1.0.0 ... linking ... done. + Loading package bytestring-0.9.2.1 ... linking ... done. + Loading package unix-2.5.1.0 ... linking ... done. + Loading package directory-1.1.0.2 ... linking ... done. + Loading package process-1.1.0.1 ... linking ... done. + Loading package array-0.4.0.0 ... linking ... done. + Loading package deepseq-1.3.0.0 ... linking ... done. + Loading package time-1.4 ... linking ... done. + Loading package containers-0.4.2.1 ... linking ... done. + Loading package text-0.11.2.0 ... linking ... done. + Loading package blaze-builder-0.3.1.0 ... linking ... done. + Loading package blaze-markup-0.5.1.1 ... linking ... done. + Loading package blaze-html-0.5.1.0 ... linking ... done. + Loading package hashable-1.1.2.5 ... linking ... done. + Loading package case-insensitive-0.4.0.3 ... linking ... done. + Loading package primitive-0.5.0.1 ... linking ... done. + Loading package vector-0.10.0.1 ... linking ... done. + Loading package random-1.0.1.1 ... linking ... done. + Loading package dlist-0.5 ... linking ... done. + Loading package data-default-0.5.0 ... linking ... done. + Loading package transformers-0.3.0.0 ... linking ... done. + Loading package mtl-2.1.1 ... linking ... done. + Loading package parsec-3.1.2 ... linking ... done. + Loading package network-2.3.0.13 ... linking ... done. + Loading package failure-0.2.0.1 ... linking ... done. + Loading package template-haskell ... linking ... done. + Loading package shakespeare-1.0.2 ... linking ... done. + Loading package hamlet-1.1.1.1 ... linking ... done. + Loading package http-types-0.7.3.0.1 ... linking ... done. + Loading package base-unicode-symbols-0.2.2.4 ... linking ... done. + Loading package transformers-base-0.4.1 ... linking ... done. + Loading package monad-control-0.3.1.4 ... linking ... done. + Loading package lifted-base-0.2 ... linking ... done. + Loading package resourcet-0.4.3 ... linking ... done. + Loading package semigroups-0.8.4.1 ... linking ... done. + Loading package void-0.5.8 ... linking ... done. + Loading package conduit-0.5.4.1 ... linking ... done. + Loading package unordered-containers-0.2.2.1 ... linking ... done. + Loading package vault-0.2.0.1 ... linking ... done. + Loading package wai-1.3.0.1 ... linking ... done. + Loading package date-cache-0.3.0 ... linking ... done. + Loading package unix-time-0.1.2 ... linking ... done. + Loading package fast-logger-0.3.1 ... linking ... done. + Loading package attoparsec-0.10.2.0 ... linking ... done. + Loading package cookie-0.4.0.1 ... linking ... done. + Loading package shakespeare-css-1.0.2 ... linking ... done. + Loading package syb-0.3.6.1 ... linking ... done. + Loading package aeson-0.6.0.2 ... linking ... done. + Loading package shakespeare-js-1.1.0 ... linking ... done. + Loading package ansi-terminal-0.5.5 ... linking ... done. + Loading package blaze-builder-conduit-0.5.0.2 ... linking ... done. + Loading package stringsearch-0.3.6.4 ... linking ... done. + Loading package byteorder-1.0.3 ... linking ... done. + Loading package wai-logger-0.3.0 ... linking ... done. + Loading package zlib-0.5.3.3 ... linking ... done. + Loading package zlib-bindings-0.1.1.1 ... linking ... done. + Loading package zlib-conduit-0.5.0.2 ... linking ... done. + Loading package wai-extra-1.3.0.4 ... linking ... done. + Loading package monad-logger-0.2.1 ... linking ... done. + Loading package cereal-0.3.5.2 ... linking ... done. + Loading package base64-bytestring-1.0.0.0 ... linking ... done. + Loading package cipher-aes-0.1.2 ... linking ... done. + Loading package entropy-0.2.1 ... linking ... done. + Loading package largeword-1.0.3 ... linking ... done. + Loading package tagged-0.4.4 ... linking ... done. + Loading package crypto-api-0.10.2 ... linking ... done. + Loading package cpu-0.1.1 ... linking ... done. + Loading package crypto-pubkey-types-0.1.1 ... linking ... done. + Loading package cryptocipher-0.3.5 ... linking ... done. + Loading package cprng-aes-0.2.4 ... linking ... done. + Loading package skein-0.1.0.9 ... linking ... done. + Loading package clientsession-0.8.0.1 ... linking ... done. + Loading package path-pieces-0.1.2 ... linking ... done. + Loading package shakespeare-i18n-1.0.0.2 ... linking ... done. + Loading package yesod-routes-1.1.1.1 ... linking ... done. + Loading package yesod-core-1.1.5 ... linking ... done. + [157 of 279] Compiling Assistant.Types.DaemonStatus ( Assistant/Types/DaemonStatus.hs, tmp/Assistant/Types/DaemonStatus.o ) + [158 of 279] Compiling Assistant.Monad ( Assistant/Monad.hs, tmp/Assistant/Monad.o ) + [159 of 279] Compiling Assistant.Types.NamedThread ( Assistant/Types/NamedThread.hs, tmp/Assistant/Types/NamedThread.o ) + [160 of 279] Compiling Assistant.Common ( Assistant/Common.hs, tmp/Assistant/Common.o ) + [161 of 279] Compiling Assistant.XMPP ( Assistant/XMPP.hs, tmp/Assistant/XMPP.o ) + [162 of 279] Compiling Assistant.XMPP.Buddies ( Assistant/XMPP/Buddies.hs, tmp/Assistant/XMPP/Buddies.o ) + [163 of 279] Compiling Assistant.NetMessager ( Assistant/NetMessager.hs, tmp/Assistant/NetMessager.o ) + + Assistant/NetMessager.hs:12:1: + Warning: The import of `Types.Remote' is redundant + except perhaps to import instances from `Types.Remote' + To import instances alone, use: import Types.Remote() + + Assistant/NetMessager.hs:13:1: + Warning: The import of `Git' is redundant + except perhaps to import instances from `Git' + To import instances alone, use: import Git() + + Assistant/NetMessager.hs:20:1: + Warning: The import of `Data.Text' is redundant + except perhaps to import instances from `Data.Text' + To import instances alone, use: import Data.Text() + [164 of 279] Compiling Assistant.Pushes ( Assistant/Pushes.hs, tmp/Assistant/Pushes.o ) + [165 of 279] Compiling Assistant.ScanRemotes ( Assistant/ScanRemotes.hs, tmp/Assistant/ScanRemotes.o ) + [166 of 279] Compiling Assistant.Install ( Assistant/Install.hs, tmp/Assistant/Install.o ) + [167 of 279] Compiling Assistant.XMPP.Client ( Assistant/XMPP/Client.hs, tmp/Assistant/XMPP/Client.o ) + [168 of 279] Compiling Assistant.Commits ( Assistant/Commits.hs, tmp/Assistant/Commits.o ) + [169 of 279] Compiling Assistant.BranchChange ( Assistant/BranchChange.hs, tmp/Assistant/BranchChange.o ) + [170 of 279] Compiling Assistant.Changes ( Assistant/Changes.hs, tmp/Assistant/Changes.o ) + [171 of 279] Compiling Assistant.WebApp.Types ( Assistant/WebApp/Types.hs, tmp/Assistant/WebApp/Types.o ) + Loading package unix-compat-0.4.0.0 ... linking ... done. + Loading package file-embed-0.0.4.6 ... linking ... done. + Loading package system-filepath-0.4.7 ... linking ... done. + Loading package system-fileio-0.3.10 ... linking ... done. + Loading package cryptohash-0.7.8 ... linking ... done. + Loading package crypto-conduit-0.4.0.1 ... linking ... done. + Loading package http-date-0.0.2 ... linking ... done. + Loading package mime-types-0.1.0.0 ... linking ... done. + Loading package wai-app-static-1.3.0.4 ... linking ... done. + Loading package yesod-static-1.1.1.1 ... linking ... done. + [172 of 279] Compiling Assistant.WebApp ( Assistant/WebApp.hs, tmp/Assistant/WebApp.o ) + Loading package network-conduit-0.6.1.1 ... linking ... done. + Loading package safe-0.3.3 ... linking ... done. + Loading package simple-sendfile-0.2.8 ... linking ... done. + Loading package warp-1.3.4.4 ... linking ... done. + Loading package yaml-0.8.1 ... linking ... done. + Loading package yesod-default-1.1.2 ... linking ... done. + [173 of 279] Compiling Assistant.WebApp.OtherRepos ( Assistant/WebApp/OtherRepos.hs, tmp/Assistant/WebApp/OtherRepos.o ) + [174 of 279] Compiling Limit ( Limit.hs, tmp/Limit.o ) + [175 of 279] Compiling Option ( Option.hs, tmp/Option.o ) + [176 of 279] Compiling Seek ( Seek.hs, tmp/Seek.o ) + [177 of 279] Compiling Command ( Command.hs, tmp/Command.o ) + [178 of 279] Compiling CmdLine ( CmdLine.hs, tmp/CmdLine.o ) + [179 of 279] Compiling Command.ConfigList ( Command/ConfigList.hs, tmp/Command/ConfigList.o ) + [180 of 279] Compiling Command.InAnnex ( Command/InAnnex.hs, tmp/Command/InAnnex.o ) + [181 of 279] Compiling Command.DropKey ( Command/DropKey.hs, tmp/Command/DropKey.o ) + [182 of 279] Compiling Command.SendKey ( Command/SendKey.hs, tmp/Command/SendKey.o ) + [183 of 279] Compiling Command.RecvKey ( Command/RecvKey.hs, tmp/Command/RecvKey.o ) + [184 of 279] Compiling Command.TransferInfo ( Command/TransferInfo.hs, tmp/Command/TransferInfo.o ) + [185 of 279] Compiling Command.Commit ( Command/Commit.hs, tmp/Command/Commit.o ) + [186 of 279] Compiling Command.Add ( Command/Add.hs, tmp/Command/Add.o ) + [187 of 279] Compiling Command.Unannex ( Command/Unannex.hs, tmp/Command/Unannex.o ) + [188 of 279] Compiling Command.FromKey ( Command/FromKey.hs, tmp/Command/FromKey.o ) + [189 of 279] Compiling Command.ReKey ( Command/ReKey.hs, tmp/Command/ReKey.o ) + [190 of 279] Compiling Command.Fix ( Command/Fix.hs, tmp/Command/Fix.o ) + [191 of 279] Compiling Command.Describe ( Command/Describe.hs, tmp/Command/Describe.o ) + [192 of 279] Compiling Command.InitRemote ( Command/InitRemote.hs, tmp/Command/InitRemote.o ) + [193 of 279] Compiling Command.Unlock ( Command/Unlock.hs, tmp/Command/Unlock.o ) + [194 of 279] Compiling Command.Lock ( Command/Lock.hs, tmp/Command/Lock.o ) + [195 of 279] Compiling Command.PreCommit ( Command/PreCommit.hs, tmp/Command/PreCommit.o ) + [196 of 279] Compiling Command.Log ( Command/Log.hs, tmp/Command/Log.o ) + [197 of 279] Compiling Command.Merge ( Command/Merge.hs, tmp/Command/Merge.o ) + [198 of 279] Compiling Command.Group ( Command/Group.hs, tmp/Command/Group.o ) + [199 of 279] Compiling Command.Ungroup ( Command/Ungroup.hs, tmp/Command/Ungroup.o ) + [200 of 279] Compiling Command.Import ( Command/Import.hs, tmp/Command/Import.o ) + [201 of 279] Compiling Logs.Unused ( Logs/Unused.hs, tmp/Logs/Unused.o ) + [202 of 279] Compiling Command.AddUnused ( Command/AddUnused.hs, tmp/Command/AddUnused.o ) + [203 of 279] Compiling Command.Find ( Command/Find.hs, tmp/Command/Find.o ) + [204 of 279] Compiling Logs.PreferredContent ( Logs/PreferredContent.hs, tmp/Logs/PreferredContent.o ) + [205 of 279] Compiling Annex.Wanted ( Annex/Wanted.hs, tmp/Annex/Wanted.o ) + [206 of 279] Compiling Command.Whereis ( Command/Whereis.hs, tmp/Command/Whereis.o ) + [207 of 279] Compiling Command.Trust ( Command/Trust.hs, tmp/Command/Trust.o ) + [208 of 279] Compiling Command.Untrust ( Command/Untrust.hs, tmp/Command/Untrust.o ) + [209 of 279] Compiling Command.Semitrust ( Command/Semitrust.hs, tmp/Command/Semitrust.o ) + [210 of 279] Compiling Command.Dead ( Command/Dead.hs, tmp/Command/Dead.o ) + [211 of 279] Compiling Command.Vicfg ( Command/Vicfg.hs, tmp/Command/Vicfg.o ) + [212 of 279] Compiling Command.Map ( Command/Map.hs, tmp/Command/Map.o ) + [213 of 279] Compiling Command.Init ( Command/Init.hs, tmp/Command/Init.o ) + [214 of 279] Compiling Command.Uninit ( Command/Uninit.hs, tmp/Command/Uninit.o ) + [215 of 279] Compiling Command.Version ( Command/Version.hs, tmp/Command/Version.o ) + [216 of 279] Compiling Upgrade.V1 ( Upgrade/V1.hs, tmp/Upgrade/V1.o ) + [217 of 279] Compiling Upgrade.V0 ( Upgrade/V0.hs, tmp/Upgrade/V0.o ) + [218 of 279] Compiling Upgrade ( Upgrade.hs, tmp/Upgrade.o ) + [219 of 279] Compiling Command.Upgrade ( Command/Upgrade.hs, tmp/Command/Upgrade.o ) + [220 of 279] Compiling Command.Drop ( Command/Drop.hs, tmp/Command/Drop.o ) + [221 of 279] Compiling Command.Move ( Command/Move.hs, tmp/Command/Move.o ) + [222 of 279] Compiling Command.Copy ( Command/Copy.hs, tmp/Command/Copy.o ) + [223 of 279] Compiling Command.Get ( Command/Get.hs, tmp/Command/Get.o ) + [224 of 279] Compiling Command.TransferKey ( Command/TransferKey.hs, tmp/Command/TransferKey.o ) + [225 of 279] Compiling Command.DropUnused ( Command/DropUnused.hs, tmp/Command/DropUnused.o ) + [226 of 279] Compiling Command.Fsck ( Command/Fsck.hs, tmp/Command/Fsck.o ) + [227 of 279] Compiling Command.Reinject ( Command/Reinject.hs, tmp/Command/Reinject.o ) + [228 of 279] Compiling Command.Migrate ( Command/Migrate.hs, tmp/Command/Migrate.o ) + [229 of 279] Compiling Command.Unused ( Command/Unused.hs, tmp/Command/Unused.o ) + [230 of 279] Compiling Command.Status ( Command/Status.hs, tmp/Command/Status.o ) + [231 of 279] Compiling Command.Sync ( Command/Sync.hs, tmp/Command/Sync.o ) + [232 of 279] Compiling Command.Help ( Command/Help.hs, tmp/Command/Help.o ) + [233 of 279] Compiling Command.AddUrl ( Command/AddUrl.hs, tmp/Command/AddUrl.o ) + [234 of 279] Compiling Assistant.DaemonStatus ( Assistant/DaemonStatus.hs, tmp/Assistant/DaemonStatus.o ) + [235 of 279] Compiling Assistant.Sync ( Assistant/Sync.hs, tmp/Assistant/Sync.o ) + [236 of 279] Compiling Assistant.MakeRemote ( Assistant/MakeRemote.hs, tmp/Assistant/MakeRemote.o ) + [237 of 279] Compiling Assistant.XMPP.Git ( Assistant/XMPP/Git.hs, tmp/Assistant/XMPP/Git.o ) + [238 of 279] Compiling Command.XMPPGit ( Command/XMPPGit.hs, tmp/Command/XMPPGit.o ) + [239 of 279] Compiling Assistant.Threads.NetWatcher ( Assistant/Threads/NetWatcher.hs, tmp/Assistant/Threads/NetWatcher.o ) + [240 of 279] Compiling Assistant.NamedThread ( Assistant/NamedThread.hs, tmp/Assistant/NamedThread.o ) + [241 of 279] Compiling Assistant.WebApp.Notifications ( Assistant/WebApp/Notifications.hs, tmp/Assistant/WebApp/Notifications.o ) + + Assistant/WebApp/Notifications.hs:39:11: + No instances for (Text.Julius.ToJavascript String, + Text.Julius.ToJavascript Text) + arising from a use of `Text.Julius.toJavascript' + Possible fix: + add instance declarations for + (Text.Julius.ToJavascript String, Text.Julius.ToJavascript Text) + In the first argument of `Text.Julius.Javascript', namely + `Text.Julius.toJavascript delay' + In the expression: + Text.Julius.Javascript (Text.Julius.toJavascript delay) + In the first argument of `Data.Monoid.mconcat', namely + `[Text.Julius.Javascript + ((Data.Text.Lazy.Builder.fromText . Text.Shakespeare.pack') + "function longpoll_"), + Text.Julius.Javascript (Text.Julius.toJavascript ident), + Text.Julius.Javascript + ((Data.Text.Lazy.Builder.fromText . Text.Shakespeare.pack') + "() {\ + \\tlongpoll(longpoll_"), + Text.Julius.Javascript (Text.Julius.toJavascript ident), ....]' + make: *** [git-annex] Error 1 + +> Reproduced this and confirmed it's fixed in git. --[[Joey]] [[done]] diff --git a/doc/bugs/3.20121112:_build_error_in_assistant/comment_1_b42f40ffd83321ab5cc0ef24ced15e98._comment b/doc/bugs/3.20121112:_build_error_in_assistant/comment_1_b42f40ffd83321ab5cc0ef24ced15e98._comment new file mode 100644 index 0000000000..9690885f07 --- /dev/null +++ b/doc/bugs/3.20121112:_build_error_in_assistant/comment_1_b42f40ffd83321ab5cc0ef24ced15e98._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 1" + date="2012-11-17T20:21:12Z" + content=""" +This looks rather like a bug in Yesod. I've made a change in git (b0a76592c313b4c8f51918d6469c40d1fd16a2b1) that *may* avoid the problem. +"""]] diff --git a/doc/bugs/3.20121112:_build_error_in_assistant/comment_2_b1d2aa10ea84c5c370b3e76507fc8761._comment b/doc/bugs/3.20121112:_build_error_in_assistant/comment_2_b1d2aa10ea84c5c370b3e76507fc8761._comment new file mode 100644 index 0000000000..0d4f3666c3 --- /dev/null +++ b/doc/bugs/3.20121112:_build_error_in_assistant/comment_2_b1d2aa10ea84c5c370b3e76507fc8761._comment @@ -0,0 +1,476 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkHZw2Vx0VPtb9XM8vum7nEnh6gHGSCQvM" + nickname="Andrew" + subject="comment 2" + date="2012-11-24T18:48:23Z" + content=""" +Not quite. Though this might just be an artifact of me disabling WITH_WEBDAV on account of that not compiling on OS X (can't find module Network.Protocol.HTTP.DAV). +OS: OS X 10.8.0 + + % git branch -v + * master d1ba407 Added a comment: git annex fix + % grep \"FEATURES?=\" Makefile + FEATURES?=$(GIT_ANNEX_LOCAL_FEATURES) -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_DNS + $ make + ghc -O2 -Wall -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_DNS -DWITH_KQUEUE -threaded --make configure + [1 of 7] Compiling Utility.Exception ( Utility/Exception.hs, tmp/Utility/Exception.o ) + [2 of 7] Compiling Utility.Misc ( Utility/Misc.hs, tmp/Utility/Misc.o ) + [3 of 7] Compiling Utility.Process ( Utility/Process.hs, tmp/Utility/Process.o ) + [4 of 7] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, tmp/Utility/SafeCommand.o ) + [5 of 7] Compiling Build.TestConfig ( Build/TestConfig.hs, tmp/Build/TestConfig.o ) + [6 of 7] Compiling Build.Configure ( Build/Configure.hs, tmp/Build/Configure.o ) + [7 of 7] Compiling Main ( configure.hs, tmp/Main.o ) + Linking configure ... + ./configure + checking version... 3.20121113 + checking git... yes + checking git version... 1.7.10.2 (Apple Git-33) + checking cp -a... yes + checking cp -p... yes + checking cp --reflink=auto... no + checking uuid generator... uuidgen + checking xargs -0... yes + checking rsync... yes + checking curl... yes + checking wget... no + checking bup... no + checking gpg... no + checking lsof... yes + checking ssh connection caching... yes + checking sha1... sha1sum + checking sha256.../bin/sh: sha256sum: command not found + gsha256sum + checking sha512.../bin/sh: sha512sum: command not found + gsha512sum + checking sha224.../bin/sh: sha224sum: command not found + gsha224sum + checking sha384.../bin/sh: sha384sum: command not found + gsha384sum + hsc2hs Utility/Touch.hsc + Touch.hsc:117:2: warning: #warning \"utimensat and lutimes not available; building without symlink timestamp preservation support\" + Touch.hsc:117:2: warning: #warning \"utimensat and lutimes not available; building without symlink timestamp preservation support\" + Touch.hsc:117:2: warning: #warning \"utimensat and lutimes not available; building without symlink timestamp preservation support\" + hsc2hs Utility/Mounts.hsc + cc -Wall -c -o Utility/libdiskfree.o Utility/libdiskfree.c + Utility/libdiskfree.c:53:6: warning: 'statfs64' is deprecated: first deprecated in Mac OS X 10.6 [-Wdeprecated-declarations] + if (STATCALL(path, &buf) != 0) + ^ + Utility/libdiskfree.c:16:19: note: expanded from macro 'STATCALL' + # define STATCALL statfs64 + ^ + /usr/include/sys/mount.h:381:5: note: 'statfs64' declared here + int statfs64(const char *, struct statfs64 *) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_5,__MAC_10_6,__IPHONE_NA,__IPHONE_NA); + ^ + 1 warning generated. + cc -Wall -c -o Utility/libmounts.o Utility/libmounts.c + cc -Wall -c -o Utility/libkqueue.o Utility/libkqueue.c + install -d tmp + ghc -O2 -Wall -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_DNS -DWITH_KQUEUE -threaded --make git-annex -o tmp/git-annex Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o + + Assistant/Threads/NetWatcher.hs:26:0: + warning: #warning Building without dbus support; will poll for network connection changes + + Assistant/Threads/MountWatcher.hs:33:0: + warning: #warning Building without dbus support; will use mtab polling + [ 1 of 285] Compiling Utility.Dot ( Utility/Dot.hs, tmp/Utility/Dot.o ) + [ 2 of 285] Compiling Utility.Mounts ( Utility/Mounts.hs, tmp/Utility/Mounts.o ) + [ 3 of 285] Compiling Utility.Yesod ( Utility/Yesod.hs, tmp/Utility/Yesod.o ) + [ 4 of 285] Compiling Utility.Tense ( Utility/Tense.hs, tmp/Utility/Tense.o ) + [ 5 of 285] Compiling Utility.Verifiable ( Utility/Verifiable.hs, tmp/Utility/Verifiable.o ) + [ 6 of 285] Compiling Assistant.Types.TransferSlots ( Assistant/Types/TransferSlots.hs, tmp/Assistant/Types/TransferSlots.o ) + [ 7 of 285] Compiling Types.StandardGroups ( Types/StandardGroups.hs, tmp/Types/StandardGroups.o ) + [ 8 of 285] Compiling Utility.Percentage ( Utility/Percentage.hs, tmp/Utility/Percentage.o ) + [ 9 of 285] Compiling Utility.Observed ( Utility/Observed.hs, tmp/Utility/Observed.o ) + [ 10 of 285] Compiling Utility.Base64 ( Utility/Base64.hs, tmp/Utility/Base64.o ) + [ 11 of 285] Compiling Utility.DataUnits ( Utility/DataUnits.hs, tmp/Utility/DataUnits.o ) + [ 12 of 285] Compiling Utility.JSONStream ( Utility/JSONStream.hs, tmp/Utility/JSONStream.o ) + [ 13 of 285] Compiling Messages.JSON ( Messages/JSON.hs, tmp/Messages/JSON.o ) + [ 14 of 285] Compiling Build.SysConfig ( Build/SysConfig.hs, tmp/Build/SysConfig.o ) + [ 15 of 285] Compiling Types.KeySource ( Types/KeySource.hs, tmp/Types/KeySource.o ) + [ 16 of 285] Compiling Types.Meters ( Types/Meters.hs, tmp/Types/Meters.o ) + [ 17 of 285] Compiling Utility.State ( Utility/State.hs, tmp/Utility/State.o ) + [ 18 of 285] Compiling Types.UUID ( Types/UUID.hs, tmp/Types/UUID.o ) + [ 19 of 285] Compiling Types.Messages ( Types/Messages.hs, tmp/Types/Messages.o ) + [ 20 of 285] Compiling Types.Group ( Types/Group.hs, tmp/Types/Group.o ) + [ 21 of 285] Compiling Types.TrustLevel ( Types/TrustLevel.hs, tmp/Types/TrustLevel.o ) + [ 22 of 285] Compiling Types.BranchState ( Types/BranchState.hs, tmp/Types/BranchState.o ) + [ 23 of 285] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, tmp/Utility/PartialPrelude.o ) + [ 24 of 285] Compiling Utility.HumanTime ( Utility/HumanTime.hs, tmp/Utility/HumanTime.o ) + [ 25 of 285] Compiling Utility.Format ( Utility/Format.hs, tmp/Utility/Format.o ) + [ 26 of 285] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, tmp/Utility/FileSystemEncoding.o ) + [ 27 of 285] Compiling Utility.Touch ( Utility/Touch.hs, tmp/Utility/Touch.o ) + + Utility/Touch.hsc:17:1: + Warning: The import of `Utility.FileSystemEncoding' is redundant + except perhaps to import instances from `Utility.FileSystemEncoding' + To import instances alone, use: import Utility.FileSystemEncoding() + + Utility/Touch.hsc:19:1: + Warning: The import of `Foreign' is redundant + except perhaps to import instances from `Foreign' + To import instances alone, use: import Foreign() + + Utility/Touch.hsc:21:1: + Warning: The import of `Control.Monad' is redundant + except perhaps to import instances from `Control.Monad' + To import instances alone, use: import Control.Monad() + [ 28 of 285] Compiling Utility.Applicative ( Utility/Applicative.hs, tmp/Utility/Applicative.o ) + [ 29 of 285] Compiling Utility.Monad ( Utility/Monad.hs, tmp/Utility/Monad.o ) + [ 30 of 285] Compiling Utility.Exception ( Utility/Exception.hs, tmp/Utility/Exception.o ) + [ 31 of 285] Compiling Utility.Misc ( Utility/Misc.hs, tmp/Utility/Misc.o ) + [ 32 of 285] Compiling Utility.Process ( Utility/Process.hs, tmp/Utility/Process.o ) + [ 33 of 285] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, tmp/Utility/SafeCommand.o ) + [ 34 of 285] Compiling Utility.Network ( Utility/Network.hs, tmp/Utility/Network.o ) + [ 35 of 285] Compiling Utility.SRV ( Utility/SRV.hs, tmp/Utility/SRV.o ) + + Utility/SRV.hs:88:1: Warning: Defined but not used: `lookupSRVHost' + + Utility/SRV.hs:94:1: Warning: Defined but not used: `parseSrvHost' + [ 36 of 285] Compiling Git.Types ( Git/Types.hs, tmp/Git/Types.o ) + [ 37 of 285] Compiling Utility.UserInfo ( Utility/UserInfo.hs, tmp/Utility/UserInfo.o ) + [ 38 of 285] Compiling Utility.Path ( Utility/Path.hs, tmp/Utility/Path.o ) + [ 39 of 285] Compiling Utility.TempFile ( Utility/TempFile.hs, tmp/Utility/TempFile.o ) + [ 40 of 285] Compiling Utility.Directory ( Utility/Directory.hs, tmp/Utility/Directory.o ) + [ 41 of 285] Compiling Utility.FreeDesktop ( Utility/FreeDesktop.hs, tmp/Utility/FreeDesktop.o ) + [ 42 of 285] Compiling Utility.OSX ( Utility/OSX.hs, tmp/Utility/OSX.o ) + + Utility/OSX.hs:10:1: + Warning: The import of `Utility.Path' is redundant + except perhaps to import instances from `Utility.Path' + To import instances alone, use: import Utility.Path() + [ 43 of 285] Compiling Assistant.Install.AutoStart ( Assistant/Install/AutoStart.hs, tmp/Assistant/Install/AutoStart.o ) + [ 44 of 285] Compiling Common ( Common.hs, tmp/Common.o ) + [ 45 of 285] Compiling Utility.FileMode ( Utility/FileMode.hs, tmp/Utility/FileMode.o ) + [ 46 of 285] Compiling Git ( Git.hs, tmp/Git.o ) + [ 47 of 285] Compiling Git.FilePath ( Git/FilePath.hs, tmp/Git/FilePath.o ) + [ 48 of 285] Compiling Utility.Matcher ( Utility/Matcher.hs, tmp/Utility/Matcher.o ) + [ 49 of 285] Compiling Utility.Gpg ( Utility/Gpg.hs, tmp/Utility/Gpg.o ) + [ 50 of 285] Compiling Types.Crypto ( Types/Crypto.hs, tmp/Types/Crypto.o ) + [ 51 of 285] Compiling Types.Key ( Types/Key.hs, tmp/Types/Key.o ) + [ 52 of 285] Compiling Types.Backend ( Types/Backend.hs, tmp/Types/Backend.o ) + [ 53 of 285] Compiling Types.Remote ( Types/Remote.hs, tmp/Types/Remote.o ) + [ 54 of 285] Compiling Meters ( Meters.hs, tmp/Meters.o ) + [ 55 of 285] Compiling Git.Sha ( Git/Sha.hs, tmp/Git/Sha.o ) + [ 56 of 285] Compiling Utility.CoProcess ( Utility/CoProcess.hs, tmp/Utility/CoProcess.o ) + [ 57 of 285] Compiling Git.Command ( Git/Command.hs, tmp/Git/Command.o ) + [ 58 of 285] Compiling Git.Ref ( Git/Ref.hs, tmp/Git/Ref.o ) + [ 59 of 285] Compiling Git.Branch ( Git/Branch.hs, tmp/Git/Branch.o ) + [ 60 of 285] Compiling Git.UpdateIndex ( Git/UpdateIndex.hs, tmp/Git/UpdateIndex.o ) + [ 61 of 285] Compiling Git.Queue ( Git/Queue.hs, tmp/Git/Queue.o ) + [ 62 of 285] Compiling Git.HashObject ( Git/HashObject.hs, tmp/Git/HashObject.o ) + [ 63 of 285] Compiling Git.CatFile ( Git/CatFile.hs, tmp/Git/CatFile.o ) + [ 64 of 285] Compiling Git.UnionMerge ( Git/UnionMerge.hs, tmp/Git/UnionMerge.o ) + [ 65 of 285] Compiling Git.Url ( Git/Url.hs, tmp/Git/Url.o ) + [ 66 of 285] Compiling Git.Construct ( Git/Construct.hs, tmp/Git/Construct.o ) + [ 67 of 285] Compiling Git.Config ( Git/Config.hs, tmp/Git/Config.o ) + [ 68 of 285] Compiling Git.SharedRepository ( Git/SharedRepository.hs, tmp/Git/SharedRepository.o ) + [ 69 of 285] Compiling Git.Version ( Git/Version.hs, tmp/Git/Version.o ) + [ 70 of 285] Compiling Git.CheckAttr ( Git/CheckAttr.hs, tmp/Git/CheckAttr.o ) + [ 71 of 285] Compiling Annex ( Annex.hs, tmp/Annex.o ) + [ 72 of 285] Compiling Types.Option ( Types/Option.hs, tmp/Types/Option.o ) + [ 73 of 285] Compiling Types ( Types.hs, tmp/Types.o ) + [ 74 of 285] Compiling Messages ( Messages.hs, tmp/Messages.o ) + [ 75 of 285] Compiling Types.Command ( Types/Command.hs, tmp/Types/Command.o ) + [ 76 of 285] Compiling Locations ( Locations.hs, tmp/Locations.o ) + [ 77 of 285] Compiling Common.Annex ( Common/Annex.hs, tmp/Common/Annex.o ) + [ 78 of 285] Compiling Fields ( Fields.hs, tmp/Fields.o ) + [ 79 of 285] Compiling Annex.BranchState ( Annex/BranchState.hs, tmp/Annex/BranchState.o ) + [ 80 of 285] Compiling Annex.CatFile ( Annex/CatFile.hs, tmp/Annex/CatFile.o ) + [ 81 of 285] Compiling Annex.Perms ( Annex/Perms.hs, tmp/Annex/Perms.o ) + [ 82 of 285] Compiling Crypto ( Crypto.hs, tmp/Crypto.o ) + [ 83 of 285] Compiling Annex.Exception ( Annex/Exception.hs, tmp/Annex/Exception.o ) + [ 84 of 285] Compiling Annex.Journal ( Annex/Journal.hs, tmp/Annex/Journal.o ) + [ 85 of 285] Compiling Annex.Branch ( Annex/Branch.hs, tmp/Annex/Branch.o ) + [ 86 of 285] Compiling Usage ( Usage.hs, tmp/Usage.o ) + [ 87 of 285] Compiling Annex.CheckAttr ( Annex/CheckAttr.hs, tmp/Annex/CheckAttr.o ) + [ 88 of 285] Compiling Remote.Helper.Special ( Remote/Helper/Special.hs, tmp/Remote/Helper/Special.o ) + [ 89 of 285] Compiling Logs.Presence ( Logs/Presence.hs, tmp/Logs/Presence.o ) + [ 90 of 285] Compiling Logs.Location ( Logs/Location.hs, tmp/Logs/Location.o ) + [ 91 of 285] Compiling Logs.Web ( Logs/Web.hs, tmp/Logs/Web.o ) + [ 92 of 285] Compiling Remote.Helper.Chunked ( Remote/Helper/Chunked.hs, tmp/Remote/Helper/Chunked.o ) + [ 93 of 285] Compiling Annex.LockPool ( Annex/LockPool.hs, tmp/Annex/LockPool.o ) + [ 94 of 285] Compiling Logs.Transfer ( Logs/Transfer.hs, tmp/Logs/Transfer.o ) + [ 95 of 285] Compiling Backend.SHA ( Backend/SHA.hs, tmp/Backend/SHA.o ) + [ 96 of 285] Compiling Backend.WORM ( Backend/WORM.hs, tmp/Backend/WORM.o ) + [ 97 of 285] Compiling Backend.URL ( Backend/URL.hs, tmp/Backend/URL.o ) + [ 98 of 285] Compiling Assistant.Types.ScanRemotes ( Assistant/Types/ScanRemotes.hs, tmp/Assistant/Types/ScanRemotes.o ) + [ 99 of 285] Compiling Assistant.Types.ThreadedMonad ( Assistant/Types/ThreadedMonad.hs, tmp/Assistant/Types/ThreadedMonad.o ) + [100 of 285] Compiling Assistant.Types.TransferQueue ( Assistant/Types/TransferQueue.hs, tmp/Assistant/Types/TransferQueue.o ) + [101 of 285] Compiling Assistant.Types.Pushes ( Assistant/Types/Pushes.hs, tmp/Assistant/Types/Pushes.o ) + [102 of 285] Compiling Assistant.Types.BranchChange ( Assistant/Types/BranchChange.hs, tmp/Assistant/Types/BranchChange.o ) + [103 of 285] Compiling Logs.UUIDBased ( Logs/UUIDBased.hs, tmp/Logs/UUIDBased.o ) + [104 of 285] Compiling Logs.Remote ( Logs/Remote.hs, tmp/Logs/Remote.o ) + [105 of 285] Compiling Logs.Group ( Logs/Group.hs, tmp/Logs/Group.o ) + [106 of 285] Compiling Utility.DiskFree ( Utility/DiskFree.hs, tmp/Utility/DiskFree.o ) + [107 of 285] Compiling Utility.Url ( Utility/Url.hs, tmp/Utility/Url.o ) + [108 of 285] Compiling Utility.CopyFile ( Utility/CopyFile.hs, tmp/Utility/CopyFile.o ) + [109 of 285] Compiling Utility.Rsync ( Utility/Rsync.hs, tmp/Utility/Rsync.o ) + [110 of 285] Compiling Git.LsFiles ( Git/LsFiles.hs, tmp/Git/LsFiles.o ) + [111 of 285] Compiling Git.AutoCorrect ( Git/AutoCorrect.hs, tmp/Git/AutoCorrect.o ) + [112 of 285] Compiling Git.CurrentRepo ( Git/CurrentRepo.hs, tmp/Git/CurrentRepo.o ) + [113 of 285] Compiling Locations.UserConfig ( Locations/UserConfig.hs, tmp/Locations/UserConfig.o ) + [114 of 285] Compiling Git.Merge ( Git/Merge.hs, tmp/Git/Merge.o ) + [115 of 285] Compiling Utility.Parallel ( Utility/Parallel.hs, tmp/Utility/Parallel.o ) + [116 of 285] Compiling Git.Remote ( Git/Remote.hs, tmp/Git/Remote.o ) + [117 of 285] Compiling Assistant.Ssh ( Assistant/Ssh.hs, tmp/Assistant/Ssh.o ) + [118 of 285] Compiling Assistant.Pairing ( Assistant/Pairing.hs, tmp/Assistant/Pairing.o ) + [119 of 285] Compiling Assistant.Types.NetMessager ( Assistant/Types/NetMessager.hs, tmp/Assistant/Types/NetMessager.o ) + [120 of 285] Compiling Utility.NotificationBroadcaster ( Utility/NotificationBroadcaster.hs, tmp/Utility/NotificationBroadcaster.o ) + [121 of 285] Compiling Assistant.Types.Buddies ( Assistant/Types/Buddies.hs, tmp/Assistant/Types/Buddies.o ) + [122 of 285] Compiling Utility.TSet ( Utility/TSet.hs, tmp/Utility/TSet.o ) + [123 of 285] Compiling Assistant.Types.Commits ( Assistant/Types/Commits.hs, tmp/Assistant/Types/Commits.o ) + [124 of 285] Compiling Assistant.Types.Changes ( Assistant/Types/Changes.hs, tmp/Assistant/Types/Changes.o ) + [125 of 285] Compiling Utility.WebApp ( Utility/WebApp.hs, tmp/Utility/WebApp.o ) + [126 of 285] Compiling Utility.Daemon ( Utility/Daemon.hs, tmp/Utility/Daemon.o ) + [127 of 285] Compiling Utility.ThreadScheduler ( Utility/ThreadScheduler.hs, tmp/Utility/ThreadScheduler.o ) + [128 of 285] Compiling Utility.LogFile ( Utility/LogFile.hs, tmp/Utility/LogFile.o ) + [129 of 285] Compiling Git.Filename ( Git/Filename.hs, tmp/Git/Filename.o ) + [130 of 285] Compiling Git.LsTree ( Git/LsTree.hs, tmp/Git/LsTree.o ) + [131 of 285] Compiling Utility.Types.DirWatcher ( Utility/Types/DirWatcher.hs, tmp/Utility/Types/DirWatcher.o ) + [132 of 285] Compiling Utility.Kqueue ( Utility/Kqueue.hs, tmp/Utility/Kqueue.o ) + [133 of 285] Compiling Utility.DirWatcher ( Utility/DirWatcher.hs, tmp/Utility/DirWatcher.o ) + [134 of 285] Compiling Utility.Lsof ( Utility/Lsof.hs, tmp/Utility/Lsof.o ) + [135 of 285] Compiling Config ( Config.hs, tmp/Config.o ) + [136 of 285] Compiling Annex.UUID ( Annex/UUID.hs, tmp/Annex/UUID.o ) + [137 of 285] Compiling Logs.UUID ( Logs/UUID.hs, tmp/Logs/UUID.o ) + [138 of 285] Compiling Backend ( Backend.hs, tmp/Backend.o ) + [139 of 285] Compiling Remote.Helper.Hooks ( Remote/Helper/Hooks.hs, tmp/Remote/Helper/Hooks.o ) + [140 of 285] Compiling Remote.Helper.Encryptable ( Remote/Helper/Encryptable.hs, tmp/Remote/Helper/Encryptable.o ) + [141 of 285] Compiling Creds ( Creds.hs, tmp/Creds.o ) + [142 of 285] Compiling Remote.Helper.AWS ( Remote/Helper/AWS.hs, tmp/Remote/Helper/AWS.o ) + [143 of 285] Compiling Annex.Queue ( Annex/Queue.hs, tmp/Annex/Queue.o ) + [144 of 285] Compiling Annex.Content ( Annex/Content.hs, tmp/Annex/Content.o ) + [145 of 285] Compiling Remote.S3 ( Remote/S3.hs, tmp/Remote/S3.o ) + [146 of 285] Compiling Remote.Directory ( Remote/Directory.hs, tmp/Remote/Directory.o ) + [147 of 285] Compiling Remote.Rsync ( Remote/Rsync.hs, tmp/Remote/Rsync.o ) + [148 of 285] Compiling Remote.Web ( Remote/Web.hs, tmp/Remote/Web.o ) + [149 of 285] Compiling Remote.Glacier ( Remote/Glacier.hs, tmp/Remote/Glacier.o ) + [150 of 285] Compiling Remote.Hook ( Remote/Hook.hs, tmp/Remote/Hook.o ) + [151 of 285] Compiling Upgrade.V2 ( Upgrade/V2.hs, tmp/Upgrade/V2.o ) + [152 of 285] Compiling Annex.Ssh ( Annex/Ssh.hs, tmp/Annex/Ssh.o ) + [153 of 285] Compiling Remote.Helper.Ssh ( Remote/Helper/Ssh.hs, tmp/Remote/Helper/Ssh.o ) + [154 of 285] Compiling Remote.Bup ( Remote/Bup.hs, tmp/Remote/Bup.o ) + [155 of 285] Compiling Annex.Version ( Annex/Version.hs, tmp/Annex/Version.o ) + [156 of 285] Compiling Init ( Init.hs, tmp/Init.o ) + [157 of 285] Compiling Checks ( Checks.hs, tmp/Checks.o ) + [158 of 285] Compiling Remote.Git ( Remote/Git.hs, tmp/Remote/Git.o ) + [159 of 285] Compiling Remote.List ( Remote/List.hs, tmp/Remote/List.o ) + [160 of 285] Compiling Logs.Trust ( Logs/Trust.hs, tmp/Logs/Trust.o ) + [161 of 285] Compiling Remote ( Remote.hs, tmp/Remote.o ) + [162 of 285] Compiling Assistant.Alert ( Assistant/Alert.hs, tmp/Assistant/Alert.o ) + [163 of 285] Compiling Assistant.Types.DaemonStatus ( Assistant/Types/DaemonStatus.hs, tmp/Assistant/Types/DaemonStatus.o ) + [164 of 285] Compiling Assistant.Monad ( Assistant/Monad.hs, tmp/Assistant/Monad.o ) + [165 of 285] Compiling Assistant.Types.NamedThread ( Assistant/Types/NamedThread.hs, tmp/Assistant/Types/NamedThread.o ) + [166 of 285] Compiling Assistant.Common ( Assistant/Common.hs, tmp/Assistant/Common.o ) + [167 of 285] Compiling Assistant.XMPP ( Assistant/XMPP.hs, tmp/Assistant/XMPP.o ) + [168 of 285] Compiling Assistant.XMPP.Buddies ( Assistant/XMPP/Buddies.hs, tmp/Assistant/XMPP/Buddies.o ) + [169 of 285] Compiling Assistant.NetMessager ( Assistant/NetMessager.hs, tmp/Assistant/NetMessager.o ) + [170 of 285] Compiling Assistant.Pushes ( Assistant/Pushes.hs, tmp/Assistant/Pushes.o ) + [171 of 285] Compiling Assistant.ScanRemotes ( Assistant/ScanRemotes.hs, tmp/Assistant/ScanRemotes.o ) + [172 of 285] Compiling Assistant.Install ( Assistant/Install.hs, tmp/Assistant/Install.o ) + [173 of 285] Compiling Assistant.XMPP.Client ( Assistant/XMPP/Client.hs, tmp/Assistant/XMPP/Client.o ) + [174 of 285] Compiling Assistant.Commits ( Assistant/Commits.hs, tmp/Assistant/Commits.o ) + [175 of 285] Compiling Assistant.BranchChange ( Assistant/BranchChange.hs, tmp/Assistant/BranchChange.o ) + [176 of 285] Compiling Assistant.Changes ( Assistant/Changes.hs, tmp/Assistant/Changes.o ) + [177 of 285] Compiling Assistant.WebApp.Types ( Assistant/WebApp/Types.hs, tmp/Assistant/WebApp/Types.o ) + Loading package ghc-prim ... linking ... done. + Loading package integer-gmp ... linking ... done. + Loading package base ... linking ... done. + Loading object (static) Utility/libdiskfree.o ... done + Loading object (static) Utility/libmounts.o ... done + Loading object (static) Utility/libkqueue.o ... done + final link ... done + Loading package bytestring-0.9.2.1 ... linking ... done. + Loading package zlib-0.5.4.0 ... linking ... done. + Loading package array-0.4.0.0 ... linking ... done. + Loading package deepseq-1.3.0.0 ... linking ... done. + Loading package primitive-0.5.0.1 ... linking ... done. + Loading package vector-0.10.0.1 ... linking ... done. + Loading package transformers-0.3.0.0 ... linking ... done. + Loading package text-0.11.2.3 ... linking ... done. + Loading package old-locale-1.0.0.4 ... linking ... done. + Loading package time-1.4 ... linking ... done. + Loading package random-1.0.1.1 ... linking ... done. + Loading package mtl-2.1.2 ... linking ... done. + Loading package parsec-3.1.3 ... linking ... done. + Loading package pretty-1.1.1.0 ... linking ... done. + Loading package filepath-1.3.0.0 ... linking ... done. + Loading package old-time-1.1.0.0 ... linking ... done. + Loading package unix-2.5.1.1 ... linking ... done. + Loading package directory-1.1.0.2 ... linking ... done. + Loading package process-1.1.0.1 ... linking ... done. + Loading package containers-0.4.2.1 ... linking ... done. + Loading package base64-bytestring-1.0.0.0 ... linking ... done. + Loading package cereal-0.3.5.2 ... linking ... done. + Loading package base-unicode-symbols-0.2.2.4 ... linking ... done. + Loading package transformers-base-0.4.1 ... linking ... done. + Loading package monad-control-0.3.1.4 ... linking ... done. + Loading package lifted-base-0.2 ... linking ... done. + Loading package resourcet-0.4.4 ... linking ... done. + Loading package semigroups-0.8.4.1 ... linking ... done. + Loading package void-0.5.8 ... linking ... done. + Loading package conduit-0.5.4.1 ... linking ... done. + Loading package entropy-0.2.1 ... linking ... done. + Loading package largeword-1.0.3 ... linking ... done. + Loading package tagged-0.4.4 ... linking ... done. + Loading package crypto-api-0.10.2 ... linking ... done. + Loading package crypto-conduit-0.4.1 ... linking ... done. + Loading package cryptohash-0.7.8 ... linking ... done. + Loading package template-haskell ... linking ... done. + Loading package file-embed-0.0.4.6 ... linking ... done. + Loading package blaze-builder-0.3.1.0 ... linking ... done. + Loading package hashable-1.1.2.5 ... linking ... done. + Loading package case-insensitive-0.4.0.3 ... linking ... done. + Loading package http-types-0.7.3.0.1 ... linking ... done. + Loading package system-filepath-0.4.7 ... linking ... done. + Loading package unix-compat-0.4.0.0 ... linking ... done. + Loading package network-2.4.0.1 ... linking ... done. + Loading package unordered-containers-0.2.2.1 ... linking ... done. + Loading package vault-0.2.0.1 ... linking ... done. + Loading package wai-1.3.0.1 ... linking ... done. + Loading package blaze-markup-0.5.1.1 ... linking ... done. + Loading package blaze-html-0.5.1.0 ... linking ... done. + Loading package attoparsec-0.10.2.0 ... linking ... done. + Loading package http-date-0.0.3 ... linking ... done. + Loading package mime-types-0.1.0.0 ... linking ... done. + Loading package system-fileio-0.3.10 ... linking ... done. + Loading package wai-app-static-1.3.0.4 ... linking ... done. + Loading package dlist-0.5 ... linking ... done. + Loading package syb-0.3.7 ... linking ... done. + Loading package aeson-0.6.0.2 ... linking ... done. + Loading package cpu-0.1.1 ... linking ... done. + Loading package crypto-pubkey-types-0.2.0 ... linking ... done. + Loading package cryptocipher-0.3.6 ... linking ... done. + Loading package cprng-aes-0.2.4 ... linking ... done. + Loading package skein-0.1.0.10 ... linking ... done. + Loading package clientsession-0.8.0.1 ... linking ... done. + Loading package data-default-0.5.0 ... linking ... done. + Loading package cookie-0.4.0.1 ... linking ... done. + Loading package failure-0.2.0.1 ... linking ... done. + Loading package date-cache-0.3.0 ... linking ... done. + Loading package unix-time-0.1.2 ... linking ... done. + Loading package fast-logger-0.3.1 ... linking ... done. + Loading package shakespeare-1.0.2 ... linking ... done. + Loading package hamlet-1.1.1.1 ... linking ... done. + Loading package monad-logger-0.2.1 ... linking ... done. + Loading package path-pieces-0.1.2 ... linking ... done. + Loading package shakespeare-css-1.0.2 ... linking ... done. + Loading package shakespeare-i18n-1.0.0.2 ... linking ... done. + Loading package shakespeare-js-1.1.0 ... linking ... done. + Loading package ansi-terminal-0.5.5 ... linking ... done. + Loading package blaze-builder-conduit-0.5.0.3 ... linking ... done. + Loading package stringsearch-0.3.6.4 ... linking ... done. + Loading package byteorder-1.0.3 ... linking ... done. + Loading package wai-logger-0.3.0 ... linking ... done. + Loading package zlib-bindings-0.1.1.2 ... linking ... done. + Loading package zlib-conduit-0.5.0.3 ... linking ... done. + Loading package wai-extra-1.3.0.4 ... linking ... done. + Loading package yesod-routes-1.1.1.1 ... linking ... done. + Loading package yesod-core-1.1.6 ... linking ... done. + Loading package yesod-static-1.1.1.1 ... linking ... done. + [178 of 285] Compiling Assistant.WebApp ( Assistant/WebApp.hs, tmp/Assistant/WebApp.o ) + Loading package network-conduit-0.6.1.1 ... linking ... done. + Loading package safe-0.3.3 ... linking ... done. + Loading package simple-sendfile-0.2.8 ... linking ... done. + Loading package warp-1.3.5 ... linking ... done. + Loading package yaml-0.8.1.1 ... linking ... done. + Loading package yesod-default-1.1.2 ... linking ... done. + [179 of 285] Compiling Assistant.WebApp.OtherRepos ( Assistant/WebApp/OtherRepos.hs, tmp/Assistant/WebApp/OtherRepos.o ) + [180 of 285] Compiling Limit ( Limit.hs, tmp/Limit.o ) + [181 of 285] Compiling Option ( Option.hs, tmp/Option.o ) + [182 of 285] Compiling Seek ( Seek.hs, tmp/Seek.o ) + [183 of 285] Compiling Command ( Command.hs, tmp/Command.o ) + [184 of 285] Compiling CmdLine ( CmdLine.hs, tmp/CmdLine.o ) + [185 of 285] Compiling Command.ConfigList ( Command/ConfigList.hs, tmp/Command/ConfigList.o ) + [186 of 285] Compiling Command.InAnnex ( Command/InAnnex.hs, tmp/Command/InAnnex.o ) + [187 of 285] Compiling Command.DropKey ( Command/DropKey.hs, tmp/Command/DropKey.o ) + [188 of 285] Compiling Command.SendKey ( Command/SendKey.hs, tmp/Command/SendKey.o ) + [189 of 285] Compiling Command.RecvKey ( Command/RecvKey.hs, tmp/Command/RecvKey.o ) + [190 of 285] Compiling Command.TransferInfo ( Command/TransferInfo.hs, tmp/Command/TransferInfo.o ) + [191 of 285] Compiling Command.Commit ( Command/Commit.hs, tmp/Command/Commit.o ) + [192 of 285] Compiling Command.Add ( Command/Add.hs, tmp/Command/Add.o ) + [193 of 285] Compiling Command.Unannex ( Command/Unannex.hs, tmp/Command/Unannex.o ) + [194 of 285] Compiling Command.FromKey ( Command/FromKey.hs, tmp/Command/FromKey.o ) + [195 of 285] Compiling Command.ReKey ( Command/ReKey.hs, tmp/Command/ReKey.o ) + [196 of 285] Compiling Command.Fix ( Command/Fix.hs, tmp/Command/Fix.o ) + [197 of 285] Compiling Command.Describe ( Command/Describe.hs, tmp/Command/Describe.o ) + [198 of 285] Compiling Command.InitRemote ( Command/InitRemote.hs, tmp/Command/InitRemote.o ) + [199 of 285] Compiling Command.Unlock ( Command/Unlock.hs, tmp/Command/Unlock.o ) + [200 of 285] Compiling Command.Lock ( Command/Lock.hs, tmp/Command/Lock.o ) + [201 of 285] Compiling Command.PreCommit ( Command/PreCommit.hs, tmp/Command/PreCommit.o ) + [202 of 285] Compiling Command.Log ( Command/Log.hs, tmp/Command/Log.o ) + [203 of 285] Compiling Command.Merge ( Command/Merge.hs, tmp/Command/Merge.o ) + [204 of 285] Compiling Command.Group ( Command/Group.hs, tmp/Command/Group.o ) + [205 of 285] Compiling Command.Ungroup ( Command/Ungroup.hs, tmp/Command/Ungroup.o ) + [206 of 285] Compiling Command.Import ( Command/Import.hs, tmp/Command/Import.o ) + [207 of 285] Compiling Logs.Unused ( Logs/Unused.hs, tmp/Logs/Unused.o ) + [208 of 285] Compiling Command.AddUnused ( Command/AddUnused.hs, tmp/Command/AddUnused.o ) + [209 of 285] Compiling Command.Find ( Command/Find.hs, tmp/Command/Find.o ) + [210 of 285] Compiling Logs.PreferredContent ( Logs/PreferredContent.hs, tmp/Logs/PreferredContent.o ) + [211 of 285] Compiling Annex.Wanted ( Annex/Wanted.hs, tmp/Annex/Wanted.o ) + [212 of 285] Compiling Command.Whereis ( Command/Whereis.hs, tmp/Command/Whereis.o ) + [213 of 285] Compiling Command.Trust ( Command/Trust.hs, tmp/Command/Trust.o ) + [214 of 285] Compiling Command.Untrust ( Command/Untrust.hs, tmp/Command/Untrust.o ) + [215 of 285] Compiling Command.Semitrust ( Command/Semitrust.hs, tmp/Command/Semitrust.o ) + [216 of 285] Compiling Command.Dead ( Command/Dead.hs, tmp/Command/Dead.o ) + [217 of 285] Compiling Command.Vicfg ( Command/Vicfg.hs, tmp/Command/Vicfg.o ) + [218 of 285] Compiling Command.Map ( Command/Map.hs, tmp/Command/Map.o ) + [219 of 285] Compiling Command.Init ( Command/Init.hs, tmp/Command/Init.o ) + [220 of 285] Compiling Command.Uninit ( Command/Uninit.hs, tmp/Command/Uninit.o ) + [221 of 285] Compiling Command.Version ( Command/Version.hs, tmp/Command/Version.o ) + [222 of 285] Compiling Upgrade.V1 ( Upgrade/V1.hs, tmp/Upgrade/V1.o ) + [223 of 285] Compiling Upgrade.V0 ( Upgrade/V0.hs, tmp/Upgrade/V0.o ) + [224 of 285] Compiling Upgrade ( Upgrade.hs, tmp/Upgrade.o ) + [225 of 285] Compiling Command.Upgrade ( Command/Upgrade.hs, tmp/Command/Upgrade.o ) + [226 of 285] Compiling Command.Drop ( Command/Drop.hs, tmp/Command/Drop.o ) + [227 of 285] Compiling Command.Move ( Command/Move.hs, tmp/Command/Move.o ) + [228 of 285] Compiling Command.Copy ( Command/Copy.hs, tmp/Command/Copy.o ) + [229 of 285] Compiling Command.Get ( Command/Get.hs, tmp/Command/Get.o ) + [230 of 285] Compiling Command.TransferKey ( Command/TransferKey.hs, tmp/Command/TransferKey.o ) + [231 of 285] Compiling Command.DropUnused ( Command/DropUnused.hs, tmp/Command/DropUnused.o ) + [232 of 285] Compiling Command.Fsck ( Command/Fsck.hs, tmp/Command/Fsck.o ) + [233 of 285] Compiling Command.Reinject ( Command/Reinject.hs, tmp/Command/Reinject.o ) + [234 of 285] Compiling Command.Migrate ( Command/Migrate.hs, tmp/Command/Migrate.o ) + [235 of 285] Compiling Command.Unused ( Command/Unused.hs, tmp/Command/Unused.o ) + [236 of 285] Compiling Command.Status ( Command/Status.hs, tmp/Command/Status.o ) + [237 of 285] Compiling Command.Sync ( Command/Sync.hs, tmp/Command/Sync.o ) + [238 of 285] Compiling Command.Help ( Command/Help.hs, tmp/Command/Help.o ) + [239 of 285] Compiling Command.AddUrl ( Command/AddUrl.hs, tmp/Command/AddUrl.o ) + [240 of 285] Compiling Assistant.DaemonStatus ( Assistant/DaemonStatus.hs, tmp/Assistant/DaemonStatus.o ) + [241 of 285] Compiling Assistant.Sync ( Assistant/Sync.hs, tmp/Assistant/Sync.o ) + [242 of 285] Compiling Assistant.MakeRemote ( Assistant/MakeRemote.hs, tmp/Assistant/MakeRemote.o ) + [243 of 285] Compiling Assistant.XMPP.Git ( Assistant/XMPP/Git.hs, tmp/Assistant/XMPP/Git.o ) + [244 of 285] Compiling Command.XMPPGit ( Command/XMPPGit.hs, tmp/Command/XMPPGit.o ) + [245 of 285] Compiling Assistant.Threads.NetWatcher ( Assistant/Threads/NetWatcher.hs, tmp/Assistant/Threads/NetWatcher.o ) + [246 of 285] Compiling Assistant.NamedThread ( Assistant/NamedThread.hs, tmp/Assistant/NamedThread.o ) + [247 of 285] Compiling Assistant.WebApp.Notifications ( Assistant/WebApp/Notifications.hs, tmp/Assistant/WebApp/Notifications.o ) + [248 of 285] Compiling Assistant.WebApp.SideBar ( Assistant/WebApp/SideBar.hs, tmp/Assistant/WebApp/SideBar.o ) + [249 of 285] Compiling Assistant.WebApp.Configurators.Ssh ( Assistant/WebApp/Configurators/Ssh.hs, tmp/Assistant/WebApp/Configurators/Ssh.o ) + [250 of 285] Compiling Assistant.WebApp.Configurators.S3 ( Assistant/WebApp/Configurators/S3.hs, tmp/Assistant/WebApp/Configurators/S3.o ) + [251 of 285] Compiling Assistant.WebApp.Documentation ( Assistant/WebApp/Documentation.hs, tmp/Assistant/WebApp/Documentation.o ) + [252 of 285] Compiling Assistant.WebApp.Configurators.XMPP ( Assistant/WebApp/Configurators/XMPP.hs, tmp/Assistant/WebApp/Configurators/XMPP.o ) + [253 of 285] Compiling Assistant.Pairing.Network ( Assistant/Pairing/Network.hs, tmp/Assistant/Pairing/Network.o ) + [254 of 285] Compiling Assistant.Pairing.MakeRemote ( Assistant/Pairing/MakeRemote.hs, tmp/Assistant/Pairing/MakeRemote.o ) + [255 of 285] Compiling Assistant.TransferQueue ( Assistant/TransferQueue.hs, tmp/Assistant/TransferQueue.o ) + [256 of 285] Compiling Assistant.Threads.Merger ( Assistant/Threads/Merger.hs, tmp/Assistant/Threads/Merger.o ) + [257 of 285] Compiling Assistant.TransferSlots ( Assistant/TransferSlots.hs, tmp/Assistant/TransferSlots.o ) + [258 of 285] Compiling Assistant.Threads.Transferrer ( Assistant/Threads/Transferrer.hs, tmp/Assistant/Threads/Transferrer.o ) + [259 of 285] Compiling Assistant.Threads.DaemonStatus ( Assistant/Threads/DaemonStatus.hs, tmp/Assistant/Threads/DaemonStatus.o ) + [260 of 285] Compiling Assistant.Threads.Pusher ( Assistant/Threads/Pusher.hs, tmp/Assistant/Threads/Pusher.o ) + [261 of 285] Compiling Assistant.Threads.MountWatcher ( Assistant/Threads/MountWatcher.hs, tmp/Assistant/Threads/MountWatcher.o ) + [262 of 285] Compiling Assistant.Threads.ConfigMonitor ( Assistant/Threads/ConfigMonitor.hs, tmp/Assistant/Threads/ConfigMonitor.o ) + [263 of 285] Compiling Assistant.Threads.PairListener ( Assistant/Threads/PairListener.hs, tmp/Assistant/Threads/PairListener.o ) + [264 of 285] Compiling Assistant.Threads.XMPPClient ( Assistant/Threads/XMPPClient.hs, tmp/Assistant/Threads/XMPPClient.o ) + [265 of 285] Compiling Assistant.WebApp.Utility ( Assistant/WebApp/Utility.hs, tmp/Assistant/WebApp/Utility.o ) + [266 of 285] Compiling Assistant.WebApp.Configurators.Edit ( Assistant/WebApp/Configurators/Edit.hs, tmp/Assistant/WebApp/Configurators/Edit.o ) + [267 of 285] Compiling Assistant.WebApp.Configurators.Local ( Assistant/WebApp/Configurators/Local.hs, tmp/Assistant/WebApp/Configurators/Local.o ) + [268 of 285] Compiling Assistant.WebApp.Configurators ( Assistant/WebApp/Configurators.hs, tmp/Assistant/WebApp/Configurators.o ) + [269 of 285] Compiling Assistant.WebApp.DashBoard ( Assistant/WebApp/DashBoard.hs, tmp/Assistant/WebApp/DashBoard.o ) + [270 of 285] Compiling Assistant.WebApp.Configurators.Pairing ( Assistant/WebApp/Configurators/Pairing.hs, tmp/Assistant/WebApp/Configurators/Pairing.o ) + [271 of 285] Compiling Assistant.Threads.WebApp ( Assistant/Threads/WebApp.hs, tmp/Assistant/Threads/WebApp.o ) + + Assistant/Threads/WebApp.hs:47:1: Not in scope: `getAddBoxComR' + + Assistant/Threads/WebApp.hs:47:1: Not in scope: `getEnableWebDAVR' + make: *** [git-annex] Error 1 +"""]] diff --git a/doc/bugs/3.20121112:_build_error_in_assistant/comment_3_b38e40d36bba95b16afbce68e7f25a80._comment b/doc/bugs/3.20121112:_build_error_in_assistant/comment_3_b38e40d36bba95b16afbce68e7f25a80._comment new file mode 100644 index 0000000000..26bd0194b8 --- /dev/null +++ b/doc/bugs/3.20121112:_build_error_in_assistant/comment_3_b38e40d36bba95b16afbce68e7f25a80._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 3" + date="2012-11-25T18:36:49Z" + content=""" +The webdav library should not be hard to install, but I've gotten the webapp to build without it. +"""]] diff --git a/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04.mdwn b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04.mdwn new file mode 100644 index 0000000000..cd08976497 --- /dev/null +++ b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04.mdwn @@ -0,0 +1,97 @@ +What steps will reproduce the problem? + +* Start with Ubuntu 12.04 +* sudo apt-get install haskell-platform libgsasl7-dev gsasl g2hs +* cabal install git-annex --bindir=$HOME/bin + +What is the expected output? What do you see instead? + +Expected omething like "installation successful" + +Actual output, after build notices: + + +Loading package IfElse-0.85 ... linking ... done. +Loading object (static) dist/build/git-annex/git-annex-tmp/Utility/libdiskfree.o ... done +Loading object (static) dist/build/git-annex/git-annex-tmp/Utility/libmounts.o ... done +final link ... done +[157 of 279] Compiling Assistant.Types.DaemonStatus ( Assistant/Types/DaemonStatus.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/DaemonStatus.o ) +[158 of 279] Compiling Assistant.Monad ( Assistant/Monad.hs, dist/build/git-annex/git-annex-tmp/Assistant/Monad.o ) + +Assistant/Monad.hs:86:16: + Couldn't match expected type `Assistant a' + with actual type `Reader AssistantData a' + Expected type: (AssistantData -> a) -> Assistant a + Actual type: (AssistantData -> a) -> Reader AssistantData a + In the expression: reader + In an equation for `getAssistant': getAssistant = reader + +Assistant/Monad.hs:93:15: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: st <- reader threadState + In the expression: + do { st <- reader threadState; + liftIO $ runThreadState st a } + +Assistant/Monad.hs:99:14: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: d <- reader id + In the expression: + do { d <- reader id; + liftIO $ io $ runAssistant d a } + +Assistant/Monad.hs:105:14: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: d <- reader id + In the expression: + do { d <- reader id; + return $ runAssistant d a } + +Assistant/Monad.hs:110:14: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: d <- reader id + In the expression: + do { d <- reader id; + return $ \ v -> runAssistant d $ a v } + +Assistant/Monad.hs:115:14: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: d <- reader id + In the expression: + do { d <- reader id; + return $ \ v1 v2 -> runAssistant d (a v1 v2) } + +Assistant/Monad.hs:120:12: + Couldn't match expected type `Assistant a0' + with actual type `Reader r0 a1' + In the return type of a call of `reader' + In the first argument of `(>>=)', namely `reader v' + In the expression: reader v >>= liftIO . io +cabal: Error: some packages failed to install: +git-annex-3.20121112 failed during the building phase. The exception was: +ExitFailure 1 + + +What version of git-annex are you using? On what operating system? + +git annex 3.20121112 +Ubuntu 12.04 (current "long term support", all packages up to date) + +Please provide any additional information below. + +No idea how important this is for git-annex in general but reporting in case it is. Thank you for working on git annex! + +> I was able to reproduce this build error when I force installed +> an old version of the haskell mtl library. So git-annex needs version +> 2.1.1 to build, and I have adjusted the build dependencies appropriately. +> [[done]] --[[Joey]] diff --git a/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_1_ce2efd2196e7682f4cdbabdb0616d449._comment b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_1_ce2efd2196e7682f4cdbabdb0616d449._comment new file mode 100644 index 0000000000..49f43149cf --- /dev/null +++ b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_1_ce2efd2196e7682f4cdbabdb0616d449._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 1" + date="2012-11-15T17:53:25Z" + content=""" +I'm not quite sure what's going on here, but my guess is it's an out of date version of the haskell mtl library. Try installing a newer one with 'cabal install mtl' +"""]] diff --git a/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_2_2a6faf662ebb85a8f1c89adcdfb9adb6._comment b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_2_2a6faf662ebb85a8f1c89adcdfb9adb6._comment new file mode 100644 index 0000000000..09e7688fe8 --- /dev/null +++ b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_2_2a6faf662ebb85a8f1c89adcdfb9adb6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U" + nickname="Carlo" + subject="Not resolved" + date="2012-11-15T21:44:11Z" + content=""" +I did as instructed, same behavior though. + +I'll wait or keep trying things, whatever works better for you. +"""]] diff --git a/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_3_37f34baa34068def1adf794d0942e462._comment b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_3_37f34baa34068def1adf794d0942e462._comment new file mode 100644 index 0000000000..88cd12df95 --- /dev/null +++ b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_3_37f34baa34068def1adf794d0942e462._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 3" + date="2012-11-18T18:13:31Z" + content=""" +My other guess would be a too old version of ghc. +"""]] diff --git a/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_4_2f8a859fef9edc8eb93bf1cc74296702._comment b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_4_2f8a859fef9edc8eb93bf1cc74296702._comment new file mode 100644 index 0000000000..781259e790 --- /dev/null +++ b/doc/bugs/3.20121112_build_fails_on_Ubuntu_12.04/comment_4_2f8a859fef9edc8eb93bf1cc74296702._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlXlJDAF_lXaxbqeBdH4EGj6jsBjjrDODM" + nickname="Antoine" + subject="comment 4" + date="2012-11-21T07:33:43Z" + content=""" +I have the same problem (sorry, duplicate bug entry) on the same platform. The ghc version is 7.4.1 and is the one distributed by apt repos. +"""]] diff --git a/doc/bugs/3.20121113_build_error___39__not_in_scope_getAddBoxComR__39__.mdwn b/doc/bugs/3.20121113_build_error___39__not_in_scope_getAddBoxComR__39__.mdwn new file mode 100644 index 0000000000..59ca6b51fa --- /dev/null +++ b/doc/bugs/3.20121113_build_error___39__not_in_scope_getAddBoxComR__39__.mdwn @@ -0,0 +1,33 @@ +What steps will reproduce the problem? + +Building from latest source, Cabal update, cabal install --only dependencies, cabal configure, Cabal build + +What is the expected output? What do you see instead? + +Error message from build + +... + +Loading package DAV-0.2 ... linking ... done. + +Loading object (static) dist/build/git-annex/git-annex-tmp/Utility/libdiskfree.o ... done + +Loading object (static) dist/build/git-annex/git-annex-tmp/Utility/libmounts.o ... done + +final link ... done + + +Assistant/Threads/WebApp.hs:47:1: Not in scope: `getAddBoxComR' + +Assistant/Threads/WebApp.hs:47:1: Not in scope: `getEnableWebDAVR' + + +What version of git-annex are you using? On what operating system? + +Latest version via git from git-annex.branchable.com + +Debian Squeeze (6.0.6) + +Please provide any additional information below. + +> I noticed this earlier and fixed it. [[done]] --[[Joey]] diff --git a/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build.mdwn b/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build.mdwn new file mode 100644 index 0000000000..5acc4d5690 --- /dev/null +++ b/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build.mdwn @@ -0,0 +1,57 @@ +What steps will reproduce the problem? + +install from a git checkout of tag 4.20130227; error comes up at the "cabal install" stage. + + +What is the expected output? What do you see instead? + +Expected successful install; got: + + + git-annex.branchable.com$ cabal install --bindir=$HOME/bin + Resolving dependencies... + Configuring testpack-2.1.2... + Building testpack-2.1.2... + Preprocessing library testpack-2.1.2... + [1 of 3] Compiling Test.QuickCheck.Instances ( src/Test/QuickCheck/Instances.hs, dist/build/Test/QuickCheck/Instances.o ) + [2 of 3] Compiling Test.QuickCheck.Tools ( src/Test/QuickCheck/Tools.hs, dist/build/Test/QuickCheck/Tools.o ) + + src/Test/QuickCheck/Tools.hs:33:9: + Warning: Fields of `MkResult' not initialised: abort + In the expression: + MkResult + {ok = Just (expected == actual), expect = True, + interrupted = False, + reason = "Result: expected " + ++ show expected ++ ", got " ++ show actual, + stamp = [], callbacks = []} + In an equation for `@=?': + expected @=? actual + = MkResult + {ok = Just (expected == actual), expect = True, + interrupted = False, + reason = "Result: expected " + ++ show expected ++ ", got " ++ show actual, + stamp = [], callbacks = []} + [3 of 3] Compiling Test.HUnit.Tools ( src/Test/HUnit/Tools.hs, dist/build/Test/HUnit/Tools.o ) + + src/Test/HUnit/Tools.hs:131:57: + `maxDiscard' is not a (visible) constructor field name + + src/Test/HUnit/Tools.hs:177:40: Not in scope: `maxDiscard' + Failed to install testpack-2.1.2 + cabal: Error: some packages failed to install: + git-annex-4.20130227 depends on testpack-2.1.2 which failed to install. + testpack-2.1.2 failed during the building phase. The exception was: + ExitFailure 1 + git-annex.branchable.com$ + + +What version of git-annex are you using? On what operating system? + +trying to compile git checkout of 4.20130227 on OS X Lion. + +Please provide any additional information below. + + +> removed dependency on testpack [[done]] --[[Joey]] diff --git a/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build/comment_1_b7140e2bf1ea9c73ecc9e214095968e7._comment b/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build/comment_1_b7140e2bf1ea9c73ecc9e214095968e7._comment new file mode 100644 index 0000000000..b4e0c69f7d --- /dev/null +++ b/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build/comment_1_b7140e2bf1ea9c73ecc9e214095968e7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-02-28T02:12:50Z" + content=""" +Pass -f-TestSuite to cabal to disable building the test suite. +"""]] diff --git a/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build/comment_2_6be87b2fb2ed828e7b4bf785729e910e._comment b/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build/comment_2_6be87b2fb2ed828e7b4bf785729e910e._comment new file mode 100644 index 0000000000..d2d1d6bac6 --- /dev/null +++ b/doc/bugs/4.20130227_won__39__t_build_on_OS_X_Lion__44___because_testpack_won__39__t_build/comment_2_6be87b2fb2ed828e7b4bf785729e910e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 2" + date="2013-02-28T03:31:32Z" + content=""" +that fixed it, thank you! + +"""]] diff --git a/doc/bugs/Allow_syncing_to_a_specific_directory_on_a_USB_remote.mdwn b/doc/bugs/Allow_syncing_to_a_specific_directory_on_a_USB_remote.mdwn new file mode 100644 index 0000000000..2bba743e49 --- /dev/null +++ b/doc/bugs/Allow_syncing_to_a_specific_directory_on_a_USB_remote.mdwn @@ -0,0 +1,9 @@ +This follows up to the [comment made by Laszlo](http://git-annex.branchable.com/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/#comment-f26d3b6b45bb66601ecfaa883ace161c) on the [recent poll](http://git-annex.branchable.com/design/assistant/polls/what_is_preventing_me_from_using_git-annex_assistant/). + +I too need to be able to select the directory on the remote drive that the annex will be synced to. + +If I just add a remote drive via the web app, it syncs the repository to `/mnt/usb/annex`, and it looks like it just creates a bare repository in that folder. I need the repository to be synced to something like `/mnt/usb/subfolder/myspecifiedfoldername` and I need that remote to be a full repository. + +My use case is that I use the USB drive to keep annexes in sync between two computers. I have multiple annexes that need to be synced between the two computers, and none of them are in a directory called `annex`. I also need to be able to plug the drive into other computers and access the files directly, without doing a `git clone` or anything like that. I have all of this setup and working fine with just plain old git annex, but the web app does not seem to support creating new repositories with this workflow. + +I think it makes a lot of sense to allow the web application to add a new remote that is simply a directory. People like me could specify the path of the directory to be on the mounted USB drive. Others may want to add a remote that is a mounted network share or something like that. diff --git a/doc/bugs/Allow_syncing_to_a_specific_directory_on_a_USB_remote/comment_1_13ecedfbb34c3564af3a790b8bf0f591._comment b/doc/bugs/Allow_syncing_to_a_specific_directory_on_a_USB_remote/comment_1_13ecedfbb34c3564af3a790b8bf0f591._comment new file mode 100644 index 0000000000..cf13d97c16 --- /dev/null +++ b/doc/bugs/Allow_syncing_to_a_specific_directory_on_a_USB_remote/comment_1_13ecedfbb34c3564af3a790b8bf0f591._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmYiJgOvC4IDYkr2KIjMlfVD9r_1Sij_jY" + nickname="Douglas" + subject="Not limited to usb drives" + date="2013-01-23T02:08:38Z" + content=""" +I have a machine with multiple hard drives. One of these drives is strictly used for backups, ie /dev/sdc holds /home and /dev/sdd is a giant drive that i keep a backup of important items that I would like to have readily available in case of a drive failure. Somewhere in \"add more repositories\" the assistant should be able to create a copy of the repository on the same computer. When I saw the add a removable drive feature i thought it might work but it will not let me select a specific directory. + +Current setup, I have the assistant managing a group of repos: + + /home/me/Videos.Annex + /home/me/Photos.Annex + /home/me/Documents.Annex + + +I access these files via /home/me/DIRECTORY. However in case my /home drive breaks I would like to know that git-annex was faithfully keeping a copy in /mnt/bigdrive as well as on me@someothermachine. I would like to use the assistant to manage: + + /mnt/bigdrive/Annex.Backups/Videos.Annex + /mnt/bigdrive/Annex.Backups/Photos.Annex + /mnt/bigdrive/Annex.Backups/Documents.Annex + + + + +"""]] diff --git a/doc/bugs/Annex_thinks_file_exists_afer_being_dropped.mdwn b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped.mdwn new file mode 100644 index 0000000000..48be21ceec --- /dev/null +++ b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped.mdwn @@ -0,0 +1,27 @@ +#### What steps will reproduce the problem? + +I've posted some code here: + https://gist.github.com/4552036 + + +#### What is the expected output? What do you see instead? + +I think I've found three bugs. If they aren't bugs then there is a usage issue that could do with some documentation improvements. + +Problem 1 - With 3 local annexes git-annex doesn't seem to search properly for them (See code) + +Problem 2 - Even after a sync an annex thinks another (local) annex has a file, even after it has been dropped (See code) + +SCARY bug - `whereis` seems to think that a locally dropped file still exists (See code) + + +#### What version of git-annex are you using? On what operating system? + +git-annex version: 3.20130114 + +OS: OSX 10.6.8 + + +#### Please provide any additional information below. + +> [[done]]; see comments --[[Joey]] diff --git a/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_1_1d100441fd1ef529eb854b350fece9ee._comment b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_1_1d100441fd1ef529eb854b350fece9ee._comment new file mode 100644 index 0000000000..7ddc9d2dde --- /dev/null +++ b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_1_1d100441fd1ef529eb854b350fece9ee._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 1" + date="2013-01-17T00:46:27Z" + content=""" +* Problem 1 - With 3 local annexes git-annex doesn't seem to search properly for them (See code) + +While test1 thinks test2 and test3 have the file, they don't; it was just dropped from them. So the check that the other repos have the file is done correctly here, and it correctly refuses to drop it. The message is confusing in that it suggests the repositories are not available. + +* Problem 2 - Even after a sync an annex thinks another (local) annex has a file, even after it has been dropped (See code) + +I have not fully analized this, but it does not have anything to do with direct mode, it's a location log bookeeping problem. + +The merge where this happens looks like this: + +
+ - 1358382328.617882s 0 620af8a6-603c-11e2-b332-f3ecf1856be4 (test2)
+ + 1358382324.420048s 1 620af8a6-603c-11e2-b332-f3ecf1856be4 (test2)
+  +1358382324.572356s 1 622cf3b6-603c-11e2-bb55-4fe0f1a02dee (test3)
+++ 1358382328.472266s 0 622cf3b6-603c-11e2-bb55-4fe0f1a02dee (test3)
+
+ +The first 2 lines are the problem, and in this merge, a newer line is deleted leaving behind an older, now incorrect line. Union merge should not be deleting lines at all, so this appears to be a bug in the union merge code. + +* SCARY bug - whereis seems to think that a locally dropped file still exists (See code) + +This is a consequence of problem 2; the bad location data gets synced back to test2. Of course fsck will fix this, and the location log's is not relied on when dropping files, so this cannot result in data loss at least. +"""]] diff --git a/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_2_166c459c2b27859cf457e17da685fe75._comment b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_2_166c459c2b27859cf457e17da685fe75._comment new file mode 100644 index 0000000000..661386d8e0 --- /dev/null +++ b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_2_166c459c2b27859cf457e17da685fe75._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 2" + date="2013-01-17T01:35:44Z" + content=""" +All right, the bug here involves differing changes to location log info for the same file being synced from two repositories and merged. Which probably explains why this bug was not noticed before. + +The union merge code generates a stream of data to feed into a single call to update-index. For each ref being merged, it calculates a union merge between that ref and the index. However, this means that the merge data for test2 is fed into update-index, and this is followed by the merge data for test3, which overwrites the previous merge data, causing the unique line from it to be lost. + +A fix, although perhaps not the most efficient way, is to run update-index once for each ref to merge, so that each merge builds on the one before. I've put this in place and can confirm problems #2 and #3 are fixed. Leaving open for the minor wording problem #1. + +Thanks for an excellent test case for this most unusual bug! (Which I should haskell-ize and add to the regression test suite, when I have time..) +"""]] diff --git a/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_3_9d985b6e7973bfaaf8b4f5349d8c13ee._comment b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_3_9d985b6e7973bfaaf8b4f5349d8c13ee._comment new file mode 100644 index 0000000000..7301dda92e --- /dev/null +++ b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_3_9d985b6e7973bfaaf8b4f5349d8c13ee._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 3" + date="2013-01-17T01:45:59Z" + content=""" +Fixed problem #1; it now no longer says to make available repositories where it has confirmed the file is not present. +"""]] diff --git a/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_4_3e084cff454b95c7170c0225a53f0c30._comment b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_4_3e084cff454b95c7170c0225a53f0c30._comment new file mode 100644 index 0000000000..a4fc5813aa --- /dev/null +++ b/doc/bugs/Annex_thinks_file_exists_afer_being_dropped/comment_4_3e084cff454b95c7170c0225a53f0c30._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://a-or-b.myopenid.com/" + ip="203.45.2.230" + subject="comment 4" + date="2013-01-17T01:57:06Z" + content=""" +No problem. Glad to be of assistance. + +This was actually a bug that I ran into whilst testing direct mode. Quantifying the commands needed to reproduce was a bit of a challenge. :-) + +"""]] diff --git a/doc/bugs/Assistant_enters_eternal_loop_and_eats_up_all_of_RAM_after_X_restart.mdwn b/doc/bugs/Assistant_enters_eternal_loop_and_eats_up_all_of_RAM_after_X_restart.mdwn new file mode 100644 index 0000000000..d3f5394310 --- /dev/null +++ b/doc/bugs/Assistant_enters_eternal_loop_and_eats_up_all_of_RAM_after_X_restart.mdwn @@ -0,0 +1,24 @@ +*What steps will reproduce the problem?* + +Log in to X, have the DE start the assistant with --autostart. Then kill X with ctrl+alt+backspace and log back in once X comes back up. + +*What is the expected output? What do you see instead?* + +It enters an eternal loop, quickly using up all of the available RAM as well as 100% of CPU. Initially noticed because the computer became extremely sluggish, at which point the assistant was using up over 7G (of the available 8G) of RAM, and all of the available power on one of the CPU cores. + +Killing the assistant and then starting it again results in it working normally again. + +*What version of git-annex are you using? On what operating system?* + +git-annex version: 3.20121010 on Debian Sid (under GNOME3/Gnome-Shell in case that's relevant). +I've also seen it happen on another computer in similar circumstances. That one on Debian Testing, with git-annex from sid (so same git-annex version). In this case X was restarted while running with /etc/init.d/gdm3 restart, and again the issue appeared after logging out and then back in. + +*Please provide any additional information below.* + +Given that the assistant isn't really using X directly, I suppose this could be due to losing its connection to the gpg and ssh agents as a side-effect of X being shut down. I'm not sure if it happens immediately after X being killed, or once I log back in again. + +> Reproduced. Root caused to a bug in the haskell dbus library, which I can +> reproduce with 2 line test case; basically anything using connectSession +> will do this when the dbus session goes away. Sent test case and +> profiling data to library author to get it fixed, and have disabled +> dbus in git-annex in the meantime. [[done]] --[[Joey]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor.mdwn b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor.mdwn new file mode 100644 index 0000000000..832ad80217 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor.mdwn @@ -0,0 +1,26 @@ +### What steps will reproduce the problem? + +Run `git annex assistant`. + + +### What is the expected output? What do you see instead? + +git-annex complains: + + dbus failed; falling back to mtab polling (ClientError {clientErrorMessage = + "Call failed: The name org.gtk.Private.GduVolumeMonitor was not provided + by any .service files", clientErrorFatal = False}) + +This is because the `gvfs-gdu-volume-monitor` daemon has been obsoleted and removed from GNOME 3.6 (maybe even earlier). + +git-annex should start using `gvfs-udisks2-volume-monitor` at bus name `org.gtk.Private.UDisks2VolumeMonitor`. + +Alternatively, git-annex should stop relying on any per-user services, and use kernel interfaces directly when available. (This way, monitoring could work even if the user wasn't logged in and/or didn't have a DBus session bus.) + + * On all Linux kernels since 2.6.15, the `/proc/self/mounts` file is pollable – you can use **select(), poll() or epoll** to detect new mounted filesystems, without having to rely on periodic checks. (Run `findmnt -p` to see it in action.) + + * On BSD systems, kqueue on `/etc/mtab`. + +### What version of git-annex are you using? On what operating system? + +git-annex 3.20130102 on Linux 3.7.1, GNOME 3.7 diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_10_0e1db417a5815ea903c1f7ccd07308c4._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_10_0e1db417a5815ea903c1f7ccd07308c4._comment new file mode 100644 index 0000000000..37e2613a23 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_10_0e1db417a5815ea903c1f7ccd07308c4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://nullroute.eu.org/~grawity/" + nickname="Mantas" + subject="comment 10" + date="2013-01-10T21:51:29Z" + content=""" +Works now. +"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_1_28b0cfcba8902c9c16dbe6c4b07984c4._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_1_28b0cfcba8902c9c16dbe6c4b07984c4._comment new file mode 100644 index 0000000000..fa71012d27 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_1_28b0cfcba8902c9c16dbe6c4b07984c4._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 1" + date="2013-01-10T17:28:30Z" + content=""" +Thanks, excellent info. + +Any chance you could provide dbus-monitor output for the events generated by org.gtk.Private.UDisks2VolumeMonitor when a volume is mounted, and unmounted? +"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_2_952b3f78da756ff5f89235db94bec67f._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_2_952b3f78da756ff5f89235db94bec67f._comment new file mode 100644 index 0000000000..2db95f8af9 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_2_952b3f78da756ff5f89235db94bec67f._comment @@ -0,0 +1,53 @@ +[[!comment format=mdwn + username="http://nullroute.eu.org/~grawity/" + nickname="Mantas" + subject="comment 2" + date="2013-01-10T17:38:47Z" + content=""" +When mounted: + +
+signal path=/org/gtk/Private/RemoteVolumeMonitor; interface=org.gtk.Private.RemoteVolumeMonitor; member=MountAdded
+   string \"org.gtk.Private.UDisks2VolumeMonitor\"
+   string \"0x1971ab0\"
+   struct {
+      string \"0x1971ab0\"
+      string \"grawpqi\"
+      string \". GThemedIcon media-removable media\"
+      string \". GThemedIcon media-removable-symbolic media-removable media\"
+      string \"\"
+      string \"file:///run/media/grawity/grawpqi\"
+      boolean true
+      string \"0x18f4e50\"
+      array [
+      ]
+      string \"gvfs.time_detected_usec.1357838999510252\"
+      array [
+      ]
+   }
+
+ +When unmounted: + +
+signal path=/org/gtk/Private/RemoteVolumeMonitor; interface=org.gtk.Private.RemoteVolumeMonitor; member=MountRemoved
+   string \"org.gtk.Private.UDisks2VolumeMonitor\"
+   string \"0x1971910\"
+   struct {
+      string \"0x1971910\"
+      string \"grawpqi\"
+      string \". GThemedIcon media-removable media\"
+      string \". GThemedIcon media-removable-symbolic media-removable media\"
+      string \"\"
+      string \"file:///run/media/grawity/grawpqi\"
+      boolean true
+      string \"\"
+      array [
+      ]
+      string \"gvfs.time_detected_usec.1357839107487969\"
+      array [
+      ]
+   }
+
+ +"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_3_d86aba42d014c4b4f708dcb5fe86e055._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_3_d86aba42d014c4b4f708dcb5fe86e055._comment new file mode 100644 index 0000000000..c1bc52f2c8 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_3_d86aba42d014c4b4f708dcb5fe86e055._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 3" + date="2013-01-10T19:08:50Z" + content=""" +I've just committed support for using the new name. I've not been able to test it yet, as I don't have a new enough gnome here. Any testing you can do much appreciated. + +Leaving this bug open until it gets tested, and also because it's certianly appealing to just use poll rather than this fragile dbus stuff. And in any case, should add OSX support. +"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_4_9aaf296ef53da317d6dc6728705d5c56._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_4_9aaf296ef53da317d6dc6728705d5c56._comment new file mode 100644 index 0000000000..6df6f40f20 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_4_9aaf296ef53da317d6dc6728705d5c56._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://nullroute.eu.org/~grawity/" + nickname="Mantas" + subject="comment 4" + date="2013-01-10T19:41:22Z" + content=""" +I'm getting: + + dbus failed; falling back to mtab polling (ClientError {clientErrorMessage = + \"Call failed: The name org.gtk.Private.RemoteVolumeMonitor was not provided + by any .service files\", clientErrorFatal = False}) + +The volume monitor's bus name should be `org.gtk.Private.UDisks2VolumeMonitor`. + +`org.gtk.Private.RemoteVolumeMonitor` is the interface name, which all Gvfs monitors implement. +"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_5_0d5f8a05a1505660f7ff1bc4ac6ff271._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_5_0d5f8a05a1505660f7ff1bc4ac6ff271._comment new file mode 100644 index 0000000000..22b1d923c4 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_5_0d5f8a05a1505660f7ff1bc4ac6ff271._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="oops.." + date="2013-01-10T20:09:56Z" + content=""" +Think I have the right name in there now. +"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_6_3dfdfd49597c85575cb689adb70d2de6._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_6_3dfdfd49597c85575cb689adb70d2de6._comment new file mode 100644 index 0000000000..3320b77daa --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_6_3dfdfd49597c85575cb689adb70d2de6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://nullroute.eu.org/~grawity/" + nickname="Mantas" + subject="comment 6" + date="2013-01-10T20:13:42Z" + content=""" +It doesn't look completely right – the *service* name (in `checkMountMonitor`) is `org.gtk.Private.UDisks2VolumeMonitor`, but the *interface* name (in `mountChanged`) is `org.gtk.Private.RemoteVolumeMonitor`, so the fix changed too much. +"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_7_943a446c60ed9d7d4f240ba7f00fe925._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_7_943a446c60ed9d7d4f240ba7f00fe925._comment new file mode 100644 index 0000000000..8f0cd64cc3 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_7_943a446c60ed9d7d4f240ba7f00fe925._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="been a while.." + date="2013-01-10T20:19:33Z" + content=""" +Think I have it now. +"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_8_9563859850fb40b1cc2c20c516c12960._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_8_9563859850fb40b1cc2c20c516c12960._comment new file mode 100644 index 0000000000..0a3ae95a75 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_8_9563859850fb40b1cc2c20c516c12960._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://nullroute.eu.org/~grawity/" + nickname="Mantas" + subject="build failure" + date="2013-01-10T20:24:09Z" + content=""" +
+$ cabal build
+Building git-annex-3.20130108...
+Preprocessing executable 'git-annex' for git-annex-3.20130108...
+[269 of 299] Compiling Assistant.Threads.MountWatcher ( Assistant/Threads/MountWatcher.hs,
+    dist/build/git-annex/git-annex-tmp/Assistant/Threads/MountWatcher.o )
+
+Assistant/Threads/MountWatcher.hs:122:17: Not in scope: `gvfs'
+
+"""]] diff --git a/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_9_cf6221c585ee3dbf039bdaea71842d9b._comment b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_9_cf6221c585ee3dbf039bdaea71842d9b._comment new file mode 100644 index 0000000000..b5b66b8ea3 --- /dev/null +++ b/doc/bugs/Assistant_uses_obsolete_GDU_volume_monitor/comment_9_cf6221c585ee3dbf039bdaea71842d9b._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 9" + date="2013-01-10T20:44:37Z" + content=""" +Realized that'd happen while away on a walk.. fixed now. + +"""]] diff --git a/doc/bugs/Build-depends_needs___39__hxt__39___added_-_3.20121127.mdwn b/doc/bugs/Build-depends_needs___39__hxt__39___added_-_3.20121127.mdwn new file mode 100644 index 0000000000..83720cca23 --- /dev/null +++ b/doc/bugs/Build-depends_needs___39__hxt__39___added_-_3.20121127.mdwn @@ -0,0 +1,36 @@ +What steps will reproduce the problem? + +Install git-annex via cabal - either from Hackage or as a manual install. (i.e. ) + +What is the expected output? What do you see instead? + +Expect a clean install. + +However, get the following error: + + Assistant/Install.hs:24:8: + Could not find module `Data.AssocList' + It is a member of the hidden package `hxt-9.3.1.1'. + Perhaps you need to add `hxt' to the build-depends in your .cabal file. + Use -v to see a list of the files searched for. + + +What version of git-annex are you using? On what operating system? + +git-annex: 3.20121127 +OS: Mac OSX 10.6.8 + +Please provide any additional information below. + +The fix seems to be as simple as adding 'htx' to the 'git-annex.cabal' file: + + Executable git-annex + Main-Is: git-annex.hs + Build-Depends: MissingH, hslogger, directory, filepath, + unix, containers, utf8-string, network (>= 2.0), mtl (>= 2.1.1), + bytestring, old-locale, time, + -- Added htx here + hxt, + pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP, + +> I removed the need for hxt, which was accidental. [[done]] --[[Joey]] diff --git a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn new file mode 100644 index 0000000000..43fb0323c4 --- /dev/null +++ b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn @@ -0,0 +1,11 @@ +While following the instructions given at the OSX build page , I get this error: + +$ make +ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make git-annex + +Utility/JSONStream.hs:14:8: + Could not find module `Text.JSON': + Use -v to see a list of the files searched for. +make: *** [git-annex] Error 1 + +> Updated the instructions. [[done]] --[[Joey]] diff --git a/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn b/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn new file mode 100644 index 0000000000..ba87b11915 --- /dev/null +++ b/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn @@ -0,0 +1,45 @@ +Applying this + +
+laplace:git-annex jtang$ git diff
+diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
+index b9630b1..bf36e59 100644
+--- a/Assistant/WebApp/Configurators.hs
++++ b/Assistant/WebApp/Configurators.hs
+@@ -101,7 +101,7 @@ checkRepositoryPath p = do
+  -
+  - If run in another directory, the user probably wants to put it there. -}
+ defaultRepositoryPath :: Bool -> IO FilePath
+-defaultRepositoryPath firstrun = do
++defaultRepositoryPath firstRun = do
+        cwd <- liftIO $ getCurrentDirectory
+        home <- myHomeDir
+        if home == cwd && firstRun
+
+ +Causes this to occur, + +
+Assistant/WebApp/Configurators.hs:114:17:
+    Couldn't match expected type `Control.Monad.Trans.RWS.Lazy.RWST
+                                    (Maybe (Env, FileEnv), WebApp, [Yesod.Form.Types.Lang])
+                                    Enctype
+                                    Ints
+                                    (GHandler WebApp WebApp)
+                                    t0'
+                with actual type `Text'
+    Expected type: String
+                   -> Control.Monad.Trans.RWS.Lazy.RWST
+                        (Maybe (Env, FileEnv), WebApp, [Yesod.Form.Types.Lang])
+                        Enctype
+                        Ints
+                        (GHandler WebApp WebApp)
+                        t0
+      Actual type: String -> Text
+    In the first argument of `(.)', namely `T.pack'
+    In the first argument of `(<$>)', namely
+      `T.pack . addTrailingPathSeparator'
+make: *** [git-annex] Error 1
+
+ +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Building_fails:_Could_not_find_module___96__Text.Blaze__39__.mdwn b/doc/bugs/Building_fails:_Could_not_find_module___96__Text.Blaze__39__.mdwn new file mode 100644 index 0000000000..b75d92e18b --- /dev/null +++ b/doc/bugs/Building_fails:_Could_not_find_module___96__Text.Blaze__39__.mdwn @@ -0,0 +1,105 @@ +What steps will reproduce the problem? + +
+dominik@Atlantis:/var/tmp$ git clone git://github.com/joeyh/git-annex.git
+Cloning into 'git-annex'...
+remote: Counting objects: 40580, done.
+remote: Compressing objects: 100% (10514/10514), done.
+remote: Total 40580 (delta 29914), reused 40502 (delta 29837)
+Receiving objects: 100% (40580/40580), 9.17 MiB | 238 KiB/s, done.
+Resolving deltas: 100% (29914/29914), done.
+dominik@Atlantis:/var/tmp$ cd git-annex/
+dominik@Atlantis:/var/tmp/git-annex$ cabal update
+Downloading the latest package list from hackage.haskell.org
+dominik@Atlantis:/var/tmp/git-annex$ cabal install --only-dependencies
+Resolving dependencies...
+All the requested packages are already installed:
+Use --reinstall if you want to reinstall anyway.
+dominik@Atlantis:/var/tmp/git-annex$ cabal configure
+Resolving dependencies...
+[ 1 of 21] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, dist/setup/Utility/FileSystemEncoding.o )
+[ 2 of 21] Compiling Utility.Applicative ( Utility/Applicative.hs, dist/setup/Utility/Applicative.o )
+[ 3 of 21] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, dist/setup/Utility/PartialPrelude.o )
+[ 4 of 21] Compiling Utility.UserInfo ( Utility/UserInfo.hs, dist/setup/Utility/UserInfo.o )
+[ 5 of 21] Compiling Utility.Monad    ( Utility/Monad.hs, dist/setup/Utility/Monad.o )
+[ 6 of 21] Compiling Utility.Path     ( Utility/Path.hs, dist/setup/Utility/Path.o )
+[ 7 of 21] Compiling Utility.OSX      ( Utility/OSX.hs, dist/setup/Utility/OSX.o )
+[ 8 of 21] Compiling Utility.Exception ( Utility/Exception.hs, dist/setup/Utility/Exception.o )
+[ 9 of 21] Compiling Utility.TempFile ( Utility/TempFile.hs, dist/setup/Utility/TempFile.o )
+[10 of 21] Compiling Utility.Misc     ( Utility/Misc.hs, dist/setup/Utility/Misc.o )
+[11 of 21] Compiling Utility.Process  ( Utility/Process.hs, dist/setup/Utility/Process.o )
+[12 of 21] Compiling Utility.FreeDesktop ( Utility/FreeDesktop.hs, dist/setup/Utility/FreeDesktop.o )
+[13 of 21] Compiling Assistant.Install.AutoStart ( Assistant/Install/AutoStart.hs, dist/setup/Assistant/Install/AutoStart.o )
+[14 of 21] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, dist/setup/Utility/SafeCommand.o )
+[15 of 21] Compiling Utility.Directory ( Utility/Directory.hs, dist/setup/Utility/Directory.o )
+[16 of 21] Compiling Common           ( Common.hs, dist/setup/Common.o )
+[17 of 21] Compiling Locations.UserConfig ( Locations/UserConfig.hs, dist/setup/Locations/UserConfig.o )
+[18 of 21] Compiling Build.TestConfig ( Build/TestConfig.hs, dist/setup/Build/TestConfig.o )
+[19 of 21] Compiling Build.Configure  ( Build/Configure.hs, dist/setup/Build/Configure.o )
+[20 of 21] Compiling Build.InstallDesktopFile ( Build/InstallDesktopFile.hs, dist/setup/Build/InstallDesktopFile.o )
+[21 of 21] Compiling Main             ( Setup.hs, dist/setup/Main.o )
+Linking ./dist/setup/setup ...
+  checking version... 3.20121018
+  checking git... yes
+  checking git version... 1.7.10.4
+  checking cp -a... yes
+  checking cp -p... yes
+  checking cp --reflink=auto... yes
+  checking uuid generator... uuidgen
+  checking xargs -0... yes
+  checking rsync... yes
+  checking curl... yes
+  checking wget... yes
+  checking bup... no
+  checking gpg... yes
+  checking lsof... yes
+  checking host... no
+  checking ssh connection caching... yes
+  checking sha1... sha1sum
+  checking sha256... sha256sum
+  checking sha512... sha512sum
+  checking sha224... sha224sum
+  checking sha384... sha384sum
+Configuring git-annex-3.20121018...
+dominik@Atlantis:/var/tmp/git-annex$ cabal build
+Building git-annex-3.20121018...
+Preprocessing executable 'git-annex' for git-annex-3.20121018...
+
+Assistant/Alert.hs:21:8:
+    Could not find module `Text.Blaze'
+    It is a member of the hidden package `blaze-markup-0.5.1.1'.
+    Perhaps you need to add `blaze-markup' to the build-depends in your .cabal file.
+    Use -v to see a list of the files searched for.
+
+ +What is the expected output? What do you see instead? + +I expect the latest git HEAD to build without an error message or provide me with a package I need to install. Instead the error above is shown. In fact the package requested is installed: + +
+dominik@Atlantis:/var/tmp/git-annex$ cabal install blaze-markup
+Resolving dependencies...
+All the requested packages are already installed:
+blaze-markup-0.5.1.1
+Use --reinstall if you want to reinstall anyway.
+
+ +What version of git-annex are you using? On what operating system? + +git HEAD, Ubuntu 12.10 + +Please provide any additional information below. + +
+$ cabal --version
+cabal-install version 0.14.0
+using version 1.14.0 of the Cabal library 
+
+$ ghc --version
+The Glorious Glasgow Haskell Compilation System, version 7.4.2
+
+$ uname -a
+Linux Atlantis 3.5.0-17-generic #28-Ubuntu SMP Tue Oct 9 19:31:23 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux
+
+ +> [[done]] --[[Joey]] diff --git a/doc/bugs/Building_fails:_Not_in_scope:___96__myHomeDir__39___.mdwn b/doc/bugs/Building_fails:_Not_in_scope:___96__myHomeDir__39___.mdwn new file mode 100644 index 0000000000..e1d2da4ce1 --- /dev/null +++ b/doc/bugs/Building_fails:_Not_in_scope:___96__myHomeDir__39___.mdwn @@ -0,0 +1,56 @@ +What steps will reproduce the problem? + +Building of the current github HEAD fails with a strange error message regarding OSX. I'm not using OSX but Ubuntu 12.10, why is cabal trying to build these files? + +
+dominik@Atlantis:/var/tmp$ git clone git://github.com/joeyh/git-annex.git
+Cloning into 'git-annex'...
+remote: Counting objects: 40243, done.
+remote: Compressing objects: 100% (10568/10568), done.
+remote: Total 40243 (delta 29647), reused 40044 (delta 29449)
+Receiving objects: 100% (40243/40243), 9.12 MiB | 184 KiB/s, done.
+Resolving deltas: 100% (29647/29647), done.
+dominik@Atlantis:/var/tmp$ cd git-annex/
+dominik@Atlantis:/var/tmp/git-annex$ cabal update
+Downloading the latest package list from hackage.haskell.org
+dominik@Atlantis:/var/tmp/git-annex$ cabal install --only-dependencies
+Resolving dependencies...
+All the requested packages are already installed:
+Use --reinstall if you want to reinstall anyway.
+dominik@Atlantis:/var/tmp/git-annex$ cabal configure
+Resolving dependencies...
+[ 1 of 21] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, dist/setup/Utility/FileSystemEncoding.o )
+[ 2 of 21] Compiling Utility.Applicative ( Utility/Applicative.hs, dist/setup/Utility/Applicative.o )
+[ 3 of 21] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, dist/setup/Utility/PartialPrelude.o )
+[ 4 of 21] Compiling Utility.UserInfo ( Utility/UserInfo.hs, dist/setup/Utility/UserInfo.o )
+[ 5 of 21] Compiling Utility.Monad    ( Utility/Monad.hs, dist/setup/Utility/Monad.o )
+[ 6 of 21] Compiling Utility.Path     ( Utility/Path.hs, dist/setup/Utility/Path.o )
+[ 7 of 21] Compiling Utility.OSX      ( Utility/OSX.hs, dist/setup/Utility/OSX.o )
+
+Utility/OSX.hs:22:17: Not in scope: `myHomeDir'
+
+ +What is the expected output? What do you see instead? + +I expect cabal to build git-annex. + +What version of git-annex are you using? On what operating system? + +github HEAD on Ubuntu 12.10 + +Please provide any additional information below. + +
+$ cabal --version
+cabal-install version 0.14.0
+using version 1.14.0 of the Cabal library 
+
+$ ghc --version
+The Glorious Glasgow Haskell Compilation System, version 7.4.2
+
+$ uname -a
+Linux Atlantis 3.5.0-17-generic #28-Ubuntu SMP Tue Oct 9 19:31:23 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux
+
+
+ +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Building_fails:__Could_not_find_module___96__Data.XML.Types__39__.mdwn b/doc/bugs/Building_fails:__Could_not_find_module___96__Data.XML.Types__39__.mdwn new file mode 100644 index 0000000000..c033534369 --- /dev/null +++ b/doc/bugs/Building_fails:__Could_not_find_module___96__Data.XML.Types__39__.mdwn @@ -0,0 +1,82 @@ +What steps will reproduce the problem? + +
+dominik@Atlantis:/var/tmp$ git clone git://github.com/joeyh/git-annex.git
+Cloning into 'git-annex'...
+remote: Counting objects: 40841, done.
+remote: Compressing objects: 100% (10648/10648), done.
+remote: Total 40841 (delta 30135), reused 40669 (delta 29964)
+Receiving objects: 100% (40841/40841), 9.21 MiB | 517 KiB/s, done.
+Resolving deltas: 100% (30135/30135), done.
+dominik@Atlantis:/var/tmp/git-annex$ cabal install --only-dependencies
+Resolving dependencies...
+All the requested packages are already installed:
+Use --reinstall if you want to reinstall anyway.
+dominik@Atlantis:/var/tmp/git-annex$ cabal configure
+Resolving dependencies...
+[ 1 of 21] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, dist/setup/Utility/FileSystemEncoding.o )
+[ 2 of 21] Compiling Utility.Applicative ( Utility/Applicative.hs, dist/setup/Utility/Applicative.o )
+[ 3 of 21] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, dist/setup/Utility/PartialPrelude.o )
+[ 4 of 21] Compiling Utility.UserInfo ( Utility/UserInfo.hs, dist/setup/Utility/UserInfo.o )
+[ 5 of 21] Compiling Utility.Monad    ( Utility/Monad.hs, dist/setup/Utility/Monad.o )
+[ 6 of 21] Compiling Utility.Path     ( Utility/Path.hs, dist/setup/Utility/Path.o )
+[ 7 of 21] Compiling Utility.OSX      ( Utility/OSX.hs, dist/setup/Utility/OSX.o )
+[ 8 of 21] Compiling Utility.Exception ( Utility/Exception.hs, dist/setup/Utility/Exception.o )
+[ 9 of 21] Compiling Utility.TempFile ( Utility/TempFile.hs, dist/setup/Utility/TempFile.o )
+[10 of 21] Compiling Utility.Misc     ( Utility/Misc.hs, dist/setup/Utility/Misc.o )
+[11 of 21] Compiling Utility.Process  ( Utility/Process.hs, dist/setup/Utility/Process.o )
+[12 of 21] Compiling Utility.FreeDesktop ( Utility/FreeDesktop.hs, dist/setup/Utility/FreeDesktop.o )
+[13 of 21] Compiling Assistant.Install.AutoStart ( Assistant/Install/AutoStart.hs, dist/setup/Assistant/Install/AutoStart.o )
+[14 of 21] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, dist/setup/Utility/SafeCommand.o )
+[15 of 21] Compiling Utility.Directory ( Utility/Directory.hs, dist/setup/Utility/Directory.o )
+[16 of 21] Compiling Common           ( Common.hs, dist/setup/Common.o )
+[17 of 21] Compiling Locations.UserConfig ( Locations/UserConfig.hs, dist/setup/Locations/UserConfig.o )
+[18 of 21] Compiling Build.TestConfig ( Build/TestConfig.hs, dist/setup/Build/TestConfig.o )
+[19 of 21] Compiling Build.Configure  ( Build/Configure.hs, dist/setup/Build/Configure.o )
+[20 of 21] Compiling Build.InstallDesktopFile ( Build/InstallDesktopFile.hs, dist/setup/Build/InstallDesktopFile.o )
+[21 of 21] Compiling Main             ( Setup.hs, dist/setup/Main.o )
+Linking ./dist/setup/setup ...
+  checking version... 3.20121018
+  checking git... yes
+  checking git version... 1.7.10.4
+  checking cp -a... yes
+  checking cp -p... yes
+  checking cp --reflink=auto... yes
+  checking uuid generator... uuidgen
+  checking xargs -0... yes
+  checking rsync... yes
+  checking curl... yes
+  checking wget... yes
+  checking bup... no
+  checking gpg... yes
+  checking lsof... yes
+  checking ssh connection caching... yes
+  checking sha1... sha1sum
+  checking sha256... sha256sum
+  checking sha512... sha512sum
+  checking sha224... sha224sum
+  checking sha384... sha384sum
+Configuring git-annex-3.20121018...
+dominik@Atlantis:/var/tmp/git-annex$ cabal build
+Building git-annex-3.20121018...
+Preprocessing executable 'git-annex' for git-annex-3.20121018...
+
+Assistant/XMPP.hs:18:8:
+    Could not find module `Data.XML.Types'
+    It is a member of the hidden package `xml-types-0.3.3'.
+    Perhaps you need to add `xml-types' to the build-depends in your .cabal file.
+    Use -v to see a list of the files searched for.
+
+ +What is the expected output? What do you see instead? + +I exepect the current git HEAD to build without errors. + +What version of git-annex are you using? On what operating system? + +git-annex HEAD from git, Ubuntu 12.10 + +Please provide any additional information below. + +> [[done]] --[[Joey]] (and tested the whole cabal build, which I usually +> only do on releases) diff --git a/doc/bugs/Building_fails:___Not_in_scope:_type_constructor_or_class___96__Html__39__.mdwn b/doc/bugs/Building_fails:___Not_in_scope:_type_constructor_or_class___96__Html__39__.mdwn new file mode 100644 index 0000000000..6459e3f2ed --- /dev/null +++ b/doc/bugs/Building_fails:___Not_in_scope:_type_constructor_or_class___96__Html__39__.mdwn @@ -0,0 +1,189 @@ +What steps will reproduce the problem? + +
+dominik@Atlantis:/var/tmp/git-annex$ cabal build
+Building git-annex-3.20121018...
+Preprocessing executable 'git-annex' for git-annex-3.20121018...
+
+Assistant/Threads/NetWatcher.hs:26:2:
+     warning: #warning Building without dbus support; will poll for network connection changes [-Wcpp]
+
+Assistant/Threads/MountWatcher.hs:33:2:
+     warning: #warning Building without dbus support; will use mtab polling [-Wcpp]
+[  1 of 270] Compiling Utility.Dot      ( Utility/Dot.hs, dist/build/git-annex/git-annex-tmp/Utility/Dot.o )
+[  2 of 270] Compiling Utility.ThreadLock ( Utility/ThreadLock.hs, dist/build/git-annex/git-annex-tmp/Utility/ThreadLock.o )
+[  3 of 270] Compiling Utility.Mounts   ( dist/build/git-annex/git-annex-tmp/Utility/Mounts.hs, dist/build/git-annex/git-annex-tmp/Utility/Mounts.o )
+[  4 of 270] Compiling Utility.Yesod    ( Utility/Yesod.hs, dist/build/git-annex/git-annex-tmp/Utility/Yesod.o )
+[  5 of 270] Compiling Utility.Tense    ( Utility/Tense.hs, dist/build/git-annex/git-annex-tmp/Utility/Tense.o )
+[  6 of 270] Compiling Utility.Verifiable ( Utility/Verifiable.hs, dist/build/git-annex/git-annex-tmp/Utility/Verifiable.o )
+[  7 of 270] Compiling Assistant.Types.TransferSlots ( Assistant/Types/TransferSlots.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/TransferSlots.o )
+[  8 of 270] Compiling Types.StandardGroups ( Types/StandardGroups.hs, dist/build/git-annex/git-annex-tmp/Types/StandardGroups.o )
+[  9 of 270] Compiling Utility.Percentage ( Utility/Percentage.hs, dist/build/git-annex/git-annex-tmp/Utility/Percentage.o )
+[ 10 of 270] Compiling Utility.Base64   ( Utility/Base64.hs, dist/build/git-annex/git-annex-tmp/Utility/Base64.o )
+[ 11 of 270] Compiling Utility.DataUnits ( Utility/DataUnits.hs, dist/build/git-annex/git-annex-tmp/Utility/DataUnits.o )
+[ 12 of 270] Compiling Utility.JSONStream ( Utility/JSONStream.hs, dist/build/git-annex/git-annex-tmp/Utility/JSONStream.o )
+[ 13 of 270] Compiling Messages.JSON    ( Messages/JSON.hs, dist/build/git-annex/git-annex-tmp/Messages/JSON.o )
+[ 14 of 270] Compiling Build.SysConfig  ( Build/SysConfig.hs, dist/build/git-annex/git-annex-tmp/Build/SysConfig.o )
+[ 15 of 270] Compiling Types.KeySource  ( Types/KeySource.hs, dist/build/git-annex/git-annex-tmp/Types/KeySource.o )
+[ 16 of 270] Compiling Utility.State    ( Utility/State.hs, dist/build/git-annex/git-annex-tmp/Utility/State.o )
+[ 17 of 270] Compiling Types.UUID       ( Types/UUID.hs, dist/build/git-annex/git-annex-tmp/Types/UUID.o )
+[ 18 of 270] Compiling Types.Messages   ( Types/Messages.hs, dist/build/git-annex/git-annex-tmp/Types/Messages.o )
+[ 19 of 270] Compiling Types.Group      ( Types/Group.hs, dist/build/git-annex/git-annex-tmp/Types/Group.o )
+[ 20 of 270] Compiling Types.TrustLevel ( Types/TrustLevel.hs, dist/build/git-annex/git-annex-tmp/Types/TrustLevel.o )
+[ 21 of 270] Compiling Types.BranchState ( Types/BranchState.hs, dist/build/git-annex/git-annex-tmp/Types/BranchState.o )
+[ 22 of 270] Compiling Utility.UserInfo ( Utility/UserInfo.hs, dist/build/git-annex/git-annex-tmp/Utility/UserInfo.o )
+[ 23 of 270] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, dist/build/git-annex/git-annex-tmp/Utility/PartialPrelude.o )
+[ 24 of 270] Compiling Utility.HumanTime ( Utility/HumanTime.hs, dist/build/git-annex/git-annex-tmp/Utility/HumanTime.o )
+[ 25 of 270] Compiling Utility.Format   ( Utility/Format.hs, dist/build/git-annex/git-annex-tmp/Utility/Format.o )
+[ 26 of 270] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, dist/build/git-annex/git-annex-tmp/Utility/FileSystemEncoding.o )
+[ 27 of 270] Compiling Utility.Touch    ( dist/build/git-annex/git-annex-tmp/Utility/Touch.hs, dist/build/git-annex/git-annex-tmp/Utility/Touch.o )
+[ 28 of 270] Compiling Utility.Applicative ( Utility/Applicative.hs, dist/build/git-annex/git-annex-tmp/Utility/Applicative.o )
+[ 29 of 270] Compiling Utility.Monad    ( Utility/Monad.hs, dist/build/git-annex/git-annex-tmp/Utility/Monad.o )
+[ 30 of 270] Compiling Utility.Path     ( Utility/Path.hs, dist/build/git-annex/git-annex-tmp/Utility/Path.o )
+[ 31 of 270] Compiling Utility.Exception ( Utility/Exception.hs, dist/build/git-annex/git-annex-tmp/Utility/Exception.o )
+[ 32 of 270] Compiling Utility.TempFile ( Utility/TempFile.hs, dist/build/git-annex/git-annex-tmp/Utility/TempFile.o )
+[ 33 of 270] Compiling Utility.Misc     ( Utility/Misc.hs, dist/build/git-annex/git-annex-tmp/Utility/Misc.o )
+[ 34 of 270] Compiling Utility.Process  ( Utility/Process.hs, dist/build/git-annex/git-annex-tmp/Utility/Process.o )
+[ 35 of 270] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, dist/build/git-annex/git-annex-tmp/Utility/SafeCommand.o )
+[ 36 of 270] Compiling Utility.Directory ( Utility/Directory.hs, dist/build/git-annex/git-annex-tmp/Utility/Directory.o )
+[ 37 of 270] Compiling Utility.Network  ( Utility/Network.hs, dist/build/git-annex/git-annex-tmp/Utility/Network.o )
+[ 38 of 270] Compiling Utility.FreeDesktop ( Utility/FreeDesktop.hs, dist/build/git-annex/git-annex-tmp/Utility/FreeDesktop.o )
+[ 39 of 270] Compiling Assistant.Install.AutoStart ( Assistant/Install/AutoStart.hs, dist/build/git-annex/git-annex-tmp/Assistant/Install/AutoStart.o )
+[ 40 of 270] Compiling Utility.SRV      ( Utility/SRV.hs, dist/build/git-annex/git-annex-tmp/Utility/SRV.o )
+[ 41 of 270] Compiling Git.Types        ( Git/Types.hs, dist/build/git-annex/git-annex-tmp/Git/Types.o )
+[ 42 of 270] Compiling Common           ( Common.hs, dist/build/git-annex/git-annex-tmp/Common.o )
+[ 43 of 270] Compiling Utility.FileMode ( Utility/FileMode.hs, dist/build/git-annex/git-annex-tmp/Utility/FileMode.o )
+[ 44 of 270] Compiling Git              ( Git.hs, dist/build/git-annex/git-annex-tmp/Git.o )
+[ 45 of 270] Compiling Git.FilePath     ( Git/FilePath.hs, dist/build/git-annex/git-annex-tmp/Git/FilePath.o )
+[ 46 of 270] Compiling Utility.Matcher  ( Utility/Matcher.hs, dist/build/git-annex/git-annex-tmp/Utility/Matcher.o )
+[ 47 of 270] Compiling Utility.Gpg      ( Utility/Gpg.hs, dist/build/git-annex/git-annex-tmp/Utility/Gpg.o )
+[ 48 of 270] Compiling Types.Crypto     ( Types/Crypto.hs, dist/build/git-annex/git-annex-tmp/Types/Crypto.o )
+[ 49 of 270] Compiling Types.Key        ( Types/Key.hs, dist/build/git-annex/git-annex-tmp/Types/Key.o )
+[ 50 of 270] Compiling Types.Backend    ( Types/Backend.hs, dist/build/git-annex/git-annex-tmp/Types/Backend.o )
+[ 51 of 270] Compiling Types.Remote     ( Types/Remote.hs, dist/build/git-annex/git-annex-tmp/Types/Remote.o )
+[ 52 of 270] Compiling Git.Sha          ( Git/Sha.hs, dist/build/git-annex/git-annex-tmp/Git/Sha.o )
+[ 53 of 270] Compiling Utility.CoProcess ( Utility/CoProcess.hs, dist/build/git-annex/git-annex-tmp/Utility/CoProcess.o )
+[ 54 of 270] Compiling Git.Command      ( Git/Command.hs, dist/build/git-annex/git-annex-tmp/Git/Command.o )
+[ 55 of 270] Compiling Git.Ref          ( Git/Ref.hs, dist/build/git-annex/git-annex-tmp/Git/Ref.o )
+[ 56 of 270] Compiling Git.Branch       ( Git/Branch.hs, dist/build/git-annex/git-annex-tmp/Git/Branch.o )
+[ 57 of 270] Compiling Git.UpdateIndex  ( Git/UpdateIndex.hs, dist/build/git-annex/git-annex-tmp/Git/UpdateIndex.o )
+[ 58 of 270] Compiling Git.Queue        ( Git/Queue.hs, dist/build/git-annex/git-annex-tmp/Git/Queue.o )
+[ 59 of 270] Compiling Git.HashObject   ( Git/HashObject.hs, dist/build/git-annex/git-annex-tmp/Git/HashObject.o )
+[ 60 of 270] Compiling Git.CatFile      ( Git/CatFile.hs, dist/build/git-annex/git-annex-tmp/Git/CatFile.o )
+[ 61 of 270] Compiling Git.UnionMerge   ( Git/UnionMerge.hs, dist/build/git-annex/git-annex-tmp/Git/UnionMerge.o )
+[ 62 of 270] Compiling Git.Url          ( Git/Url.hs, dist/build/git-annex/git-annex-tmp/Git/Url.o )
+[ 63 of 270] Compiling Git.Construct    ( Git/Construct.hs, dist/build/git-annex/git-annex-tmp/Git/Construct.o )
+[ 64 of 270] Compiling Git.Config       ( Git/Config.hs, dist/build/git-annex/git-annex-tmp/Git/Config.o )
+[ 65 of 270] Compiling Git.SharedRepository ( Git/SharedRepository.hs, dist/build/git-annex/git-annex-tmp/Git/SharedRepository.o )
+[ 66 of 270] Compiling Git.Version      ( Git/Version.hs, dist/build/git-annex/git-annex-tmp/Git/Version.o )
+[ 67 of 270] Compiling Git.CheckAttr    ( Git/CheckAttr.hs, dist/build/git-annex/git-annex-tmp/Git/CheckAttr.o )
+[ 68 of 270] Compiling Annex            ( Annex.hs, dist/build/git-annex/git-annex-tmp/Annex.o )
+[ 69 of 270] Compiling Types.Option     ( Types/Option.hs, dist/build/git-annex/git-annex-tmp/Types/Option.o )
+[ 70 of 270] Compiling Types            ( Types.hs, dist/build/git-annex/git-annex-tmp/Types.o )
+[ 71 of 270] Compiling Messages         ( Messages.hs, dist/build/git-annex/git-annex-tmp/Messages.o )
+[ 72 of 270] Compiling Types.Command    ( Types/Command.hs, dist/build/git-annex/git-annex-tmp/Types/Command.o )
+[ 73 of 270] Compiling Locations        ( Locations.hs, dist/build/git-annex/git-annex-tmp/Locations.o )
+[ 74 of 270] Compiling Common.Annex     ( Common/Annex.hs, dist/build/git-annex/git-annex-tmp/Common/Annex.o )
+[ 75 of 270] Compiling Fields           ( Fields.hs, dist/build/git-annex/git-annex-tmp/Fields.o )
+[ 76 of 270] Compiling Annex.BranchState ( Annex/BranchState.hs, dist/build/git-annex/git-annex-tmp/Annex/BranchState.o )
+[ 77 of 270] Compiling Annex.CatFile    ( Annex/CatFile.hs, dist/build/git-annex/git-annex-tmp/Annex/CatFile.o )
+[ 78 of 270] Compiling Annex.Perms      ( Annex/Perms.hs, dist/build/git-annex/git-annex-tmp/Annex/Perms.o )
+[ 79 of 270] Compiling Crypto           ( Crypto.hs, dist/build/git-annex/git-annex-tmp/Crypto.o )
+[ 80 of 270] Compiling Annex.Exception  ( Annex/Exception.hs, dist/build/git-annex/git-annex-tmp/Annex/Exception.o )
+[ 81 of 270] Compiling Annex.Journal    ( Annex/Journal.hs, dist/build/git-annex/git-annex-tmp/Annex/Journal.o )
+[ 82 of 270] Compiling Annex.Branch     ( Annex/Branch.hs, dist/build/git-annex/git-annex-tmp/Annex/Branch.o )
+[ 83 of 270] Compiling Usage            ( Usage.hs, dist/build/git-annex/git-annex-tmp/Usage.o )
+[ 84 of 270] Compiling Annex.CheckAttr  ( Annex/CheckAttr.hs, dist/build/git-annex/git-annex-tmp/Annex/CheckAttr.o )
+[ 85 of 270] Compiling Remote.Helper.Special ( Remote/Helper/Special.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Special.o )
+[ 86 of 270] Compiling Logs.Presence    ( Logs/Presence.hs, dist/build/git-annex/git-annex-tmp/Logs/Presence.o )
+[ 87 of 270] Compiling Logs.Location    ( Logs/Location.hs, dist/build/git-annex/git-annex-tmp/Logs/Location.o )
+[ 88 of 270] Compiling Logs.Web         ( Logs/Web.hs, dist/build/git-annex/git-annex-tmp/Logs/Web.o )
+[ 89 of 270] Compiling Annex.LockPool   ( Annex/LockPool.hs, dist/build/git-annex/git-annex-tmp/Annex/LockPool.o )
+[ 90 of 270] Compiling Logs.Transfer    ( Logs/Transfer.hs, dist/build/git-annex/git-annex-tmp/Logs/Transfer.o )
+[ 91 of 270] Compiling Backend.SHA      ( Backend/SHA.hs, dist/build/git-annex/git-annex-tmp/Backend/SHA.o )
+[ 92 of 270] Compiling Backend.WORM     ( Backend/WORM.hs, dist/build/git-annex/git-annex-tmp/Backend/WORM.o )
+[ 93 of 270] Compiling Backend.URL      ( Backend/URL.hs, dist/build/git-annex/git-annex-tmp/Backend/URL.o )
+[ 94 of 270] Compiling Assistant.Ssh    ( Assistant/Ssh.hs, dist/build/git-annex/git-annex-tmp/Assistant/Ssh.o )
+[ 95 of 270] Compiling Assistant.Types.ThreadedMonad ( Assistant/Types/ThreadedMonad.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/ThreadedMonad.o )
+[ 96 of 270] Compiling Assistant.Types.ScanRemotes ( Assistant/Types/ScanRemotes.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/ScanRemotes.o )
+[ 97 of 270] Compiling Assistant.Types.TransferQueue ( Assistant/Types/TransferQueue.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/TransferQueue.o )
+[ 98 of 270] Compiling Assistant.Types.BranchChange ( Assistant/Types/BranchChange.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/BranchChange.o )
+[ 99 of 270] Compiling Assistant.Pairing ( Assistant/Pairing.hs, dist/build/git-annex/git-annex-tmp/Assistant/Pairing.o )
+[100 of 270] Compiling Logs.UUIDBased   ( Logs/UUIDBased.hs, dist/build/git-annex/git-annex-tmp/Logs/UUIDBased.o )
+[101 of 270] Compiling Logs.Remote      ( Logs/Remote.hs, dist/build/git-annex/git-annex-tmp/Logs/Remote.o )
+[102 of 270] Compiling Logs.Group       ( Logs/Group.hs, dist/build/git-annex/git-annex-tmp/Logs/Group.o )
+[103 of 270] Compiling Utility.DiskFree ( Utility/DiskFree.hs, dist/build/git-annex/git-annex-tmp/Utility/DiskFree.o )
+[104 of 270] Compiling Utility.Url      ( Utility/Url.hs, dist/build/git-annex/git-annex-tmp/Utility/Url.o )
+[105 of 270] Compiling Utility.CopyFile ( Utility/CopyFile.hs, dist/build/git-annex/git-annex-tmp/Utility/CopyFile.o )
+[106 of 270] Compiling Utility.Rsync    ( Utility/Rsync.hs, dist/build/git-annex/git-annex-tmp/Utility/Rsync.o )
+[107 of 270] Compiling Git.LsFiles      ( Git/LsFiles.hs, dist/build/git-annex/git-annex-tmp/Git/LsFiles.o )
+[108 of 270] Compiling Git.AutoCorrect  ( Git/AutoCorrect.hs, dist/build/git-annex/git-annex-tmp/Git/AutoCorrect.o )
+[109 of 270] Compiling Git.CurrentRepo  ( Git/CurrentRepo.hs, dist/build/git-annex/git-annex-tmp/Git/CurrentRepo.o )
+[110 of 270] Compiling Git.Merge        ( Git/Merge.hs, dist/build/git-annex/git-annex-tmp/Git/Merge.o )
+[111 of 270] Compiling Utility.WebApp   ( Utility/WebApp.hs, dist/build/git-annex/git-annex-tmp/Utility/WebApp.o )
+[112 of 270] Compiling Utility.Daemon   ( Utility/Daemon.hs, dist/build/git-annex/git-annex-tmp/Utility/Daemon.o )
+[113 of 270] Compiling Locations.UserConfig ( Locations/UserConfig.hs, dist/build/git-annex/git-annex-tmp/Locations/UserConfig.o )
+[114 of 270] Compiling Utility.TSet     ( Utility/TSet.hs, dist/build/git-annex/git-annex-tmp/Utility/TSet.o )
+[115 of 270] Compiling Assistant.Types.Pushes ( Assistant/Types/Pushes.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/Pushes.o )
+[116 of 270] Compiling Assistant.Types.Commits ( Assistant/Types/Commits.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/Commits.o )
+[117 of 270] Compiling Assistant.Types.Changes ( Assistant/Types/Changes.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/Changes.o )
+[118 of 270] Compiling Utility.NotificationBroadcaster ( Utility/NotificationBroadcaster.hs, dist/build/git-annex/git-annex-tmp/Utility/NotificationBroadcaster.o )
+[119 of 270] Compiling Utility.Parallel ( Utility/Parallel.hs, dist/build/git-annex/git-annex-tmp/Utility/Parallel.o )
+[120 of 270] Compiling Utility.ThreadScheduler ( Utility/ThreadScheduler.hs, dist/build/git-annex/git-annex-tmp/Utility/ThreadScheduler.o )
+[121 of 270] Compiling Utility.LogFile  ( Utility/LogFile.hs, dist/build/git-annex/git-annex-tmp/Utility/LogFile.o )
+[122 of 270] Compiling Git.Filename     ( Git/Filename.hs, dist/build/git-annex/git-annex-tmp/Git/Filename.o )
+[123 of 270] Compiling Git.LsTree       ( Git/LsTree.hs, dist/build/git-annex/git-annex-tmp/Git/LsTree.o )
+[124 of 270] Compiling Utility.Types.DirWatcher ( Utility/Types/DirWatcher.hs, dist/build/git-annex/git-annex-tmp/Utility/Types/DirWatcher.o )
+[125 of 270] Compiling Utility.INotify  ( Utility/INotify.hs, dist/build/git-annex/git-annex-tmp/Utility/INotify.o )
+[126 of 270] Compiling Utility.DirWatcher ( Utility/DirWatcher.hs, dist/build/git-annex/git-annex-tmp/Utility/DirWatcher.o )
+[127 of 270] Compiling Utility.Lsof     ( Utility/Lsof.hs, dist/build/git-annex/git-annex-tmp/Utility/Lsof.o )
+[128 of 270] Compiling Config           ( Config.hs, dist/build/git-annex/git-annex-tmp/Config.o )
+[129 of 270] Compiling Annex.UUID       ( Annex/UUID.hs, dist/build/git-annex/git-annex-tmp/Annex/UUID.o )
+[130 of 270] Compiling Logs.UUID        ( Logs/UUID.hs, dist/build/git-annex/git-annex-tmp/Logs/UUID.o )
+[131 of 270] Compiling Backend          ( Backend.hs, dist/build/git-annex/git-annex-tmp/Backend.o )
+[132 of 270] Compiling Remote.Helper.Hooks ( Remote/Helper/Hooks.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Hooks.o )
+[133 of 270] Compiling Remote.Helper.Encryptable ( Remote/Helper/Encryptable.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Encryptable.o )
+[134 of 270] Compiling Annex.Queue      ( Annex/Queue.hs, dist/build/git-annex/git-annex-tmp/Annex/Queue.o )
+[135 of 270] Compiling Annex.Content    ( Annex/Content.hs, dist/build/git-annex/git-annex-tmp/Annex/Content.o )
+[136 of 270] Compiling Remote.S3        ( Remote/S3.hs, dist/build/git-annex/git-annex-tmp/Remote/S3.o )
+[137 of 270] Compiling Remote.Directory ( Remote/Directory.hs, dist/build/git-annex/git-annex-tmp/Remote/Directory.o )
+[138 of 270] Compiling Remote.Rsync     ( Remote/Rsync.hs, dist/build/git-annex/git-annex-tmp/Remote/Rsync.o )
+[139 of 270] Compiling Remote.Web       ( Remote/Web.hs, dist/build/git-annex/git-annex-tmp/Remote/Web.o )
+[140 of 270] Compiling Remote.Hook      ( Remote/Hook.hs, dist/build/git-annex/git-annex-tmp/Remote/Hook.o )
+[141 of 270] Compiling Upgrade.V2       ( Upgrade/V2.hs, dist/build/git-annex/git-annex-tmp/Upgrade/V2.o )
+[142 of 270] Compiling Annex.Ssh        ( Annex/Ssh.hs, dist/build/git-annex/git-annex-tmp/Annex/Ssh.o )
+[143 of 270] Compiling Remote.Helper.Ssh ( Remote/Helper/Ssh.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Ssh.o )
+[144 of 270] Compiling Remote.Bup       ( Remote/Bup.hs, dist/build/git-annex/git-annex-tmp/Remote/Bup.o )
+[145 of 270] Compiling Annex.Version    ( Annex/Version.hs, dist/build/git-annex/git-annex-tmp/Annex/Version.o )
+[146 of 270] Compiling Init             ( Init.hs, dist/build/git-annex/git-annex-tmp/Init.o )
+[147 of 270] Compiling Checks           ( Checks.hs, dist/build/git-annex/git-annex-tmp/Checks.o )
+[148 of 270] Compiling Remote.Git       ( Remote/Git.hs, dist/build/git-annex/git-annex-tmp/Remote/Git.o )
+[149 of 270] Compiling Remote.List      ( Remote/List.hs, dist/build/git-annex/git-annex-tmp/Remote/List.o )
+[150 of 270] Compiling Logs.Trust       ( Logs/Trust.hs, dist/build/git-annex/git-annex-tmp/Logs/Trust.o )
+[151 of 270] Compiling Remote           ( Remote.hs, dist/build/git-annex/git-annex-tmp/Remote.o )
+[152 of 270] Compiling Assistant.Alert  ( Assistant/Alert.hs, dist/build/git-annex/git-annex-tmp/Assistant/Alert.o )
+
+Assistant/Alert.hs:60:26:
+    Not in scope: type constructor or class `Html'
+
+Assistant/Alert.hs:66:21: Not in scope: `preEscapedText'
+
+Assistant/Alert.hs:68:26:
+    Not in scope: type constructor or class `Html'
+
+Assistant/Alert.hs:69:19: Not in scope: `preEscapedText'
+
+ +What is the expected output? What do you see instead? + +The current git HEAD should build and not throw an error. + +What version of git-annex are you using? On what operating system? + +git-annex HEAD from git, Ubuntu 12.10. + +Please provide any additional information below. + +> Hmm, seems that Blaze's API is not stable, and I should avoid using it +> directly. Converted this code to using Hamlet instead for its html +> generation. [[done]] --[[Joey]] diff --git a/doc/bugs/Building_in_cabal_using_--bindir___126____47__bin_breaks_the_desktop_link.mdwn b/doc/bugs/Building_in_cabal_using_--bindir___126____47__bin_breaks_the_desktop_link.mdwn new file mode 100644 index 0000000000..85a311716c --- /dev/null +++ b/doc/bugs/Building_in_cabal_using_--bindir___126____47__bin_breaks_the_desktop_link.mdwn @@ -0,0 +1,14 @@ +What steps will reproduce the problem? + +Download the sourcecode, build using 'cabal build', then install using 'cabal install --bindir ~/bin'. + +What is the expected output? What do you see instead? + +The .desktop file contains `~/bin/git-annex webapp` as command which is of course a invalid command as ~ is not expanded when running the desktop file. + +What version of git-annex are you using? On what operating system? +Latest Head from git, Ubuntu 12.04 + +Please provide any additional information below. +I'm not sure whether this is a bug or not. I just ran into problems because I did not expect the cabal build process to create my desktop file but instead thought that git-annex will create it by it-self taking its own path. Perhaps it would make sense to produce an error if the bindir is invalid. An automatic expansion of '~' in the build script would be even better. + diff --git a/doc/bugs/Building_in_cabal_using_--bindir___126____47__bin_breaks_the_desktop_link/comment_1_c0f0a2878070ed86900815c6b6a5fa5e._comment b/doc/bugs/Building_in_cabal_using_--bindir___126____47__bin_breaks_the_desktop_link/comment_1_c0f0a2878070ed86900815c6b6a5fa5e._comment new file mode 100644 index 0000000000..82dfa0d481 --- /dev/null +++ b/doc/bugs/Building_in_cabal_using_--bindir___126____47__bin_breaks_the_desktop_link/comment_1_c0f0a2878070ed86900815c6b6a5fa5e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.60" + subject="comment 1" + date="2012-09-15T16:39:19Z" + content=""" +Something is going on that I don't understand. When I run `cabal install --bindir ~/bin`, my shell passes `/home/joey/bin` to cabal, so of course it works. +"""]] diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn new file mode 100644 index 0000000000..13980dd292 --- /dev/null +++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn @@ -0,0 +1,17 @@ +Just issuing the command `cabal install` results in the following error message. + + Command/Add.hs:54:3: + No instance for (Control.Monad.IO.Control.MonadControlIO + (Control.Monad.State.Lazy.StateT Annex.AnnexState IO)) + arising from a use of `handle' at Command/Add.hs:54:3-24 + +Adding the dependency for `monadIO` to `git-annex.cabal` should fix this? +-- Thomas + +> No, it's already satisfied by `monad-control` being listed as a +> dependency in the cabal file. Your system might be old/new/or broken, +> perhaps it's time to provide some details about the version of haskell +> and of `monad-control` you have installed? --[[Joey]] + +>> Closing as apparently user error or a broken system. +>> If you see this problem please do say. [[done]] --[[Joey]] diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment new file mode 100644 index 0000000000..8e38205f00 --- /dev/null +++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment @@ -0,0 +1,75 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s" + nickname="Thomas" + subject="comment 1" + date="2011-08-08T09:04:20Z" + content=""" +I use Debian Squeeze, I have the Debian package cabal-install 0.8.0-1 installed. + + $ git clone git://git-annex.branchable.com/ + $ cd git-annex.branchable.com + $ cabal update + $ cabal install cabal-install + +This installed: Cabal-1.10.2.0, zlib-0.5.3.1, cabal-install 0.10.2. +No version of monad-control or monadIO installed. + + $ ~/.cabal/bin/cabal install + Registering QuickCheck-2.4.1.1... + Registering Crypto-4.2.3... + Registering base-unicode-symbols-0.2.2.1... + Registering deepseq-1.1.0.2... + Registering hxt-charproperties-9.1.0... + Registering hxt-regex-xmlschema-9.0.0... + Registering hxt-unicode-9.0.1... + Registering hxt-9.1.2... + Registering stm-2.2.0.1... + Registering hS3-0.5.6... + Registering transformers-0.2.2.0... + Registering monad-control-0.2.0.1... + [1 of 1] Compiling Main ( Setup.hs, dist/setup/Main.o ) + Linking ./dist/setup/setup ... + ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make configure + [1 of 2] Compiling TestConfig ( TestConfig.hs, TestConfig.o ) + [2 of 2] Compiling Main ( configure.hs, configure.o ) + Linking configure ... + ./configure + checking version... 3.20110720 + checking cp -a... yes + checking cp -p... yes + checking cp --reflink=auto... yes + checking uuid generator... uuid + checking xargs -0... yes + checking rsync... yes + checking curl... yes + checking bup... yes + checking gpg... yes + checking sha1... sha1sum + checking sha256... sha256sum + checking sha512... sha512sum + checking sha224... sha224sum + checking sha384... sha384sum + + ... + + Command/Add.hs:54:3: + No instance for (Control.Monad.IO.Control.MonadControlIO + (Control.Monad.State.Lazy.StateT Annex.AnnexState IO)) + arising from a use of `handle' at Command/Add.hs:54:3-24 + Possible fix: + add an instance declaration for + (Control.Monad.IO.Control.MonadControlIO + (Control.Monad.State.Lazy.StateT Annex.AnnexState IO)) + In the first argument of `($)', namely `handle (undo file key)' + In a stmt of a 'do' expression: + handle (undo file key) $ moveAnnex key file + In the expression: + do { handle (undo file key) $ moveAnnex key file; + next $ cleanup file key } + cabal: Error: some packages failed to install: + git-annex-3.20110719 failed during the building phase. The exception was: + ExitFailure 1 + +After I added a depencency for monadIO to the git-annex.cabal file, it installed correctly. +-- Thomas +"""]] diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment new file mode 100644 index 0000000000..adf7a34e66 --- /dev/null +++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-08-17T04:56:30Z" + content=""" +Finally got a chance to try to reproduce this. I followed your recipe exactly in a clean squeeze chroot. monadIO was not installed, but git-annex built ok, using monad-control. +"""]] diff --git a/doc/bugs/Calls_to_rsync_don__39__t_always_use__annex-rsync-options.mdwn b/doc/bugs/Calls_to_rsync_don__39__t_always_use__annex-rsync-options.mdwn new file mode 100644 index 0000000000..df1163b465 --- /dev/null +++ b/doc/bugs/Calls_to_rsync_don__39__t_always_use__annex-rsync-options.mdwn @@ -0,0 +1,35 @@ +What steps will reproduce the problem? + +Add a rsync special remote - one that you need a username/password to access (stored in text file $HOME/.rsync.password): + + $ git annex initremote myrsync type=rsync rsyncurl=rsync://username@rsync.example.com/myrsync encryption=none + $ git annex describe myrsync "rsync server" + $ git config remote.myrsync.annex-rsync-options "--password-file=$HOME/.rsync.password" + +Copy a file to the remote: + + $ git annex -d copy my-file --to myrsync + +What is the expected output? What do you see instead? + +Expect to see the file copied over to the rsync remote, but the check doesn't use the annex-rsync-options and asks for a password. The debug output is: + + copy my-file (checking myrsync...) [2012-10-28 01:01:01 EST] call: sh ["-c","rsync --quiet 'rsync://username@rsync.example.com/myrsync/[...SNIP...]' 2>/dev/null"] + +However the actual copy does use annex-rsync-options and the copy works: + + [2012-10-28 01:01:05 EST] read: rsync ["--password-file=/home/blah/.rsync.password","--progress","--recursive","--partial","--partial-dir=.rsync-partial","/home/blah/annex/.git/annex/tmp/rsynctmp/12345/","rsync://username@rsync.example.com/myrsync"] + + +What version of git-annex are you using? On what operating system? + +git-annex: 3.20121017 + +OS: Ubuntu 12.04 + +Please provide any additional information below. + +I think this fix is as easy as including the annex-rsync-options wherever rsync is called. + +> I belive there was only the one place this was neglected. [[done]] +> --[[Joey]] diff --git a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn new file mode 100644 index 0000000000..ea56c37320 --- /dev/null +++ b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn @@ -0,0 +1,27 @@ +Hi there, + +After updating to 3.20111203 (on Arch Linux) I noticed I was not able to use `git annex get` from a SSH remote (server running Arch Linux, same version of git-annex): "requested key is not present". Same behavior with current master (commit 6cf28585). I had no issue with the previous version (3.20111122). + +On this server, I was able to track down the issue using `git-annex-shell inannex` and `strace`: + + $ strace -f -o log git-annex-shell inannex ~/photos-annex.git WORM-s369360-m1321602916--2011-11-17.jpg + $ echo $? + 1 + $ tail -n20 log + [...] + 25623 chdir("/home/schnouki/git-annex") = 0 + 25623 stat("/home/schnouki/photos-annex.git/annex/objects/082/676/WORM-s369360-m1321602916--2011-11-17.jpg/WORM-s369360-m1321602916--2011-11-17.jpg", {st_mode=S_IFREG|0400, st_size=369360, ...}) = 0 + 25623 open("annex/objects/082/676/WORM-s369360-m1321602916--2011-11-17.jpg/WORM-s369360-m1321602916--2011-11-17.jpg", O_RDONLY) = -1 ENOENT (No such file or directory) + [...] + +Note there is a call to `stat()` with the full path to the requested file, and *then* a call to `open()` with a relative path -- which calls this call to fail, and git-annex-shell to return 1. With 3.20111122, there was no call to `stat()`, just a successful call to `open()` with a full absolute path. + +Using `git bisect` I was able to determine that this bug appeared in commit 64672c62 ("refactor"). Reverting it makes `git-annex-shell` work as expected, but I'm sure there are better ways to fix this. However I don't know enough Haskell to do it myself. + +Could you please try to fix this in a future version? + +> Thanks for a very good bug report. +> +> I've fixed this stupid mistake introduced in the code refactoring. +> [[done]] +> --[[Joey]] diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client.mdwn b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client.mdwn new file mode 100644 index 0000000000..1b90fb3f4f --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client.mdwn @@ -0,0 +1,19 @@ +Note: This may not be a bug maybe the software doesn't work the way I think. + +What steps will reproduce the problem? + +Using the webapp, I created a normal repo on my desktop. I then added a 'Removable drive' repo on my usb stick. + +I want all my files to be synced and accessibles on both repos so I set the 'Removable drive' repo to 'client'. + + +What is the expected output? What do you see instead? + +When I look in the 'annex' folder on my usb stick. I see a git repo (annex, branches, hooks) instead of seeing the files in the same way the 'annex' repo on my desktop is. + + +What version of git-annex are you using? On what operating system? + +I'm using 9e57edff287ac53fc4b1cefef7271e9ed17f2285 (Fri Feb 22 15:19:25 2013 +0000). + +Ubuntu 12.10 x86_64 diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_1_25eb2d7d0a9cdd1c55df0cec68472723._comment b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_1_25eb2d7d0a9cdd1c55df0cec68472723._comment new file mode 100644 index 0000000000..282e78298b --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_1_25eb2d7d0a9cdd1c55df0cec68472723._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.253.75" + subject="comment 1" + date="2013-02-22T21:46:59Z" + content=""" +You have a bare git repository on your USB stick. The assistant uses bare repositories on USB sticks currently, although recent changes to support FAT make it possible to use non-bare repositories. + +For this to really work the way you want it to, the assistant would need to start updating local non-bare repositories whenever it pushed changes to them. Currently the assistant only updates the repository it's running in, so a non-bare repository on USB would lag behind and show an old version of the directory until manually updated with `git annex sync`. +"""]] diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_2_9e9b96e5113a50533251e946c2560d81._comment b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_2_9e9b96e5113a50533251e946c2560d81._comment new file mode 100644 index 0000000000..da1fcbc0f2 --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_2_9e9b96e5113a50533251e946c2560d81._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="Xyem" + ip="87.194.19.134" + subject="comment 2" + date="2013-02-27T00:02:01Z" + content=""" +Struggling to get the assistant to behave properly, including a similar situation to the above. + +I start the assistant with 'git annex webapp' and create an annex from the webapp. +Adding an SSH remote (even if just a different directory on localhost) with \"Remote server\", creates the remote directory with what looks like what the contents of .git should be (annex, branches, hooks, objects etc.), regardless of the group chosen. + +Judging from your above comment, this means it is creating a bare repository. Why would it be doing this for an SSH remote, where git-annex-shell is available? + +System: Arch Linux +git-annex version: 3.20130216 +Installed with: yaourt -S git-annex-bin +"""]] diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_3_6b091198ddd6ed709b076df1296aeb77._comment b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_3_6b091198ddd6ed709b076df1296aeb77._comment new file mode 100644 index 0000000000..48d1b26040 --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_3_6b091198ddd6ed709b076df1296aeb77._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 3" + date="2013-02-27T01:12:29Z" + content=""" +Xyem, if you use git to clone a repo, instead of letting the assistant do it for you, you can get a non-bare repo. See the walkthrough for some examples. You can keep its working tree up to date with \"git annex sync\" run in that directory. + +Only helpful if you don't mind using git annex at the command line, I know. + +"""]] diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_4_118b588685b535cca4c02eb6ef297c67._comment b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_4_118b588685b535cca4c02eb6ef297c67._comment new file mode 100644 index 0000000000..8ad0de3a5c --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_4_118b588685b535cca4c02eb6ef297c67._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="Xyem" + ip="87.194.19.134" + subject="comment 4" + date="2013-02-27T12:16:45Z" + content=""" +@edheil: +Yes, I have found that creating the repositories outside of the assistant (or directly on each machine through the assistant) and adding the remotes manually doesn't have this issue.. but as I am not setting this up for myself, so dropping to the command line is an issue (she's not averse to it, but also isn't a power user). + +It seems that the assistant/webapp is replacing dropbox functionality, while ignoring the functionality of git-annex. Perhaps I am misunderstanding the goal of it..? + +My intended use case in the situation where this issue comes up is: + +Repository on desktop has a copy of all files in indirect mode. +Repository on netbook only has a copy of files added locally or fetched manually. +Dropping a file in the netbook annex should cause it to be uploaded to the desktop repository (when it is available). Files should be fetchable/droppable (and unlockable) using the webapp from the desktop/netbook repository respectively. + +But it looks like this is (currently) unachievable as the webapp provides no method of fetching/dropping files, the assistant does not create the remote repositories correctly and requires dropping to the command line if worked around. Eep! + +Hope this doesn't come off as sour complaining. I was introduced to git-annex by a friend about a week ago (she only learned about it about 2-3 weeks ago and *loves* it) and it is a very nice and flexible tool which fits my own use case perfectly, meeting and exceeding every expectation.. until it came to the assistant and webapp (which makes me wonder why Joey is working on an Android port..). +"""]] diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_5_5cead277493e1c020e16be6f9245fe33._comment b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_5_5cead277493e1c020e16be6f9245fe33._comment new file mode 100644 index 0000000000..26b472dc75 --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_5_5cead277493e1c020e16be6f9245fe33._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 5" + date="2013-02-27T17:05:07Z" + content=""" +There is a way to fetch/drop files if you don't want to use the command line, though it's not nearly as flexible as using a command line. If you've got a machine set as \"client\" then dragging files into a subdirectory named \"archive\" drops them, and dragging them out of that subdirectory fetches them. + +Honestly, moving files in and out of directories, and preferred content settings, have both been subject to the occasional regression, so there have been times this hasn't worked right, but in theory, that's how you do it. + +Unfortunately if you've got more than one \"client\" machine, this means adding/dropping files on one will affect them all, as file movements propagate. +"""]] diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_6_0f135f97c2808dce094628dc6608e617._comment b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_6_0f135f97c2808dce094628dc6608e617._comment new file mode 100644 index 0000000000..3fadb6219f --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_6_0f135f97c2808dce094628dc6608e617._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 6" + date="2013-02-27T17:06:45Z" + content=""" +also... I've never used the feature where you have two local machines running git-annex and you tell one to share a repo with the other through the web interface, but maybe it would be helpful for this scenario? Apologies if you've already tried that. +"""]] diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_7_1d6f47f9e6cf935f19d68af6d5aa92fa._comment b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_7_1d6f47f9e6cf935f19d68af6d5aa92fa._comment new file mode 100644 index 0000000000..03d9950bca --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_7_1d6f47f9e6cf935f19d68af6d5aa92fa._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 7" + date="2013-02-27T17:43:45Z" + content=""" +The way this is supposed to (and does) work is that you use the webapp to create a repository on each computer, and then you pair them together using either local pairing or xmpp pairing. + +Today's release of git-annex also adds UI in the webapp to create additional local repositories that are connected to the current repository. However these are not really intended to be put on removable drives since the assistant needs to be running on them to keep them up-to-date. +"""]] diff --git a/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_8_c5758fdb32348b9cd804ff17d27864e1._comment b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_8_c5758fdb32348b9cd804ff17d27864e1._comment new file mode 100644 index 0000000000..1ef87e6a65 --- /dev/null +++ b/doc/bugs/Can__39__t_access_files_from___39__Removable_drive__39___repo_even_if_set_as_client/comment_8_c5758fdb32348b9cd804ff17d27864e1._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="Xyem" + ip="87.194.19.134" + subject="comment 8" + date="2013-02-27T18:36:16Z" + content=""" +@Joey: +As my use case is 1 person using multiple machines, I didn't think local/XMPP pairing was appropriate. + +Local pairing because the netbook leaves the network the desktop machine is on, which is when transferring the files is wanted (otherwise, they would just use the desktop...). +XMPP pairing description implies that it is for 2 different people (thus 2 different addresses). Can it be used where both annexes are using the same address? Or does the address also allow you to include the location (i.e. name@host.com/netbook)? + +It definitely sounds like I am misunderstanding how this is supposed to work so I apologise for that. Please keep up the excellent work. git-annex is one of those tools I wish I had learned about a long time ago! + +@edheil: Interesting workaround, thanks. +"""]] diff --git a/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared.mdwn b/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared.mdwn new file mode 100644 index 0000000000..31f37f6e59 --- /dev/null +++ b/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared.mdwn @@ -0,0 +1,54 @@ +I'm trying to transfer 3 files to my rsync.net remote. It was set up a few months ago with encryption=shared. + +Here's what I get: + + % LC_ALL=C git annex copy --to rsn 2012-11-26* + copy 2012-11-26 1.jpg (gpg) (checking rsn...) (to rsn...) gpg: no valid OpenPGP data found. + gpg: decrypt_message failed: Unknown system error + failed + copy 2012-11-26 2.jpg (checking rsn...) (to rsn...) gpg: no valid OpenPGP data found. + gpg: decrypt_message failed: Unknown system error + failed + copy 2012-11-26 3.jpg (checking rsn...) (to rsn...) gpg: no valid OpenPGP data found. + gpg: decrypt_message failed: Unknown system error + failed + git-annex: copy: 3 failed + + +Here's the output with `--debug`: + + % LC_ALL=C git annex --debug copy --to rsn 2012-11-26* + [2012-11-27 18:28:22 CET] read: git ["--git-dir=/home/schnouki/Photos/.git","--work-tree=/home/schnouki/Photos","show-ref","git-annex"] + [2012-11-27 18:28:22 CET] read: git ["--git-dir=/home/schnouki/Photos/.git","--work-tree=/home/schnouki/Photos","show-ref","--hash","refs/heads/git-annex"] + [2012-11-27 18:28:22 CET] read: git ["--git-dir=/home/schnouki/Photos/.git","--work-tree=/home/schnouki/Photos","log","refs/heads/git-annex..0164d6150fcd56e035926c72c9a519114735d2a1","--oneline","-n1"] + [2012-11-27 18:28:22 CET] read: git ["--git-dir=/home/schnouki/Photos/.git","--work-tree=/home/schnouki/Photos","log","refs/heads/git-annex..04bb9b87872200d5712b17463fc81d4de27b9acf","--oneline","-n1"] + [2012-11-27 18:28:22 CET] read: git ["--git-dir=/home/schnouki/Photos/.git","--work-tree=/home/schnouki/Photos","log","refs/heads/git-annex..59c38c176971cd6323817e568f4c32305ba708b0","--oneline","-n1"] + [2012-11-27 18:28:22 CET] chat: git ["--git-dir=/home/schnouki/Photos/.git","--work-tree=/home/schnouki/Photos","cat-file","--batch"] + [2012-11-27 18:28:22 CET] read: git ["--git-dir=/home/schnouki/Photos/.git","--work-tree=/home/schnouki/Photos","ls-files","--cached","-z","--","2012-11-26 1.jpg","2012-11-26 2.jpg","2012-11-26 3.jpg"] + copy 2012-11-26 1.jpg (gpg) (checking rsn...) [2012-11-27 18:28:22 CET] read: rsync ["rsn:photos/123/3a1/GPGHMACSHA1--4f4b2440da1f41aa42a590dc558fbfff39e44b5e/GPGHMACSHA1--4f4b2440da1f41aa42a590dc558fbfff39e44b5e"] + [2012-11-27 18:28:22 CET] read: rsync ["rsn:photos/jg/8p/GPGHMACSHA1--4f4b2440da1f41aa42a590dc558fbfff39e44b5e/GPGHMACSHA1--4f4b2440da1f41aa42a590dc558fbfff39e44b5e"] + (to rsn...) [2012-11-27 18:28:22 CET] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--passphrase-fd","11","--decrypt"] + gpg: no valid OpenPGP data found. + gpg: decrypt_message failed: Unknown system error + failed + copy 2012-11-26 2.jpg (checking rsn...) [2012-11-27 18:28:22 CET] read: rsync ["rsn:photos/61b/da7/GPGHMACSHA1--9c17b4b7414b830a0e7b86cbe08afd1e0878bdbd/GPGHMACSHA1--9c17b4b7414b830a0e7b86cbe08afd1e0878bdbd"] + [2012-11-27 18:28:23 CET] read: rsync ["rsn:photos/Z1/9V/GPGHMACSHA1--9c17b4b7414b830a0e7b86cbe08afd1e0878bdbd/GPGHMACSHA1--9c17b4b7414b830a0e7b86cbe08afd1e0878bdbd"] + (to rsn...) [2012-11-27 18:28:23 CET] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--passphrase-fd","10","--decrypt"] + gpg: no valid OpenPGP data found. + gpg: decrypt_message failed: Unknown system error + failed + copy 2012-11-26 3.jpg (checking rsn...) [2012-11-27 18:28:23 CET] read: rsync ["rsn:photos/340/2ec/GPGHMACSHA1--ce18f116ae33176c8387cecf9d62b0e694501a6e/GPGHMACSHA1--ce18f116ae33176c8387cecf9d62b0e694501a6e"] + [2012-11-27 18:28:23 CET] read: rsync ["rsn:photos/8f/V0/GPGHMACSHA1--ce18f116ae33176c8387cecf9d62b0e694501a6e/GPGHMACSHA1--ce18f116ae33176c8387cecf9d62b0e694501a6e"] + (to rsn...) [2012-11-27 18:28:23 CET] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--passphrase-fd","12","--decrypt"] + gpg: no valid OpenPGP data found. + gpg: decrypt_message failed: Unknown system error + failed + git-annex: copy: 3 failed + +I'm using git-annex 3.20121127. I think I was able to transfer files to that remote with the previous release. + +Any idea how to fix this? Am I doing something wrong? + +> Damn. This was a completely stupid bug, calling "decrypt" where it was +> supposed to call "encrypt". [[done]] in git; I'll have to +> make a new release to fix this. --[[Joey]] diff --git a/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared/comment_1_ca7ec2041bbec330476fb040b1e66a92._comment b/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared/comment_1_ca7ec2041bbec330476fb040b1e66a92._comment new file mode 100644 index 0000000000..36061bac2b --- /dev/null +++ b/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared/comment_1_ca7ec2041bbec330476fb040b1e66a92._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://schnouki.net/" + nickname="Schnouki" + subject="comment 1" + date="2012-11-27T17:39:38Z" + content=""" +Downgraded to 3.20121112 and the transfer went fine. In both cases I used the [prebuilt tarball](http://downloads.kitenet.net/git-annex/linux/). So it must be a regression in the current release. +"""]] diff --git a/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared/comment_2_c476847665a5320214721497d8fad15b._comment b/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared/comment_2_c476847665a5320214721497d8fad15b._comment new file mode 100644 index 0000000000..def9d19c88 --- /dev/null +++ b/doc/bugs/Can__39__t_transfer_files_to_rsync_remote_with_encryption__61__shared/comment_2_c476847665a5320214721497d8fad15b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://schnouki.net/" + nickname="Schnouki" + subject="comment 2" + date="2012-11-27T22:33:54Z" + content=""" +Thanks for the quick fix! :) +"""]] diff --git a/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1.mdwn b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1.mdwn new file mode 100644 index 0000000000..f4ae47b32f --- /dev/null +++ b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1.mdwn @@ -0,0 +1,18 @@ +What steps will reproduce the problem? + +cabal install git-annex + +What is the expected output? What do you see instead? + +I get this: + + Assistant/WebApp/Configurators/Local.hs:55:11: + `fieldEnctype' is not a (visible) field of constructor `Field' + +What version of git-annex are you using? On what operating system? + +20121127 + +Please provide any additional information below. + +> [[done]]; see comments. --[[Joey]] diff --git a/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_1_b25859c159d62f2e92b92f505535131b._comment b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_1_b25859c159d62f2e92b92f505535131b._comment new file mode 100644 index 0000000000..f01f4382a8 --- /dev/null +++ b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_1_b25859c159d62f2e92b92f505535131b._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0" + nickname="Tobias" + subject="comment 1" + date="2012-11-27T15:48:33Z" + content=""" +I was told to my yesod was too old. + +Either upgrade yesod with cabal (After a \"cabal install yesod\" it worked on one of my computers, still failing on the other...) + +Or use the Makefile (failed on both the computers i tested). + + +"""]] diff --git a/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_2_4c9eab9120718457fdc1ae9051e44bca._comment b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_2_4c9eab9120718457fdc1ae9051e44bca._comment new file mode 100644 index 0000000000..cca43f63e3 --- /dev/null +++ b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_2_4c9eab9120718457fdc1ae9051e44bca._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0" + nickname="Tobias" + subject="comment 2" + date="2012-11-27T16:03:06Z" + content=""" +For reference here is a run of: + +cabal install yesod --force-reinstalls ; make + +http://pastebin.com/3Tr5BA0u + +Tried adding -DWITH_OLD_YESOD to the FEATURES in Makefile and got this after a \"make clean; make\": + +http://pastebin.com/GV84YgjZ +"""]] diff --git a/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_3_61aec9801e1f76db4a286536ffacc3ed._comment b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_3_61aec9801e1f76db4a286536ffacc3ed._comment new file mode 100644 index 0000000000..32cdf66769 --- /dev/null +++ b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_3_61aec9801e1f76db4a286536ffacc3ed._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0" + nickname="Tobias" + subject="comment 3" + date="2012-11-27T16:06:22Z" + content=""" +For the machine that could compile and install the latest git-annex after updating yesod. \"make\" still fails. + +The first run here is without -DWITH_OLD_YESOD, the second is with. + +http://pastebin.com/T3RpPTX +"""]] diff --git a/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_4_6381ff0ea419831d9bbed27511cad1e9._comment b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_4_6381ff0ea419831d9bbed27511cad1e9._comment new file mode 100644 index 0000000000..fb58e28ce8 --- /dev/null +++ b/doc/bugs/Cannot_build_the_latest_with_GHC_7.6.1/comment_4_6381ff0ea419831d9bbed27511cad1e9._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0" + nickname="Tobias" + subject="comment 4" + date="2012-11-27T16:10:31Z" + content=""" +cabal install yesod --force-reinstalls + +Then \"cabal configure, cabal build\" worked on my secondary machine... + +Woohooo, i have the latest git-annex running on both my machines now. Though neither can compile using the Makefile. + +Sorry for the spam here, just hoping it will help someone else jump through less hoops. + + +"""]] diff --git a/doc/bugs/Cannot_clone_an_annex.mdwn b/doc/bugs/Cannot_clone_an_annex.mdwn new file mode 100644 index 0000000000..62e9ed27f1 --- /dev/null +++ b/doc/bugs/Cannot_clone_an_annex.mdwn @@ -0,0 +1,67 @@ +I have an annex that I use to store my digital photos. I had a few false +starts creating this annex, but now it's looking good on my server: + + root@titan.local:/tank/Media/Pictures# git annex status + supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 2 + 00000000-0000-0000-0000-000000000001 -- web + be88bc5a-17e2-11e2-a99b-d388d4437350 -- here (titan) + untrusted repositories: 0 + dead repositories: 5 + 0A9F3136-A12A-43C7-9BE2-33F59954FD52 -- vulcan + 57349F02-E497-4420-9230-6B15D8AB14EE -- vulcan + 6195C912-2707-4B75-AC8C-11C51FAA8FE0 -- vulcan + D51DEDC4-9255-4A99-8520-2B1CED337674 -- hermes + EE327B34-3E20-4B5B-8F0E-D500CBC9738D -- hermes + transfers in progress: none + available local disk space: unknown + local annex keys: 20064 + local annex size: 217 gigabytes + known annex keys: 21496 + known annex size: 217 gigabytes + bloom filter size: 16 mebibytes (4% full) + backend usage: + SHA256E: 41560 + root@titan.local:/tank/Media/Pictures# git annex unused + unused . (checking for unused data...) ok + +It passes `git annex fsck` without any problems. However, when I "git clone" +this annex to my desktop machine and then do a `git annex sync`, I see this: + + Vulcan /Volumes/tank/Media/Pictures (master) $ git annex status + supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 5 + 00000000-0000-0000-0000-000000000001 -- web + 0A9F3136-A12A-43C7-9BE2-33F59954FD52 -- vulcan + 274D3474-7A25-44CD-8368-CF11C451014F -- here (vulcan) + EE327B34-3E20-4B5B-8F0E-D500CBC9738D -- hermes + be88bc5a-17e2-11e2-a99b-d388d4437350 -- titan + untrusted repositories: 0 + dead repositories: 3 + 57349F02-E497-4420-9230-6B15D8AB14EE -- vulcan + 6195C912-2707-4B75-AC8C-11C51FAA8FE0 -- vulcan + D51DEDC4-9255-4A99-8520-2B1CED337674 -- hermes + transfers in progress: none + available local disk space: 1 terabyte (+1 megabyte reserved) + local annex keys: 0 + local annex size: 0 bytes + known annex keys: 21025 + known annex size: 217 gigabytes + bloom filter size: 16 mebibytes (0% full) + backend usage: + SHA256: 18707 + SHA256E: 2318 + +Where did all these `SHA256` keys come from? + +Why doesn't the known annex keys size match? + +Further, I cannot `git annex get` on most of the files, because it says that +the `SHA256` key is not present. + +It looks like I'll have to rollback my ZFS snapshots and start over, but I'm +wondering: how was I even able to create this situation? diff --git a/doc/bugs/Cannot_clone_an_annex/comment_1_b40a2652361a79c6c6eab0fc21be8e46._comment b/doc/bugs/Cannot_clone_an_annex/comment_1_b40a2652361a79c6c6eab0fc21be8e46._comment new file mode 100644 index 0000000000..44b93b1c2f --- /dev/null +++ b/doc/bugs/Cannot_clone_an_annex/comment_1_b40a2652361a79c6c6eab0fc21be8e46._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.23" + subject="comment 1" + date="2012-10-22T13:53:06Z" + content=""" +The SHA256 keys are symlinks in your git repository with \"SHA256-\" as the prefix on the file linked to. You should be able to determine the rest by looking at the history of those files in git. +"""]] diff --git a/doc/bugs/Committer_crashed.mdwn b/doc/bugs/Committer_crashed.mdwn new file mode 100644 index 0000000000..caa8b1c50b --- /dev/null +++ b/doc/bugs/Committer_crashed.mdwn @@ -0,0 +1,32 @@ +# What steps will reproduce the problem? + +Editing a text file with vim + +#What is the expected output? What do you see instead? + + # On branch master + # Changes not staged for commit: + + # (use "git add ..." to update what will be committed) + # (use "git checkout -- ..." to discard changes in working directory) + # + # typechange: test + # + # Untracked files: + # (use "git add ..." to include in what will be committed) + # + # .test.swp + + no changes added to commit (use "git add" and/or "git commit -a") + + /.test.swp still has writers, not adding + + Committer crashed: ./test~: createLink: does not exist (No such file or directory) + +# What version of git-annex are you using? On what operating system? + +3.20130107 prebuilt tar ball on Debian testing + +> Could also fail in `getFileStatus`. In either case it's a race +> with the file being deleted while it's still in the process of being +> locked down. Fixed this [[done]] --[[Joey]] diff --git a/doc/bugs/Complete_failure_trying_to_unannex_a_large_annex.mdwn b/doc/bugs/Complete_failure_trying_to_unannex_a_large_annex.mdwn new file mode 100644 index 0000000000..b59dd8b5d7 --- /dev/null +++ b/doc/bugs/Complete_failure_trying_to_unannex_a_large_annex.mdwn @@ -0,0 +1,53 @@ +I really don't know what's happened here, I just did `git annex unannex .` in a very large annex: + + unannex Inbox/Lolcat.JPG (Recording state in git...) + ok + unannex Inbox/Lolcat.jpg (Recording state in git...) + ok + unannex Inbox/May 2012 Photo Stream/120502_0004.JPG (Recording state in git...) + ok + unannex Inbox/May 2012 Photo Stream/120518_0005.JPG (Recording state in git...) + ok + unannex Inbox/May 2012 Photo Stream/120523_0006.JPG (Recording state in git...) + ok + unannex Inbox/May 2012 Photo Stream/120523_0007.JPG (Recording state in git...) + ok + unannex Inbox/My boyfriend of 7 years and I are both physicists. Here's how he proposed to me. - Imgur.jpg (Recording state in git...) + ok + unannex Inbox/Nov 2012 Photo Stream/121102_0035.JPG (Recording state in git...) + ok + unannex Inbox/Nov 2012 Photo Stream/121102_0036.JPG (Recording state in git...) + ok + unannex Inbox/Nov 2012 Photo Stream/121102_0037.JPG (Recording state in git...) + ok + unannex Inbox/Nov 2012 Photo Stream/121102_0038.JPG (Recording state in git...) + ok + unannex Inbox/Nov 2012 Photo Stream/121102_0039.JPG (Recording state in git...) + ok + unannex Inbox/Nov 2012 Photo Stream/121103_0040.JPG (Recording state in git...) + ok + unannex Inbox/Nov 2012 Photo Stream/121104_0041.JPG (Recording state in git...) + ok + unannex Inbox/Nov 2012 Photo Stream/121105_0042.JPG (Recording state in git...) + error: bad index file sha1 signature + fatal: index file corrupt + git-annex: failed to read sha from git write-tree + git-annex: git commit [Param "-q",Params "-m",Param "content removed from git annex",Param "--",File "Inbox/Nov 2012 Photo Stream/121105_0042.JPG"] failed + Vulcan:~/Pictures $ ga unannex . + unannex Inbox/Nov 2012 Photo Stream/121109_0043.JPG error: bad index file sha1 signature + fatal: index file corrupt + + git-annex: fd:12: hClose: resource vanished (Broken pipe) + failed + git-annex: pre-commit: 1 failed + git-annex: git commit [Param "-q",Params "-m",Param "content removed from git annex",Param "--",File "Inbox/Nov 2012 Photo Stream/121109_0043.JPG"] failed + Vulcan:~/Pictures $ ga -F unannex . + unannex Inbox/Nov 2012 Photo Stream/121124_0044.JPG error: bad index file sha1 signature + fatal: index file corrupt + + git-annex: fd:12: hClose: resource vanished (Broken pipe) + failed + git-annex: pre-commit: 1 failed + git-annex: git commit [Param "-q",Params "-m",Param "content removed from git annex",Param "--",File "Inbox/Nov 2012 Photo Stream/121124_0044.JPG"] failed + +I guess now I'll just try to unlink the symlinks by hand, and drop the `.git` directory? diff --git a/doc/bugs/ControlPath_too_long_for_Unix_domain_socket.mdwn b/doc/bugs/ControlPath_too_long_for_Unix_domain_socket.mdwn new file mode 100644 index 0000000000..9cf4531643 --- /dev/null +++ b/doc/bugs/ControlPath_too_long_for_Unix_domain_socket.mdwn @@ -0,0 +1,53 @@ +What steps will reproduce the problem? +Pairing an existing git annex repository with a fresh repository on another computer in the git-annex webapp + + +What is the expected output? What do you see instead? +Expected result is that the two machines sync correctly. + +What i see are some "ControlPath too long for unix domain socket" errors from ssh, but the computers do actually sync properly. + +Even though the data is synced properly, either the sender(or both of the clients) don't actually realize this. And the queue circles, all the transfers are being redone constantly(On every start of git-annex webapp on the original repository at least). + + +What version of git-annex are you using? On what operating system? +Latest git master as of this post. Debian sid and Ubuntu 12.04 + + +Please provide any additional information below. + + +stdout snippet from git-annex webapp: + + + ControlPath "/home/alansmithee/Desktop/annex/.git/annex/ssh/alansmithee@git-annex-debbook.local-alansmithee.dxpXHVCkLhsxvWaH" too long for Unix domain socket + SHA256-s51233--0b4c59b3ab03b1ca6d95d4084fa6ff7220cf26695b6e3dd575f78af3dec6b701 + 51233 100% 5.43MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 30 bytes received 51385 bytes 102830.00 bytes/sec + total size is 51233 speedup is 1.00 + ok + (Recording state in git...) + + ControlPath "/home/alansmithee/Desktop/annex/.git/annex/ssh/alansmithee@git-annex-debbook.local-alansmithee.9ZQwEjraTxi20B6W" too long for Unix domain socket + SHA256-s47883982--4b7cbb49506dcdd223a9db7b400cc41fc2e3ebbf5b2b17b75c9334bb949b6754 + 47883982 100% 1.34MB/s 0:00:34 (xfer#1, to-check=0/1) + + sent 30 bytes received 47889978 bytes 1388116.17 bytes/sec + total size is 47883982 speedup is 1.00 + ok + (Recording state in git...) + + +This data appears on both the sending and receiving git-annex stdout. At least for the initial sync. For later syncs it only appears on the sender, though the client system is using a lot of resources. + +> I've made git-annex detect if the control path would be too long, +> and disable ssh connection caching. It also tries a relative path +> to the file, which tends to make it shorter, and I think would +> keep ssh connection caching working in your example. +> +> Please test and see if it works, and also if the "looping" problem +> still happens. --[[Joey]] + +>> Closing; I'm pretty sure the looping is just transfer retrying, to be +>> expected if they fail. [[done]] --[[Joey]] diff --git a/doc/bugs/ControlPath_too_long_for_Unix_domain_socket/comment_1_60f58e205604eebe668b1e05dcfbf9a7._comment b/doc/bugs/ControlPath_too_long_for_Unix_domain_socket/comment_1_60f58e205604eebe668b1e05dcfbf9a7._comment new file mode 100644 index 0000000000..0cba1a6939 --- /dev/null +++ b/doc/bugs/ControlPath_too_long_for_Unix_domain_socket/comment_1_60f58e205604eebe668b1e05dcfbf9a7._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.8.30" + subject="comment 1" + date="2012-09-13T22:00:53Z" + content=""" +You can work around the ControlPath too long by running in your git repo: + +git config annex.sshcaching false + +While I could reproduce that part of your problem, I did *not* see any data actually be transferred when the ControlPath was too long. For example: + +
+joey@gnu:~/tmp/t/1>git annex transferkey --to foo SHA256E-s29--66ffceb8b7a1af9c2eb896f26fb8f3bdab0a606da59a7a2d71ae4f7d78ad13c4/SHA256E-s29--66ffceb8b7a1af9c2eb896f26fb8f3bdab0a606da59a7a2d71ae4f7d78ad13c4
+
+ControlPath too long
+rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+rsync error: error in rsync protocol data stream (code 12) at io.c(605) [sender=3.0.9]
+
+  rsync failed -- run git annex again to resume file transfer
+failed
+git-annex: transferkey: 1 failed
+
+"""]] diff --git a/doc/bugs/Could_not_find_module_Data.Default.mdwn b/doc/bugs/Could_not_find_module_Data.Default.mdwn new file mode 100644 index 0000000000..64202de425 --- /dev/null +++ b/doc/bugs/Could_not_find_module_Data.Default.mdwn @@ -0,0 +1,33 @@ +**What steps will reproduce the problem?** + +Manually building git-annex from git. + +**What is the expected output? What do you see instead?** + + $ cabal update + ... + $ cabal install --only-dependencies + ... + $ cabal configure + ... + $ cabal build + Building git-annex-3.20120826... + Preprocessing executable 'git-annex' for git-annex-3.20120826... + + Utility/Yesod.hs:15:8: + Could not find module `Data.Default' + It is a member of the hidden package `data-default-0.5.0'. + Perhaps you need to add `data-default' to the build-depends in your .cabal file. + Use -v to see a list of the files searched for. + $ + +**What version of git-annex are you using? On what operating system?** + +commit e7d728672a5fc923be9ab1d6fe4b65f2058b49c7 +Arch Linux + +**Please provide any additional information below.** + +When I add data-default to git-annex.cabal's Build-Deps it works fine. + +> Thanks, [[done]]. --[[Joey]] diff --git a/doc/bugs/Could_not_resolve_dependencies.mdwn b/doc/bugs/Could_not_resolve_dependencies.mdwn new file mode 100644 index 0000000000..9eb98f31ef --- /dev/null +++ b/doc/bugs/Could_not_resolve_dependencies.mdwn @@ -0,0 +1,40 @@ +I'm not able to install git-annex with cabal. + +What steps will reproduce the problem? + + bbigras@bbigras-VirtualBox:~$ cabal update + Downloading the latest package list from hackage.haskell.org + bbigras@bbigras-VirtualBox:~$ cabal install git-annex --bindir=$HOME/bin + Resolving dependencies... + cabal: Could not resolve dependencies: + trying: git-annex-3.20130207 (user goal) + trying: git-annex-3.20130207:+webdav + trying: git-annex-3.20130207:+webapp + trying: git-annex-3.20130207:+assistant + trying: yesod-1.1.8.2 (dependency of git-annex-3.20130207:+assistant) + trying: yesod-auth-1.1.5.2 (dependency of yesod-1.1.8.2) + trying: authenticate-1.3.2.4 (dependency of yesod-auth-1.1.5.2) + trying: xml-conduit-1.1.0.1 (dependency of authenticate-1.3.2.4) + next goal: DAV (dependency of git-annex-3.20130207:+webdav) + rejecting: DAV-0.3 (conflict: xml-conduit==1.1.0.1, DAV => xml-conduit>=1.0 && + <=1.1) + rejecting: DAV-0.2, 0.1, 0.0.1, 0.0 (conflict: git-annex-3.20130207:webdav => + DAV(>=0.3)) + bbigras@bbigras-VirtualBox:~$ + + +What version of git-annex are you using? On what operating system? + +Ubuntu 12.10 x86_64 + +cabal-install version 0.14.0 +using version 1.14.0 of the Cabal library + +> The Haskell DAV library needs to be updated to build with +> the newer version of xml-conduit. Library skew of this sort +> is common when using cabal. +> +> You can work around this by building git-annex without webdav: +> `cabal configure --flags=-WebDAV` +> +> This is not a git-annex bug. [[done]] --[[Joey]] diff --git a/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh.mdwn b/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh.mdwn new file mode 100644 index 0000000000..38f54d2b61 --- /dev/null +++ b/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh.mdwn @@ -0,0 +1,43 @@ +What steps will reproduce the problem? + +I create a new annex, added in a bunch of files. + +I cloned this annex to another machine, where I already had those files, so I copied them into a directory named "foo", did "git annex add foo", and then did "git rm -r foo", and git commit'd my clone of the annex. + +Then I try to "git annex sync" with the remote. + +What is the expected output? What do you see instead? + +I don't know, I've never used git-annex before. This is what I get each time: + + Hermes ~/Products/tmp/Movies (master) $ ga sync + git-annex-shell: Prelude.(!!): index too large + +What version of git-annex are you using? On what operating system? + +It's the 'master' as of yesterday: c504f4025fec49e62601fbd4a3cd8f1270c7d221 + +I'm on OS X 10.8.2, using GHC 7.6.1. The annex in question has 38G in a few hundred files. + +Please provide any additional information below. + +I'm willing to help track this down! + +> I've got it, October 9th's release +> included commit bc649a35bacbecef93e378b1497f6a05b30bf452, which included a +> change to a `segment` function. It was supposed to be a +> rewrite in terms of a more general version, but it introduced a bug +> in what it returned in an edge case and this in turn led git-annex-shell's +> parameter parser to fail in a code path that was never reachable before. +> +> It'd fail both when a new repo was running `git-annex-shell configlist`, +> and in `git-annex-shell commit`, although this latter crash was less +> noticible and I'm sure you saw the former. +> +> Fixed the reversion; fixed insufficient guards around the partial code +> (which I cannot see a way to entirely eliminate sadly; look at +> GitAnnexShell.hs's `partitionParams` and weep or let me know if you have +> any smart ideas..); added a regression test to check the non-obvious +> behavior of segment with an empty segment. I'll be releasing a new +> version with this fix as soon as I have bandwidth, ie tomorrow. +> [[done]] --[[Joey]] diff --git a/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh/comment_1_9705f295ad8101f3f0ede18e590b56ef._comment b/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh/comment_1_9705f295ad8101f3f0ede18e590b56ef._comment new file mode 100644 index 0000000000..660b52e7f6 --- /dev/null +++ b/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh/comment_1_9705f295ad8101f3f0ede18e590b56ef._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="I can confirm the fix" + date="2012-10-16T05:53:15Z" + content=""" +It's working great now, thanks Joey! +"""]] diff --git a/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh/comment_2_0d751d81ac618f8d7e3f1dd20c830542._comment b/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh/comment_2_0d751d81ac618f8d7e3f1dd20c830542._comment new file mode 100644 index 0000000000..5622bd6da1 --- /dev/null +++ b/doc/bugs/Crash_trying_to_sync_with_a_repo_over_ssh/comment_2_0d751d81ac618f8d7e3f1dd20c830542._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.246.8" + subject="comment 2" + date="2012-10-16T05:56:10Z" + content=""" +And I found a clean way to avoid that partial !! too. :) Down to 22 of the little monsters to clean up now. +"""]] diff --git a/doc/bugs/Crash_when_adding_jabber_account_.mdwn b/doc/bugs/Crash_when_adding_jabber_account_.mdwn new file mode 100644 index 0000000000..aac1b3321f --- /dev/null +++ b/doc/bugs/Crash_when_adding_jabber_account_.mdwn @@ -0,0 +1,30 @@ +*What steps will reproduce the problem?* + +1. Start git-annex webapp +2. Configuration +3. Configure Jabber Account +4. Insert user and pass +5. Click "User this account" + +Tryed 4 times, all the same. + + +*What is the expected output? What do you see instead?* + +On Chrome I get "Error 101 (net::ERR_CONNECTION_RESET): The connection was reset." or "Error 324 (net::ERR_EMPTY_RESPONSE): The server closed the connection without sending any data." + +On the terminal where git-annex was running I get "Segmentation fault (core dumped)" + + +*What version of git-annex are you using? On what operating system?* + +git-annex version: I downloaded 3.20130107 (twice to be sure), but for some reason 'git-annex version' reports 3.20130102 + +OS: Ubuntu 12.04.1 LTS 3.2.0-35-generic-pae #55-Ubuntu SMP Wed Dec 5 18:04:39 UTC 2012 i686 i686 i386 GNU/Linux + + +*Please provide any additional information below.* + +On dmesg: +[45773.212717] git-annex[26779]: segfault at b724e840 ip 09699150 sp b4cfd038 error 7 in git-annex[8048000+1762000] + diff --git a/doc/bugs/Crash_when_adding_jabber_account_/comment_1_2dc61ebcfa8919fb839656999c155c52._comment b/doc/bugs/Crash_when_adding_jabber_account_/comment_1_2dc61ebcfa8919fb839656999c155c52._comment new file mode 100644 index 0000000000..c32ee75dec --- /dev/null +++ b/doc/bugs/Crash_when_adding_jabber_account_/comment_1_2dc61ebcfa8919fb839656999c155c52._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 1" + date="2013-01-14T16:39:00Z" + content=""" +Are you using the 64 bit or the 32 bit build? Did you download the standalone tarball? How are you running git-annex exactly? (Using runshell, or by hand?) + +A segfault here seems likely to involve the Haskell GNUTLS binding. At least, the only other time git-annex has segfaulted, which also involved jabber, it was a bug in the GNUTLS binding. +"""]] diff --git a/doc/bugs/Crash_when_adding_jabber_account_/comment_2_e49af3b8a937d82eda1509b6f67b21d4._comment b/doc/bugs/Crash_when_adding_jabber_account_/comment_2_e49af3b8a937d82eda1509b6f67b21d4._comment new file mode 100644 index 0000000000..03f7a20699 --- /dev/null +++ b/doc/bugs/Crash_when_adding_jabber_account_/comment_2_e49af3b8a937d82eda1509b6f67b21d4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawn-4QWOFI0QQ4n-yu-oFq3PHeuSPUv_-b4" + nickname="Rodrigo" + subject="comment 2" + date="2013-01-14T22:10:44Z" + content=""" +32 bit. Yes, the standalone tarball. From terminal, I'm simply running: git-annex webapp +"""]] diff --git a/doc/bugs/Creating_an_S3_repository_with_an_invalid_name_throws_an_exception.mdwn b/doc/bugs/Creating_an_S3_repository_with_an_invalid_name_throws_an_exception.mdwn new file mode 100644 index 0000000000..48b30ecebd --- /dev/null +++ b/doc/bugs/Creating_an_S3_repository_with_an_invalid_name_throws_an_exception.mdwn @@ -0,0 +1,18 @@ +What steps will reproduce the problem? + +Attempt to create an S3 repository called 'S3 (encrypted)' via the web app. + +What is the expected output? What do you see instead? + +Expected to either accept this input, or tell me that it's invalid. Instead, I see: + +Internal Server Error +user error (openTCPConnection: host lookup failure for "S3 (encrypted)-610526e6-5ce2-11e2-997e-970cbca93f19.s3.amazonaws.com") + +What version of git-annex are you using? On what operating system? + +3.20130107 On Fedora 17 (64-bit). + +Please provide any additional information below. + +> [[done]], the name will now be sanitized. --[[Joey]] diff --git a/doc/bugs/Creating_an_encrypted_S3_does_not_check_for_presence_of_GPG.mdwn b/doc/bugs/Creating_an_encrypted_S3_does_not_check_for_presence_of_GPG.mdwn new file mode 100644 index 0000000000..aab03cd7fe --- /dev/null +++ b/doc/bugs/Creating_an_encrypted_S3_does_not_check_for_presence_of_GPG.mdwn @@ -0,0 +1,17 @@ +What steps will reproduce the problem? + +Don't have gpg installed/on your $PATH, and attempt to create an encrypted S3 remote via the web interface. + +What is the expected output? What do you see instead? + +Expected to be told to install GPG. Actual output was a Yesod error: + +Internal Server Error +user error (gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--gen-random","--armor","1","512"] exited 127) + +What version of git-annex are you using? On what operating system? + +3.20130107 on Fedora 17 (64-bit). + +Please provide any additional information below. + diff --git a/doc/bugs/DS__95__Store_not_gitignored.mdwn b/doc/bugs/DS__95__Store_not_gitignored.mdwn new file mode 100644 index 0000000000..269eb417aa --- /dev/null +++ b/doc/bugs/DS__95__Store_not_gitignored.mdwn @@ -0,0 +1,26 @@ +What steps will reproduce the problem? + +Create a new git repo on OS X. create a .gitignore file containing the line ".DS_Store". Check it into git. do "git annex init" to make the directory into a git annex repo. do "git annex direct" to put it in direct mode. do "git annex assistant" to start the assistant and watch it with "git annex webapp". + +Open the directory in the finder. Drag in some files and watch the assistant as it checks them in. Play with your window's viewing preferences, maybe view things as "icons" so that the finder is forced to store metadata about where you dragged the icons in the window in the .DS_Store file. As you do this, watch the webapp. .DS_Store will start to be checked into git annex, and rechecked in frequently as it is updated. Git annex assistant is not respecting the .gitignore file. Is there some other way to tell git annex assistant that certain files should be ignored than .gitignore? + + +What is the expected output? What do you see instead? + +.DS_Store files are ignored by the assistant. + + +What version of git-annex are you using? On what operating system? + +git-annex version: 3.20130124 + +OS X Lion + +Please provide any additional information below. + +> Assistant does not support .gitignore yet. Requires an efficient query +> interface for ignores, which git does not provide. +> +> However, I've added a special case, OSX only ignore for .DS_Store files. +> [[done]] --[[Joey]] + diff --git a/doc/bugs/DS__95__Store_not_gitignored/comment_1_b93ac0ea3be82c361ceb4352e742ba39._comment b/doc/bugs/DS__95__Store_not_gitignored/comment_1_b93ac0ea3be82c361ceb4352e742ba39._comment new file mode 100644 index 0000000000..0bc821f872 --- /dev/null +++ b/doc/bugs/DS__95__Store_not_gitignored/comment_1_b93ac0ea3be82c361ceb4352e742ba39._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 1" + date="2013-01-28T14:31:53Z" + content=""" +outstanding! Thank you. +"""]] diff --git a/doc/bugs/DS__95__Store_not_gitignored/comment_2_4136e1f4aba7aa7562dafcf6a213e10c._comment b/doc/bugs/DS__95__Store_not_gitignored/comment_2_4136e1f4aba7aa7562dafcf6a213e10c._comment new file mode 100644 index 0000000000..209e0545a6 --- /dev/null +++ b/doc/bugs/DS__95__Store_not_gitignored/comment_2_4136e1f4aba7aa7562dafcf6a213e10c._comment @@ -0,0 +1,58 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmRFKwny4rArBaz-36xTcsJYqKIgdDaw5Q" + nickname="Andrew" + subject="comment 2" + date="2013-01-28T23:26:27Z" + content=""" +This actually explains an interesting quirk (I wouldn't necessarily call it a bug) that results because of annex's handling of .gitignore: + + % › git annex import ~/Downloads/BoxcarMac.dmg + import BoxcarMac.dmg (checksum...) ok + (Recording state in git...) + The following paths are ignored by one of your .gitignore files: + BoxcarMac.dmg + Use -f if you really want to add them. + fatal: no files added + + git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=/Users/akraut/Desktop/annexes/test/.git\",\"--work-tree=/Users/akraut/Desktop/annexes/test\",\"add\",\"--\"] exited 1) + failed + git-annex: import: 1 failed + + % › ls -l + total 8 + lrwxr-xr-x 1 akraut staff 198 Jan 28 15:01 BoxcarMac.dmg -> .git/annex/objects/K4/Q8/SHA256E-s6240024--2d3b032f29c8411f81f9379bd79abfa713b66b9783559ef48cd945ab418e97a3.dmg/SHA256E-s6240024--2d3b032f29c8411f81f9379bd79abfa713b66b9783559ef48cd945ab418e97a3.dmg + -rw-r--r-- 1 akraut staff 0 Jan 28 14:56 README + lrwxr-xr-x 1 akraut staff 200 Jan 28 14:58 Wireshark 1.8.4 Intel 64.dmg -> .git/annex/objects/vj/55/SHA256E-s21772874--eb01484d832a9dc5b8fdecacdccabc4ef28fb17a2e20bc2837ccc43a69df30c5.dmg/SHA256E-s21772874--eb01484d832a9dc5b8fdecacdccabc4ef28fb17a2e20bc2837ccc43a69df30c5.dmg + + % › cp ~/Downloads/Wireshark\ 1.8.4\ Intel\ 64.dmg . + + % › git annex add Wireshark\ 1.8.4\ Intel\ 64.dmg + (Recording state in git...) + + % › ls -l + total 21264 + -rw-r--r-- 1 akraut staff 0 Jan 28 14:56 README + -rw-r--r-- 1 akraut staff 21772874 Jan 28 14:58 Wireshark 1.8.4 Intel 64.dmg + + % › git annex import Wireshark\ 1.8.4\ Intel\ 64.dmg + import Wireshark 1.8.4 Intel 64.dmg git-annex: not overwriting existing Wireshark 1.8.4 Intel 64.dmg (use --force to override) + + % › git annex import --force Wireshark\ 1.8.4\ Intel\ 64.dmg + import Wireshark 1.8.4 Intel 64.dmg (checksum...) ok + (Recording state in git...) + + % › ls + README Wireshark 1.8.4 Intel 64.dmg@ + + % › git annex sync + commit + [master 0a17811] git-annex automatic sync + 1 file changed, 1 insertion(+) + create mode 120000 Wireshark 1.8.4 Intel 64.dmg + ok + + % › ls -l + total 4 + -rw-r--r-- 1 akraut staff 0 Jan 28 14:56 README + lrwxr-xr-x 1 akraut staff 200 Jan 28 14:58 Wireshark 1.8.4 Intel 64.dmg -> .git/annex/objects/vj/55/SHA256E-s21772874--eb01484d832a9dc5b8fdecacdccabc4ef28fb17a2e20bc2837ccc43a69df30c5.dmg/SHA256E-s21772874--eb01484d832a9dc5b8fdecacdccabc4ef28fb17a2e20bc2837ccc43a69df30c5.dmg +"""]] diff --git a/doc/bugs/Detection_assumes_that_shell_is_bash.mdwn b/doc/bugs/Detection_assumes_that_shell_is_bash.mdwn new file mode 100644 index 0000000000..46b159e804 --- /dev/null +++ b/doc/bugs/Detection_assumes_that_shell_is_bash.mdwn @@ -0,0 +1,20 @@ +###What steps will reproduce the problem?### + +"Adding a remote server using ssh" and try to add a remote server where the account has ex. tcsh as loginshell + +###What is the expected output? What do you see instead?### + +To discover remote programs, it dumps away some born-shell code like: +"echo git-annex-probe loggedin;if which git-annex-shell; then echo git-annex-probe git-annex-shell; fi;if which rsync; then echo git-annex-probe rsync; fi;if which ~/.ssh/git-annex-shell; then echo git-annex-probe ~/.ssh/git-annex-shell; fi" + +just wrap it with a bash -c '..' and you know that its interpreted by bash. + +###What version of git-annex are you using? On what operating system?### + +git-annex version: 3.20121017 + +###Please provide any additional information below.### + +Not everyone has bash as there login-shell. + +[[!tag /design/assistant]] diff --git a/doc/bugs/Disconcerting_warning_from_git-annex.mdwn b/doc/bugs/Disconcerting_warning_from_git-annex.mdwn new file mode 100644 index 0000000000..169dc26d14 --- /dev/null +++ b/doc/bugs/Disconcerting_warning_from_git-annex.mdwn @@ -0,0 +1,6 @@ +I did a "git annex add" of a bunch of files on a storage server with low IOPS, and saw this: + + git-annex: /tank/Media/Pictures/.git/annex/tmp/430_32b_SHA256E-s4464838--c1785a76ee1451f602e93c99c147e214705004e541de8256d74a3be3717d15e5.jpg.log: openBinaryFile: resource busy (file is locked) +failed + +How is that even possible, when the server is doing nothing else? diff --git a/doc/bugs/Disconcerting_warning_from_git-annex/comment_1_58cebd377bfdf247b6c4fee27a3ba461._comment b/doc/bugs/Disconcerting_warning_from_git-annex/comment_1_58cebd377bfdf247b6c4fee27a3ba461._comment new file mode 100644 index 0000000000..3076b38de1 --- /dev/null +++ b/doc/bugs/Disconcerting_warning_from_git-annex/comment_1_58cebd377bfdf247b6c4fee27a3ba461._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.211" + subject="comment 1" + date="2013-01-05T21:13:09Z" + content=""" +Is this a Solaris system? +"""]] diff --git a/doc/bugs/Disconcerting_warning_from_git-annex/comment_2_dc7407044d4c739d05248300c58d8ef2._comment b/doc/bugs/Disconcerting_warning_from_git-annex/comment_2_dc7407044d4c739d05248300c58d8ef2._comment new file mode 100644 index 0000000000..96d495e66c --- /dev/null +++ b/doc/bugs/Disconcerting_warning_from_git-annex/comment_2_dc7407044d4c739d05248300c58d8ef2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="comment 2" + date="2013-01-20T03:38:43Z" + content=""" +No, it's CentOS 6.3 x86_64., +"""]] diff --git a/doc/bugs/Displayed_copy_speed_is_wrong.mdwn b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn new file mode 100644 index 0000000000..cf3b31cf48 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn @@ -0,0 +1,8 @@ +When copying data to my remote, I regularly see speeds in excess of 100 MB/s on my home DSL line. + + 2073939 100% 176.96MB/s 0:00:00 (xfer#1, to-check=0/1) + +This is definitely not correct. + +> Closing, as rsync does this to show you when it's making your life +> faster than it would be w/o rsync. [[done]] --[[Joey]] diff --git a/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment b/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment new file mode 100644 index 0000000000..62a595be77 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T01:37:29Z" + content=""" +That is displayed by rsync. It's not unheard of for rsync to resume a transfer and display extremely high speeds. +"""]] diff --git a/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment b/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment new file mode 100644 index 0000000000..28305d3ac8 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-03T08:56:48Z" + content=""" +Pity. Mark as done/upstream (or similar) for house-keeping? +"""]] diff --git a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn new file mode 100644 index 0000000000..85e2433829 --- /dev/null +++ b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn @@ -0,0 +1,79 @@ +What steps will reproduce the problem? + + $ git annex initremote rsyncremote type=rsync rsyncurl=myuser@rsync.hidrive.strato.com:/users/myuser/git-annex/Music/ encryption=0xC597DECC177AFD7C + $ git annex get --from rsyncremote "file" + +What is the expected output? What do you see instead? + +I expect that the requested file is copied as for every other remote, but instead I get this error: + +---------------------------------------- + get (from rsyncremote...) (gpg) + rsync: change_dir "/users/myuser/git-annex/Music/0e5/a5b/'GPGHMACSHA1--3afd32ab8e70ac329262adeb770c330b0845b1e0" failed: No such file or directory (2) + + sent 8 bytes received 10 bytes 7.20 bytes/sec + total size is 0 speedup is 0.00 + rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1518) [Receiver=3.0.9] + + rsync failed -- run git annex again to resume file transfer + + rsync: change_dir "/users/myuser/git-annex/Music/8k/QZ/'GPGHMACSHA1--3afd32ab8e70ac329262adeb770c330b0845b1e0" failed: No such file or directory (2) + + sent 8 bytes received 10 bytes 36.00 bytes/sec + total size is 0 speedup is 0.00 + rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1518) [Receiver=3.0.9] + + rsync failed -- run git annex again to resume file transfer +failed + git-annex: get: 1 failed +---------------------------------------- + +I can verify that the directory /users/myuser/git-annex/Music/0e5/a5b/GPGHMACSHA1--3afd32ab8e70ac329262adeb770c330b0845b1e0 exists in the rsync remote, without the ' character. + +What version of git-annex are you using? On what operating system? + +I tried versions 3.20120315 and 3.20120430 on Gentoo linux. + + $ uname -a + Linux odin 3.3.1-gentoo-odin #1 SMP Sat Apr 7 21:18:11 CEST 2012 x86_64 Intel(R) Core(TM) i5 CPU M 560 @ 2.67GHz GenuineIntel GNU/Linux + + $ ghc --version + The Glorious Glasgow Haskell Compilation System, version 7.4.1 + +Please provide any additional information below. + +The rsync remote config in .git/config: + + [remote "rsyncremote"] + annex-rsyncurl = myuser@rsync.hidrive.strato.com:/users/myuser/git-annex/Music/ + annex-uuid = "UUID" + +> Here's what the --debug flag shows is being run: --[[Joey]] + + Running: rsync ["--progress","--inplace","joey@localhost:/tmp/Music/d98/a3c/'GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c/GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c'","/home/joey/tmp/x/.git/annex/tmp/GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c"] + +> But, this works for me, here, despite containing the quoting! +> That's because here it's using rsync over ssh, which actually requires +> that quoting. Are you using rsync +> over the rsync protocol? If so, the workaround is to explicitly make +> the rsyncurl start with `rsync://` +> +> And if this is the case, I need +> to adjust the code in git-annex that determines if it's using ssh or +> the rsync protocol. It assumes that (and this is what the rsync man +> says AFAICS) that the rsync protocol is only used if the url starts +> with `rsync://` or contains `::`. +> +>> Nope, it is indeed using rsync over ssh as git-annex thought. +> +> Hmm, I see that `hidrive.strato.com` is some kind of rsync provider? +> Perhaps they do something with rsync over ssh that +> avoids the need for shell quoting. For example, they might pass incoming +> ssh connections directly into rsync, bypassing the shell +> -- which avoids the need for this quoting. Any details you can provide +> about them would probably be useful then. Ie, do they really use rsync +> over ssh, is it really a `rsync.net` type rsync provider? +> --[[Joey]] +> +>> This was the case, and the shellescape=no config option has been added +>> to rsync special remotes to deal with it. [[done]] --[[Joey]] diff --git a/doc/bugs/Error_creating_remote_repository_using_ssh_on_OSX.mdwn b/doc/bugs/Error_creating_remote_repository_using_ssh_on_OSX.mdwn new file mode 100644 index 0000000000..cf7dcc88e2 --- /dev/null +++ b/doc/bugs/Error_creating_remote_repository_using_ssh_on_OSX.mdwn @@ -0,0 +1,36 @@ +What steps will reproduce the problem? + +1. Click "Remote server: Set up a repository on a remote server using ssh." +2. Enter hostname and different username than currently logged in user +3. Click check this server + + +What is the expected output? + +> I expected to see the next step in the remote repo creration process. + +What do you see instead? + + +> Failed to ssh to the server. Transcript: ssh_askpass: exec(/usr/libexec/ssh-askpass): No such file or directory Permission denied, please try again. ssh_askpass: exec(/usr/libexec/ssh-askpass): No such file or directory Permission denied, please try again. ssh_askpass: exec(/usr/libexec/ssh-askpass): No such file or directory Permission denied (publickey,password). + + +What version of git-annex are you using? + +> git-annex: Version: 3.20130114 + + +On what operating system? + +> OSX: 10.8.2 + + + +Please provide any additional information below. + +> I mentioned "with a different username" because the assistant will allow me to create a remote repository on the same target machine when I use my normal username. I think this is most likely because I have a ssh-key setup for the account on the remote machine. However I do not want to assume anything and send you down the wrong OSX rabbit hole. + +> After a little research it seems that OSX does not have a ssh-askpass + +[[!tag /design/assistant/OSX]] +[[!meta title="ssh-askpass not available on OSX"]] diff --git a/doc/bugs/Error_creating_remote_repository_using_ssh_on_OSX/comment_1_559555934d79ae6be383063abcaae22e._comment b/doc/bugs/Error_creating_remote_repository_using_ssh_on_OSX/comment_1_559555934d79ae6be383063abcaae22e._comment new file mode 100644 index 0000000000..8b701250c7 --- /dev/null +++ b/doc/bugs/Error_creating_remote_repository_using_ssh_on_OSX/comment_1_559555934d79ae6be383063abcaae22e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.3.125" + subject="comment 1" + date="2013-02-05T19:45:24Z" + content=""" +It should be possible to use SSHPassKey as described here: + +However, this all seems pretty crummy, and so perhaps it would be worth it for the webapp to do its own ssh password prompting when setting up a remote. +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__.mdwn b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__.mdwn new file mode 100644 index 0000000000..d0b2c15389 --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__.mdwn @@ -0,0 +1,31 @@ +Running 3.20121112 on Debian Squeeze. + +Since adding a certain directory of files (just a bunch of PDFs) yesterday I am getting errors when I try to use `git annex drop .` when the files aren't present, rather doing nothing or saying 'ok', as it used to do/should do. The errors are of the form `git-annex: fd:10: hGetLine: end of file` and sometimes of the form `git-annex: fd:17: hFlush: resource vanished (Broken pipe)`. In my `daemon.log`, I have the errors + + (scanning...) Already up-to-date. + Already up-to-date. + TransferScanner crashed: fd:26: hGetLine: end of file + Already up-to-date. + (started...) git-annex: fd:25: hGetLine: end of file + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + git-annex: fd:24: hFlush: resource vanished (Broken pipe) + [many more repetitions] + +If I `git annex get` the files and then drop them again, a further attempt at a drop gives all these errors again. + +> So in summary, a git-annex built against the old version of git in +> debian stable fails to work with a newer version of git, and rebuilding +> fixes it. FWIW, the git-annex backport to stable does not have this +> problem, because it checks git version at runtime. But I want to avoid +> the overhead of that check in git-annex mainline, because this old git +> version is well, very old and increasingly unlikely to be used. So, +> I don't think any changes to git-annex are warrented. [[done]] --[[Joey]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_10_8742f7ac27b5f4ad6261d04a174a691c._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_10_8742f7ac27b5f4ad6261d04a174a691c._comment new file mode 100644 index 0000000000..921a200243 --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_10_8742f7ac27b5f4ad6261d04a174a691c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.14.162" + subject="comment 10" + date="2012-12-06T22:00:45Z" + content=""" +Can you show me a transcript of git-check-attr? This is sounding like a git bug. + +(The build error is due to too old a version of yesod.) +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_11_b8e720340000537de6713c49b7733b2f._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_11_b8e720340000537de6713c49b7733b2f._comment new file mode 100644 index 0000000000..db964e58c4 --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_11_b8e720340000537de6713c49b7733b2f._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="spwhitton" + ip="163.1.167.50" + subject="comment 11" + date="2012-12-06T22:17:37Z" + content=""" +Upgrading yesod then git-annex removes the problem I was seeing. Thanks for your help with that. Here is a `git-check-attr` session: + + backups $ git check-attr --stdin annex.backend annex.numcopies -- + athena-etc.tar.gz + athena-etc.tar.gz: annex.backend: unspecified + athena-etc.tar.gz: annex.numcopies: 2 + athena-mail.gz + athena-mail.gz: annex.backend: unspecified + athena-mail.gz: annex.numcopies: 2 + athena-web.tar.gz + athena-web.tar.gz: annex.backend: unspecified + athena-web.tar.gz: annex.numcopies: 2 + [^D pressed] + backups $ +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_1_489fa3a717519cd5d8b4c1a9d143d8c6._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_1_489fa3a717519cd5d8b4c1a9d143d8c6._comment new file mode 100644 index 0000000000..9024643170 --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_1_489fa3a717519cd5d8b4c1a9d143d8c6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.253.113" + subject="comment 1" + date="2012-12-06T15:04:12Z" + content=""" +Can you run it with --debug ? +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_2_b0796d3b1913e1b6f7b34d75a591be42._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_2_b0796d3b1913e1b6f7b34d75a591be42._comment new file mode 100644 index 0000000000..696bdb5aa9 --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_2_b0796d3b1913e1b6f7b34d75a591be42._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="spwhitton" + ip="163.1.167.50" + subject="comment 2" + date="2012-12-06T15:13:02Z" + content=""" +Sure, I've added --debug to the assistant autostart file and am running `git annex --debug drop`. For the first type of error given above I get output like + + [2012-12-06 15:08:50 GMT] read: git [\"--git-dir=/home/swhitton/var/.git\",\"--work-tree=/home/swhitton/var\",\"ls-files\",\"--cached\",\"-z\",\"--\",\".\"] + [2012-12-06 15:08:50 GMT] chat: git [\"--git-dir=/home/swhitton/var/.git\",\"--work-tree=/home/swhitton/var\",\"check-attr\",\"-z\",\"--stdin\",\"annex.backend\",\"annex.numcopies\",\"--\"] + + git-annex: fd:10: hGetLine: end of file + failed + +and for the \"resource vanished\" errors I get no further output. There is also nothing further in `daemon.log`. +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_3_d8ca17ccaa5ee48d590736af8e77d88a._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_3_d8ca17ccaa5ee48d590736af8e77d88a._comment new file mode 100644 index 0000000000..c59d60047e --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_3_d8ca17ccaa5ee48d590736af8e77d88a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.253.113" + subject="comment 3" + date="2012-12-06T15:21:25Z" + content=""" +If you run `git annex find` instead of `git annex drop`, does it have any errors? +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_4_aa7a690aaf75d21f52051a31d7fce70e._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_4_aa7a690aaf75d21f52051a31d7fce70e._comment new file mode 100644 index 0000000000..a1baa47cd6 --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_4_aa7a690aaf75d21f52051a31d7fce70e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="spwhitton" + ip="163.1.167.50" + subject="comment 4" + date="2012-12-06T15:23:02Z" + content=""" +No, `git annex find --debug .` outputs just `[2012-12-06 15:22:14 GMT] read: git [\"--git-dir=/home/swhitton/var/.git\",\"--work-tree=/home/swhitton/var\",\"ls-files\",\"--cached\",\"-z\",\"--\",\".\"]`. +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_5_dc235dc2d024b7f340721bb578630e00._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_5_dc235dc2d024b7f340721bb578630e00._comment new file mode 100644 index 0000000000..af77507713 --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_5_dc235dc2d024b7f340721bb578630e00._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.253.113" + subject="comment 5" + date="2012-12-06T15:26:27Z" + content=""" +What version of git do you have installed? + +How did you install this version of git-annex on squeeze? +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_6_5d1e6ea5b5725c773acc6e288add812c._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_6_5d1e6ea5b5725c773acc6e288add812c._comment new file mode 100644 index 0000000000..e208b31696 --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_6_5d1e6ea5b5725c773acc6e288add812c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="spwhitton" + ip="163.1.167.50" + subject="comment 6" + date="2012-12-06T15:29:46Z" + content=""" +git is at 1.7.10.4 which I got from backports. I only upgraded a couple of days ago (to solve a strange error I was having with cloning to a removable drive); sorry for not mentioning that. I installed git-annex using cabal run as root. +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_7_6389b4f03ebc916358bc6674398d70c4._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_7_6389b4f03ebc916358bc6674398d70c4._comment new file mode 100644 index 0000000000..9d61f4ac0d --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_7_6389b4f03ebc916358bc6674398d70c4._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.253.113" + subject="comment 7" + date="2012-12-06T15:39:26Z" + content=""" +The problem seems to involve git check-attr exiting unexpectedly. Does this happen only in a specific directory of files, or can you reproduce it elsewhere? + +You may be able to reproduce it exiting by running: + +`git check-attr --stdin annex.backend annex.numcopies --` + +And then entering filenames, pressing enter, and it should answer back with two attribute values and keep running. +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_8_bcacc9fb3751042968118ebe33802e27._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_8_bcacc9fb3751042968118ebe33802e27._comment new file mode 100644 index 0000000000..8e4e5cf91a --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_8_bcacc9fb3751042968118ebe33802e27._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.253.113" + subject="comment 8" + date="2012-12-06T15:41:31Z" + content=""" +Have you rebuilt git-annex since you upgraded git? + +This seems most likely to be the problem; the version of git in squeeze requires git-annex behave differently when using git check-attr, and if you have a version of git-annex built for it, it will do things the new git dislikes. +"""]] diff --git a/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_9_6d4c9f0e133ebd94fc11346df446402e._comment b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_9_6d4c9f0e133ebd94fc11346df446402e._comment new file mode 100644 index 0000000000..a48e98ebcf --- /dev/null +++ b/doc/bugs/Error_when_dropping___34__hGetLine:_end_of_file__34__/comment_9_6d4c9f0e133ebd94fc11346df446402e._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="spwhitton" + ip="163.1.167.50" + subject="comment 9" + date="2012-12-06T21:37:52Z" + content=""" +Yes, running git-attr like that does indeed produce the problem. I attempted to rebuild git-annex using cabal, but the build failed with the following error: + + [273 of 290] Compiling Assistant.WebApp.Configurators.Local ( Assistant/WebApp/Configurators/Local.hs, dist/build/git-annex/git-annex-tmp/Assistant/WebApp/Configurators/Local.o ) + + Assistant/WebApp/Configurators/Local.hs:55:11: + `fieldEnctype' is not a (visible) field of constructor `Field' + cabal: Error: some packages failed to install: + git-annex-3.20121127.1 failed during the building phase. The exception was: + ExitFailure 1 +"""]] diff --git a/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn new file mode 100644 index 0000000000..34d05c0b19 --- /dev/null +++ b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn @@ -0,0 +1,21 @@ +I just noticed that if you move a git-annex symlink to a location ignored by git, it simply works. Upon committing that change, however, part of git-annex's `fix` function apparently tries to `git-add` the symlink. This fails because the new, ignored location requires a `git-add --force`. + +Considering that git proper doesn't fail or warn, I think git-annex shouldn't either. + +This is the error message: + + $ git mv annexed-file ignored-dir/ + $ git commit + fix ignored-dir/annexed-file ok + (Recording state in git...) + The following paths are ignored by one of your .gitignore files: + ignored-dir + Use -f if you really want to add them. + fatal: no files added + Command xargs ["-0","git","--git-dir=/home/[...]/repo/.git","--work-tree=/home/[...]/repo","add","--"] failed; exit code 123 + + git-annex: user error (Command xargs ["-0","git","--git-dir=/home/[...]/repo/.git","--work-tree=/home/[...]/repo","add","--"] failed; exit code 123) + failed + git-annex: 1 failed + +> Weird edge case.. ok, fixed. [[done]] --[[Joey]] diff --git a/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn new file mode 100644 index 0000000000..21293af547 --- /dev/null +++ b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -0,0 +1,46 @@ +I'm importing a directory where some files are hard links of each other. + +This is confusing git-annex. Here's a small test of that: + +
+paulproteus@pathi:/tmp$ mkdir annex-test
+paulproteus@pathi:/tmp$ cd annex-test
+paulproteus@pathi:/tmp/annex-test$ git init
+Initialized empty Git repository in /tmp/annex-test/.git/
+paulproteus@pathi:/tmp/annex-test$ git annex init testing
+init testing ok
+paulproteus@pathi:/tmp/annex-test$ echo '* annex.backend=SHA1' >> .gitattributes 
+paulproteus@pathi:/tmp/annex-test$ git commit .gitattributes -m 'Default to sha1'
+[master dd54b41] Default to sha1
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+paulproteus@pathi:/tmp/annex-test$ echo "Look at me" > file1
+paulproteus@pathi:/tmp/annex-test$ cp -l file1 file2
+paulproteus@pathi:/tmp/annex-test$ git annex add file1
+add file1 (checksum...) ok
+(Recording state in git...)
+paulproteus@pathi:/tmp/annex-test$ git commit -m 'So far, so good'
+[master eb43084] So far, so good
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 100644 .git-annex/9a3/f1f/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c.log
+ create mode 120000 file1
+paulproteus@pathi:/tmp/annex-test$ git annex add file2
+add file2 (checksum...) 
+  git-annex: .git/annex/objects/PM/7p/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c: createSymbolicLink: already exists (File exists)
+git-annex: 1 failed
+paulproteus@pathi:/tmp/annex-test$ 
+
+ +When trying to make a small test case for this bug, I noticed that if file1 and file2 have the same contents but are not hard links of each other, they both get annexed just fine. + +I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. + + +-- Asheesh. + +> The same thing happens anytime the key for a file collides with a key +> already in the annex, AFAICS. (Including when the files have the same +> content but are not hard links... unless you're using WORM backend.) +> +> I've fixed this bug. The first file in wins. See commit for some +> interesting discussion about why it should not check for hash collisions +> in this situation. [[done]] --[[Joey]] diff --git a/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn b/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn new file mode 100644 index 0000000000..428d62ab17 --- /dev/null +++ b/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn @@ -0,0 +1,26 @@ +Utility/WebApp.hs, didn't quite have the right definition to use 'open' instead of 'xdg-open' on OSX, the follow fixes that + +
+diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
+index 6936c66..0593dda 100644
+--- a/Utility/WebApp.hs
++++ b/Utility/WebApp.hs
+@@ -42,7 +42,7 @@ localhost = "localhost"
+ runBrowser :: String -> IO Bool
+ runBrowser url = boolSystem cmd [Param url]
+        where
+-#if MAC
++#if OSX
+                cmd = "open"
+ #else
+                cmd = "xdg-open"
+
+ +> [[done]], thanks + +I guess I should really clone the repo and submit a stream of minor changes +:P, @joeyh please let me know if you're getting annoyed with copy and +pasting the small fixes from the bug/forums section. + +> If you're going to be writing some patches, a git repo I can pull from +> would make my life easier. --[[Joey]] diff --git a/doc/bugs/GIT_DIR_support_incomplete.mdwn b/doc/bugs/GIT_DIR_support_incomplete.mdwn new file mode 100644 index 0000000000..1b9738c4f7 --- /dev/null +++ b/doc/bugs/GIT_DIR_support_incomplete.mdwn @@ -0,0 +1,17 @@ +`GIT_DIR` support isn't right. Git does not look for `GIT_DIR/.git`; +git-annex does. + +Also, to support this scenario, support for core.worktree needs to be added +as well: + + mkdir repo workdir + git --work-tree=$PWD/workdir --git-dir=$PWD/repo init + export GIT_DIR=$PWD/repo + git status + # ok + git annex init "new repo" + # fail + +--[[Joey]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/GPG_passphrase_repeated_prompt.mdwn b/doc/bugs/GPG_passphrase_repeated_prompt.mdwn new file mode 100644 index 0000000000..085aede920 --- /dev/null +++ b/doc/bugs/GPG_passphrase_repeated_prompt.mdwn @@ -0,0 +1,24 @@ +#### What steps will reproduce the problem? + +1. Create a new repository with a directory +2. Add files +3. Select "Store your data in the cloud" with the "Remote server" option +4. Enter host, user, directory +5. Select "Use an encrypted rsync repository on the server" (Will there be an option for unencrypted later?) +6. GPG Passphrase prompt comes up for every file + +#### What is the expected output? What do you see instead? + +I expect to enter a passphase once and then it will sync all files with the remote server. + +Instead, it begins syncing the files to the server but prompts for a GPG passphase for every single file. + +#### What version of git-annex are you using? On what operating system? + +3.20121017 precompiled binary on Arch Linux + +#### Please provide any additional information below. + +Not sure if I'm just missing a setting for GPG, but I would think I should only need to use the web app to configure the remote server. + +[[!tag /design/assistant]] diff --git a/doc/bugs/GPG_passphrase_repeated_prompt/comment_1_6ef1c9725befc84ad57bce196ef630ef._comment b/doc/bugs/GPG_passphrase_repeated_prompt/comment_1_6ef1c9725befc84ad57bce196ef630ef._comment new file mode 100644 index 0000000000..016ecb9945 --- /dev/null +++ b/doc/bugs/GPG_passphrase_repeated_prompt/comment_1_6ef1c9725befc84ad57bce196ef630ef._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.194" + subject="you should install a gpg agent" + date="2012-10-31T17:39:53Z" + content=""" +A gpg agent will cache your passphrase. It is beyond the scope of the git-annex package to provide one (though it does bundle gpg), but it should be easy to install gpg-agent on your distribution. + +That's all that's needed for normal git-annex use, but the assistant does seem to have a larger problem in this area, since it can need to unlock a remote's key at any time to sync files from it. Since a separate +process is spawned for each transfer, this defeats git-annex's normal in-process caching of encryption keys of remotes. So I think it needs to unlock any encrypted special remotes at startup, or when first accessing them, and pass the cached keys to the transfer processes it spawns. This is now on my todo list. + +However, none of the special remotes set up by the assistant will use +password protected gpg keys, even when it's using encryption it's using a +non-password protected shared key. So only encrypted special remotes set up +at the command line cause this problem. +"""]] diff --git a/doc/bugs/Install_of_git-annex-3.20121112_fails.mdwn b/doc/bugs/Install_of_git-annex-3.20121112_fails.mdwn new file mode 100644 index 0000000000..f4f1c6c7de --- /dev/null +++ b/doc/bugs/Install_of_git-annex-3.20121112_fails.mdwn @@ -0,0 +1,20 @@ +What steps will reproduce the problem? + +- rm -rf ~/.ghc/ && cabal update && cabal install git-annex --bindir=$HOME/bin + +What is the expected output? What do you see instead? + +- I would like to have the latest release installed + +What version of git-annex are you using? On what operating system? + +- git-annex-3.20121112 +- Ubuntu 12.04 LTS +- The Glorious Glasgow Haskell Compilation System, version 7.4.1 + +Please provide any additional information below. + +I use it heavily on 4 machines since a month and I really like it. + +> closing since this is a cabal library problem, and not something that +> can be fixed by any change to git-annex. [[done]] --[[Joey]] diff --git a/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_1_80fc80151d4390bd8a4332f30723962e._comment b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_1_80fc80151d4390bd8a4332f30723962e._comment new file mode 100644 index 0000000000..b3889eca6c --- /dev/null +++ b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_1_80fc80151d4390bd8a4332f30723962e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 1" + date="2012-11-13T17:08:14Z" + content=""" +You forgot to say how it fails. +"""]] diff --git a/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_2_2613320a41a74dc757a3277c8c328bd0._comment b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_2_2613320a41a74dc757a3277c8c328bd0._comment new file mode 100644 index 0000000000..fa2b326e98 --- /dev/null +++ b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_2_2613320a41a74dc757a3277c8c328bd0._comment @@ -0,0 +1,62 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnldTTAP8PAifJUmqhRar6RAWNWlRcencw" + nickname="Marco" + subject="The failure" + date="2012-11-13T17:24:40Z" + content=""" +Configuring certificate-1.2.2... +Building certificate-1.2.2... +Preprocessing library certificate-1.2.2... +[1 of 9] Compiling Data.Certificate.KeyRSA ( Data/Certificate/KeyRSA.hs, dist/build/Data/Certificate/KeyRSA.o ) + +Data/Certificate/KeyRSA.hs:29:27: + Constructor `RSA.PrivateKey' does not have field `RSA.private_size' + In the expression: + RSA.PrivateKey + {RSA.private_size = calculate_modulus p_modulus 1, + RSA.private_n = p_modulus, RSA.private_d = priv_exp, + RSA.private_p = p_p1, RSA.private_q = p_p2, + RSA.private_dP = p_exp1, RSA.private_dQ = p_exp2, + RSA.private_qinv = p_coef} + In an equation for `privkey': + privkey + = RSA.PrivateKey + {RSA.private_size = calculate_modulus p_modulus 1, + RSA.private_n = p_modulus, RSA.private_d = priv_exp, + RSA.private_p = p_p1, RSA.private_q = p_p2, + RSA.private_dP = p_exp1, RSA.private_dQ = p_exp2, + RSA.private_qinv = p_coef} + In an equation for `parsePrivate': + parsePrivate + [Start Sequence, + IntVal 0, + IntVal p_modulus, + IntVal pub_exp, + IntVal priv_exp, + IntVal p_p1, + IntVal p_p2, + IntVal p_exp1, + IntVal p_exp2, + IntVal p_coef, + End Sequence] + = Right (pubkey, privkey) + where + privkey + = RSA.PrivateKey + {RSA.private_size = calculate_modulus p_modulus 1, + RSA.private_n = p_modulus, RSA.private_d = priv_exp, + RSA.private_p = p_p1, RSA.private_q = p_p2, + RSA.private_dP = p_exp1, RSA.private_dQ = p_exp2, + RSA.private_qinv = p_coef} + pubkey + = RSA.PublicKey + {RSA.public_size = calculate_modulus p_modulus 1, + RSA.public_n = p_modulus, RSA.public_e = pub_exp} + calculate_modulus n i + = if (2 ^ (i * 8)) > n then i else calculate_modulus n (i + 1) +cabal: Error: some packages failed to install: +authenticate-1.3.2 depends on certificate-1.2.2 which failed to install. +certificate-1.2.2 failed during the building phase. The exception was: +ExitFailure 1 +git-annex-3.20121112 depends on certificate-1.2.2 which failed to install. +"""]] diff --git a/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_3_c364764d0c56e8dc3cac276905d99841._comment b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_3_c364764d0c56e8dc3cac276905d99841._comment new file mode 100644 index 0000000000..e3478d03dd --- /dev/null +++ b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_3_c364764d0c56e8dc3cac276905d99841._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 3" + date="2012-11-13T17:54:30Z" + content=""" +Hmm, the current version of certificate is 1.3.1, and I installed it without trouble just now. I don't know why cabal is installing a much older version there. + +In any case, this is not a bug in git annex, but some problem with the library that a library it uses depends on. You can use the pre-built standalone binary until cabal sorts itself out, I suppose. +"""]] diff --git a/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_4_f1057340dfa978071d3bbc9e2af1e612._comment b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_4_f1057340dfa978071d3bbc9e2af1e612._comment new file mode 100644 index 0000000000..2741daacf9 --- /dev/null +++ b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_4_f1057340dfa978071d3bbc9e2af1e612._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnldTTAP8PAifJUmqhRar6RAWNWlRcencw" + nickname="Marco" + subject="Assistant depends on Yesod" + date="2012-11-13T18:34:19Z" + content=""" +Thank you Joey. Than I tried a workaround. On one machine I don't need the Webapp. + +I was using + +cabal install --only-dependencies -f-Webapp +cabal configure -f-Webapp + +mp/Utility/libdiskfree.o dist/build/git-annex/git-annex-tmp/Utility/libmounts.o ./git-annex.hs + +Assistant/Alert.hs:21:8: + Could not find module `Yesod' + Use -v to see a list of the files searched for. +"""]] diff --git a/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_5_9007b1a3abd647945604968db19cb841._comment b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_5_9007b1a3abd647945604968db19cb841._comment new file mode 100644 index 0000000000..666eef28be --- /dev/null +++ b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_5_9007b1a3abd647945604968db19cb841._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="youch!" + date="2012-11-13T19:37:23Z" + content=""" +The assistant is indeed supposed to build w/o Yesod. I've fixed that in git! +"""]] diff --git a/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_6_0bb3ac5375f29ce9d3d0be93879267e3._comment b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_6_0bb3ac5375f29ce9d3d0be93879267e3._comment new file mode 100644 index 0000000000..2aaf9cf43d --- /dev/null +++ b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_6_0bb3ac5375f29ce9d3d0be93879267e3._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0" + nickname="Tobias" + subject="Bah, i missed this before making my own bug report" + date="2012-11-13T20:56:50Z" + content=""" +This line worked for me on ubuntu 12.10: + +cabal install --only-dependencies ./ --constraint=certificate==1.2.2 --constraint=crypto-pubkey-types==0.1.1 + +"""]] diff --git a/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_7_ae4443b8cd069080d1f77fca16aa8b04._comment b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_7_ae4443b8cd069080d1f77fca16aa8b04._comment new file mode 100644 index 0000000000..99124ebe5f --- /dev/null +++ b/doc/bugs/Install_of_git-annex-3.20121112_fails/comment_7_ae4443b8cd069080d1f77fca16aa8b04._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnldTTAP8PAifJUmqhRar6RAWNWlRcencw" + nickname="Marco" + subject="Works" + date="2012-11-14T07:01:34Z" + content=""" +Thanks Joey for fixing this. Now I got a working version. + +Btw would it make sense to reference a stable Yesod version in the cabal file? I'm new to the haskell universe so I don't know what problems would come up with it. +"""]] diff --git a/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X.mdwn b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X.mdwn new file mode 100644 index 0000000000..575a636840 --- /dev/null +++ b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X.mdwn @@ -0,0 +1,25 @@ +What steps will reproduce the problem? + +* Start with a clean setup. +* Allow webapp to start; use it to create annex in ~/Documents/annex That works. +* Go to add remote repo. Removable drive. +* Select "/Volumes/G-DRIVE slim". Click next. + +What is the expected output? What do you see instead? +Expected is something like "done". What I see is + +Internal Server Error + +git config [Param "annex.uuid",Param "6898F314-7817-4CD5-B1C3-588C55522A3B"] failed + +What version of git-annex are you using? On what operating system? + +git-annex version 3.20130107, OS X Mountain Lion. No MacPorts/homebrew/fink installed. gcc / git are installed. + +Please provide any additional information below. + +Maybe something to do with the drive name having spaces? "/Volumes/git-annex" worked fine. + +> Good thought in the comment. I was able to reproduce the failure +> if the removable drive already had an "annex" directory that was not +> a git repo. I've made it handle this case. [[done]] --[[Joey]] diff --git a/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_1_b2ef077d87a9da624f20649c21401b5b._comment b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_1_b2ef077d87a9da624f20649c21401b5b._comment new file mode 100644 index 0000000000..f46292c195 --- /dev/null +++ b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_1_b2ef077d87a9da624f20649c21401b5b._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 1" + date="2013-01-14T16:07:18Z" + content=""" +Spaces in the name is a good guess, but does not seem to cause the problem. I just successfully set up a USB drive that has spaces in the name. (Also, git-annex is carefully coded to avoid such problems..) + +It seems that the repository is created, but then running `git config` in it fails for some reason. One thing you could do is look at `~/Documents/annex/.git/annex/daemon.log`. It should have any error message output by the command. + +Or, you could try, in a shell: + +cd \"/Volumes/G-DRIVE slim/annex\" +git config annex.uuid 6898F314-7817-4CD5-B1C3-588C55522A3B + +And see how that is failing. +"""]] diff --git a/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_2_ef849e25b0264808bff800d9d3836119._comment b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_2_ef849e25b0264808bff800d9d3836119._comment new file mode 100644 index 0000000000..ec78676532 --- /dev/null +++ b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_2_ef849e25b0264808bff800d9d3836119._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="arafel" + ip="94.174.218.177" + subject="comment 2" + date="2013-01-14T20:50:50Z" + content=""" +It did seem unlikely that you hadn't tested that. :-) + +Oddly, there doesn't seem to be a daemon.log file in that directory. I don't have the drive handy to test at the moment - when I do I'll try the commands you suggest and see what the output is. +"""]] diff --git a/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_3_ae3cbd0eb69cbeb9b349e0060d056d43._comment b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_3_ae3cbd0eb69cbeb9b349e0060d056d43._comment new file mode 100644 index 0000000000..f2e5394ce0 --- /dev/null +++ b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_3_ae3cbd0eb69cbeb9b349e0060d056d43._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="arafel" + ip="94.174.218.177" + subject="comment 3" + date="2013-01-14T21:19:12Z" + content=""" +Pauls-MacBook-Pro:annex Paul$ cd /Volumes/G-DRIVE\ slim/annex/ + +Pauls-MacBook-Pro:annex Paul$ git config annex.uuid 6898F314-7817-4CD5-B1C3-588C55522A3B + +error: could not lock config file .git/config: No such file or directory + +Pauls-MacBook-Pro:annex Paul$ ls -a + +. .. + +Looks like it created the directory but never got as far as 'git init'. Is it safe to just run that command, and will that (followed by the config) be enough? Or is there more to it than that? e.g. creating the remotes +"""]] diff --git a/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_4_0ff2897805928b14829b7b369a3aed91._comment b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_4_0ff2897805928b14829b7b369a3aed91._comment new file mode 100644 index 0000000000..7e6b621c84 --- /dev/null +++ b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_4_0ff2897805928b14829b7b369a3aed91._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 4" + date="2013-01-14T22:54:30Z" + content=""" +Can you do this: + +* `rmdir /Volumes/G-DRIVE\ slim/annex/` + +* Stop any git-annex assistant you have running. + +* At the console, run \"git annex webapp\", and reproduce the bug again. When it fails you should see some error message at the console where you started it. + +(If you're using the git-annex.app built for OSX, you should instead run the `git-annex-webapp` script included in the app.) +"""]] diff --git a/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_5_414a45573aeb5201f4d80433955669d5._comment b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_5_414a45573aeb5201f4d80433955669d5._comment new file mode 100644 index 0000000000..f5423f04fa --- /dev/null +++ b/doc/bugs/Internal_server_error_adding_USB_drive_on_OS_X/comment_5_414a45573aeb5201f4d80433955669d5._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="arafel" + ip="94.174.218.177" + subject="comment 5" + date="2013-01-14T23:22:19Z" + content=""" +Hm. After removing the directory, I can't get the problem to happen. I've tried both running the webapp from the CLI, and the git-annex app from Spotlight, and it works. + +A thought - does annex check for the existence of .git in the target folder, or does it just assume (if the folder exists) that it's been set up with git? That's the only thing I can think of. From this experience it seems that would be a worthwhile check if it doesn't already do it. + +Other than that we may just have to close the bug as unreproducible. I'm not sure what happened there; I'm not aware of doing anything differently the other times. Apologies for wasting your time! +"""]] diff --git a/doc/bugs/Interrupted_switch_to_direct_mode_can_cause_all_following_switches_to_fail.mdwn b/doc/bugs/Interrupted_switch_to_direct_mode_can_cause_all_following_switches_to_fail.mdwn new file mode 100644 index 0000000000..432c72083b --- /dev/null +++ b/doc/bugs/Interrupted_switch_to_direct_mode_can_cause_all_following_switches_to_fail.mdwn @@ -0,0 +1,50 @@ +### What steps will reproduce the problem? + +How to reproduce: + +1. Check that the files involved are not available in the local repository. +2. Force a switch to direct mode to fail, for example, due to a 'exotic' filename +(first bug). +3. Solve that problem. +4. Make another switch to direct mode, but call that command in a subdirectory of +the git-tree. Which will because some annexed file does not exists (second bug). +Note: all subsequent switches to direct mode will now fail, no matter where started. + + +Example: + +mkdir test1 test2 && cd test1 && git init . && git annex init +mkdir umlaut something\ else +date > umlaut/this_has_a_$'\201' +date > something\ else/problem +git annex add . +git commit -m "Init" +cd ../test2/ + +git clone ../test1/ . && git annex init +git annex move --from origin . + +cd ../test1/ +git annex direct #aborts with: commitBuffer: invalid argument (invalid character) +cd umlaut +git mv this_has_a_$'\201' this_has_a_o +git commit -m "fix" +git annex direct # fails with getSymbolicLinkStatus: does not exist (No such file or directory) + +### What is the expected output? What do you see instead? + +Expected is a repository switched to direct mode. +The result is a repository which can't be switched to direct mode. + +### What version of git-annex are you using? On what operating system? + +git-annex version: 3.20130114 +OS: Arch Linux (3.7.2-2-ck) + +### Please provide any additional information below. + +The second bug seams to be very specific about folder- and/or filename. +I believe that it has something to do with the space in the foldername. + +> Fixed both bugs. You should be able to upgrade git-annex and re-run the command +> and end up with a working direct-mode repository. [[done]] --[[Joey]] diff --git a/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__.mdwn b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__.mdwn new file mode 100644 index 0000000000..64826c02ac --- /dev/null +++ b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__.mdwn @@ -0,0 +1,19 @@ +What steps will reproduce the problem? + +Adding files to a local annex set up to sync to a remote S3 one + + +What is the expected output? What do you see instead? + +It syncs, but maxes out the uplink + + +What version of git-annex are you using? On what operating system? + +3.20121112 on Debian testing + + +Please provide any additional information below. + +The man page lists how to configure rate limiting for rsync, not sure how to do it for this + diff --git a/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_1_ef97e735ce308f7bcc03f5d9fda588bf._comment b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_1_ef97e735ce308f7bcc03f5d9fda588bf._comment new file mode 100644 index 0000000000..67de2becf4 --- /dev/null +++ b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_1_ef97e735ce308f7bcc03f5d9fda588bf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 1" + date="2012-11-13T15:10:45Z" + content=""" +There's always trickle.. + + trickle -u 50 git annex ... +"""]] diff --git a/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_2_539b89de8743e435386b86119d1e982f._comment b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_2_539b89de8743e435386b86119d1e982f._comment new file mode 100644 index 0000000000..122bafbba5 --- /dev/null +++ b/doc/bugs/Is_there_any_way_to_rate_limit_uploads_to_an_S3_backend__63__/comment_2_539b89de8743e435386b86119d1e982f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 2" + date="2012-11-13T17:09:31Z" + content=""" +It's not there yet. I don't think it will be hard to add, most of the same things need to be done to support upload progress bars with S3. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn new file mode 100644 index 0000000000..ba2148454b --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn @@ -0,0 +1,24 @@ +I was dumping ~gigs of files of approximately 3-6megs a pop (my music collection) so I could track the files that I want to listen to when I'm on the go. I had the git watch command running from the assistant branch. + +I was getting something along the lines of... + + /Users/jtang/annex/.git/annex/tmp/: openTempFile: resource exhausted (Too many open files) + +and + + git-annex: createPipe: resource exhausted (Too many open files) + +I also noticed that I somehow ended up with 256 ssh-agent's running on one of my machines, I'm not sure if the two issues are related or not, I had not noticed this type of behaviour up until recently. + +Also this was appearing in the logs + + x00:annex jtang$ tail -f .git/annex/daemon.log + (scanning...) Already up-to-date. + kqueue: Too many open files + +To be precise, I suspect that the kqueue limit is 256, I had 325 files in the 'queue', I ended up doing a _git annex add_ manually and all was fine. + +[[!meta title="kqueue system limits"]] + +> This affects BSD systems that use Kqueue. It no longer affects OSX, +> since we use FSEvents there instead. --[[Joey]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment new file mode 100644 index 0000000000..d30cddca50 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 1" + date="2012-06-25T15:42:48Z" + content=""" +Yes, this is a known problem with kqueue, it has to keep every directory in the tree open. On [[design/assistant/inotify]] I have a note that it may need to fork off extra watcher processes to deal with this. Of course that adds significant complication. + +In the meantime, you may be able to increase your system's maximum allowed number of open files per process somehow. + +(I doubt that the ssh-agent is related; git-annex does not use ssh-agent directly anyway..) +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment new file mode 100644 index 0000000000..17dcf76343 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2012-06-29T12:02:48Z" + content=""" +Doing, + + sudo sysctl -w kern.maxfilesperproc=400000 + +Somewhat works for me, git-annex watch at least starts up and takes a while to scan the directory, but it's not ideal. Also, creating files seems to work okay, when I remove a file the changes don't seem to get pushed across my other repos, running a sync on the remote repo fixes things. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment new file mode 100644 index 0000000000..eb886acf6b --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 3" + date="2012-07-04T12:32:44Z" + content=""" +Jimmy, sounds like I could use something like this to get the current limit: + + sysctl kern.maxfilesperproc + +Probably prints \"sysctl kern.maxfilesperproc = 256\" or such.. can you verify? +Once I have the limit, I can make the kqueue code use subset of it, and print out a message when it needs to be increased, like the inotify code does. + +(Also, the kqueue code only opens directories, not files, so unless you have 400000 directories, that's +a little high.) + +--- + +On file removal not propigating, does this still happen? When you remove a file does a git commit automatically happen, or is that broken with kqueue? +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment new file mode 100644 index 0000000000..509752bf18 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 4" + date="2012-07-08T18:13:58Z" + content=""" +On kFreeBSD, I get this: + + $ sysctl kern.maxfilesperproc + kern.maxfilesperproc: 11095 + +But ulimit still has 1024 limit, so you'd need to adjust both, as root. Messy.. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment new file mode 100644 index 0000000000..30ea6b310e --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2012-07-04T13:17:05Z" + content=""" +In relation to the system limits, + + laplace:~ jtang$ sysctl kern.maxfilesperproc + kern.maxfilesperproc: 10240 + +Also, the maxfiles for the whole system is + + laplace:~ jtang$ sysctl kern.maxfiles + kern.maxfiles: 12288 + +the above was the defaults as far as I recall. What you probably would be interested is the ulimits that the user see + + laplace:~ jtang$ ulimit -a + core file size (blocks, -c) 0 + data seg size (kbytes, -d) unlimited + file size (blocks, -f) unlimited + max locked memory (kbytes, -l) unlimited + max memory size (kbytes, -m) unlimited + open files (-n) 256 + pipe size (512 bytes, -p) 1 + stack size (kbytes, -s) 8192 + cpu time (seconds, -t) unlimited + max user processes (-u) 709 + virtual memory (kbytes, -v) unlimited + +I would imagine the limit that you are looking for is 256. Hope this helps. + +---- + +On the point about deletions not being propagated, it does do a commit. I suspect that the kqueue code is just not picking up the changes and pushing the changes out. The watch command on a single annex with no remotes functions as expected. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment new file mode 100644 index 0000000000..cd5c73a7a5 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2012-06-25T22:36:39Z" + content=""" +On the system limits side, I think if you want to make it more approachable by more users then adjusting system limits might scare users away. On the note of the ssh-agents spawning like no tomorrow on my machine, it turned out that i had a symlink from my .bashrc to .bash_profile, I guess I should not be too lazy and have two seperate files. +"""]] diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_7_c7776d5b2d073e0d2ae36515185c25aa._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_7_c7776d5b2d073e0d2ae36515185c25aa._comment new file mode 100644 index 0000000000..b65dc4dfcc --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_7_c7776d5b2d073e0d2ae36515185c25aa._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="Just encountered this today" + date="2012-11-29T19:21:50Z" + content=""" +& it's killing my assistant. + +Found a workaround here: https://github.com/mockko/livereload/issues/1 + +This fixes it for now: + +ulimit -n 4096 + +Just posting here in case anybody else runs into this issue and, like me, finds themselves here via search engine. + +"""]] diff --git a/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie.mdwn b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie.mdwn new file mode 100644 index 0000000000..dd92591e62 --- /dev/null +++ b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie.mdwn @@ -0,0 +1,23 @@ +What steps will reproduce the problem? + +Run the git-annex assitant, and then "sudo kill" it. + +What is the expected output? What do you see instead? + +I expect it to die, instead I end up with: + + 14604 ?? S 0:00.64 ga assistant + 14623 ?? Z 0:00.00 (git) + 14624 ?? Z 0:00.00 (git) + 14936 ?? Z 0:00.00 (git-annex) + +The only way to clear these zombies is to reboot. Perhaps there is some resource not being correctly terminated under exceptional conditions? + +Note that on OpenIndiana the problem is even more severe: Aborting git-annex at the wrong time leaves behind both zombie processes and lock files which cause the machine to suddenly halt if I try to access them in any way (via mv, rsync, etc)! + +What version of git-annex are you using? On what operating system? + +4d1e0c9 on OS X 10.8.2. + +Please provide any additional information below. + diff --git a/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_1_d5fba6c061fb21795021ea83070dbfa2._comment b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_1_d5fba6c061fb21795021ea83070dbfa2._comment new file mode 100644 index 0000000000..8d7eea6c52 --- /dev/null +++ b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_1_d5fba6c061fb21795021ea83070dbfa2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="Correction" + date="2012-10-20T05:34:47Z" + content=""" +If I \"sudo kill -9\" all the processes in the above group after deleting the annex directories, then they do go away without needing to reboot. +"""]] diff --git a/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_2_12cba707239018989e8d5b6f456fa754._comment b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_2_12cba707239018989e8d5b6f456fa754._comment new file mode 100644 index 0000000000..2b552c6aa4 --- /dev/null +++ b/doc/bugs/It_is_very_easy_to_turn_git-annex_into_a_zombie/comment_2_12cba707239018989e8d5b6f456fa754._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.23" + subject="comment 2" + date="2012-10-22T14:01:42Z" + content=""" +It's not at all clear to me from this what process you killed. + +(I can't comment on files that somehow crash a kernel on access.. That's not in any UNIX spec I'm aware of.) +"""]] diff --git a/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__.mdwn b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__.mdwn new file mode 100644 index 0000000000..f0809c568b --- /dev/null +++ b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__.mdwn @@ -0,0 +1,21 @@ +What steps will reproduce the problem? + + $ git annex -j sync | json_reformat + +What is the expected output? What do you see instead? + +Expecting valid JSON, instead this happens: + + $ git annex -j sync | json_reformat + lexical error: invalid char in json text. + {"command":"commit","file":""# On branch master nothing to c + (right here) ------^ + $ + + +What version of git-annex are you using? On what operating system? + +Newest standalone (3.20121126), Linux i386. The "json_reformat" program is from the "yajl-tools" .deb package. + +> [[done]]; I've updated the --json documentation to note that it only +> works with some query commands. --[[Joey]] diff --git a/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_1_380a49b3c132f9f529729a1cb5a69621._comment b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_1_380a49b3c132f9f529729a1cb5a69621._comment new file mode 100644 index 0000000000..f0eff45c1d --- /dev/null +++ b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_1_380a49b3c132f9f529729a1cb5a69621._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 1" + date="2012-11-27T21:13:08Z" + content=""" +Yeah, so git-annex has --json as a option available to any command, but the set of commands where it's actually useful is rather smaller, and certianly does not include this one. In general there are quite a lot of places where third-party program output is allowed to show through to provide necessary progress or debugging output, and that of course makes the json mode invalid. +"""]] diff --git a/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_2_282f5f89fb4a46e1fad0980e0b2994a0._comment b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_2_282f5f89fb4a46e1fad0980e0b2994a0._comment new file mode 100644 index 0000000000..b7b4d786e3 --- /dev/null +++ b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_2_282f5f89fb4a46e1fad0980e0b2994a0._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://sunny256.sunbase.org/" + nickname="sunny256" + subject="comment 2" + date="2012-11-27T21:30:48Z" + content=""" +Yep, because of that I was in doubt if I should report it as a bug. Maybe it could be closed, as it's not a bug in git-annex, but git output leaking into the JSON. If there's not an easy way for git-annex to encapsulate the output or redirect the git output to stderr. +"""]] diff --git a/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_3_7ff98958146b7f6396226bdd878ec86e._comment b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_3_7ff98958146b7f6396226bdd878ec86e._comment new file mode 100644 index 0000000000..e3b27dd0ed --- /dev/null +++ b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_3_7ff98958146b7f6396226bdd878ec86e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 3" + date="2012-11-27T21:32:26Z" + content=""" +I'd rather not abuse stderr. + +Do you have an actual use case for the json output for git annex sync? If there's a good one, the git output could be suppressed. +"""]] diff --git a/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_4_f9e460a09e7e5f53c16c20ded2649201._comment b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_4_f9e460a09e7e5f53c16c20ded2649201._comment new file mode 100644 index 0000000000..d773ef2c7a --- /dev/null +++ b/doc/bugs/JSON_output_broken_with___34__git_annex_sync__34__/comment_4_f9e460a09e7e5f53c16c20ded2649201._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://sunny256.sunbase.org/" + nickname="sunny256" + subject="comment 4" + date="2012-11-27T21:40:05Z" + content=""" +Actually not for the sync command, as I don't parse it in any scripts. Just meant as a heads-up, and not an especially important. Feel free to close it. :) +"""]] diff --git a/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss.mdwn b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss.mdwn new file mode 100644 index 0000000000..630db722b6 --- /dev/null +++ b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss.mdwn @@ -0,0 +1,50 @@ +## What steps will reproduce the problem? + +Take a large sub-directory in a repository (e.g. `ccash`) with some files within, + + $ tar -xzf ccash.tar.gz + $ du -sh ccash + 59M ccash + $ ls -l ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar + -rw-r--r-- 1 dietz dietz 1748 Jul 27 2011 ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java + -rw-r--r-- 1 dietz dietz 313898 May 22 18:36 ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar + +Annex it, + + $ git annex add ccash + ... + $ ls -l ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar + lrwxrwxrwx 1 dietz dietz 215 Jul 27 2011 ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java -> ../../../../../../../../../../../.git/annex/objects/mv/zf/SHA256-s1748--5c0d1cbf104214b6d0ab85c53a85cadb975ec208f42a7b33a76d85e175352486/SHA256-s1748--5c0d1cbf104214b6d0ab85c53a85cadb975ec208f42a7b33a76d85e175352486 + lrwxrwxrwx 1 dietz dietz 210 Jul 27 2011 ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar -> ../../../../../../../../.git/annex/objects/8G/gQ/SHA256-s313898--593552ffea3c5823c6602478b5002a7c525fd904a3c44f1abe4065c22edfac73/SHA256-s313898--593552ffea3c5823c6602478b5002a7c525fd904a3c44f1abe4065c22edfac73 + +Unannex it (before or after committing), + + $ git annex unannex ccash + +Note that some fraction of the files will still be symbolic links, now pointing to non-existent files. This data has apparently been lost forever. + + $ ls -l ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar + -rw-r--r-- 1 dietz dietz 1748 Jul 27 2011 ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java + lrwxrwxrwx 1 dietz dietz 210 Jul 27 2011 ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar -> ../../../../../../../../.git/annex/objects/8G/gQ/SHA256-s313898--593552ffea3c5823c6602478b5002a7c525fd904a3c44f1abe4065c22edfac73/SHA256-s313898--593552ffea3c5823c6602478b5002a7c525fd904a3c44f1abe4065c22edfac73 + +It is unclear why some files are affected while others are not. That being said, unannexing small numbers of files at a time appears to avoid the issue, + + $ tar -zxf ccash.tar.gz + $ git annex add ccash + $ ls -l ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar + lrwxrwxrwx 1 dietz dietz 215 Jul 27 2011 ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java -> ../../../../../../../../../../../.git/annex/objects/mv/zf/SHA256-s1748--5c0d1cbf104214b6d0ab85c53a85cadb975ec208f42a7b33a76d85e175352486/SHA256-s1748--5c0d1cbf104214b6d0ab85c53a85cadb975ec208f42a7b33a76d85e175352486 + lrwxrwxrwx 1 dietz dietz 210 Jul 27 2011 ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar -> ../../../../../../../../.git/annex/objects/8G/gQ/SHA256-s313898--593552ffea3c5823c6602478b5002a7c525fd904a3c44f1abe4065c22edfac73/SHA256-s313898--593552ffea3c5823c6602478b5002a7c525fd904a3c44f1abe4065c22edfac73 + $ git annex unannex ccash/trunk/DataProvider/WebContent/WEB-INF + ... + $ ls -l ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar + lrwxrwxrwx 1 dietz dietz 215 Jul 27 2011 ccash/trunk/annotationinterface/src/edu/byu/nlp/annotationinterface/java/BasicAnnotation.java -> ../../../../../../../../../../../.git/annex/objects/mv/zf/SHA256-s1748--5c0d1cbf104214b6d0ab85c53a85cadb975ec208f42a7b33a76d85e175352486/SHA256-s1748--5c0d1cbf104214b6d0ab85c53a85cadb975ec208f42a7b33a76d85e175352486 + -rw-r--r-- 1 dietz dietz 313898 Jul 27 2011 ccash/trunk/DataProvider/WebContent/WEB-INF/lib/dom4j.jar + +For this reason, it seems likely this is due to some sort of race condition. + + +## What version of git-annex are you using? On what operating system? + +This is on Ubuntu 12.04 with git-annex revision a1e2bc4. + + diff --git a/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_1_fbb410a54bb0bd82d0953ef58a88600e._comment b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_1_fbb410a54bb0bd82d0953ef58a88600e._comment new file mode 100644 index 0000000000..14172a3e5a --- /dev/null +++ b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_1_fbb410a54bb0bd82d0953ef58a88600e._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 1" + date="2012-09-06T02:28:00Z" + content=""" + +Here is a quick script which reproduces the issue on another Ubuntu 12.04 machine, + + mkdir hi + cd hi + wget \"http://downloads.sourceforge.net/project/free-cad/FreeCAD%20Source/freecad-0.11.3729.tar.gz\" + + git init + git annex init + tar -zxf freecad-0.11.3729.tar.gz + git annex add FreeCAD-0.11.3729 + git annex unannex FreeCAD-0.11.3729 + echo \"The following links are broken:\" + find -L . -type l + +This results in dozens of dead symlinks. + +"""]] diff --git a/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_2_8007c9ba42a951a4426255ec3c37d961._comment b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_2_8007c9ba42a951a4426255ec3c37d961._comment new file mode 100644 index 0000000000..e1f600d886 --- /dev/null +++ b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_2_8007c9ba42a951a4426255ec3c37d961._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.236" + subject="comment 2" + date="2012-09-06T14:55:58Z" + content=""" +What's going on here is you have multiple files with the same content, so the symlinks point to the same annexed file. When unannex processes the first symlink, it moves the annexed file to replace it. This breaks the other symlink that pointed to it. Notice that if you then re-add the file to the annex, the broken symlink automatically gets fixed -- there's no actual data loss going on here. + +This problem can be avoided by using `git annex unannex --fast`, which makes hardlinks to the annexed file. +But then you are also left with the hard links in `.git/annex/objects`.. `git annex unused` can find and remove them. + +It may make sense to make the current \"--fast\" behavior the default for unannex.. +"""]] diff --git a/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_3_73ecd4cb8ee58a8dfe7cab0e893dbe5b._comment b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_3_73ecd4cb8ee58a8dfe7cab0e893dbe5b._comment new file mode 100644 index 0000000000..2a799fac0c --- /dev/null +++ b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_3_73ecd4cb8ee58a8dfe7cab0e893dbe5b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 3" + date="2012-09-06T16:04:42Z" + content=""" +Frankly, even the --fast behavior has an element of surprise to it. For example, one might have two files with identical content. Upon annexing and unannex they suddenly become a hard link to the same file, correct? If this is the case, changes to one will result in changes to the other. I would consider this a very nasty sort of surprise. +"""]] diff --git a/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_4_e8a10886a564f35414c30a04335d9d32._comment b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_4_e8a10886a564f35414c30a04335d9d32._comment new file mode 100644 index 0000000000..72b4c5c7f6 --- /dev/null +++ b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_4_e8a10886a564f35414c30a04335d9d32._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.8.30" + subject="comment 4" + date="2012-09-09T16:53:35Z" + content=""" +Perhaps the solution is to make --fast the default and to make it copy files when the content in the annex already has a hard link to it. +"""]] diff --git a/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_5_6a318edfe45c80343d017dc7b4837acb._comment b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_5_6a318edfe45c80343d017dc7b4837acb._comment new file mode 100644 index 0000000000..fbc30f17bd --- /dev/null +++ b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_5_6a318edfe45c80343d017dc7b4837acb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 5" + date="2012-09-09T20:47:04Z" + content=""" +That sounds far more reasonable. +"""]] diff --git a/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_6_f7a1d9f9d40aff531d873a95d2196edd._comment b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_6_f7a1d9f9d40aff531d873a95d2196edd._comment new file mode 100644 index 0000000000..295411d251 --- /dev/null +++ b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_6_f7a1d9f9d40aff531d873a95d2196edd._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 6" + date="2012-09-19T23:32:35Z" + content=""" +Has any progress been made here? While this issue may not result in data loss, the behavior documented in this bug is certainly surprising and does not instill confidence in new users. +"""]] diff --git a/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_7_1724ffdf986301bf37ef7a6d16b6ea8a._comment b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_7_1724ffdf986301bf37ef7a6d16b6ea8a._comment new file mode 100644 index 0000000000..fd235321aa --- /dev/null +++ b/doc/bugs/Large_unannex_operations_result_in_stale_symlinks_and_data_loss/comment_7_1724ffdf986301bf37ef7a6d16b6ea8a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.14.141" + subject="comment 7" + date="2012-09-23T18:02:45Z" + content=""" +If unannex makes the file a hard link to the annexed content, it will be mode 444 or so. But if the user changes the permissions and modifys it, that will corrupt the content still in the annex! + +So the current --fast behavior seems no worse than the proposed behavior. And it's not at all clear to me that this would be a better default behavior for unannex than the current behavior, which at least ensures that data left in the annex (and referred to by another annexed file) cannot be corrupted. +"""]] diff --git a/doc/bugs/Local_pairing_fails:_PairListener_crashed.mdwn b/doc/bugs/Local_pairing_fails:_PairListener_crashed.mdwn new file mode 100644 index 0000000000..371b923cf8 --- /dev/null +++ b/doc/bugs/Local_pairing_fails:_PairListener_crashed.mdwn @@ -0,0 +1,18 @@ +What steps will reproduce the problem? + +Attempting to pair between a local repository and a repository on a remote computer on my LAN. Pairing is initiated from my local machine and I'm interacting with the webapp on the remote machine via firefox running over an ssh -X connection. Pairing appears to work up to a point: I enter the secret at one end, the pairing request shows up at the other end. I then enter the secret at that end. + +What is the expected output? What do you see instead? + +Pairing should complete successfully. Instead I get the error message "PairListener crashed: bad comment in public key", followed by the public key. The pairing process then does not move beyond the 'awaiting pairing' pages. + +What version of git-annex are you using? On what operating system? + +Local Machine: 3.20121127, Debian Wheezy/Sid (the only package from unstable is git-annex). +Remote Machine: 3.20121113, Arch Linux (I installed the version from: https://aur.archlinux.org/packages/git-annex-bin/, which is supposedly the same as above, but reports the version specified here). + +Please provide any additional information below. + +None as yet. Let me know if there are any log files, etc. that I can post. + +> So it was the period in the hostname! [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/Local_pairing_fails:_PairListener_crashed/comment_1_d9c5d2147cf6d8d8477eb13b72081d46._comment b/doc/bugs/Local_pairing_fails:_PairListener_crashed/comment_1_d9c5d2147cf6d8d8477eb13b72081d46._comment new file mode 100644 index 0000000000..4dd77f04d5 --- /dev/null +++ b/doc/bugs/Local_pairing_fails:_PairListener_crashed/comment_1_d9c5d2147cf6d8d8477eb13b72081d46._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.72" + subject="comment 1" + date="2012-12-04T17:38:47Z" + content=""" +On the machine that didn't crash, run: + +ssh-keygen -P \"\" -f test; cat test.pub + +Probably the non-alphanumeric character is part of the machine's hostname, or perhaps your username. We'll see.. +"""]] diff --git a/doc/bugs/Local_pairing_fails:_PairListener_crashed/comment_2_60a21105145ac228f486bc4beb2ea54d._comment b/doc/bugs/Local_pairing_fails:_PairListener_crashed/comment_2_60a21105145ac228f486bc4beb2ea54d._comment new file mode 100644 index 0000000000..851dfe9f80 --- /dev/null +++ b/doc/bugs/Local_pairing_fails:_PairListener_crashed/comment_2_60a21105145ac228f486bc4beb2ea54d._comment @@ -0,0 +1,32 @@ +[[!comment format=mdwn + username="robconnolly" + ip="118.90.115.19" + subject="comment 2" + date="2012-12-06T02:29:56Z" + content=""" +Hi Joey, + +Thanks for your response. Here is the output: + +[robert@laforge ~]$ ssh-keygen -P \"\" -f test; cat test.pub +Generating public/private rsa key pair. +Your identification has been saved in test. +Your public key has been saved in test.pub. +The key fingerprint is: +84:43:aa:f0:ba:a7:f0:11:41:55:3b:c4:63:6a:71:e1 robert@laforge.enterprise +The key's randomart image is: ++--[ RSA 2048]----+ +| ...o=. | +| . .==o | +|. . .=E.. | +| o oo + | +| +. S | +| . . | +|o . | +|.o.. | +|oo. | ++-----------------+ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDIeitbarby1thLPEyVpeT/vDaWt0MJwLPvhT7TEZv0FedYalvz/mm3enULHiYdxkjWGBaO/EEzfmz5bjhoJx2tsqSLsUj/UZVv0LxsSfZwTrk0SPuUerzWvAJAXKvtPobDXJoAxsNvzSJN392BMOLKqcjAQTqPA3vd5+EjUVIgJxguz5poGrP0ZRMQD5LmsDrPGUWNJ/EHaVkrqGobAE4DVr8vhJTY41h5Gk2ipnzx+GD6CDZTSNFwl8RorjFAV3mYTTjHc2Cz7GxTLSmwF0+qFZ/UkiO2tD1M37Y4/cPLY7GpVfKvpPHTC8pJUpvj+B4uujHxvZkBJLLz9XhH9KsZ robert@laforge.enterprise + +Let me know if you need any more info. +"""]] diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn new file mode 100644 index 0000000000..c359b52103 --- /dev/null +++ b/doc/bugs/Lost_S3_Remote.mdwn @@ -0,0 +1,59 @@ +Somehow I've lost my S3 remote... git-annex knows it's there, but its not associating it with the git remote in .git/config + + $ git-annex whereis pebuilder.iso + whereis pebuilder.iso (3 copies) + 3b6fc6f6-3025-11e1-b496-33bffbc0f3ed -- housebackup (external seagate drive on /mnt/back/RemoteStore) + 6b1326d8-2abb-11e1-8f43-979159a7f900 -- synology + 9b297772-2ab2-11e1-a86f-2fd669cb2417 -- Amazon S3 + ok + +Amazon S3 is the description from the remote. My .git/config file contains this block: + + [remote "cloud"] + annex-s3 = true + annex-uuid = 9b297772-2ab2-11e1-a86f-2fd669cb2417 + annex-cost = 70 + +The UUID matches... But I cannot access it... see below: + + [39532:39531 - 0:626] 08:20:38 [vivitron@tronlap:o +3] ~/annex/ISO + $ git-annex get pebuilder.iso --from=cloud + git-annex: there is no git remote named "cloud" + + [39532:39531 - 0:627] 08:20:56 [vivitron@tronlap:o +3] ~/annex/ISO + $ git-annex get pebuilder.iso --from="Amazon S3" + git-annex: there is no git remote named "Amazon S3" + + [39532:39531 - 0:628] 08:21:01 [vivitron@tronlap:o +3] ~/annex/ISO + $ git-annex get pebuilder.iso --from=9b297772-2ab2-11e1-a86f-2fd669cb2417 + git-annex: there is no git remote named "9b297772-2ab2-11e1-a86f-2fd669cb2417" + + [39532:39531 - 0:629] 08:21:08 [vivitron@tronlap:o +3] ~/annex/ISO + $ + +git remote lists "cloud" as a remote: + + $ git remote + all + cloud + cs + es3 + origin + +git-annex status lists S3 support: + + $ git-annex status + supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL + supported remote types: git S3 bup directory rsync web hook + + + +I appreciate any help.... I've tested versions 3.20111211, 3.20111231, and 3.20120105 + + $ git --version + git version 1.7.8.1 + + + +> [[done]]; I've fixed the build system so this confusing thing cannot +> happen anymore. --[[Joey]] diff --git a/doc/bugs/Lost_S3_Remote/comment_1_6e80e6db6671581d471fc9a54181c04c._comment b/doc/bugs/Lost_S3_Remote/comment_1_6e80e6db6671581d471fc9a54181c04c._comment new file mode 100644 index 0000000000..b4f7bdc3cf --- /dev/null +++ b/doc/bugs/Lost_S3_Remote/comment_1_6e80e6db6671581d471fc9a54181c04c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2012-01-06T03:04:35Z" + content=""" +Despite `status` listing S3 support, your git-annex is actually built with S3stub, probably because it failed to find the necessary S3 module at build time. Rebuild git-annex and watch closely, you'll see \"** building without S3 support\". Look above that for the error and fix it. + +It was certianly a bug that it showed S3 as supported when built without it. I've fixed that. +"""]] diff --git a/doc/bugs/Lost_S3_Remote/comment_2_c99c65882a3924f4890e500f9492b442._comment b/doc/bugs/Lost_S3_Remote/comment_2_c99c65882a3924f4890e500f9492b442._comment new file mode 100644 index 0000000000..6112bc0890 --- /dev/null +++ b/doc/bugs/Lost_S3_Remote/comment_2_c99c65882a3924f4890e500f9492b442._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2012-01-06T03:08:28Z" + content=""" +BTW, you'll want to \"make clean\", since the S3stub hack symlinks a file into place and it will continue building with S3stub even if you fix the problem until you clean. +"""]] diff --git a/doc/bugs/Lost_S3_Remote/comment_3_1e434d5a20a692cd9dc7f6f8f20f30dd._comment b/doc/bugs/Lost_S3_Remote/comment_3_1e434d5a20a692cd9dc7f6f8f20f30dd._comment new file mode 100644 index 0000000000..69063966c6 --- /dev/null +++ b/doc/bugs/Lost_S3_Remote/comment_3_1e434d5a20a692cd9dc7f6f8f20f30dd._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M" + nickname="Dustin" + subject="Thank you!" + date="2012-01-06T03:38:27Z" + content=""" +make clean and rebuild worked... Thank you +"""]] diff --git a/doc/bugs/Makefile_is_missing_dependancies.mdwn b/doc/bugs/Makefile_is_missing_dependancies.mdwn new file mode 100644 index 0000000000..3e9d6e903c --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies.mdwn @@ -0,0 +1,47 @@ +
+From e45c73e66fc18d27bdf5797876fbeb07786a4af1 Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Tue, 22 Mar 2011 22:24:07 +0000
+Subject: [PATCH] Touch up Makefile to depend on StatFS.hs
+
+---
+ Makefile |    2 +-
+ 1 files changed, 1 insertions(+), 1 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 08e2f59..4ae8392 100644
+--- a/Makefile
++++ b/Makefile
+@@ -15,7 +15,7 @@ SysConfig.hs: configure.hs TestConfig.hs
+        hsc2hs $<
+        perl -i -pe 's/^{-# INCLUDE.*//' $@
+ 
+-$(bins): SysConfig.hs Touch.hs
++$(bins): SysConfig.hs Touch.hs StatFS.hs
+        $(GHCMAKE) $@
+ 
+ git-annex.1: doc/git-annex.mdwn
+-- 
+1.7.4.1
+
+
+ + +StatFS.hs never gets depended on and compiled, the makefile was just missing something + +> Thanks, [[done]]! Interested to hear if StatFS.hs works on OSX (no warning) or +> is a no-op (with warning). --[[Joey]] + +>> +>> for now it gives a warning, it looks like it should be easy enough to add OSX +>> support, I guess it's a case of just digging around documentation to find the equivalent +>> calls/headers. I'll give it a go at making this feature work on OSX and get back to you. +>> + +
+jtang@exia:~/develop/git-annex $ make
+hsc2hs StatFS.hsc
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+
diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment new file mode 100644 index 0000000000..ab8493a7a8 --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment @@ -0,0 +1,47 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2011-03-23T08:21:30Z" + content=""" +Just did some minor digging around and checking, this seems to satisfy the compilers etc... I have yet to confirm that it *really* is working as expected. Also it might be better to check for a darwin operating system instead of apple I think, though I don't know of any one really using a pure darwin OS. But for now it works (I think) + +
+From fbfe27c2e19906ac02e3673b91bffa920f6dae5d Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Wed, 23 Mar 2011 08:15:39 +0000
+Subject: [PATCH] Define (__APPLE__) in StatFS
+
+At least on OSX 10.6.6 it appears to have the same defintions as
+FreeBSD. The build process doesn't complain and the code is enabled,
+this needs to be tested and checked more.
+---
+ StatFS.hsc |    4 ++--
+ 1 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/StatFS.hsc b/StatFS.hsc
+index 8b453dc..45fd7e4 100644
+--- a/StatFS.hsc
++++ b/StatFS.hsc
+@@ -53,7 +53,7 @@ import Foreign.C.String
+ import Data.ByteString (useAsCString)
+ import Data.ByteString.Char8 (pack)
+ 
+-#if defined (__FreeBSD__)
++#if defined (__FreeBSD__) || defined(__APPLE__)
+ # include 
+ # include 
+ #else
+@@ -84,7 +84,7 @@ data CStatfs
+ #ifdef UNKNOWN
+ #warning free space checking code not available for this OS
+ #else
+-#if defined(__FreeBSD__)
++#if defined(__FreeBSD__) || defined(__APPLE__)
+ foreign import ccall unsafe \"sys/mount.h statfs\"
+ #else
+ foreign import ccall unsafe \"sys/vfs.h statfs64\"
+-- 
+1.7.4.1
+
+"""]] diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment new file mode 100644 index 0000000000..d355514a31 --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-03-23T15:05:12Z" + content=""" +There's a simple test -- just configure annex.diskreserve to be say, 10 megabytes less than the total free space on your disk. Then try to git annex get a 11 mb file, and a 9 mb file. :) +"""]] diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment new file mode 100644 index 0000000000..6b4cf5789c --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-23T15:13:33Z" + content=""" +Alternatively, you can just load it up in ghci and see if it reports numbers that make sense: + +
+joey@gnu:~/src/git-annex>make StatFS.hs
+hsc2hs StatFS.hsc
+perl -i -pe 's/^{-# INCLUDE.*//' StatFS.hs
+joey@gnu:~/src/git-annex>ghci StatFS.hs
+GHCi, version 6.12.1: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.5 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 7427989, fsStatByteCount = 30425042944, fsStatBytesFree = 2528489472, fsStatBytesAvailable = 2219384832, fsStatBytesUsed = 27896553472})
+
+"""]] diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment new file mode 100644 index 0000000000..c3ad2dafd6 --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2011-03-23T16:02:34Z" + content=""" +Ok, well it looks like it isn't doing anything useful at all. + +
+jtang@x00:~/develop/git-annex $ make StatFS.hs                                                                                                                                    
+hsc2hs StatFS.hsc
+perl -i -pe 's/^{-# INCLUDE.*//' StatFS.hs
+jtang@x00:~/develop/git-annex $ ghci StatFS.hs                                                                                                                                    
+GHCi, version 6.12.3: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+Loading package ffi-1.0 ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.7 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 0, fsStatBlockCount = 1048576, fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0, fsStatBytesUsed = 0})
+*StatFS> s <- getFileSystemStats \"/\"
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 0, fsStatBlockCount = 1048576, fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0, fsStatBytesUsed = 0})
+*StatFS> 
+
+"""]] diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment new file mode 100644 index 0000000000..149aeeb75a --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment @@ -0,0 +1,59 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 5" + date="2011-03-23T16:14:22Z" + content=""" +Actually I may have just been stupid and should have read the man page on statfs... + +
+jtang@x00:~/develop/git-annex $ git diff
+diff --git a/StatFS.hsc b/StatFS.hsc
+index 8b453dc..e10b2dd 100644
+--- a/StatFS.hsc
++++ b/StatFS.hsc
+@@ -53,7 +53,7 @@ import Foreign.C.String
+ import Data.ByteString (useAsCString)
+ import Data.ByteString.Char8 (pack)
+ 
+-#if defined (__FreeBSD__)
++#if defined (__FreeBSD__) || defined (__APPLE__)
+ # include 
+ # include 
+ #else
+@@ -84,8 +84,8 @@ data CStatfs
+ #ifdef UNKNOWN
+ #warning free space checking code not available for this OS
+ #else
+-#if defined(__FreeBSD__)
+-foreign import ccall unsafe \"sys/mount.h statfs\"
++#if defined(__FreeBSD__) || defined (__APPLE__)
++foreign import ccall unsafe \"sys/mount.h statfs64\"
+ #else
+ foreign import ccall unsafe \"sys/vfs.h statfs64\"
+ #endif
+
+ +yields this... + +
+jtang@x00:~/develop/git-annex $ ghci StatFS.hs                                                                                                                                    
+GHCi, version 6.12.3: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+Loading package ffi-1.0 ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.7 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 244106668, fsStatByteCount = 999860912128, fsStatBytesFree = 423097798656, fsStatBytesAvailable = 422835654656, fsStatBytesUsed = 576763113472})
+*StatFS> 
+
+ + +we could just stick another if defined (__APPLE__) instead of what I previously had and it looks like it will do the right thing on OSX. + + +"""]] diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment new file mode 100644 index 0000000000..714459fbe8 --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 6" + date="2011-03-23T16:23:56Z" + content=""" +I forgot to mention that the statfs64 stuff in OSX seems to be deprecated, see http://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man2/statfs64.2.html + +on a slightly different note, is anonymous pushing to the \"wiki\" over git allowed? I'd prefer to be able to edit stuff inline for updating some of my own comments if I can :P +"""]] diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment new file mode 100644 index 0000000000..8ba8e8d1f6 --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 7" + date="2011-03-23T16:57:56Z" + content=""" +Try the changes I've pushed to use statfs64 on apple. + +There is actually a standardized statvfs that I'd rather use, but after the last time that I tried going with the POSIX option first only to find it was not broadly implemented, I was happy to find some already existing code that worked for some OSs. + +(While ikiwiki supports anonymous git push, it's a feature we have not rolled out on Branchable.com yet, and anyway, ikiwiki disallows editing existing comments that way. I would, however, be happy to git pull changes from somewhere.) +"""]] diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment new file mode 100644 index 0000000000..63d188bcce --- /dev/null +++ b/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 8" + date="2011-03-23T17:03:51Z" + content=""" +The latest change looks good, it seems to be returning sensible numbers for me. Just tried it out on a few different mount points and it appears to be working. +"""]] diff --git a/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn b/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn new file mode 100644 index 0000000000..2253c0f524 --- /dev/null +++ b/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn @@ -0,0 +1,35 @@ +Seems commit 6cecc26206c4a539999b04664136c6f785211a41 missed on dependancy, that is blaze-markup + +
+Assistant/Threads/WebApp.hs:25:8:
+    Could not find module `Text.Blaze.Renderer.String'
+    It is a member of the hidden package `blaze-markup-0.5.1.0'.
+    Perhaps you need to add `blaze-markup' to the build-depends in your .cabal file.
+    Use -v to see a list of the files searched for.
+cabal: Error: some packages failed to install:
+git-annex-3.20120721 failed during the building phase. The exception was:
+ExitFailure 1
+
+ +This should fix it + +
+x00:git-annex jtang$ git diff
+diff --git a/git-annex.cabal b/git-annex.cabal
+index c7d9bf5..4f98d2a 100644
+--- a/git-annex.cabal
++++ b/git-annex.cabal
+@@ -76,7 +76,7 @@ Executable git-annex
+   if flag(Webapp)
+     Build-Depends: yesod, yesod-static, case-insensitive, http-types,
+      transformers, wai, wai-logger, warp, blaze-builder, blaze-html,
+-     crypto-api, hamlet
++     blaze-markup, crypto-api, hamlet
+     CPP-Options: -DWITH_WEBAPP
+ 
+   if (os(darwin))
+
+ +> [[done]].. interestingly, cabal had not complained about there here, +> as in my version, it's in blaze, not blaze-markup. Added it anyway. +> --[[Joey]] diff --git a/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn new file mode 100644 index 0000000000..00f4253073 --- /dev/null +++ b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn @@ -0,0 +1,15 @@ +Running the 'assistant' branch, I occassionally get + +To myhost1:/Users/jtang/annex + ! [rejected] master -> synced/master (non-fast-forward) +error: failed to push some refs to 'myhost1:/Users/jtang/annex' +hint: Updates were rejected because a pushed branch tip is behind its remote +hint: counterpart. Check out this branch and merge the remote changes +hint: (e.g. 'git pull') before pushing again. +hint: See the 'Note about fast-forwards' in 'git push --help' for details. +(Recording state in git...) + +manually running a 'git annex sync' usually fixes it, I guess once the sync command runs periodically this problem will go away, is this even OSX specific? I don't quite get the behaviour that is described in [[design/assistant/blog/day_15__its_aliiive]]. + +> With my changes today, I've seen it successfully recover from this +> situation. [[done]] --[[Joey]] diff --git a/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX/comment_1_377525e70640751e1ead445aeed15efa._comment b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX/comment_1_377525e70640751e1ead445aeed15efa._comment new file mode 100644 index 0000000000..77481789ce --- /dev/null +++ b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX/comment_1_377525e70640751e1ead445aeed15efa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 1" + date="2012-06-25T15:45:18Z" + content=""" +This is indeed the problem I've planned to work on today, as I see it too when things don't start off perfectly in sync. +"""]] diff --git a/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana.mdwn b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana.mdwn new file mode 100644 index 0000000000..ee188eb2ed --- /dev/null +++ b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana.mdwn @@ -0,0 +1,36 @@ +Version 3.20120825 built on my OpenIndiana system just fine, but the latest release gives me this during setup: + + Linking /tmp/git-annex-3.20121017-13013/git-annex-3.20121017/dist/setup/setup ... + checking version... 3.20121017 + checking git... yes + checking git version... 1.7.8.2 + checking cp -a... yes + checking cp -p... yes + checking cp --reflink=auto... yes + checking uuid generator... uuid -m + checking xargs -0... yes + checking rsync... yes + checking curl... yes + checking wget... yes + checking bup... no + checking gpg... no + checking lsof... no + checking ssh connection caching... yes + checking sha1... sha1sum + checking sha256... sha256sum + checking sha512... sha512sum + checking sha224... sha224sum + checking sha384... sha384sum + Configuring git-annex-3.20121017... + Building git-annex-3.20121017... + Preprocessing executable 'git-annex' for git-annex-3.20121017... + In file included from Mounts.hsc:25:0: + Utility/libmounts.h:13:3: warning: #warning mounts listing code not available for this OS [-Wcpp] + + Utility/libkqueue.c:13:23: + fatal error: sys/event.h: No such file or directory + compilation terminated. + +Is it possible to remove the new requirement? Thanks! + +> [[done]] --[[Joey]] diff --git a/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_1_f3c336ecfee51e074ea3a9fc95301de5._comment b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_1_f3c336ecfee51e074ea3a9fc95301de5._comment new file mode 100644 index 0000000000..bb1dc688fc --- /dev/null +++ b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_1_f3c336ecfee51e074ea3a9fc95301de5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 1" + date="2012-10-17T17:42:30Z" + content=""" +kqueue is used by the new assistant and watch features. It seems something else would need to be used for Solaris. I have modified the Makefile to (try to) detect Solaris and disable these features. I assumed your uname would be \"Solaris\", so you may need to modify it slightly. +"""]] diff --git a/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_2_102c0e998934e84deca92fd1c90145fa._comment b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_2_102c0e998934e84deca92fd1c90145fa._comment new file mode 100644 index 0000000000..0a2a2641c2 --- /dev/null +++ b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_2_102c0e998934e84deca92fd1c90145fa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="uname on OpenIndiana" + date="2012-10-17T22:19:04Z" + content=""" +The uname is actually SunOS, surprisingly enough. I'll give it a try! +"""]] diff --git a/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_3_1449dd796ce9f2209f085d4b017a5f33._comment b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_3_1449dd796ce9f2209f085d4b017a5f33._comment new file mode 100644 index 0000000000..8feff78d95 --- /dev/null +++ b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_3_1449dd796ce9f2209f085d4b017a5f33._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="Doesn't quite work" + date="2012-10-17T22:23:16Z" + content=""" +Your Makefile change, even with the uname corrected, had no impact at all. I'm running \"cabal install\". When I try just \"make\", I get this error: + + + Annex.hs:69:28: + No instance for (MonadBase IO (StateT AnnexState IO)) + arising from a use of `liftBase' + Possible fix: + add an instance declaration for + (MonadBase IO (StateT AnnexState IO)) + In the second argument of `(.)', namely `liftBase' + In the expression: Annex . liftBase + In an equation for `liftBase': liftBase = Annex . liftBase +"""]] diff --git a/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_4_c4aa8a4379b2c056ca9b7afcff412bbc._comment b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_4_c4aa8a4379b2c056ca9b7afcff412bbc._comment new file mode 100644 index 0000000000..eb64afee27 --- /dev/null +++ b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_4_c4aa8a4379b2c056ca9b7afcff412bbc._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.23" + subject="comment 4" + date="2012-10-18T02:04:47Z" + content=""" +You might be able to use the cabal file now. (Not sure.. the OS name used there is different from uname.) + +cabal may have better luck, the make error suggests that you have too old a version of the haskell transformers-base library installed. +"""]] diff --git a/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_5_6ca4dd2ad51182edf7198f38b336b9b6._comment b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_5_6ca4dd2ad51182edf7198f38b336b9b6._comment new file mode 100644 index 0000000000..f37b1d227c --- /dev/null +++ b/doc/bugs/Most_recent_git-annex_will_not_build_on_OpenIndiana/comment_5_6ca4dd2ad51182edf7198f38b336b9b6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="Works great!" + date="2012-10-18T05:34:06Z" + content=""" +This can be moved to done. +"""]] diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn new file mode 100644 index 0000000000..722dac50b8 --- /dev/null +++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn @@ -0,0 +1,31 @@ +I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. The name scheme for an annex should be the same as for git repos themselves. + +> What do you mean by "an annex remote"? git-annex uses the same +> remotes configuration as does git. If you put invalid +> stuff in .git/config it might handle it slightly different than +> git, I don't know. Examples needed. --[[Joey]] + +>> What I mean is this: + + % cd 1 + % git init + % git annex init "my:colon" + % [...] + % cd ../2 + % git init + % git annex init "second" + % git remote add "my:colon" ../1 + fatal: 'my:colon' is not a valid remote name + +>> -- RichiH + +>>> I see.. Git annex init does not specifiy a remote's name, it specifies +>>> an arbitrary human-readable description of the repository, which will +>>> be displayed when there is no configured remote corresponding to the +>>> repository. So this is not a bug unless some documentation of that is +>>> unclear. --[[Joey]] + +>>>> Nobody spoke up to say it's unclear, so closing as PEBKAC :) +>>>> [[done]] --[[Joey]] + +>>>>> I still think git-annex should follow the same rules as git in this regard, but if your design decision is different, I won't try to argue the point :) -- RichiH diff --git a/doc/bugs/Need_to_manually_install_c2hs_-_3.20121127_and_previous.mdwn b/doc/bugs/Need_to_manually_install_c2hs_-_3.20121127_and_previous.mdwn new file mode 100644 index 0000000000..924ce06c7b --- /dev/null +++ b/doc/bugs/Need_to_manually_install_c2hs_-_3.20121127_and_previous.mdwn @@ -0,0 +1,37 @@ +What steps will reproduce the problem? + +Install git-annex via cabal - either from Hackage or as a manual install. (i.e. ) + +What is the expected output? What do you see instead? + +Expect a clean install. + +However, get the following error: + + Configuring gnuidn-0.2... + cabal: The program c2hs is required but it could not be found. + Failed to install gnuidn-0.2 + cabal: Error: some packages failed to install: + git-annex-3.20121127 depends on gnuidn-0.2 which failed to install. + gnuidn-0.2 failed during the configure step. The exception was: + ExitFailure 1 + network-protocol-xmpp-0.4.4 depends on gnuidn-0.2 which failed to install. + +What version of git-annex are you using? On what operating system? + +git-annex: 3.20121127 (and previous versions) + +OS: Mac OSX 10.6.8 + + +Please provide any additional information below. + +The fix seems as easy as + + cabal install c2hs + +Should c2hs be included as a dep got git-annex or is this a bug in gnuidn? + +> Apparently cabal does not support automatically installing programs +> needed for the build. I've updated the cabal installation instructions +> to document the need to install c2hs. [[done]] --[[Joey]] diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn new file mode 100644 index 0000000000..8df3bde481 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn @@ -0,0 +1,12 @@ +My local git index got corrupted and I needed to clone and annex get all data from my main repo. + +Some files were never copied anywhere so I am stuck with symlinks to nowhere. + +I tried to copy over the symlink with a copy of the actual file, which did not work. Trying to unlock, copying over the symlink, and relock did not work, either. + +Then, I copied the annex object to the correct place in .git/annex/objects/..., set all modes, re-ran fsck and the file re-appeared. + + +Long story short, I think there should be a `git annex reinject $file` or similar which will take a file, either one replacing the symlink or with an arbitrary path, and put it into the correct place in the object store. Called normally, it should reject all reinjects where the checksum does not match. With --force, this should be overridden. For reasons of safety, WORM should always require --force. + +> [[closing|done]], seems addressed --[[Joey]] diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment new file mode 100644 index 0000000000..9688012a47 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T01:46:16Z" + content=""" +Have you seen [[walkthrough/recover_data_from_lost+found]]? The method described there will also work in this scenario. +"""]] diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment new file mode 100644 index 0000000000..c9b74d98f0 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-03T09:00:17Z" + content=""" +I did not. Thanks :) + +This still means that you can't re-inject a new version of a file unless you have the old one if you are using a SHA* backend, but that might be a corner case anyway. +"""]] diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment new file mode 100644 index 0000000000..9c56452e53 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-05-14T16:28:36Z" + content=""" +To re-inject new content for a file, you really want to get a new key for the file. Otherwise, other repos that have the old file will never get the new content. So: + +
+git rm file
+mv ~/newcontent file
+git annex add file
+
+"""]] diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment new file mode 100644 index 0000000000..fcca0561d4 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-05-14T16:29:35Z" + content=""" +Now available as `git-annex reinject`. +"""]] diff --git a/doc/bugs/No_progress_bars_with_S3.mdwn b/doc/bugs/No_progress_bars_with_S3.mdwn new file mode 100644 index 0000000000..907b3cb115 --- /dev/null +++ b/doc/bugs/No_progress_bars_with_S3.mdwn @@ -0,0 +1,21 @@ +## What steps will reproduce the problem? + +Add new data to a repository with an S3 special remote. Monitor the repository with the web app. + + +## What is the expected output? What do you see instead? + +I expect a changing status bar and percentage. Instead I see no changes when an upload becomes active. + + +## What version of git-annex are you using? On what operating system? + +3.20130102 on Arch 64-bit. + + +## Please provide any additional information below. + + +When uploading local data to an S3 remote, I see no progress bars. The progress bar area on active uploads stays the same grey as the bar on queued uploads. The status does not change from "0% of...". The uploads are completing, but this makes it very difficult to judge their activity. + +The only remotes I currently have setup are S3 special remotes, so I cannot say whether progress bars are working for uploads to other remote types. diff --git a/doc/bugs/No_progress_bars_with_S3/comment_1_33a601201a9fdd2357f1c03e32fa6b9c._comment b/doc/bugs/No_progress_bars_with_S3/comment_1_33a601201a9fdd2357f1c03e32fa6b9c._comment new file mode 100644 index 0000000000..90d00807de --- /dev/null +++ b/doc/bugs/No_progress_bars_with_S3/comment_1_33a601201a9fdd2357f1c03e32fa6b9c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 1" + date="2013-01-14T16:31:06Z" + content=""" +How large are your files? IIRC the S3 progress bar updated with a rather large granularity. +"""]] diff --git a/doc/bugs/No_progress_bars_with_S3/comment_2_52361805ced99c22d663b3b1e8a5b221._comment b/doc/bugs/No_progress_bars_with_S3/comment_2_52361805ced99c22d663b3b1e8a5b221._comment new file mode 100644 index 0000000000..04f823e0b1 --- /dev/null +++ b/doc/bugs/No_progress_bars_with_S3/comment_2_52361805ced99c22d663b3b1e8a5b221._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="annexuser" + ip="24.16.193.140" + subject="comment 2" + date="2013-01-14T21:26:18Z" + content=""" +4MB to 8MB +"""]] diff --git a/doc/bugs/No_progress_bars_with_S3/comment_3_5903c1c40c4562f4fbaccd1640fedb18._comment b/doc/bugs/No_progress_bars_with_S3/comment_3_5903c1c40c4562f4fbaccd1640fedb18._comment new file mode 100644 index 0000000000..c5ed8246fd --- /dev/null +++ b/doc/bugs/No_progress_bars_with_S3/comment_3_5903c1c40c4562f4fbaccd1640fedb18._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="annexuser" + ip="50.125.41.100" + subject="comment 3" + date="2013-01-19T20:40:27Z" + content=""" +Progress bars do work with a USB remote. +"""]] diff --git a/doc/bugs/No_progress_bars_with_S3/comment_4_80799c33e513384894b390fe34ab312a._comment b/doc/bugs/No_progress_bars_with_S3/comment_4_80799c33e513384894b390fe34ab312a._comment new file mode 100644 index 0000000000..6c00c43d32 --- /dev/null +++ b/doc/bugs/No_progress_bars_with_S3/comment_4_80799c33e513384894b390fe34ab312a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="annexuser" + ip="50.125.41.100" + subject="comment 4" + date="2013-01-19T20:41:48Z" + content=""" +I should also mention that I've uploaded a handful of 30MB files, and I've seen no progress bars on them. +"""]] diff --git a/doc/bugs/No_version_information_from_cli.mdwn b/doc/bugs/No_version_information_from_cli.mdwn new file mode 100644 index 0000000000..a0d30db414 --- /dev/null +++ b/doc/bugs/No_version_information_from_cli.mdwn @@ -0,0 +1,18 @@ +git-annex does not listen to -v, --version or version. + +At the very least, it should return both the version of the binary and the version of the object store it supports. +If it supports several annex versions, they should be listed in a comma-separated fashion. +If git-annex is called from within an annex, it should print the version of the local object store. + +Sample: + + % git annex version + git-annex version : 0.24 + default object store version : 3 + supported object store versions : 2,3 + local object store version : 2 + % + +The above might look like overkill, but it's in a form that will, most likely, never need to be extended. + +> Great idea, [[done]] --[[Joey]] diff --git a/doc/bugs/OSX_alias_permissions_and_versions_problem.mdwn b/doc/bugs/OSX_alias_permissions_and_versions_problem.mdwn new file mode 100644 index 0000000000..f4ebc9d1e9 --- /dev/null +++ b/doc/bugs/OSX_alias_permissions_and_versions_problem.mdwn @@ -0,0 +1,37 @@ +What steps will reproduce the problem? + +Use assistant and create repository the a folder in home dir. +Use textedit and save a new txt to the repository folder. + +What is the expected output? What do you see instead? + +The alias solution is broken. It should work more like Dropbox. +Textedit saves the file initially, but it is immediately locked. +Since it autosaves, it asks to unlock or duplicate. +Then gives the error: +"The file “Untitled 16.txt” cannot be unlocked." + +If the file exists: +The document “Untitled 14” could not be saved as “Untitled 14.txt”. You don’t have permission. + +If you open a file from the repository (now replaced by a symlink) with textedit, there are other problems: +- The filename will not be correct (will show the sha hash). +- It will ask to unlock, then give the error "You don’t have permission to write to the folder that the file “SHA256E-s8--8985d9832de2e28b5e1af64258c391a34d7528709ef916bac496e698c139020c.txt” is in." + +What version of git-annex are you using? On what operating system? + +OSX Lion +git-annex version: 3.20120924 + +Please provide any additional information below. + +Even if you fix these problems, automatic versioning in lion will probably don't work, and the symlinks seem a hackish solution and don't seem intuitive or easy to the end user. +The sync should be transparent but it's not, and it's error prone. It would even be best to keep file copies in the git repo and sync them with the original folder than make symlinks. + +Dropbox even allows to put a symlink in the dropbox directory, and it will sync the file. + +[[!tag /design/assistant/OSX]] + +> Now the assistant creates new repositories using direct mode on OSX. +> In direct mode, there is no locking of files; they can be modified +> directly. [[done]] --[[Joey]] diff --git a/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_1_4fabe32e7e626e6ca23aa0b6f449c4c6._comment b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_1_4fabe32e7e626e6ca23aa0b6f449c4c6._comment new file mode 100644 index 0000000000..74a20d08e3 --- /dev/null +++ b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_1_4fabe32e7e626e6ca23aa0b6f449c4c6._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlatTbI0K-qydpeYHl37iseqPNvERcdIMk" + nickname="Tiago" + subject="comment 1" + date="2012-09-26T23:38:51Z" + content=""" +After looking more into the git annex working, I understand the use of symlinks, but they are not always correctly handled by OSX. + + +Apple's Timemachine uses hardlinks for example... + + +Not being able to easily edit files that are in the repo without problems or quirks makes it unusable as a dropbox replacement and version control, which is a shame. +"""]] diff --git a/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_2_064d60fcc8366a70958540bc145e611a._comment b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_2_064d60fcc8366a70958540bc145e611a._comment new file mode 100644 index 0000000000..78125262e8 --- /dev/null +++ b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_2_064d60fcc8366a70958540bc145e611a._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlatTbI0K-qydpeYHl37iseqPNvERcdIMk" + nickname="Tiago" + subject="comment 2" + date="2012-09-26T23:50:55Z" + content=""" +It seems many people complain about symlink behavior in Lion and Mountain Lion. +I never used symlinks to files often, only to folders. +I hope you can solve this problems and make it work correctly in OSX, or use hardlinks instead. +Git annex sound awesome, but the people using the assistant will want KISS behavior, and don't even know what a symbolic link is. +"""]] diff --git a/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_3_6c72d4f40ea0a9566a1185901beff5ba._comment b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_3_6c72d4f40ea0a9566a1185901beff5ba._comment new file mode 100644 index 0000000000..8ef5e6cc56 --- /dev/null +++ b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_3_6c72d4f40ea0a9566a1185901beff5ba._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlatTbI0K-qydpeYHl37iseqPNvERcdIMk" + nickname="Tiago" + subject="comment 3" + date="2012-09-26T23:57:18Z" + content=""" +Ok, so it seems Sharebox solves all this problems...you might want to integrate sharebox into the assistant, so non-techie people can treat the repos as mutable. +Maybe make a mutable/unmutable toggle in the gui or somethink like that? + +What I don't understand is the need for kqueue, if the files are not supposed to be written to without unlocking, why not use FSEvents and only monitor the folders for new files. + +http://git-annex.branchable.com/news/sharebox_a_FUSE_filesystem_for_git-annex + +"""]] diff --git a/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_4_8a11f404bb72a1aeb2290744cce2d00d._comment b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_4_8a11f404bb72a1aeb2290744cce2d00d._comment new file mode 100644 index 0000000000..aeacce31ef --- /dev/null +++ b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_4_8a11f404bb72a1aeb2290744cce2d00d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlatTbI0K-qydpeYHl37iseqPNvERcdIMk" + nickname="Tiago" + subject="comment 4" + date="2012-09-27T00:06:31Z" + content=""" +Seems sharebox development stopped a year ago...now that you are working on the assistant, which sound awesome, I think you really should consider pick up the sharebox development and integrate it. + +Too bad I missed kickstarter, but I might donate now. +I just find it wierd that it said \"Like Dropbox\" on the title, but sharebox-like functionality is not on the roadmap, and being able to easily edit the files is a big feature of dropbox. + +"""]] diff --git a/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_5_30888607199d6a48b76d0c48f5aa4f64._comment b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_5_30888607199d6a48b76d0c48f5aa4f64._comment new file mode 100644 index 0000000000..d95d8f149f --- /dev/null +++ b/doc/bugs/OSX_alias_permissions_and_versions_problem/comment_5_30888607199d6a48b76d0c48f5aa4f64._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="I agree" + date="2012-10-18T07:11:20Z" + content=""" +I really can't think of any way that Git annex + the assistant is like Dropbox. The Annex is not usable for even a single of the uses cases for which I also use Dropbox. I think this \"branding\" of annex should be dropped, or else it actually needs to acquire the features of Dropbox. +"""]] diff --git a/doc/bugs/OSX_app_issues.mdwn b/doc/bugs/OSX_app_issues.mdwn new file mode 100644 index 0000000000..95a34eff42 --- /dev/null +++ b/doc/bugs/OSX_app_issues.mdwn @@ -0,0 +1,6 @@ +This is a collection of problem reports for the standalone OSX app. +If you have a problem using it, post it here. --[[Joey]] + +(Some things that should be fixed now have been moved to [[old]].) + +[[!tag /design/assistant/OSX]] diff --git a/doc/bugs/OSX_app_issues/comment_10_54d8f3e429df9a9958370635c890abf0._comment b/doc/bugs/OSX_app_issues/comment_10_54d8f3e429df9a9958370635c890abf0._comment new file mode 100644 index 0000000000..cad356c675 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_10_54d8f3e429df9a9958370635c890abf0._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.3.194" + subject="comment 10" + date="2013-01-19T16:13:20Z" + content=""" +The app uses the version of git included in it, because using the system's installed version if there is one is likely to run into other version incompatabilities. + +You can either install git-annex using cabal and homebrew, as documented, or you could go in and delete +all the git programs out of the app, and then it'd use the system's git stuff instead. +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_11_bb2ceb95a844449795addee6986d0763._comment b/doc/bugs/OSX_app_issues/comment_11_bb2ceb95a844449795addee6986d0763._comment new file mode 100644 index 0000000000..b3bbf60770 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_11_bb2ceb95a844449795addee6986d0763._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlYy4BrJyV1PdfqzevCVziXRp89iUH6Xzw" + nickname="Christopher" + subject="Code signing errors in log on starting git-annex.app" + date="2013-01-19T22:30:32Z" + content=""" +When I run via the App and set up a fresh repo, i get some Console.app spam (looks like one per file added to the repo dir... or maybe one per git command?): + + 2013-01-19 2:44:55.000 PM kernel[0]: CODE SIGNING: cs_invalid_page(0x1008b4000): p=73995[git] clearing CS_VALID + 2013-01-19 2:44:55.000 PM kernel[0]: CODE SIGNING: cs_invalid_page(0x10f99e000): p=73996[git] clearing CS_VALID + 2013-01-19 2:44:55.000 PM kernel[0]: CODE SIGNING: cs_invalid_page(0x102b44000): p=73997[git] clearing CS_VALID + 2013-01-19 2:44:55.000 PM kernel[0]: CODE SIGNING: cs_invalid_page(0x1029f4000): p=73998[git] clearing CS_VALID + ... + +and nothing seems to work. The page address and the pid increment steadily with each line. I'm using 10.8.2 (12C60) on a Mac Pro, and grabbed: + + /git-annex/OSX/current/10.8.2_Mountain_Lion/git-annex.dmg.bz2 + +(published 14-Jan-2013 15:19) + +It seems to be a code signing issue, perhaps with the vendored git binaries. While things are sort-of working, the web app shows the files flying by really fast. + +Using a fresh repo via `git annex webapp` works great (I built that after much teeth-knashing, brew install/link cycles, and then cabal install git-annex). + +I am very excited for this to work, this is exactly what I've been waiting for to replace dropbox. Came very close to writing it myself a few times (and in Haskell no less!!). +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_12_f3bc5a4e4895ac9351786f0bdd8005ba._comment b/doc/bugs/OSX_app_issues/comment_12_f3bc5a4e4895ac9351786f0bdd8005ba._comment new file mode 100644 index 0000000000..460465b81b --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_12_f3bc5a4e4895ac9351786f0bdd8005ba._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmYiJgOvC4IDYkr2KIjMlfVD9r_1Sij_jY" + nickname="Douglas" + subject="Error creating remote repository using ssh on OSX" + date="2013-01-25T13:18:40Z" + content=""" +There is an issue with creating remote repositories using ssh (the problem may require using a different account name.) I filed the following bug: + + + +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_2_fd560811c57df5cbc3976639642b8b19._comment b/doc/bugs/OSX_app_issues/comment_2_fd560811c57df5cbc3976639642b8b19._comment new file mode 100644 index 0000000000..2107390bee --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_2_fd560811c57df5cbc3976639642b8b19._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkN91jAhoesnVI9TtWANaBPaYjd1V9Pag8" + nickname="Benjamin" + subject="Package for older OS X" + date="2012-11-17T12:36:45Z" + content=""" +Is there an option to provide application bundle for older versions of OS X? The last time I tried the bundle wouldn't work under 10.5. If no specific features from newer OS X versions are required, it could be enough to add a simple switch when building. +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_4_4cda124b57ddc87645d5822f14ed5c59._comment b/doc/bugs/OSX_app_issues/comment_4_4cda124b57ddc87645d5822f14ed5c59._comment new file mode 100644 index 0000000000..7581409030 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_4_4cda124b57ddc87645d5822f14ed5c59._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkBTVYS5lTecuenAB01eHgfUxE20vWVpU4" + nickname="Peng" + subject="Large Mountain Loin package size" + date="2012-12-28T22:17:43Z" + content=""" +I see that git-annex package file is 489.9MB on Mac Mountain Loin (git-annex.dmg.bz2). The file git-annex.dmg.bz2 is only of size 24M. Is there any way to reduce the package size? +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_5_0d1df34f83a8dac9c438d93806236818._comment b/doc/bugs/OSX_app_issues/comment_5_0d1df34f83a8dac9c438d93806236818._comment new file mode 100644 index 0000000000..89078a7da1 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_5_0d1df34f83a8dac9c438d93806236818._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 5" + date="2013-01-02T19:52:28Z" + content=""" +The build for today's release is 24 mb. +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_6_bc44d5aea5f77e331a32913ada293730._comment b/doc/bugs/OSX_app_issues/comment_6_bc44d5aea5f77e331a32913ada293730._comment new file mode 100644 index 0000000000..187197e811 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_6_bc44d5aea5f77e331a32913ada293730._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="https://www.jsilence.org/" + nickname="jsilence" + subject="Setting up a repository fails on OSX 10.6" + date="2013-01-05T11:17:18Z" + content=""" +I installed Haskell via MacPorts and managed to compile git-annex as described via cabal. + +On the initial start, git-annex webapp starts just fine and the webapp opens in the browser. When I try to configure a local repository, the webapp crashes. + + 1 jsilence@zeo ~ % git-annex webapp + (Recording state in git...) + WebApp crashed: watch mode is not available on this system + + Launching web browser on file:///var/folders/6b/6bWnFAnbFXSPCLPvCnKNrE+++TI/-Tmp-/webapp1003.html + jsilence@zeo ~ % + +\"watch mode is not suported\" suggests that lsof is not in the PATH. lsof resides in /usr/sbin and can be found just fine: + + jsilence@zeo ~ % which lsof + /usr/sbin/lsof + +Any help would be appreciated. + +-jsl + +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_7_93e0bb53ac2d7daef53426fbdc5f92d9._comment b/doc/bugs/OSX_app_issues/comment_7_93e0bb53ac2d7daef53426fbdc5f92d9._comment new file mode 100644 index 0000000000..fccd9fb3f7 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_7_93e0bb53ac2d7daef53426fbdc5f92d9._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkurjhi0CRJvgm7QNaZDWS9hitBtavqIpc" + nickname="Bret" + subject="git-annex.app Not working on 32 bit machines" + date="2012-11-03T19:18:47Z" + content=""" +I tried running the git-annex.app on my Core Duo Macbook pro, and it does not run at all. I get an error on my system.log + +`Nov 3 12:13:26 Bret-Mac [0x0-0x15015].com.branchable.git-annex[155]: /Applications/git-annex.app/Contents/MacOS/runshell: line 52: /Applications/git-annex.app/Contents/MacOS/bin/git-annex: Bad CPU type in executable +Nov 3 12:13:26 Bret-Mac com.apple.launchd.peruser.501[92] ([0x0-0x15015].com.branchable.git-annex[155]): Exited with exit code: 1` + +It works on my 64 bit machine, and this has become quite the problem for a while now, where people with newer macs dont compile back for a 32bit machine. + +Is there any hope for a pre-compiled binary that works on a 32 bit machine? +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_7_acd73cc5c4caa88099e2d2f19947aadf._comment b/doc/bugs/OSX_app_issues/comment_7_acd73cc5c4caa88099e2d2f19947aadf._comment new file mode 100644 index 0000000000..bdbdfc9ab9 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_7_acd73cc5c4caa88099e2d2f19947aadf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.211" + subject="comment 7" + date="2013-01-05T17:48:52Z" + content=""" +@jsilence, this problem does not involve lsof. There was a typo in the cabal file that prevented cabal building it with hfsevents. I'm releasing a new version to cabal with this typo fixed. +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_8_141eac2f3fb25fe18b4268786f00ad6a._comment b/doc/bugs/OSX_app_issues/comment_8_141eac2f3fb25fe18b4268786f00ad6a._comment new file mode 100644 index 0000000000..f7b1f22818 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_8_141eac2f3fb25fe18b4268786f00ad6a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 8" + date="2012-11-07T16:08:00Z" + content=""" +I've been updating my haskell platform install recently, i used to try and get the builder to spit out 32/64bit binaries, but recently it's just become too messy, I've just migrated to a full 64bit build system. I'm afraid I won't be able to provide 32bit builds any more. +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_8_f4d5b2645d7f29b80925159efb94a998._comment b/doc/bugs/OSX_app_issues/comment_8_f4d5b2645d7f29b80925159efb94a998._comment new file mode 100644 index 0000000000..3a4ca0b57a --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_8_f4d5b2645d7f29b80925159efb94a998._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmOsimKUgz6rxpmsS_nrBQGavEYyUpDlsE" + nickname="Tim" + subject="OS X 10.8.2" + date="2013-01-11T00:07:54Z" + content=""" +Double click on the app, give permission for it to run and ... nothing +"""]] diff --git a/doc/bugs/OSX_app_issues/comment_9_2e6dfca0fd8df04066769653724eae28._comment b/doc/bugs/OSX_app_issues/comment_9_2e6dfca0fd8df04066769653724eae28._comment new file mode 100644 index 0000000000..8b9fcbc686 --- /dev/null +++ b/doc/bugs/OSX_app_issues/comment_9_2e6dfca0fd8df04066769653724eae28._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmRFKwny4rArBaz-36xTcsJYqKIgdDaw5Q" + nickname="Andrew" + subject="Prefer the system/path git binaries if they're a newer version" + date="2013-01-19T06:20:38Z" + content=""" +I have used homebrew to install git v1.8.0.2 but git-annex.app packages git v1.7.10.2. git 1.7 crashes due to some newer directives in my global git config. + + error: Malformed value for push.default: simple + error: Must be one of nothing, matching, tracking or current. + fatal: bad config file line 38 in /Users/akraut/.gitconfig + + git-annex: fd:13: hGetLine: end of file + failed + git-annex: webapp: 1 failed + +"""]] diff --git a/doc/bugs/OSX_app_issues/old.mdwn b/doc/bugs/OSX_app_issues/old.mdwn new file mode 100644 index 0000000000..42f77125d1 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old.mdwn @@ -0,0 +1 @@ +These issues should be fixed now. diff --git a/doc/bugs/OSX_app_issues/old/comment_10_bb823dc3cd6dc914ed14c176afa0b2f3._comment b/doc/bugs/OSX_app_issues/old/comment_10_bb823dc3cd6dc914ed14c176afa0b2f3._comment new file mode 100644 index 0000000000..857cf3a628 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_10_bb823dc3cd6dc914ed14c176afa0b2f3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://wiggy.net/" + nickname="Wichert" + subject="Re: Trying to add remote server after failed attempt blocks forever" + date="2012-11-28T21:26:36Z" + content=""" +It appears to not just wait forever: there was also a **git config --null --list** process taking all CPU. +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_11_a30e69fed14b0809184ffe05358ab871._comment b/doc/bugs/OSX_app_issues/old/comment_11_a30e69fed14b0809184ffe05358ab871._comment new file mode 100644 index 0000000000..a25514ba95 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_11_a30e69fed14b0809184ffe05358ab871._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 11" + date="2012-11-29T19:55:17Z" + content=""" +I've dealt with these ssh issues by including ssh in the app bundle. + +OTOH, I don't know how `git config --null --list` could possibly take any appreciable amount of CPU to run. +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_12_23d47b3696e537d60df1d383f33f19e4._comment b/doc/bugs/OSX_app_issues/old/comment_12_23d47b3696e537d60df1d383f33f19e4._comment new file mode 100644 index 0000000000..73a5d1345f --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_12_23d47b3696e537d60df1d383f33f19e4._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnqQyhiNXdPIWWSuu232luY5nc-h5RS8bE" + nickname="Arve" + subject="Make Repository hangs (git consuming 100% cpu)" + date="2012-12-01T01:26:04Z" + content=""" +When starting git-annex from Applications first time (Finder or spotlight), pressing \"Make Repository\" hangs. + +\"ps aux | grep git\" shows +arve 1723 100.0 0.0 2459668 3988 ?? R 2:00AM 0:31.30 git init --quiet /Users/arve/Desktop/annex/ + +Strange enough, if i start the app from terminal (/Applications/git-annex.app/Contents/MacOS/git-annex-webapp), the creation of repository works. Though, if I kill it and start git-annex from spotlight, the web frontend doesn't show, and \"git config --null --list\" is consuming 100% cpu. + +My OSX version is 10.8.1 +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_13_be5738b42b13ec8cd828c5fa66f030e8._comment b/doc/bugs/OSX_app_issues/old/comment_13_be5738b42b13ec8cd828c5fa66f030e8._comment new file mode 100644 index 0000000000..8a9ee061a3 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_13_be5738b42b13ec8cd828c5fa66f030e8._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 13" + date="2012-12-01T18:50:31Z" + content=""" +So it works with a controlling console, and git commands are somehow misbehaving without a controlling console. Very strange. + +Any chance you can `dtrace -p` the stuck git processes to see what they're doing or what resource they're blocked on? +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_14_5783a4716cd104e1f1c276aa0b9cb153._comment b/doc/bugs/OSX_app_issues/old/comment_14_5783a4716cd104e1f1c276aa0b9cb153._comment new file mode 100644 index 0000000000..a88a0047e5 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_14_5783a4716cd104e1f1c276aa0b9cb153._comment @@ -0,0 +1,41 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY" + nickname="Vincent" + subject="OS/X build 2012-12-12" + date="2012-12-13T00:59:51Z" + content=""" +I installed this today from the .dmg.bz2, md5sum 1bb50b3ee5eda3cd7f4b4a70cdae1855 on OS/X 10.8.2 + +uname -a +Darwin foo 12.2.0 Darwin Kernel Version 12.2.0: Sat Aug 25 00:48:52 PDT 2012; root:xnu-2050.18.24~1/RELEASE_X86_64 x86_64 + +I installed the app to the Applications folder. + +I had chrome and firefox running, recent versions. + +Double-click and it opens a new chrome window. This came up behind the existing (iconified) window. A nit, but just so you know. + +The configuration part of the app is shown, so far so good. +I type in the path I want it to use (~/work/annex) and press the create button. +It hangs forever trying to access localhost:55163 + + $ ps aux|grep git + me 85291 100.0 0.0 2460884 4160 ?? R 11:42am 12:03.72 git init --quiet /Users/me/work/annex/ + me 85233 0.0 0.3 2687204 44064 ?? S 11:42am 0:00.44 git-annex webapp -psn_0_50204638 + me 85226 0.0 0.0 2433432 868 ?? S 11:42am 0:00.00 /bin/sh /Applications/git-annex.app/Contents/MacOS/git-annex-webapp -psn_0_50204638 + me 85515 0.0 0.0 2432768 620 s000 S+ 11:54am 0:00.00 grep git + + $ netstat -an |grep 55163 + tcp4 0 0 127.0.0.1.55163 127.0.0.1.55207 CLOSE_WAIT + tcp4 0 0 127.0.0.1.55163 127.0.0.1.55206 CLOSE_WAIT + tcp4 0 0 127.0.0.1.55163 127.0.0.1.55205 CLOSE_WAIT + tcp4 0 0 127.0.0.1.55163 127.0.0.1.55201 ESTABLISHED + tcp4 0 0 127.0.0.1.55201 127.0.0.1.55163 ESTABLISHED + tcp4 0 0 127.0.0.1.55163 127.0.0.1.55199 CLOSE_WAIT + tcp4 0 0 127.0.0.1.55163 127.0.0.1.55197 CLOSE_WAIT + tcp4 0 0 127.0.0.1.55163 *.* LISTEN + +I was plugged into wired ethernet, no other interfaces up, no VPN. + +I have macports but no haskell packages, which ghc returns nothing. +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_14_e126d87a263f3aa6261f72ee7ff086fc._comment b/doc/bugs/OSX_app_issues/old/comment_14_e126d87a263f3aa6261f72ee7ff086fc._comment new file mode 100644 index 0000000000..30e9c40cbe --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_14_e126d87a263f3aa6261f72ee7ff086fc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnqQyhiNXdPIWWSuu232luY5nc-h5RS8bE" + nickname="Arve" + subject="comment 14" + date="2012-12-01T22:33:21Z" + content=""" +\"dtrace -p pid\" gives \"dtrace: no probes specified.\" I've tried to read man dtrace, and some online resources, dtrace is new to me. Any directions? +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_15_56c7fcafc7dca8be28ebf9e37a8f6b71._comment b/doc/bugs/OSX_app_issues/old/comment_15_56c7fcafc7dca8be28ebf9e37a8f6b71._comment new file mode 100644 index 0000000000..1cf4916f57 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_15_56c7fcafc7dca8be28ebf9e37a8f6b71._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY" + nickname="Vincent" + subject="comment 15" + date="2012-12-13T01:04:44Z" + content=""" +following up to #14. + + dtruss -p + +shows the same symptom as reported earlier + SYSCALL(args) = return + workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22 + workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22 + workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22 + workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22 + workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22 + workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22 + ... + workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22 + dtrace: 339527 drops on CPU 0 + +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_15_e58bd3d66f0f43c159d2b37172f152de._comment b/doc/bugs/OSX_app_issues/old/comment_15_e58bd3d66f0f43c159d2b37172f152de._comment new file mode 100644 index 0000000000..2ae1926464 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_15_e58bd3d66f0f43c159d2b37172f152de._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 15" + date="2012-12-01T22:49:54Z" + content=""" +Seems the command I was thinking of is really `dtruss -p` +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_16_01f2c968bad66b0ff0c09eb468325deb._comment b/doc/bugs/OSX_app_issues/old/comment_16_01f2c968bad66b0ff0c09eb468325deb._comment new file mode 100644 index 0000000000..5c076012f1 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_16_01f2c968bad66b0ff0c09eb468325deb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnqQyhiNXdPIWWSuu232luY5nc-h5RS8bE" + nickname="Arve" + subject="comment 16" + date="2012-12-01T22:53:29Z" + content=""" +Returns alot of \"workq_kernreturn(0x1, 0x107D27000, 0x0) = -1 Err#22\" and occasionally \"dtrace: 36244 drops on CPU 1\" +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_16_0b7cd3d5952c5abf36a89a68a4afc1e7._comment b/doc/bugs/OSX_app_issues/old/comment_16_0b7cd3d5952c5abf36a89a68a4afc1e7._comment new file mode 100644 index 0000000000..0b110ce528 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_16_0b7cd3d5952c5abf36a89a68a4afc1e7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.8.117" + subject="comment 16" + date="2012-12-13T22:40:59Z" + content=""" +Today's daily build of the OSX app has a further change that *might* help. I removed the system library and frameworks from the bundle. +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_17_82d9963e1fbf17644ce697e5a43943f5._comment b/doc/bugs/OSX_app_issues/old/comment_17_82d9963e1fbf17644ce697e5a43943f5._comment new file mode 100644 index 0000000000..f710878c28 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_17_82d9963e1fbf17644ce697e5a43943f5._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 17" + date="2012-12-01T23:08:50Z" + content=""" +Doesn't say anything to me.. + +Using nohup might help, or at least get us an error message to see. Edit `git-annex.app/Contents/MacOS/git-annex-webapp` and make the last line: + +
+nohup \"$base/runshell\" git-annex webapp \"$@\"
+
+ +It'll put a nohup.out log file in your home directory, or somewhere like that. +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_17_c2de94a48e7958b9efffd89dda9144ff._comment b/doc/bugs/OSX_app_issues/old/comment_17_c2de94a48e7958b9efffd89dda9144ff._comment new file mode 100644 index 0000000000..83c2023db1 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_17_c2de94a48e7958b9efffd89dda9144ff._comment @@ -0,0 +1,59 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY" + nickname="Vincent" + subject="OS/X daily 2012-12-13" + date="2012-12-14T09:22:42Z" + content=""" +Thanks for the update - I tried again, similar results. + +same platform. installed image md5sum 1bb50b3ee5eda3cd7f4b4a70cdae1855 + +Procedure was the same. + - download, bunzip2, mount, drag app to Applications. + - chrome had one tab open, iconified + - double-click application icon + +Chrome opens up and shows the config window. Type in the same path +after checking that the final element of the path did not already exist. + + ps aux|grep git + me 89194 99.0 0.0 2460884 4160 ?? R 8:03pm 0:12.58 git init --quiet /Users/me/me/annex/ + me 89245 0.2 0.0 2423356 220 s001 R+ 8:07pm 0:00.00 grep git + me 89182 0.0 0.3 2668772 44208 ?? S 8:03pm 0:00.30 git-annex webapp -psn_0_55022710 + me 89177 0.0 0.0 2433432 868 ?? S 8:03pm 0:00.00 /bin/sh /Applications/git-annex.app/Contents/MacOS/git-annex-webapp -psn_0_55022710 + +I ran dtruss on the two processes of interest, including when I sent them kill -9 in case that showed anything of interest. +Mail me if you need that but the gist is git init was doing + + workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22 + workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22 + workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22 + ... + workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22 + dtrace: 2006687 drops on CPU 0 + workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22 + +and the git-annex webapp was doing stuff like + + psynch_cvwait(0x7FD7F1418888, 0x2BE010002BF00, 0x600) = -1 Err#260 + sigreturn(0x7FFF543DD810, 0x1E, 0x2) = 0 Err#-2 + __pthread_canceled(0x0, 0x2BE010002BF00, 0x7FFF543DD8C8) = -1 Err#22 + psynch_cvwait(0x7FD7F1418888, 0x2C8010002C900, 0x600) = -1 Err#260 + sigreturn(0x7FFF543DD810, 0x1E, 0x2) = 0 Err#-2 + __pthread_canceled(0x0, 0x2C8010002C900, 0x7FFF543DD8C8) = -1 Err#22 + psynch_cvwait(0x7FD7F1418888, 0x2D0010002D100, 0x600) = -1 Err#260 + sigreturn(0x7FFF543DD810, 0x1E, 0x2) = 0 Err#-2 + __pthread_canceled(0x0, 0x2D0010002D100, 0x7FFF543DD8C8) = -1 Err#22 + psynch_cvwait(0x7FD7F1418888, 0x2D7010002D800, 0x600) = -1 Err#260 + sigreturn(0x7FFF543DD810, 0x1E, 0x10DCEDC00) = 0 Err#-2 + __pthread_canceled(0x0, 0x2D7010002D800, 0x7FFF543DD8C8) = -1 Err#22 + psynch_cvwait(0x7FD7F14189D8, 0x230100002400, 0x2300) = 0 0 + read(0x6, \"\377\0\", 0x1000) = 1 0 + setitimer(0x0, 0x116903E50, 0x0) = 0 0 + write(0x7, \"\377\0\", 0x1) = 1 0 + sigreturn(0x7FFF543DD810, 0x1E, 0x0) = 0 Err#-2 + __pthread_canceled(0x0, 0x2BC010002BD00, 0x7FFF543DD8C8) = -1 Err#22 + +which may be unrelated browser event loop stuff I guess. + +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_18_29af9df9ea295d114574e76e15b8e737._comment b/doc/bugs/OSX_app_issues/old/comment_18_29af9df9ea295d114574e76e15b8e737._comment new file mode 100644 index 0000000000..9a2bcfcd0c --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_18_29af9df9ea295d114574e76e15b8e737._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnqQyhiNXdPIWWSuu232luY5nc-h5RS8bE" + nickname="Arve" + subject="comment 18" + date="2012-12-02T00:02:51Z" + content=""" +Adding nohup doesn't produce any output (that I can find; sudo find / -name nohup.out) :/ +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_18_88ddc846eb4e4a2d54028a3412ba28d6._comment b/doc/bugs/OSX_app_issues/old/comment_18_88ddc846eb4e4a2d54028a3412ba28d6._comment new file mode 100644 index 0000000000..45c14024eb --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_18_88ddc846eb4e4a2d54028a3412ba28d6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.8.117" + subject="comment 18" + date="2012-12-14T20:55:12Z" + content=""" +I've built the app on 10.8.2, let's hope that'll finally put this problem to rest. + +Temporarily available here: + +(This build currently lacks XMPP support.) +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_19_6d6341b05123cd317c4eac96353c8662._comment b/doc/bugs/OSX_app_issues/old/comment_19_6d6341b05123cd317c4eac96353c8662._comment new file mode 100644 index 0000000000..70e7e7daad --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_19_6d6341b05123cd317c4eac96353c8662._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.244" + subject="comment 19" + date="2012-12-07T16:13:42Z" + content=""" +This hang seems not to occur on OSX 10.7, which is the version the autobuilder is running. So this is some sort of incompatability. + +So, building git-annex from source should avoid the problem. I'd like to get a version of the app built for 10.8, have not yet been able to arrange that. +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_19_aff4ab761c4d196732baa046af45fe24._comment b/doc/bugs/OSX_app_issues/old/comment_19_aff4ab761c4d196732baa046af45fe24._comment new file mode 100644 index 0000000000..76fc993e41 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_19_aff4ab761c4d196732baa046af45fe24._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY" + nickname="Vincent" + subject="comment 19" + date="2012-12-16T05:27:06Z" + content=""" +downloaded the 10.8.2 build, md5 9fc31ec6dcf0088d3723d1b25110f7f7. + +git --init works instantaneously. + +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_20_43bd5985d8a3a5e7f826a34e5dd9216e._comment b/doc/bugs/OSX_app_issues/old/comment_20_43bd5985d8a3a5e7f826a34e5dd9216e._comment new file mode 100644 index 0000000000..8e01d948e0 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_20_43bd5985d8a3a5e7f826a34e5dd9216e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.8.117" + subject="comment 20" + date="2012-12-17T16:23:05Z" + content=""" +@Vincent OMG, it works!?! + +\o/ +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_3_08613b2e2318680508483d204a43da76._comment b/doc/bugs/OSX_app_issues/old/comment_3_08613b2e2318680508483d204a43da76._comment new file mode 100644 index 0000000000..24b6bde51c --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_3_08613b2e2318680508483d204a43da76._comment @@ -0,0 +1,76 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + nickname="edheil" + subject="No luck running it on OS X Lion." + date="2012-11-21T06:07:55Z" + content=""" +here's the crash info: + +
+Process:         git-annex [84369]
+Path:            /Applications/git-annex.app/Contents/MacOS/bin/git-annex
+Identifier:      git-annex
+Version:         ??? (???)
+Code Type:       X86-64 (Native)
+Parent Process:  sh [84364]
+
+Date/Time:       2012-11-21 00:27:03.068 -0500
+OS Version:      Mac OS X 10.7.5 (11G63)
+Report Version:  9
+
+Crashed Thread:  0
+
+Exception Type:  EXC_BREAKPOINT (SIGTRAP)
+Exception Codes: 0x0000000000000002, 0x0000000000000000
+
+Application Specific Information:
+dyld: launch, loading dependent libraries
+
+Dyld Error Message:
+  Library not loaded: /opt/local/lib/libgss.3.dylib
+  Referenced from: /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libgsasl.7.dylib
+  Reason: image not found
+
+Binary Images:
+       0x105baa000 -        0x107b89fe7 +git-annex (??? - ???) <45311C82-015C-3F87-9F9B-01325EFBD0D9> /Applications/git-annex.app/Contents/MacOS/bin/git-annex
+       0x10822d000 -        0x10823eff7 +libz.1.dylib (1.2.7 - compatibility 1.0.0) <57016CC1-AD54-337E-A983-457933B24D35> /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libz.1.dylib
+       0x108245000 -        0x10827dff7 +libpcre.1.dylib (2.1.0 - compatibility 2.0.0) <431BD758-FA7B-38B3-AB7E-6511EC06152E> /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libpcre.1.dylib
+       0x108283000 -        0x1083b3ff7 +libxml2.2.dylib (11.0.0 - compatibility 11.0.0) <0663F820-D436-3304-B12F-9158901087EB> /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libxml2.2.dylib
+       0x1083e9000 -        0x108400fef +libgsasl.7.dylib (16.6.0 - compatibility 16.0.0) <41503EE1-D58B-385C-AC2E-BEAA7D0D4E38> /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libgsasl.7.dylib
+       0x10840a000 -        0x1084a1fff +libgnutls.26.dylib (49.3.0 - compatibility 49.0.0) <0320352A-3336-3B6B-A7DE-F3069669AD27> /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libgnutls.26.dylib
+       0x1084c3000 -        0x1084f1ff7 +libidn.11.dylib (18.8.0 - compatibility 18.0.0) <97073970-9370-3F85-B943-1B989EA41148> /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libidn.11.dylib
+       0x1084fc000 -        0x1085f5ff7 +libiconv.2.dylib (8.1.0 - compatibility 8.0.0) <1B8D243B-F617-301E-97B1-EE78A72617AB> /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libiconv.2.dylib
+       0x108606000 -        0x108606fff +libcharset.1.dylib (2.0.0 - compatibility 2.0.0)  /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libcharset.1.dylib
+       0x10860c000 -        0x108665fef +libgmp.10.dylib (11.5.0 - compatibility 11.0.0)  /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libgmp.10.dylib
+       0x108675000 -        0x1086a2fe7 +libSystem.B.dylib (159.1.0 - compatibility 1.0.0) <7BEBB139-50BB-3112-947A-F4AA168F991C> /Applications/git-annex.app/Contents/MacOS/usr/lib/libSystem.B.dylib
+       0x1086b4000 -        0x1086c8fef +libgcc_s.1.dylib (??? - ???) <3C5BF0B8-B1E9-3B41-B52F-F7499687217C> /Applications/git-annex.app/Contents/MacOS/opt/local/lib/gcc47/libgcc_s.1.dylib
+       0x1086d8000 -        0x1086f5ff7 +liblzma.5.dylib (6.4.0 - compatibility 6.0.0) <1D682E06-EB89-34CA-855A-AEF611C4DF86> /usr/local/lib/liblzma.5.dylib
+    0x7fff657aa000 -     0x7fff657debaf  dyld (195.6 - ???) <0CD1B35B-A28F-32DA-B72E-452EAD609613> /usr/lib/dyld
+    0x7fff8b669000 -     0x7fff8b672ff7  libsystem_notify.dylib (80.1.0 - compatibility 1.0.0)  /usr/lib/system/libsystem_notify.dylib
+    0x7fff8b6e4000 -     0x7fff8b6e5ff7  libsystem_sandbox.dylib (??? - ???) <2A09E4DA-F47C-35CB-B70C-E0492BA9F20E> /usr/lib/system/libsystem_sandbox.dylib
+    0x7fff8c000000 -     0x7fff8c006ff7  libunwind.dylib (30.0.0 - compatibility 1.0.0) <1E9C6C8C-CBE8-3F4B-A5B5-E03E3AB53231> /usr/lib/system/libunwind.dylib
+    0x7fff8c1c4000 -     0x7fff8c1c5ff7  libremovefile.dylib (21.1.0 - compatibility 1.0.0) <739E6C83-AA52-3C6C-A680-B37FE2888A04> /usr/lib/system/libremovefile.dylib
+    0x7fff8cf13000 -     0x7fff8cf4efff  libsystem_info.dylib (??? - ???) <35F90252-2AE1-32C5-8D34-782C614D9639> /usr/lib/system/libsystem_info.dylib
+    0x7fff8dbc3000 -     0x7fff8dbc8fff  libcache.dylib (47.0.0 - compatibility 1.0.0) <1571C3AB-BCB2-38CD-B3B2-C5FC3F927C6A> /usr/lib/system/libcache.dylib
+    0x7fff8dbc9000 -     0x7fff8dbd0fff  libcopyfile.dylib (85.1.0 - compatibility 1.0.0) <0AB51EE2-E914-358C-AC19-47BC024BDAE7> /usr/lib/system/libcopyfile.dylib
+    0x7fff8dbdf000 -     0x7fff8dbedfff  libdispatch.dylib (187.10.0 - compatibility 1.0.0) <8E03C652-922A-3399-93DE-9EA0CBFA0039> /usr/lib/system/libdispatch.dylib
+    0x7fff8dcf2000 -     0x7fff8dcf7ff7  libsystem_network.dylib (??? - ???) <5DE7024E-1D2D-34A2-80F4-08326331A75B> /usr/lib/system/libsystem_network.dylib
+    0x7fff8e1bb000 -     0x7fff8e298fef  libsystem_c.dylib (763.13.0 - compatibility 1.0.0) <41B43515-2806-3FBC-ACF1-A16F35B7E290> /usr/lib/system/libsystem_c.dylib
+    0x7fff8e6e2000 -     0x7fff8e6eafff  libsystem_dnssd.dylib (??? - ???) <584B321E-5159-37CD-B2E7-82E069C70AFB> /usr/lib/system/libsystem_dnssd.dylib
+    0x7fff8fab6000 -     0x7fff8fab8fff  libquarantine.dylib (36.7.0 - compatibility 1.0.0) <8D9832F9-E4A9-38C3-B880-E5210B2353C7> /usr/lib/system/libquarantine.dylib
+    0x7fff8fc3e000 -     0x7fff8fc80ff7  libcommonCrypto.dylib (55010.0.0 - compatibility 1.0.0)  /usr/lib/system/libcommonCrypto.dylib
+    0x7fff90fa3000 -     0x7fff90fa9fff  libmacho.dylib (800.0.0 - compatibility 1.0.0) <165514D7-1BFA-38EF-A151-676DCD21FB64> /usr/lib/system/libmacho.dylib
+    0x7fff90faa000 -     0x7fff90fabfff  libunc.dylib (24.0.0 - compatibility 1.0.0) <337960EE-0A85-3DD0-A760-7134CF4C0AFF> /usr/lib/system/libunc.dylib
+    0x7fff910b4000 -     0x7fff910b8fff  libmathCommon.A.dylib (2026.0.0 - compatibility 1.0.0)  /usr/lib/system/libmathCommon.A.dylib
+    0x7fff916b9000 -     0x7fff916bdfff  libdyld.dylib (195.6.0 - compatibility 1.0.0)  /usr/lib/system/libdyld.dylib
+    0x7fff916be000 -     0x7fff916defff  libsystem_kernel.dylib (1699.32.7 - compatibility 1.0.0) <66C9F9BD-C7B3-30D4-B1A0-03C8A6392351> /usr/lib/system/libsystem_kernel.dylib
+    0x7fff916df000 -     0x7fff916e0fff  libdnsinfo.dylib (395.11.0 - compatibility 1.0.0) <853BAAA5-270F-3FDC-B025-D448DB72E1C3> /usr/lib/system/libdnsinfo.dylib
+    0x7fff929f8000 -     0x7fff929fdfff  libcompiler_rt.dylib (6.0.0 - compatibility 1.0.0) <98ECD5F6-E85C-32A5-98CD-8911230CB66A> /usr/lib/system/libcompiler_rt.dylib
+    0x7fff93a3c000 -     0x7fff93a3cfff  libkeymgr.dylib (23.0.0 - compatibility 1.0.0) <61EFED6A-A407-301E-B454-CD18314F0075> /usr/lib/system/libkeymgr.dylib
+    0x7fff97139000 -     0x7fff9713aff7  libsystem_blocks.dylib (53.0.0 - compatibility 1.0.0) <8BCA214A-8992-34B2-A8B9-B74DEACA1869> /usr/lib/system/libsystem_blocks.dylib
+    0x7fff9724f000 -     0x7fff9726cfff  libxpc.dylib (77.19.0 - compatibility 1.0.0) <9F57891B-D7EF-3050-BEDD-21E7C6668248> /usr/lib/system/libxpc.dylib
+    0x7fff97cfe000 -     0x7fff97d08ff7  liblaunch.dylib (392.39.0 - compatibility 1.0.0) <8C235D13-2928-30E5-9E12-2CC3D6324AE2> /usr/lib/system/liblaunch.dylib
+
+ + +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_6_12bd83e7e2327c992448e87bdb85d17e._comment b/doc/bugs/OSX_app_issues/old/comment_6_12bd83e7e2327c992448e87bdb85d17e._comment new file mode 100644 index 0000000000..62851c15b1 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_6_12bd83e7e2327c992448e87bdb85d17e._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/6xTna_B_h.ECb6_ftC2dYLytAEwrv36etg_054U-#4c1e7" + nickname="Fake" + subject="libncurses on 10.7" + date="2012-10-17T21:24:24Z" + content=""" +I'm getting an error from gpg when I try to set up a repository on a remote server with encrypted rsync. Looks like libncurses in /usr/lib is 32 bit: + + Dyld Error Message: + Library not loaded: /opt/local/lib/libncurses.5.dylib + Referenced from: /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libreadline.6.2.dylib + Reason: no suitable image found. Did find: + /usr/lib/libncurses.5.dylib: mach-o, but wrong architecture + /usr/lib/libncurses.5.dylib: mach-o, but wrong architecture +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_6_cea97dbbfb566a9fe463365ca4511119._comment b/doc/bugs/OSX_app_issues/old/comment_6_cea97dbbfb566a9fe463365ca4511119._comment new file mode 100644 index 0000000000..6c968aa1ea --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_6_cea97dbbfb566a9fe463365ca4511119._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://nico.kaiser.me/" + nickname="Nico Kaiser" + subject="git-annex crashing on OS X 10.8.2" + date="2012-11-27T08:01:29Z" + content=""" + $ /Applications/git-annex.app/Contents/MacOS/git-annex-webapp + dyld: Symbol not found: _OBJC_CLASS_$_NSObject + Referenced from: /usr/bin/open + Expected in: /Applications/git-annex.app/Contents/MacOS/usr/lib/libobjc.A.dylib + in /usr/bin/open + WebApp crashed: failed to start web browser + + Launching web browser on file:///var/folders/8g/_fvs7jf572l4fj03q5mhrq9r0000gn/T/webapp1196.html + +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_7_911f187d46890093a54859032ada2442._comment b/doc/bugs/OSX_app_issues/old/comment_7_911f187d46890093a54859032ada2442._comment new file mode 100644 index 0000000000..636627959a --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_7_911f187d46890093a54859032ada2442._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 7" + date="2012-11-27T21:13:18Z" + content=""" +@Nico, that's actually kind of promising; people have been reporting much earlier crashes with OSX. This later crash is one I think I can do something about! :) + +So, it looks like the standalone app build needs to run the web browser in a clean environment that doesn't use any of its bundled libraries. I've committed that change and it will be in a release later today. +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_8_08b091a58106ca6050ac669579ed9ff4._comment b/doc/bugs/OSX_app_issues/old/comment_8_08b091a58106ca6050ac669579ed9ff4._comment new file mode 100644 index 0000000000..d32d9a024c --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_8_08b091a58106ca6050ac669579ed9ff4._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://wiggy.net/" + nickname="Wichert" + subject="Adding a remote server repository fails" + date="2012-11-28T21:17:23Z" + content=""" +The error message I get is: + +> Failed to ssh to the server. Transcript: dyld: Symbol not found: ___progname Referenced from: /usr/bin/ssh Expected in: /Users/wichert/Applications/git-annex.app/Contents/MacOS/usr/lib/libSystem.B.dylib in /usr/bin/ssh + +"""]] diff --git a/doc/bugs/OSX_app_issues/old/comment_9_8464c839cb169a4c6e72bebdc2065e9a._comment b/doc/bugs/OSX_app_issues/old/comment_9_8464c839cb169a4c6e72bebdc2065e9a._comment new file mode 100644 index 0000000000..42aaa9bfe4 --- /dev/null +++ b/doc/bugs/OSX_app_issues/old/comment_9_8464c839cb169a4c6e72bebdc2065e9a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://wiggy.net/" + nickname="Wichert" + subject="Trying to add remote server after failed attempt blocks forever" + date="2012-11-28T21:23:54Z" + content=""" +Out of curiosity I checked what would happen if I tried to add a remote server repo again after it just failed with the missing library error (see previous comment). Surprisingly the webapp is now waiting forever at the *Testing server ...* message. +"""]] diff --git a/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__.mdwn b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__.mdwn new file mode 100644 index 0000000000..ac5c66951f --- /dev/null +++ b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__.mdwn @@ -0,0 +1,26 @@ +**What steps will reproduce the problem?** + +Either double click on the app or from the terminal + + $ open /Applications/git-annex.app + +**What is the expected output? What do you see instead?** + +I'd expect to see git-annex run. "git-annex" doesn't run and what I see (in the terminal) is: + + LSOpenURLsWithRole() failed with error -10810 for the file /Applications/git-annex.app. + +**What version of git-annex are you using? On what operating system?** + +*git-annex*: 3.20121017 + +*git-annex.app*: ??? + +*OS*: OSX 10.6.8 64 bit + + +**Please provide any additional information below.** + +[[!tag /design/assistant/OSX]] + +> This was fixed a while ago. [[done]] --[[Joey]] diff --git a/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_1_0dfa839f1ba689b23f811787515b8cff._comment b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_1_0dfa839f1ba689b23f811787515b8cff._comment new file mode 100644 index 0000000000..896ae26ce8 --- /dev/null +++ b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_1_0dfa839f1ba689b23f811787515b8cff._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://a-or-b.myopenid.com/" + ip="220.244.41.108" + subject="comment 1" + date="2012-10-28T06:46:40Z" + content=""" +My best advice to anyone running into this is simply install using cabal. It works. :-) +"""]] diff --git a/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_2_612b947eb5474f6d792a833e33105665._comment b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_2_612b947eb5474f6d792a833e33105665._comment new file mode 100644 index 0000000000..0732cbfcfe --- /dev/null +++ b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_2_612b947eb5474f6d792a833e33105665._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2012-10-28T20:11:43Z" + content=""" +I sort of got a feeling that this build might have issues on OSX 10.6, I create the OSX builds on 10.7. +"""]] diff --git a/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_3_549b8bcae6f1f8b21932b734e32fbdd1._comment b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_3_549b8bcae6f1f8b21932b734e32fbdd1._comment new file mode 100644 index 0000000000..c5ada3d1d0 --- /dev/null +++ b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_3_549b8bcae6f1f8b21932b734e32fbdd1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnKgoTWEDuAqxBT7LnQVNJBmI0wajSgORA" + nickname="Martin" + subject="Same on 10.8.2" + date="2012-10-31T20:17:17Z" + content=""" +I get the same error on OSX 10.8.2. +"""]] diff --git a/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_4_23078dfea127fa3ef20696eb10ce964c._comment b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_4_23078dfea127fa3ef20696eb10ce964c._comment new file mode 100644 index 0000000000..6b0278fe85 --- /dev/null +++ b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_4_23078dfea127fa3ef20696eb10ce964c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlfvsw_0TFUvMcHVBRw1D1UmAaOJ3VsNSU" + nickname="chee" + subject="comment 4" + date="2012-11-16T03:14:50Z" + content=""" +Same error here, 10.8.2 + +https://snaek.org/resources/ff40b1c2fd9486614c7df72aaeca9755fb43d6ff.png +"""]] diff --git a/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_5_7da5ef8325b8787bbf1c6e2c17b1142e._comment b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_5_7da5ef8325b8787bbf1c6e2c17b1142e._comment new file mode 100644 index 0000000000..2f7c1184bf --- /dev/null +++ b/doc/bugs/OSX_git-annex.app_error:__LSOpenURLsWithRole__40____41__/comment_5_7da5ef8325b8787bbf1c6e2c17b1142e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 5" + date="2012-11-26T23:52:52Z" + content=""" +The OSX app has been updated for today's release, and includes a lot of missing libraries. I don't understand this error message, but on the off chance it was caused by some of the missing libraries, it could conceviably be fixed. Your testing of it would be appreciated. +"""]] diff --git a/doc/bugs/Old_repository_stuck.mdwn b/doc/bugs/Old_repository_stuck.mdwn new file mode 100644 index 0000000000..46b153eb29 --- /dev/null +++ b/doc/bugs/Old_repository_stuck.mdwn @@ -0,0 +1,9 @@ +I had created a test repository a time ago with an old version of git-annex. I didn't really used it so I simply deleted the directory by hand. Now I've installed a new version of git-annex and the old repository stills appears on the webapp, but there is no interface to delete it. + +* Old git-annex version: don't remember +* New git-annex version: I downloaded 3.20130107 (twice to be sure), but for some reason 'git-annex version' reports 3.20130102 +* OS: Ubuntu 12.04.1 LTS 3.2.0-35-generic-pae #55-Ubuntu SMP Wed Dec 5 18:04:39 UTC 2012 i686 i686 i386 GNU/Linux + +> This is [[fixed|done]] in git; assuming the repo was showing up +> in the upper-right menu for switching amoung local repositories. +> --[[Joey]] diff --git a/doc/bugs/Out_of_memory_error_in_fsck_whereis_find_and_status_cmds.mdwn b/doc/bugs/Out_of_memory_error_in_fsck_whereis_find_and_status_cmds.mdwn new file mode 100644 index 0000000000..610c0e9d1a --- /dev/null +++ b/doc/bugs/Out_of_memory_error_in_fsck_whereis_find_and_status_cmds.mdwn @@ -0,0 +1,78 @@ +Before I start on what's gone wrong, many thanks for a great program: finally a way of finding out where I've put all those files, and I enjoyed your talk in Australia. Not quite to New Zealand, but a good start :-) + +To play with git-annex, I decided to convert my ~/Downloads directory to a git-annex repository, as there were a wide variety of files in it, mostly easily replaceable, and also handy to have on multiple machines. In hindsight, probably wasn't a great idea, as I'd regularly forget it was a git-annex repo and move files and directories out manually, which caused all sorts of fun trying to sort out the dead symlinks. Then I after one update, the repository format changed, from WORM to xxx256 format which meant there were now both sorts in the GA object store. + +More recently I'd tried converting the repo to direct mode, you get the idea: lots of playing with git-annex commands, and now that I think about it, possibly some git commands too trying to repair missing files. + +Anyway I've ended up with a 27GB git-annex repo that now manages to kill git-annex whenever I try to check it using "git-annex fsck". + +Not only does the fsck subcommand cause it to die, but also "find", "whereis" and "status". It dies on the same file (for find/whereis/fsck). + +e.g. + + ... lots of stuff deleted ... + whereis 1wolf14.zip (2 copies) + 051f0b00-e265-11e1-894e-3b0b3f3844f2 -- Laptop + 2c4e11e0-a1b4-11e1-9a02-73e17b04c00f -- here (myPC - Downloads) + ok + git-annex: out of memory (requested 2097152 bytes) + +Now "git status" for some repo data: + + myPC:~/Downloads$ git annex status + supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + supported remote types: git S3 bup directory rsync web webdav glacier hook + repository mode: direct + trusted repositories: 0 + semitrusted repositories: 4 + 00000000-0000-0000-0000-000000000001 -- web + 051f0b00-e265-11e1-894e-3b0b3f3844f2 -- Laptop + 2c4e11e0-a1b4-11e1-9a02-73e17b04c00f -- here (myPC - Downloads) + 48fbe52a-a1b3-11e1-bb80-ebc15118871d -- netbk + untrusted repositories: 0 + dead repositories: 0 + transfers in progress: none + available local disk space: 100 gigabytes (+1 megabyte reserved) + local annex keys: 1719 + local annex size: 27 gigabytes + known annex keys: git-annex: out of memory (requested 2097152 bytes) + +It always seems to die at about 3.5GB memory usage. This is running on Ubuntu 12.04, using the latest GA release built using cabal: + + git-annex version: 4.20130227 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 4 + upgrade supported from repository versions: 0 1 2 + +There are also dead symlinks that point to directories that have meta-data but not the symlink target (manually line-wrapped): + + myPC:~/Downloads$ ls -l precise* + lrwxrwxrwx 1 nino nino 194 Oct 21 12:48 precise-dvd-i386.iso -> + .git/annex/objects/0x/Xz/SHA256-s3590631424-- + b08ecdd4846948ec076b23afae7f87be9cfba5218fb9ba4160f26c0b8d4b5dd0/ + SHA256-s3590631424--b08ecdd4846948ec076b23afae7f87be9cfba5218fb9ba4160f26c0b8d4b5dd0 + +But looking in the symlink destination directory, there's no corresponding object, only metadata: + + myPC:~/Downloads/.git/annex/objects/0x/Xz/SHA256-s3590631424--b08ecdd4846948ec076b23afae7f87be9cfba5218fb9ba4160f26c0b8d4b5dd0$ ls -l + total 8 + -rw-rw-r-- 1 nino nino 30 Jan 8 23:15 SHA256-s3590631424--b08ecdd4846948ec076b23afae7f87be9cfba5218fb9ba4160f26c0b8d4b5dd0.cache + -rw-rw-r-- 1 nino nino 49 Jan 8 23:15 SHA256-s3590631424--b08ecdd4846948ec076b23afae7f87be9cfba5218fb9ba4160f26c0b8d4b5dd0.map + +But there is another version somewhere else. + + -r--r--r-- 1 nino nino 3590631424 Mar 18 2012 ./.git/annex/objects/Kj/wM/WORM-s3590631424-m1331991509 + --precise-dvd-i386.iso/WORM-s3590631424-m1331991509--precise-dvd-i386.iso + +This actual file does exist in the "Used" directory: + + -rw-r--r-- 1 nino nino 3.4G May 27 2012 precise-dvd-i386.iso + +I'm not so worried about the mangled repo - it's quite possibly because of clueless git/git-annex command usage - but the inability to use the fsck command is concerning + +I could just uninit everything, but as it dies prematurely, I'm not certain that all the contents would be restored. +Any thoughts on how I can get git-annex (esp. fsck) to complete would be appreciated. + +Thanks +Giovanni diff --git a/doc/bugs/Out_of_memory_error_in_fsck_whereis_find_and_status_cmds/comment_1_3aef6ca929fad198f2dda0868f2d49cb._comment b/doc/bugs/Out_of_memory_error_in_fsck_whereis_find_and_status_cmds/comment_1_3aef6ca929fad198f2dda0868f2d49cb._comment new file mode 100644 index 0000000000..d410f960b6 --- /dev/null +++ b/doc/bugs/Out_of_memory_error_in_fsck_whereis_find_and_status_cmds/comment_1_3aef6ca929fad198f2dda0868f2d49cb._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-03-03T18:34:41Z" + content=""" +git-annex is designed to run in constant memory. It should never use a lot of memory. + +What file causes the problem? You show git-annex whereis successfully processing 1wolf14.zip, so that file is apparently not the problem. It's probably the *next* file you told whereis to run on. You can get a file list with `git ls-files` + +Apparently something bad is happening when it tries to calculate the key used by the problem file. + +What does the file look like? You're using direct mode so it may or may not be a symlink. If it's a symlink, show it. If not, show the output of: `echo HEAD:./$filename | git cat-file --batch` + +---- + +The stuff about \"dead symlinks\" etc is all a distraction AFAICS. git-annex allows you to remove content from your repository in multiple ways. That content could be present in another repository (git annex whereis would tell you), or the last copy of it could have been deleted. You can run \"git annex log\" on a file to show where the content was located historically. +"""]] diff --git a/doc/bugs/Pairing_locally_shows:___34__bad_comment_in_ssh_public_key_ssh-rsa__34__.mdwn b/doc/bugs/Pairing_locally_shows:___34__bad_comment_in_ssh_public_key_ssh-rsa__34__.mdwn new file mode 100644 index 0000000000..6999d714c9 --- /dev/null +++ b/doc/bugs/Pairing_locally_shows:___34__bad_comment_in_ssh_public_key_ssh-rsa__34__.mdwn @@ -0,0 +1,23 @@ +Open two webapp sessions (in two different *screen* windows or whatever) + + name@name-Computermodel-1000:~/test/annex1$ git-annex webapp + name@name-Computermodel-1000:~/test/annex2$ git-annex webapp + +In annex1's web UI, choose *Configuration* -> *Manage repositories* -> *Local computer*. + +Fill in the secret phrase and press *Start pairing*. + +In annex2's web UI, a pairing request will show on the left notifications bar. Choose *Respond*, fill in the phrase field and press *Finish pairing*. + +I was expecting a pairing between the two running session. But I got this after pressing *Finish pairing* in annex2's web UI: + +**start of output** +# Internal Server Error + +bad comment in ssh public key ssh-rsa [very long GPG key jibber jabber] name@name-Computermodel-1000 + +**end of output** + +git-annex version: 3.20121017, Ubuntu 12.04 + +> [[Done]], allowed dash and underscore in there now. diff --git a/doc/bugs/Possible_data_loss_-_git_status___39__typechange__39___and_direct_mode.mdwn b/doc/bugs/Possible_data_loss_-_git_status___39__typechange__39___and_direct_mode.mdwn new file mode 100644 index 0000000000..4a3329326b --- /dev/null +++ b/doc/bugs/Possible_data_loss_-_git_status___39__typechange__39___and_direct_mode.mdwn @@ -0,0 +1,32 @@ +#### What steps will reproduce the problem? + +When moving to direct mode files get flagged in git as 'typechange'. + + md test-directmode; cd test-directmode ; git init; git annex init + date > test.file ; git annex add test.file + git commit -m "Initial commit" + git status # All fine + git annex direct + git status # typechange: test.file + git add test.file && git commit -m "looks like I should commit this" + # And the symlink is now broken... + +#### What is the expected output? What do you see instead? + +Surprised to see the typechange status in git. Would not expect to see anything, however, if you `git add` and then commit you can get data loss. :-( + + +#### What version of git-annex are you using? On what operating system? + +git-annex version: 3.20130114 + +OS: OSX 10.6.8 + +#### Please provide any additional information below. + +> This was the pre-commit hook, made it not run in direct mode. [[done]] +> +> However, it's normal to see typechanged files in direct mode, and many +> git commands that manipulate files in the working tree *can* result in +> data loss. This is documented on [[direct_mode]]. +> --[[Joey]] diff --git a/doc/bugs/Possible_data_loss_-_git_status___39__typechange__39___and_direct_mode/comment_1_84cb8c651584ec2887f6e1b7dc107190._comment b/doc/bugs/Possible_data_loss_-_git_status___39__typechange__39___and_direct_mode/comment_1_84cb8c651584ec2887f6e1b7dc107190._comment new file mode 100644 index 0000000000..b067063742 --- /dev/null +++ b/doc/bugs/Possible_data_loss_-_git_status___39__typechange__39___and_direct_mode/comment_1_84cb8c651584ec2887f6e1b7dc107190._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://a-or-b.myopenid.com/" + ip="203.45.2.230" + subject="comment 1" + date="2013-01-17T02:13:03Z" + content=""" +Should g-a encourage users to *not* `git add` files with the typechange attribute? +"""]] diff --git a/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn b/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn new file mode 100644 index 0000000000..05024ffe9f --- /dev/null +++ b/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn @@ -0,0 +1,18 @@ +running 'git annex sync' doesn't merge the branches as expected (from the +limited testing I have done) with git 1.7.10, the behaviour of merge has +changed, it now asks for a commit message. I would expect setting +_GIT_MERGE_AUTOEDIT=no_ should resolve this issue. + +I had to manually do a merge (or set that variable) to get the branches +back in sync again, this confused me a bit when git-annex watch was running +in the background on a remote and it did not pick up the changes. + +> Yeah, I tend to miss these since the first thing I did when this +> misfeature was being posted was to write a mail discouraging them from +> doing it (sadly ignored), and then set in ~/.environment: + + # My time is more valuable than git's new, bad default + GIT_MERGE_AUTOEDIT=no + export GIT_MERGE_AUTOEDIT + +> Anyway, I've made sync run merge with --no-edit now. [[done]] --[[Joey]] diff --git a/doc/bugs/Prevent_accidental_merges.mdwn b/doc/bugs/Prevent_accidental_merges.mdwn new file mode 100644 index 0000000000..3e30e02235 --- /dev/null +++ b/doc/bugs/Prevent_accidental_merges.mdwn @@ -0,0 +1,14 @@ +With the storage layout v3, pulling the git-annex branch into the master branch is... less than ideal. + +The fact that the two branches contain totally different data make an accidental merge worse, arguably. + +Adding a tiny binary file called .gitnomerge to both branches would solve that without any noticeable overhead. + +Yes, there is an argument to be made that this is too much hand-holding, but I still think it's worth it. + +-- Richard + +> It should be as easy to undo such an accidential merge +> as it is to undo any other git commit, right? I quite like that git-annex +> no longer adds any clutter to the master branch, and would be reluctant +> to change that. --[[Joey]] diff --git a/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment b/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment new file mode 100644 index 0000000000..3e28a28cb0 --- /dev/null +++ b/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-10-23T15:00:48Z" + content=""" +Having run into the same issue again, I still think git-annex should ensure no merges take place. The clutter introduced by a .gitnomerge is neglible, imo. +"""]] diff --git a/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn new file mode 100644 index 0000000000..f8df1f0822 --- /dev/null +++ b/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn @@ -0,0 +1,52 @@ +Hi! + +Using bup for storing seems a good idea to save space, but I still have a problem when trying to copy files to my local git repo. +I have two partitions: + +- /Data (NTFS) + +- / (ext4) + +I turned the directory /Data/Audio into a git-annex repo, and cloned it into /home/me/AudioClone. +I added the remote bup to AudioClone by doing: + + git annex initremote mybup type=bup encryption=none buprepo= + +But when I try to copy some files that I have previously got by "git annex get" by doing: + + [~/AudioClone]$ git annex copy someartist/somealbum --to mybup + +it fails and tells me: + + copy Order To Die/01 Morituri Te Salutant.flac (to mybup...) + fatal: Cannot lock the ref 'refs/heads/WORM-s7351771-m1318841909--01 Morituri Te Salutant.flac'. + Traceback (most recent call last): + File "/usr/lib/bup/cmd/bup-split", line 170, in + git.update_ref(refname, commit, oldref) + File "/usr/lib/bup/bup/git.py", line 835, in update_ref + _git_wait('git update-ref', p) + File "/usr/lib/bup/bup/git.py", line 930, in _git_wait + raise GitError('%s returned %d' % (cmd, rv)) + bup.git.GitError: git update-ref returned 128 + +for each file, **except for the album cover file**, which is a simple JPG that bup doesn't try to split. This one gets copied nicely but the big FLAC files don't. + +I tried to restart my session, in case bup adds my username to a group or something. + +(I'm using Ubuntu 11.10) + +> Apparently bup-split does not allow storing data using filenames with +> spaces in them. I can reproduce the same bug using the same filename; +> if I remove the spaces all is well. +> +> Since bup-split -n uses git branches, I guess git-annex needs to avoid +> giving it any names containing spaces, or anything else not allowed +> in a git branch name. The rules for legal git branch names are quite complex +> (see git-check-ref-format(1)) so it will take me some times to code +> this up. +> +> A workaround is to switch to the SHA256 backend +> (`git annex migrate --backend=SHA256`), which avoids spaces in its keys. +> --[[Joey]] + +>> Now fixed in git. [[done]] --[[Joey]] diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn new file mode 100644 index 0000000000..83b75fb544 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx.mdwn @@ -0,0 +1,49 @@ +Followed the instructions over here: http://git-annex.branchable.com/forum/git-annex_on_OSX/ + +and had to install the following extra packages to be able to get make to start: + +[realizes pcre-light is needed but pcre not installed on my mac] +sudo port install pcre +sudo cabal install pcre-light + +> Ah right, that is a new dependency. I've updated the forum page +> with this info. +> --[[Joey]] + +But then I got the following error: + +
+ghc -O2 -Wall --make git-annex  
+[ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o   
+
+BackendTypes.hs:71:17:  
+    No instance for (Arbitrary Char)  
+      arising from a use of `arbitrary' at BackendTypes.hs:71:17-25  
+    Possible fix: add an instance declaration for (Arbitrary Char)  
+    In a stmt of a 'do' expression: backendname <- arbitrary  
+    In the expression:  
+        do backendname <- arbitrary  
+           keyname <- arbitrary  
+             return $ Key (backendname, keyname)  
+    In the definition of `arbitrary':  
+        arbitrary = do backendname <- arbitrary  
+                       keyname <- arbitrary  
+                         return $ Key (backendname, keyname)  
+make: *** [git-annex] Error 1  
+
+ +My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary so any help would be appreciated. + +> Hmm, it seems you may be missing part of the quickcheck haskell +> library, or have a different version than me. +> +> The easy fix is probably to just edit BackendTypes.hs and delete the +> entire end of the file from line 68, "for quickcheck" down. This code +> is only used by the test suite (so "make test" will fail), +> but it should get it to build. --[[Joey]] + +--- + +Closing this bug because the above problem now has a solution documented on +the install page, and the below test suite failure problems should all be +resolved on OSX. [[done]] --[[Joey]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment b/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment new file mode 100644 index 0000000000..95a9773e2b --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 10" + date="2011-02-09T15:04:50Z" + content=""" +I don't know what these problems forking could be. Can you strace it? +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment b/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment new file mode 100644 index 0000000000..3fbe57ecd5 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 11" + date="2011-02-09T19:35:47Z" + content=""" +I got dtruss to give me a trace, the output is quite big to post here (~560kb gzip'd), do you mind if I emailed it or posted it somewhere else for you? +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment b/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment new file mode 100644 index 0000000000..beba5dc42c --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 12" + date="2011-02-09T19:47:30Z" + content=""" +joey@kitenet.net (hope I can make sense of dtruss output) +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment b/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment new file mode 100644 index 0000000000..dd25c3d0cb --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 13" + date="2011-02-09T21:59:47Z" + content=""" +The dtrace puzzlingly does not have the same errors shown above, but a set of mostly new errors. I don't know what to make of that. + +> git-annex: git-annex/.t/repo/.git/hooks/pre-commit: fileAccess: permission denied (Operation not permitted) + +This seems to be caused by it setting the execute bit on the file. I don't know why that would fail; it's just written the file and renamed it into place so clearly should be able to write to it. + +> was able to modify annexed file's sha1foo content + +This also suggests something breaking with permissions. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment b/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment new file mode 100644 index 0000000000..724fe5505a --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 14" + date="2011-02-12T21:19:24Z" + content=""" +I've been trying to dig around the trace and code, and used google to see if the forkProcess issue was a haskell thing or an OSX thing. It seems that someone may have ran into a similar issue, though I am not sure if its related. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment b/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment new file mode 100644 index 0000000000..733ec997a6 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment @@ -0,0 +1,71 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 15" + date="2011-02-13T02:45:51Z" + content=""" +It may be possible that OSX has some low resource limits, for user processes (266 per user I think) doing a + + sudo sysctl -w kern.maxproc=2048 + sudo sysctl -w kern.maxprocperuid=1024 + sudo echo \"limit maxfiles 1024 unlimited\" >> /etc/launchd.conf + sudo echo \"limit maxproc 1024 2048\" >> /etc/launchd.conf + +seems to change the behaviour of the tests abit... + +
+Testing 1:blackbox:3:git-annex unannex:1:with content                         
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes                              
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Testing 1:blackbox:4:git-annex drop:1:with remote                             
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote                        
+Testing 1:blackbox:5:git-annex get                                            
+Testing 1:blackbox:6:git-annex move                                           
+Testing 1:blackbox:7:git-annex copy                                           
+Testing 1:blackbox:8:git-annex unlock/lock                                    
+Testing 1:blackbox:9:git-annex edit/commit:0                                  
+Cases: 30  Tried: 20  Errors: 0  Failures: 2add foo ok
+ok
+Testing 1:blackbox:9:git-annex edit/commit:1                                  
+Testing 1:blackbox:10:git-annex fix                                           
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust                       
+Testing 1:blackbox:12:git-annex fsck:0                                        
+Cases: 30  Tried: 24  Errors: 0  Failures: 2  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  Bad file size; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/WORM:1297565141:20:foo
+  Bad file content; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa
+Testing 1:blackbox:12:git-annex fsck:1                                        
+### Failure in: 1:blackbox:12:git-annex fsck:1
+fsck failed to fail with content only available in untrusted (current) repository
+Testing 1:blackbox:12:git-annex fsck:2                                        
+Cases: 30  Tried: 26  Errors: 0  Failures: 3  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	58e831c2-371b-11e0-bc1f-47d738dc52ee  -- test repo
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	58e831c2-371b-11e0-bc1f-47d738dc52ee  -- test repo
+Testing 1:blackbox:13:git-annex migrate:0                                     
+Cases: 30  Tried: 27  Errors: 0  Failures: 3  git-annex: user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))
+### Failure in: 1:blackbox:13:git-annex migrate:0
+migrate annexedfile failed
+Testing 1:blackbox:13:git-annex migrate:1                                     
+### Error in:   1:blackbox:13:git-annex migrate:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:14:git-annex unused/dropunused                             
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Cases: 30  Tried: 30  Errors: 2  Failures: 4
+test: failed
+
+ + +the number of failures vary as I change the values of the maxprocs, I think I have narrowed it down to OSX just being stupid with limits thus causing the tests to fail. + +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment b/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment new file mode 100644 index 0000000000..ca1b8e8cd5 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 16" + date="2011-02-13T04:52:26Z" + content=""" +I've fixed the test suite to not accumulate all those zombie processes. Now only 2 or 3 processes should run max. Am curious to see if that clears up all the problems. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment b/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment new file mode 100644 index 0000000000..7c7200fb9e --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 17" + date="2011-02-13T10:46:54Z" + content=""" +Yeap, that did the trick. I just tested a few separate OSX 10.6.6 systems and the tests are better behaved now, only 3 failures now. + +So the tests behave better (at least we don't get resource fork errors any more) + + * after the commit c319a3 without modifying the system limits (of 266 procs per user) + * without the commit c319a3 and when I increase the system process limits to as much as OSX allows + +On all the systems I tested on, I'm down to 3 failures now. + +
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Cases: 30  Tried: 20  Errors: 0  Failures: 2add foo ok
+ok
+Cases: 30  Tried: 24  Errors: 0  Failures: 2  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  Bad file size; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/WORM:1297594011:20:foo
+  Bad file content; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa
+### Failure in: 1:blackbox:12:git-annex fsck:1
+fsck failed to fail with content only available in untrusted (current) repository
+Cases: 30  Tried: 26  Errors: 0  Failures: 3  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	90d63906-375e-11e0-8867-abb8a6368269  -- test repo
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	90d63906-375e-11e0-8867-abb8a6368269  -- test repo
+Cases: 30  Tried: 30  Errors: 0  Failures: 3
+
+ +It's the same set of failures across all the OSX systems that I have tested on. Now I just need to figure out why there are still these three failures. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment b/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment new file mode 100644 index 0000000000..df76bb3017 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="maybe killed another osx bug in the test." + date="2011-02-13T15:12:10Z" + content=""" +I think I have figured out why + + ### Failure in: 1:blackbox:3:git-annex unannex:1:with content + foo is not a symlink + +It goes back to the this piece of code (in test.hs) + + copyrepo :: FilePath -> FilePath -> IO FilePath + copyrepo old new = do + cleanup new + ensuretmpdir + Utility.boolSystem \"cp\" [\"-pr\", old, new] @? \"cp -pr failed\" + +It seems that on OSX it does not preserve the symbolic link information, basically cp is not gnu cp on OSX, doing a \"cp -a SOURCE DEST\" seem's to the right thing on OSX. I tried it out on my archlinux workstation by replacing *-pr* with just *-a* and all the tests passed on archlinux. + +I'm not sure what the implications would be with changing the test with changing the cp command. + +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment b/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment new file mode 100644 index 0000000000..090c991c3a --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 19" + date="2011-02-13T15:55:47Z" + content=""" +On second thought and after some messing (trying most of the options and combinations of options on OSX for).... I tried replacing cp with gnu cp from coreutils on my OSX install, and all the tests passed. *sigh* cp -a is preserving some permissions and attributes but not all, its not behaving in the same way as the gnu cp does... the closet thing that I have found on OSX that behaves in the same way as gnu \"cp -pr\" is to use \"ditto\". + +Just doing a \"ditto SOURCE DEST\" in the tests passes everything. I'm not sure if its a good idea to use this even though it works. Though this is just the tests, does it affect CopyFile.hs where \"cp\" is called? + +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment b/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment new file mode 100644 index 0000000000..a33fef7d99 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc" + nickname="Antoine" + subject="Got it going!" + date="2011-02-06T06:02:57Z" + content=""" +Thanks to your feedback, I got it going. + +Maybe those two should be added to the 'OSX how-to' in the forum + +[realizes pcre-light is needed but pcre not installed on my mac] +sudo port install pcre +sudo cabal install pcre-light + +[tests are failing, need haskell's quickcheck] +sudo cabal install quickcheck +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment b/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment new file mode 100644 index 0000000000..b617da9261 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 20" + date="2011-02-13T17:54:09Z" + content=""" +Outside the test suite, git-annex's actual use of cp puts fairly low demands on it. It tries to use cp -a or cp -p if available just to preserve whatever attributes it can preserve, but the worst case if that you have a symlink pointing to a file that doesn't have the original timestamp or whatever. And there's little expectation git preserves that stuff anyway. + +I will probably try to make the test suite entirely use git clone rather than cp. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment b/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment new file mode 100644 index 0000000000..91d3e89f06 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-02-06T17:39:52Z" + content=""" +Yes, I've moved it to [[install/OSX]] page where anyone can update it in this wiki, and added your improvements. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment b/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment new file mode 100644 index 0000000000..39f32c244f --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment @@ -0,0 +1,57 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="tests fail with more recent installs of haskell platform" + date="2011-02-07T12:43:43Z" + content=""" +I'm running ghc 6.12.3 with the corresponding haskell-platform package from the HP site which I installed in preference to the macports version of haskell-platform (it's quite old). it seems when you install quickcheck, the version that is installed is of version 2.4.0.1 and not 1.2.0 which git-annex depends on for its tests. + +
+jtang@x00:~ $ cabal install quickcheck --reinstall               
+Resolving dependencies...
+Configuring QuickCheck-2.4.0.1...
+Preprocessing library QuickCheck-2.4.0.1...
+
+..
+and so on..
+..
+
+
+ +it fails with this + +
+[54 of 54] Compiling Main             ( test.hs, test.o )
+
+test.hs:56:3:
+    No instance for (QuickCheck-1.2.0.1:Test.QuickCheck.Arbitrary Char)
+      arising from a use of `qctest' at test.hs:56:3-64
+    Possible fix:
+      add an instance declaration for
+      (QuickCheck-1.2.0.1:Test.QuickCheck.Arbitrary Char)
+    In the expression:
+        qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode
+    In the first argument of `TestList', namely
+        `[qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode,
+          qctest \"prop_idempotent_fileKey\" Locations.prop_idempotent_fileKey,
+          qctest
+            \"prop_idempotent_key_read_show\"
+            BackendTypes.prop_idempotent_key_read_show,
+          qctest
+            \"prop_idempotent_shellEscape\" Utility.prop_idempotent_shellEscape,
+          ....]'
+    In the second argument of `($)', namely
+        `TestList
+           [qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode,
+            qctest \"prop_idempotent_fileKey\" Locations.prop_idempotent_fileKey,
+            qctest
+              \"prop_idempotent_key_read_show\"
+              BackendTypes.prop_idempotent_key_read_show,
+            qctest
+              \"prop_idempotent_shellEscape\" Utility.prop_idempotent_shellEscape,
+            ....]'
+
+ +I'd imagine if I could downgrade, it would compile and pass the tests (I hope) + +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment b/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment new file mode 100644 index 0000000000..e245e139fb --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-02-08T19:00:14Z" + content=""" +I doubt that git-annex can be used with QuickCheck 1.2.0. The QuickCheck I've tested it with is 2.1.0.3 actually. + +I suspect you have an old version of the TestPack haskell library on your system, that is linked against QuickCheck 1.2.0. Git-annex has been tested with TestPack 2.0.0, which uses QuickCheck 2.x. + +In any case, you don't have to run 'make test' to build git-annex, and my comments above should make the main program compile, I expect. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment b/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment new file mode 100644 index 0000000000..9c83feb32f --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment @@ -0,0 +1,107 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 5" + date="2011-02-08T19:56:55Z" + content=""" +Ah, that gave me a good clue, my system just got pretty confused with a mixture of quickcheck and testpack installs. Would it be possible to put up a list of versions of the software you are using on your development environment? (at least the minimum tested version) + +I guess it shouldn't matter to most users who are going to rely on packagers to sort these dependancy issues, but it's nice to know. + +Anyway, the tests build now, and they seem to fail on my (rather messy) install of haskell platform + ghc 6.12 on osx 10.6.6. + +
+< output that passed some tests >
+Testing 1:blackbox:0:git-annex init
+Testing 1:blackbox:1:git-annex add:0
+Testing 1:blackbox:1:git-annex add:1
+Cases: 30  Tried: 9  Errors: 0  Failures: 0test: sha1sum: executeFile: does not exist (No such file or directory)
+  git-annex: : hGetLine: end of file
+### Failure in: 1:blackbox:1:git-annex add:1
+add with SHA1 failed
+Testing 1:blackbox:2:git-annex setkey/fromkey
+Cases: 30  Tried: 10  Errors: 0  Failures: 1(checksum...) test: sha1sum: executeFile: does not exist (No such file or directory)
+### Error in:   1:blackbox:2:git-annex setkey/fromkey
+: hGetLine: end of file
+Testing 1:blackbox:3:git-annex unannex:0:no content
+Cases: 30  Tried: 11  Errors: 1  Failures: 1chmod: -R: No such file or directory
+chmod: -R: No such file or directory
+Testing 1:blackbox:3:git-annex unannex:1:with content
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes
+Cases: 30  Tried: 13  Errors: 1  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:0:no remotes
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:4:git-annex drop:1:with remote
+Cases: 30  Tried: 14  Errors: 2  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:1:with remote
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote
+Cases: 30  Tried: 15  Errors: 3  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:2:untrusted remote
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:5:git-annex get
+Cases: 30  Tried: 16  Errors: 4  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:5:git-annex get
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:6:git-annex move
+Cases: 30  Tried: 17  Errors: 5  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:6:git-annex move
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:7:git-annex copy
+Cases: 30  Tried: 18  Errors: 6  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:7:git-annex copy
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:8:git-annex unlock/lock
+Cases: 30  Tried: 19  Errors: 7  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:8:git-annex unlock/lock
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:9:git-annex edit/commit:0
+Cases: 30  Tried: 20  Errors: 8  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:9:git-annex edit/commit:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:9:git-annex edit/commit:1
+Cases: 30  Tried: 21  Errors: 9  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:9:git-annex edit/commit:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:10:git-annex fix
+Cases: 30  Tried: 22  Errors: 10  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:10:git-annex fix
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust
+Cases: 30  Tried: 23  Errors: 11  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:11:git-annex trust/untrust/semitrust
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:0
+Cases: 30  Tried: 24  Errors: 12  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:1
+Cases: 30  Tried: 25  Errors: 13  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:2
+Cases: 30  Tried: 26  Errors: 14  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:2
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:13:git-annex migrate:0
+Cases: 30  Tried: 27  Errors: 15  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:13:git-annex migrate:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:13:git-annex migrate:1
+Cases: 30  Tried: 28  Errors: 16  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:13:git-annex migrate:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:14:git-annex unused/dropunused
+Cases: 30  Tried: 29  Errors: 17  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Cases: 30  Tried: 30  Errors: 18  Failures: 2
+chmod: -R: No such file or directory
+test: .t/repo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+make: *** [test] Error 1
+
+ +I assumed that since the tests built, then running them shouldn't be a problem. It looks like some argument isn't being passed about for the location of the .t directory that gets created. I will check the dependancies on my system again. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment b/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment new file mode 100644 index 0000000000..afc3088d4f --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 6" + date="2011-02-08T23:20:08Z" + content=""" +You're missing the sha1sum command, everything else is a followon error from that. Added a hint about this to [[install]], +and in the next version configure will check for sha1sum. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment b/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment new file mode 100644 index 0000000000..8d8aefcb20 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 7" + date="2011-02-09T00:45:31Z" + content=""" +That's odd, I have the md5sha1sum package installed and it still fails with pretty much the same error + +
+Testing 1:blackbox:0:git-annex init
+Cases: 30  Tried: 7  Errors: 0  Failures: 0chmod: -R: No such file or directory
+### Error in:   1:blackbox:0:git-annex init
+.t/repo/.git/annex/objects/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:1:git-annex add:0
+### Error in:   1:blackbox:1:git-annex add:0
+foo: openFile: permission denied (Permission denied)
+
+< and so on >
+
+ +the configure script finds sha1sum, builds and starts to run. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment b/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment new file mode 100644 index 0000000000..9401bd453e --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 8" + date="2011-02-09T04:10:27Z" + content=""" +The chmod errors are because your chmod does not understand the -R argument. Only the test suite uses chmod -R. I've fixed it to modify modes manually. +"""]] diff --git a/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment b/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment new file mode 100644 index 0000000000..da6d7ca178 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment @@ -0,0 +1,66 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 9" + date="2011-02-09T09:12:52Z" + content=""" +[a0826293][] fixed the last problem, there is coreutils available in macports, if they are installed you get the gnu equivalents but they are prefixed with a g (e.g. gchmod instead of chmod), I guess not everyone will have these install or prefer these on [[install/OSX]] + +Some more tests fail now... + +
+Testing 1:blackbox:3:git-annex unannex:1:with content
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Testing 1:blackbox:4:git-annex drop:1:with remote
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote
+Testing 1:blackbox:5:git-annex get
+Testing 1:blackbox:6:git-annex move
+Testing 1:blackbox:7:git-annex copy
+### Failure in: 1:blackbox:7:git-annex copy
+move --to of file already there failed
+Testing 1:blackbox:8:git-annex unlock/lock
+### Error in:   1:blackbox:8:git-annex unlock/lock
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:9:git-annex edit/commit:0
+### Error in:   1:blackbox:9:git-annex edit/commit:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:9:git-annex edit/commit:1
+### Error in:   1:blackbox:9:git-annex edit/commit:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:10:git-annex fix
+### Error in:   1:blackbox:10:git-annex fix
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust
+### Error in:   1:blackbox:11:git-annex trust/untrust/semitrust
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:0
+### Error in:   1:blackbox:12:git-annex fsck:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:1
+### Error in:   1:blackbox:12:git-annex fsck:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:2
+### Error in:   1:blackbox:12:git-annex fsck:2
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:13:git-annex migrate:0
+### Error in:   1:blackbox:13:git-annex migrate:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:13:git-annex migrate:1
+### Error in:   1:blackbox:13:git-annex migrate:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:14:git-annex unused/dropunused
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Cases: 30  Tried: 30  Errors: 11  Failures: 3
+test: failed
+make: *** [test] Error 1
+
+ +On a side note, I think I found another bug in the testing. I had tested in a virtual machine in archlinux (a very recent updated version) Please see the report here [[tests fail when there is no global .gitconfig for the user]] + +[a0826293]: http://git.kitenet.net/?p=git-annex;a=commit;h=7a0826293e0ac6c0000f49a1618c1c613b909aa1 +"""]] diff --git a/doc/bugs/Provide_64-bit_standalone_build.mdwn b/doc/bugs/Provide_64-bit_standalone_build.mdwn new file mode 100644 index 0000000000..856e22b9a1 --- /dev/null +++ b/doc/bugs/Provide_64-bit_standalone_build.mdwn @@ -0,0 +1 @@ +The 32-bit standalone build appears to require two libraries (lib32-libyaml and lib32-gsasl) that are not available on Arch Linux. [See the comments on the AUR package](https://aur.archlinux.org/packages/git-annex-bin/). I'd appreciate it if you could bring back the 64-bit build. diff --git a/doc/bugs/Provide_64-bit_standalone_build/comment_1_1850bb3eb464f1d3c122cfeb4ccaf265._comment b/doc/bugs/Provide_64-bit_standalone_build/comment_1_1850bb3eb464f1d3c122cfeb4ccaf265._comment new file mode 100644 index 0000000000..76e8d0d5c2 --- /dev/null +++ b/doc/bugs/Provide_64-bit_standalone_build/comment_1_1850bb3eb464f1d3c122cfeb4ccaf265._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-03-03T21:32:39Z" + content=""" +Thanks for the heads up. + +As far as I can see, the necessary libraries are included in the standalone build. So it should work when used as intended via `runshell`. + +But that's not what the AUR is doing. It's binary editing (!!) the git-annex binary to use different library sonames, and taking the git-annex binary and dropping it unprotected into a system it was not built for to fend for itself. +That strikes me as a technique that is unlikely to continue working, and one that I cannot commit to support. + +I don't want to cause you Arch people unnecessary work, but building a 64 bit standalone build every time I release git-annex is unnecessary work on my part, as long as the 32 bit one works everywhere when used as designed. If someone has to do this 64 bit build, why not you? This would also avoid any further breakage, since you could build it against the actual library sonames it's going to be used with on Arch. +"""]] diff --git a/doc/bugs/Proxy_support.mdwn b/doc/bugs/Proxy_support.mdwn new file mode 100644 index 0000000000..d3ab3c6012 --- /dev/null +++ b/doc/bugs/Proxy_support.mdwn @@ -0,0 +1,18 @@ +What steps will reproduce the problem? + +Adding a e.g box.com repository from behind a http proxy via webapp. + +What is the expected output? What do you see instead? + +Connection should be made. But there is an error message: + +"Internal Server Error +connect: does not exist (Connection refused): user error" + +What version of git-annex are you using? On what operating system? + +3.20121127 on Archlinux + +Please provide any additional information below. + +I don't use networkmanager if proxy information is obtained from it. There should be a fallback to environment variables. diff --git a/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn new file mode 100644 index 0000000000..3e1acd4a81 --- /dev/null +++ b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn @@ -0,0 +1,6 @@ +Currently, git annex find lists files that are present in the current repository, possibly restricted to a subdirectory. But it does not easily seem possible to get this information about a remote repository. + +I would find it useful if this command understood flags that makes it tell me what is present somewhere else (maybe "--on remote") and combinations of the flags ("--on remote1 --and --not-on remote2" or "--on disk1 --or --on disk2"). + +> Almost. You're looking for `--in remote`, which was added 2 months ago. +> [[done]] --[[Joey]] diff --git a/doc/bugs/Remote_repositories_have_to_be_setup_encrypted.mdwn b/doc/bugs/Remote_repositories_have_to_be_setup_encrypted.mdwn new file mode 100644 index 0000000000..8489ba7cce --- /dev/null +++ b/doc/bugs/Remote_repositories_have_to_be_setup_encrypted.mdwn @@ -0,0 +1,25 @@ +What steps will reproduce the problem? + +Create a new remote repository in the webapp. Get to the final phase of the setup where it asks you if you want to encrypt it, yet no other option is given to continue. + +What is the expected output? What do you see instead? + +At least two options: + +1. Use an encrypted rsync repository on the server (the existing one) +2. Use an unencrypted rsync repository on the server + +What version of git-annex are you using? On what operating system? + + $ ./git-annex version + git-annex version: 3.20130102 + + $ uname -a + Linux wintermute 3.2.0-35-generic #55-Ubuntu SMP Wed Dec 5 17:45:18 UTC 2012 i686 i686 i386 GNU/Linux + + $ lsb_release -a + Distributor ID: Ubuntu + Description: Ubuntu 12.04.1 LTS + Release: 12.04 + Codename: precise + diff --git a/doc/bugs/Remote_repositories_have_to_be_setup_encrypted/comment_1_95f73315657bc35a8d3ff9b4ba207af0._comment b/doc/bugs/Remote_repositories_have_to_be_setup_encrypted/comment_1_95f73315657bc35a8d3ff9b4ba207af0._comment new file mode 100644 index 0000000000..f54b4626b2 --- /dev/null +++ b/doc/bugs/Remote_repositories_have_to_be_setup_encrypted/comment_1_95f73315657bc35a8d3ff9b4ba207af0._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.211" + subject="comment 1" + date="2013-01-03T18:02:21Z" + content=""" +Specifically, this affects only rsync special remotes. All the other ones have a check box that allows enabling encryption or not. I didn't get around to adding that for rsync due to some complications in the code. Of course, you can use `git annex initremote` at the command line to set up non-encrypted rsync remotes. +"""]] diff --git a/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code.mdwn b/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code.mdwn new file mode 100644 index 0000000000..2746ade5c4 --- /dev/null +++ b/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code.mdwn @@ -0,0 +1,24 @@ +What steps will reproduce the problem? + +I have an Annex with about 18k files in it. If I clone it and then run `git annex get .`, it gets a few thousand files and then starts reporting: + + get 2004-2012/Originals/110414_0362.jpg (from titan...) + rsync: fork: Resource temporarily unavailable (35) + rsync error: error in IPC code (code 14) at pipe.c(63) [Receiver=3.0.9] + +I have to abort and re-run `git annex get .` several times to finally get all of the files. + +What is the expected output? What do you see instead? + +I didn't expect what I saw! I think there's a resource not being released in the `get` code. + +What version of git-annex are you using? On what operating system? + +master branch, d430fb1. + +Please provide any additional information below. + +OS X 10.8.2. The machine has tons of RAM and tons of process handles free. It's really not doing anything else but this git-annex at the time of my tests. + +> [[done]], this is a bug introduced in 3.20121009, and I've reverted the +> buggy change. --[[Joey]] diff --git a/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code/comment_1_66b21720cd1b2a4f66ef24252d3e6305._comment b/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code/comment_1_66b21720cd1b2a4f66ef24252d3e6305._comment new file mode 100644 index 0000000000..4bdc7943ba --- /dev/null +++ b/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code/comment_1_66b21720cd1b2a4f66ef24252d3e6305._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 1" + date="2012-10-17T00:41:03Z" + content=""" +The resource in question appears to be processes. Do you get a lot of zombies or something? +"""]] diff --git a/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code/comment_2_18c9f55c5af1f4f690a7727df71ab561._comment b/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code/comment_2_18c9f55c5af1f4f690a7727df71ab561._comment new file mode 100644 index 0000000000..29214814b7 --- /dev/null +++ b/doc/bugs/Resource_leak_somewhere_in_the___39__get__39___code/comment_2_18c9f55c5af1f4f690a7727df71ab561._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 2" + date="2012-10-17T01:07:00Z" + content=""" +Urk. Seems I was making some recent changes lately to clean up zombies and I accidentually let them accumulate here. Fixed that. +"""]] diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn new file mode 100644 index 0000000000..2c0037c903 --- /dev/null +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn @@ -0,0 +1,10 @@ +While using HMAC instead of "plain" hash functions is inherently more secure, it's still a bad idea to re-use keys for different purposes. + +Also, ttbomk, HMAC needs two keys, not one. Are you re-using the same key twice? + +Compability for old buckets and support for different ones can be maintained by introducing a new option and simply copying over the encryption key's identifier into this new option should it be missing. + +> Bug was filed prematurely, but was a good bit of paranoia, and gpg and +> hmac are given different secret keys [[done]] --[[Joey]] + +>> Thanks :) -- RIchiH diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment new file mode 100644 index 0000000000..320fb5ef08 --- /dev/null +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-30T14:32:34Z" + content=""" +S3 doesn't support encryption at all, yet. + +It certainly makes sense to use a different portion of the encrypted secret key for HMAC than is uses as the gpg symmetric encryption key. + +The two keys used in HMAC would be the secret key and the key/value key for the content being stored. + +There is a difficult problem with encrypting filenames in S3 buckets, and that is determining when some data in the bucket is unused for dropunused. I've considered two choices: + +1. gpg encrypt the filenames. This would allow dropunused to recover the original filenames, and is probably more robust encryption. But it would double the number of times gpg is run when moving content in/out, and to check for unused content, gpg would have to be run once for every item in the bucket, which just feels way excessive, even though it would not be prompting for a passphrase. Still, haven't ruled this out. + +2. HMAC or other hash. To determine what data was unused the same hash and secret key would have to be used to hash all filenames currently used, and then that set of hashes could be interested with the set in the bucket. But then git-annex could only say \"here are some opaque hashes of content that appears unused by anything in your current git repository, but there's no way, short of downloading it and examining it to tell what it is\". (This could be improved by keeping a local mapping between filenames and S3 keys, but maintaining and committing that would bring pain of its own.) +"""]] diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment new file mode 100644 index 0000000000..dec06c89ff --- /dev/null +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-03-30T17:01:40Z" + content=""" +After mulling this over, I think actually encrypting the filenames is preferable. + +Did you consider encrypting the symmetric key with an asymmetric one? That's what TrueCrypt etc are using to allow different people access to a shared volume. This has the added benefit that you could, potentially, add new keys for data that new people should have access to while making access to old data impossible. Or keys per subdirectory, or, or, or. + +As an aside, could the same mechanism be extended to transparently encrypt data for a remote annex repo? A friend of mine is interested to host his data with me, but he wants to encrypt his data for obvious reasons. +"""]] diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment new file mode 100644 index 0000000000..c5bb26f595 --- /dev/null +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-30T18:15:18Z" + content=""" +Yes, encrypting the symmetric key with users' regular gpg keys is the plan. + +I don't think that encryption of content in a git annex remote makes much sense; the filenames obviously cannot be encrypted there. It's more likely that the same encryption would get used for a bup remote, or with the [[special_remotes/directory]] remote I threw in today. +"""]] diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment new file mode 100644 index 0000000000..09b7a8b1ab --- /dev/null +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 4" + date="2011-03-30T18:20:56Z" + content=""" +Picking up the automagic encryption idea for annex remotes, this would allow you to host a branchable-esque git-annex hosting service. (Nexenta with ZFS is a cheap and reliable option until btrfs becomes stable in a year or five). +"""]] diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment new file mode 100644 index 0000000000..49d43ffc63 --- /dev/null +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 5" + date="2011-03-30T18:59:19Z" + content=""" +This is brain-storming only so the idea might be crap, but a branch could keep encrypted filenames while master keeps the real deal. This might fit into the whole scheme just nicely or break future stuff in a dozen places, I am not really sure yet. But at least I can't forget the idea, now. +"""]] diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment new file mode 100644 index 0000000000..d994ca77f3 --- /dev/null +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 6" + date="2011-03-30T19:02:20Z" + content=""" +OTOH, if encryption makes a bup backend more likely disregard the idea above ;) +"""]] diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn new file mode 100644 index 0000000000..2f72b09ac8 --- /dev/null +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -0,0 +1,14 @@ +S3 has memory leaks + +Sending a file to S3 causes a slow memory increase toward the file size. + +Copying the file back from S3 causes a slow memory increase toward the +file size. + +The author of hS3 is aware of the problem, and working on it. I think I +have identified the root cause of the buffering; it's done by hS3 so it can +resend the data if S3 sends it a 307 redirect. --[[Joey]] + +At least the send leak should be fixed by the patch in the s3-memory-leak +branch in git. That needs a patch to hS3, which I have sent to its author. +--[[Joey]] diff --git a/doc/bugs/S3_upload_not_using_multipart.mdwn b/doc/bugs/S3_upload_not_using_multipart.mdwn new file mode 100644 index 0000000000..d242908f9d --- /dev/null +++ b/doc/bugs/S3_upload_not_using_multipart.mdwn @@ -0,0 +1,53 @@ +What steps will reproduce the problem? + +> Try to copy/move a file greater than 5G to S3. + + git annex copy large_file.tgz --to cloud + +What is the expected output? What do you see instead? + +> Looks like git-annex may not be using the Multipart Upload API: http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html + +> Expected transfer to succeed, instead this error is output: + + copy large-file.tgz (gpg) (checking cloud...) (to cloud...) Reading passphrase from file descriptor 12 + + + Your proposed upload exceeds the maximum allowed size + failed + git-annex: copy: 1 failed + +What version of git-annex are you using? On what operating system? + +> OSX 10.8.2 + +Please provide any additional information below. + + annex [master●] % git annex status + supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + supported remote types: git S3 bup directory rsync web webdav glacier hook + repository mode: indirect + trusted repositories: 0 + semitrusted repositories: 3 + 00000000-0000-0000-0000-000000000001 -- web + BE1D8EC7-C64B-47DE-AD4E-2A50437532B4 -- cloud + E84568BA-6A4B-4AA1-B622-605B9248EDB1 -- here (eric laptop) + untrusted repositories: 0 + dead repositories: 0 + transfers in progress: none + available local disk space: 169 gigabytes (+1 megabyte reserved) + temporary directory size: 218 megabytes (clean up with git-annex unused) + local annex keys: 24 + local annex size: 8 gigabytes + known annex keys: 25 + known annex size: 8 gigabytes + bloom filter size: 16 mebibytes (0% full) + backend usage: + SHA256E: 49 + annex [master●] % git annex version + git-annex version: 3.20130114 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + diff --git a/doc/bugs/SSH:_command-line:_line_0:_Bad_configuration_option:_ControlPersist___40__SSH_too_old_on_OS_X_10.6.8__63____41__.mdwn b/doc/bugs/SSH:_command-line:_line_0:_Bad_configuration_option:_ControlPersist___40__SSH_too_old_on_OS_X_10.6.8__63____41__.mdwn new file mode 100644 index 0000000000..a3d27e404d --- /dev/null +++ b/doc/bugs/SSH:_command-line:_line_0:_Bad_configuration_option:_ControlPersist___40__SSH_too_old_on_OS_X_10.6.8__63____41__.mdwn @@ -0,0 +1,27 @@ +What steps will reproduce the problem? + +Try to get any file from a remote ssh repository on OS X 10.6.8 + +What is the expected output? What do you see instead? + +Instead of retrieving the files, I get: + + % git annex get . + get command-line: line 0: Bad configuration option: ControlPersist + command-line: line 0: Bad configuration option: ControlPersist + (not available) + Try making some of these repositories available: + 2efd46d2-0e32-11e2-95fe-f72f09c6615e -- office + +What version of git-annex are you using? On what operating system? + % git annex version + git-annex version: 3.20120925 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +I seems that this option is passed to SSH, but SSH is too old: OpenSSH_5.2p1, OpenSSL 0.9.8r 8 Feb + +> Future builds of the standalone binary will default annex.sshcaching to +> false. [[done]] --[[Joey]] diff --git a/doc/bugs/SSH:_command-line:_line_0:_Bad_configuration_option:_ControlPersist___40__SSH_too_old_on_OS_X_10.6.8__63____41__/comment_1_0c57a2196d35eb1ecfb0c51273bba05c._comment b/doc/bugs/SSH:_command-line:_line_0:_Bad_configuration_option:_ControlPersist___40__SSH_too_old_on_OS_X_10.6.8__63____41__/comment_1_0c57a2196d35eb1ecfb0c51273bba05c._comment new file mode 100644 index 0000000000..ec510ae300 --- /dev/null +++ b/doc/bugs/SSH:_command-line:_line_0:_Bad_configuration_option:_ControlPersist___40__SSH_too_old_on_OS_X_10.6.8__63____41__/comment_1_0c57a2196d35eb1ecfb0c51273bba05c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.246.8" + subject="comment 1" + date="2012-10-15T18:16:58Z" + content=""" +Here's a workaround, which you can run in the local repository: + +git config annex.sshcaching false +"""]] diff --git a/doc/bugs/Switching_from_indirect_mode_to_direct_mode_breaks_duplicates.mdwn b/doc/bugs/Switching_from_indirect_mode_to_direct_mode_breaks_duplicates.mdwn new file mode 100644 index 0000000000..55d2b13b97 --- /dev/null +++ b/doc/bugs/Switching_from_indirect_mode_to_direct_mode_breaks_duplicates.mdwn @@ -0,0 +1,30 @@ +#What steps will reproduce the problem? + +1. Create a new repository in indirect mode. + +2. Add the same file twice under a different name. Now you have two symlinks pointing to the same file under .git/annex/objects/ + +3. Switch to direct mode. The first symlink gets replaced by the actual file. The second stays unchanged, pointing to nowhere. But git annex whereis still reports it has a copy. + +4. Delete the first file. Git annex whereis still thinks it has a copy of file 2, which is not true -> data loss. + +#What is the expected output? What do you see instead? + +When switching to direct mode, both symlinks should be replaced by a copy (or at least a hardlink) of the actual file. + +> The typo that caused this bug is fixed. --[[Joey]] + +#What version of git-annex are you using? On what operating system? + +3.20130107 on Arch Linux x64 + +#Please provide any additional information below. + +The deduplication performed by git-annex is very dangerous in itself +because files with identical content become replaced by references to the +same file without the user necessarily being aware. Think of the user +making a copy of a file, than modifying it. He would expect to end up with +two files, the unchanged original and the modified copy. But what he really +gets is two symlinks pointing to the same modified file. + +> I agree, it now copies rather than hard linking. [[done]] --[[Joey]] diff --git a/doc/bugs/Syncing_creates_broken_links_instead_of_proper_files.mdwn b/doc/bugs/Syncing_creates_broken_links_instead_of_proper_files.mdwn new file mode 100644 index 0000000000..8defcfe5fa --- /dev/null +++ b/doc/bugs/Syncing_creates_broken_links_instead_of_proper_files.mdwn @@ -0,0 +1,47 @@ +What steps will reproduce the problem? + +Create two repositories by running git annex webapp. Sync them by linking them to the same xmpp account. Add files on both sides. + +What is the expected output? What do you see instead? + +I expect the same file to show up on both sides with the same contents. Instead adding a file on any side creates a broken link with the same name on the other side. For example: + +Side A: + + $ ls -la + total 20 + drwxrwxr-x 3 pedrocr pedrocr 4096 Jan 3 19:24 . + drwxr-xr-x 55 pedrocr pedrocr 4096 Jan 3 19:19 .. + lrwxrwxrwx 1 pedrocr pedrocr 178 Jan 3 19:22 bar -> .git/annex/objects/FQ/vV/SHA256E-s8--12a61f4e173fb3a11c05d6471f74728f76231b4a5fcd9667cef3af87a3ae4dc2/SHA256E-s8--12a61f4e173fb3a11c05d6471f74728f76231b4a5fcd9667cef3af87a3ae4dc2 + lrwxrwxrwx 1 pedrocr pedrocr 178 Jan 3 19:20 foo -> .git/annex/objects/g7/9v/SHA256E-s4--7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730/SHA256E-s4--7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730 + drwxrwxr-x 7 pedrocr pedrocr 4096 Jan 3 19:24 .git + -rw-r--r-- 1 pedrocr pedrocr 0 Jan 3 19:24 testing + +"foo" and "bar" are broken links that were created on Side B + +Side B: + + $ ls -la + total 24 + drwxrwxr-x 3 pedrocr pedrocr 4096 Jan 3 19:24 . + drwx------ 42 pedrocr pedrocr 4096 Jan 3 19:18 .. + -rw-r--r-- 1 pedrocr pedrocr 8 Jan 3 19:22 bar + -rw-r--r-- 1 pedrocr pedrocr 4 Jan 3 19:20 foo + drwxrwxr-x 7 pedrocr pedrocr 4096 Jan 3 19:24 .git + lrwxrwxrwx 1 pedrocr pedrocr 178 Jan 3 19:24 testing -> .git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + +In this case "testing" is a broken link and was created on Side A. + +What version of git-annex are you using? On what operating system? + + $ ./git-annex version + git-annex version: 3.20130102 + + $ uname -a + Linux wintermute 3.2.0-35-generic #55-Ubuntu SMP Wed Dec 5 17:45:18 UTC 2012 i686 i686 i386 GNU/Linux + + $ lsb_release -a + Distributor ID: Ubuntu + Description: Ubuntu 12.04.1 LTS + Release: 12.04 + Codename: precise diff --git a/doc/bugs/Syncing_creates_broken_links_instead_of_proper_files/comment_1_a2bedb2e77451b02fc66fc9ef5c4405c._comment b/doc/bugs/Syncing_creates_broken_links_instead_of_proper_files/comment_1_a2bedb2e77451b02fc66fc9ef5c4405c._comment new file mode 100644 index 0000000000..146a9b1d66 --- /dev/null +++ b/doc/bugs/Syncing_creates_broken_links_instead_of_proper_files/comment_1_a2bedb2e77451b02fc66fc9ef5c4405c._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.211" + subject="comment 1" + date="2013-01-03T19:38:07Z" + content=""" +The broken links represent files whose contents have not yet arrived in the local repository. + +If you linked your repos by XMPP, they cannot ssh from one to the other to transfer file contents. +In this case, you need to set up a transfer repository, that both your repositories can access. +"""]] diff --git a/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn b/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn new file mode 100644 index 0000000000..be6db378cf --- /dev/null +++ b/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn @@ -0,0 +1,23 @@ +I only saw this just now, but the tests fail to link/build on OSX + +
+[181 of 181] Compiling Main ( git-annex.hs, tmp/Main.o )
+Linking git-annex ...
++ make -q test
++ '[' 1 = 1 ']'
++ ../maxtime 1800 make test
+[175 of 175] Compiling Main ( test.hs, tmp/Main.o )
+test.hs:175:17:
+Not in scope: data constructor `Types.Backend.KeySource'
+test.hs:175:43:
+`Types.Backend.keyFilename' is not a (visible) constructor field name
+test.hs:175:76:
+`Types.Backend.contentLocation' is not a (visible) constructor field name
+** failed to build the test suite
+make: *** [test] Error 1
++ exit 4
+
+ +this issue seems to got introduced at commit e0fdfb2e706da2cb1451193c658dc676b0530968 + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories.mdwn b/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories.mdwn new file mode 100644 index 0000000000..b8973b33bc --- /dev/null +++ b/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories.mdwn @@ -0,0 +1,21 @@ +What steps will reproduce the problem? + +After creating new remote repositories in the webapp there's no option to delete them + +What is the expected output? What do you see instead? + +Some option to delete a repository, just like I can disable sync or change the config of a remote + +What version of git-annex are you using? On what operating system? + + $ ./git-annex version + git-annex version: 3.20130102 + + $ uname -a + Linux wintermute 3.2.0-35-generic #55-Ubuntu SMP Wed Dec 5 17:45:18 UTC 2012 i686 i686 i386 GNU/Linux + + $ lsb_release -a + Distributor ID: Ubuntu + Description: Ubuntu 12.04.1 LTS + Release: 12.04 + Codename: precise diff --git a/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_1_1b80f9cfedd25e34997fa07e08d15012._comment b/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_1_1b80f9cfedd25e34997fa07e08d15012._comment new file mode 100644 index 0000000000..5087b5f4ec --- /dev/null +++ b/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_1_1b80f9cfedd25e34997fa07e08d15012._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.211" + subject="comment 1" + date="2013-01-03T19:48:46Z" + content=""" +Something I would like to support, although complicated by what \"delete\" means. At its simplest it can mean removing the git remote configuration, but leaving the remote's data as-is, so it can be added back, or other repositories can continue to access it. More complicated, it could mean completely deleting the repository, and its data -- which would probably first need to move any data that was solely in that repository off to elsewhere. +"""]] diff --git a/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_2_53499da1185c56d8fd25f86ba41d96ce._comment b/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_2_53499da1185c56d8fd25f86ba41d96ce._comment new file mode 100644 index 0000000000..e840947386 --- /dev/null +++ b/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_2_53499da1185c56d8fd25f86ba41d96ce._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="it also doesn't recognize when a local repository has been deleted manually" + date="2013-01-09T16:13:41Z" + content=""" +I created a new local repository to play with the new Direct Mode features. I ended up deleting it (deleting the directory), but when I launched the assistant, I was still given the option of \"switching repository\" to the now-deleted one. It threw the error Internal Server Error (user error (git [\"config\",\"--null\",\"--list\"] exited 126)). + +> Fixed this. --[[Joey]] + +"""]] diff --git a/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_3_3e07b8386d2c7afce2a78d24b9c260b9._comment b/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_3_3e07b8386d2c7afce2a78d24b9c260b9._comment new file mode 100644 index 0000000000..76342ec63d --- /dev/null +++ b/doc/bugs/The_webapp_doesn__39__t_allow_deleting_repositories/comment_3_3e07b8386d2c7afce2a78d24b9c260b9._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://piotr.ozarowski.pl/" + nickname="p1otr" + subject="Re: comment 1" + date="2013-01-15T09:26:21Z" + content=""" +How about asking the user? Something like: + +Do you want to... + +* only remove configuration from annex +* make sure there's a copy of all files from this repository somewhere and remove it (including remote files) +* remove remote files (and remove them from annex if there's no other copy) +"""]] diff --git a/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead.mdwn b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead.mdwn new file mode 100644 index 0000000000..6b8f66d027 --- /dev/null +++ b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead.mdwn @@ -0,0 +1,51 @@ +What steps will reproduce the problem? + +I added a (encrypted) ssh remote and everything worked fine. Now I marked the remote as dead, but git-annex still tries to upload to this remote. I recognize this because it asks for my ssh and gpg keys passwords. + +While transfering (or asking for the password), `git annex status` shows the following: +
+supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL
+supported remote types: git S3 bup directory rsync web hook
+trusted repositories: 0
+semitrusted repositories: 2
+	00000000-0000-0000-0000-000000000001 -- web
+ 	cd16b9c6-f464-11e1-9845-8749687232d2 -- here (Dell)
+untrusted repositories: 0
+dead repositories: 7
+	11379fa0-ecd6-49e2-9bec-24fc19cc7b9f -- vserver.dbruhn.de_annex
+ 	2195e036-d2ef-4357-8c89-a9aaec23ebdc -- vserver-plain
+ 	4d066ea1-fb9f-45fd-990a-5c5c836f530e -- inTmp
+ 	bb276045-6ba6-488f-88d0-39a3c5f5134d -- vserver-enc
+ 	c49f3372-3fcf-49fc-b626-73ba4454c172 -- annexBare (bareAnnex)
+ 	e52645b3-bfb6-457d-b281-967353919e29 -- AnnexUSBFAT
+ 	ea3d6acc-716c-48e8-9b6b-993b90dcc1db -- vserver2
+transfers in progress: 
+	uploading Schmidt/somefile.m4a
+
+
+ to vserver2
+available local disk space: 43 gigabytes (+1 megabyte reserved)
+temporary directory size: 389 megabytes (clean up with git-annex unused)
+local annex keys: 23
+local annex size: 396 megabytes
+known annex keys: 19
+known annex size: 396 megabytes
+bloom filter size: 16 mebibytes (0% full)
+backend usage: 
+	SHA256E: 42
+
+ +As you can see, the `vserver2` remote is marked as dead but git-annex still tries to upload. This problem keeps occuring even after restarts. + +What is the expected output? What do you see instead? + +If I do not get the `dead` status wrong, git-annex should not use these remotes. + + +What version of git-annex are you using? On what operating system? + +git-annex HEAD from yesterdays git. Ubuntu 12.10 + +Please provide any additional information below. + + diff --git a/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_1_108b3984891f82429430b503cddfb3c1._comment b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_1_108b3984891f82429430b503cddfb3c1._comment new file mode 100644 index 0000000000..f8a85cb166 --- /dev/null +++ b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_1_108b3984891f82429430b503cddfb3c1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.194" + subject="comment 1" + date="2012-11-04T20:01:35Z" + content=""" +Are you using the git-annex assistant? It's supposed to have code in it to prevent it from using dead remotes. Perhaps that is not working. + +When not using the git-annex assistant, nothing prevents git-annex from trying to continue to use dead remotes. I think this is ok; maybe the dead remote is not 100% dead yet; if it is, the remote can be removed from git's configuration to prevent it being used of course. +"""]] diff --git a/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_2_fa5b1bc26ed3e5bfe48441490c94fe3a._comment b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_2_fa5b1bc26ed3e5bfe48441490c94fe3a._comment new file mode 100644 index 0000000000..fd0d9d2ab9 --- /dev/null +++ b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_2_fa5b1bc26ed3e5bfe48441490c94fe3a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://www.dbruhn.de/" + nickname="Dominik" + subject="comment 2" + date="2012-11-08T14:13:00Z" + content=""" +Actually this happens when using the assistant. +"""]] diff --git a/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_3_0a785b5dfbf4eef30854d6bedb12b7d1._comment b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_3_0a785b5dfbf4eef30854d6bedb12b7d1._comment new file mode 100644 index 0000000000..4eeedcd6bf --- /dev/null +++ b/doc/bugs/Tries_to_upload_to_remote_although_remote_is_dead/comment_3_0a785b5dfbf4eef30854d6bedb12b7d1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.252.11.120" + subject="comment 3" + date="2012-11-13T18:00:42Z" + content=""" +I've tried & failed to reproduce this using the assistant. When I mark a repo as dead, and go into Configuration -> Manage repositories, the repository is not shown in the list of repositories it'll sync to, and it certianly doesn't upload any files to it. + + +"""]] diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn new file mode 100644 index 0000000000..8eb20baf97 --- /dev/null +++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn @@ -0,0 +1,16 @@ +The following occurs in a directory that is shared on an NFS server: + + /media/mybook/movies $ git init + Initialized empty Git repository in /media/mybook/movies/.git/ + /media/mybook/movies $ git annex init mybook-movies + init mybook-movies + git-annex: waitToSetLock: resource exhausted (No locks available) + failed + git-annex: init: 1 failed + /media/mybook/movies $ + +This happens reliably. Is there any way around it? I have shell +access on the NFS server, but it is a NAS, so I don't think it is +capable of running git-annex. + +[[done]] diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment new file mode 100644 index 0000000000..8e951ab7c9 --- /dev/null +++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-11-15T04:40:35Z" + content=""" +git-annex uses locking to avoid problems if multiple processes are run at the same time. + +I just tested on NFS, with Linux on the server and client, and it works ok. It seems your NFS client (or server) must not support fncl locking. What OS is your NAS running? +"""]] diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment new file mode 100644 index 0000000000..bd302e6bef --- /dev/null +++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-11-15T04:46:13Z" + content=""" +You might try mounting your NAS with the mount option `local_lock=all` + +This will keep the lock files on your (I assume linux) client. If you do this make sure you don't have another client using git-annex in the same NFS directory. +"""]] diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment new file mode 100644 index 0000000000..b95c795eab --- /dev/null +++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://cgray.myopenid.com/" + nickname="cgray" + subject="comment 3" + date="2011-11-15T05:14:03Z" + content=""" +I did a bit of research and my NAS had ancient NFS software on it. I upgraded that and things are now working as expected. Sorry for the noise. +"""]] diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_e42146d2dcc4052266dd61d204aeb551._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_e42146d2dcc4052266dd61d204aeb551._comment new file mode 100644 index 0000000000..59919789e9 --- /dev/null +++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_e42146d2dcc4052266dd61d204aeb551._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmnG4EuvZWse5hvgrl0XAK-U61e-0iGaao" + nickname="David" + subject="nolock option worked" + date="2012-10-01T20:31:18Z" + content=""" +I had the same problem on my NAS, updated the firmware but it didn't solve it. The remedy was to mount the NAS with the 'nolock' option. +"""]] diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn b/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn new file mode 100644 index 0000000000..d00a6720cd --- /dev/null +++ b/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn @@ -0,0 +1,21 @@ +# Calibre + +Calibre is a somewhat popular eBook management package that's also free software. + +Install via + # apt-get install calibre + +There is a somewhat unfortunate interaction between Calibre and git-annex... + +* git-annex makes its files become read-only. By the way, that's not quite obvious from the documentation; I suggest making that more prominent. +* Calibre modifies files (not quite sure of semantics, how, or why) when doing various operations, notably such as when copying a book from one's library to one's portable reading device. + +These don't play well together, sadly. + +I'd expect most of the issue to sit on the Calibre side, and have reported it as a bug. +[Calibre bug #739045](https://bugs.launchpad.net/calibre/+bug/739045) +Preliminary indication is that they're treating it as a functionality change they'll decline to fix. Which isn't entirely unreasonable - I anticipated as much, and I don't want to treat that as a bad/wrong decision. + +However, I think it's: +* Unfortunate, as fitting Calibre together with git-annex seems like a neat idea. +* Useful to make sure that this kind of "doesn't play well together" condition is documented, even if only as a bug report. diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment new file mode 100644 index 0000000000..35a2cdb3fe --- /dev/null +++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-21T13:15:03Z" + content=""" +Maybe I will run into issues myself somewhere down the road, but generally speaking, I really really like the fact that files are immutable by default. +"""]] diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment new file mode 100644 index 0000000000..719451976b --- /dev/null +++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-03-31T19:32:25Z" + content=""" +One option would be to use the new [[news/sharebox_a_FUSE_filesystem_for_git-annex]], which would hide the immutable file details from Calibre, and proxy any changes it made through to git-annex as a series of `git annex unlock; modify; git-annex lock` +"""]] diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_3_977c5f6b82f9e18cdd81d57005bb8b89._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_3_977c5f6b82f9e18cdd81d57005bb8b89._comment new file mode 100644 index 0000000000..e23490ac49 --- /dev/null +++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_3_977c5f6b82f9e18cdd81d57005bb8b89._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 3" + date="2013-01-17T20:59:03Z" + content=""" +I'd say that the best option now is to use [[direct_mode]] for repositories with files that you want to let programs modify directly. +"""]] diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_4_ff7d2e9a39dfe12b975d04650ac57cc4._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_4_ff7d2e9a39dfe12b975d04650ac57cc4._comment new file mode 100644 index 0000000000..750ba84741 --- /dev/null +++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_4_ff7d2e9a39dfe12b975d04650ac57cc4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkvSZ1AFJdY_1FeutZr_KWeqtzjZta1PNE" + nickname="Thedward" + subject="comment 4" + date="2013-01-18T16:30:59Z" + content=""" +I couldn't get calibre to work with sharebox either, but I can confirm it works fine when I use git-annex in direct mode. +"""]] diff --git a/doc/bugs/Unknown_remote_type_webdav.mdwn b/doc/bugs/Unknown_remote_type_webdav.mdwn new file mode 100644 index 0000000000..7c6f1ae367 --- /dev/null +++ b/doc/bugs/Unknown_remote_type_webdav.mdwn @@ -0,0 +1,8 @@ +When I attempt to setup a [box.com special remote](http://git-annex.branchable.com/tips/using_box.com_as_a_special_remote/) I get the following error: + + git-annex: Unknown remote type webdav + +I'm using the Linux prebuilt tarball. Does it not include webdav support? + +> The amd64 standalone tarball was indeed built without it for the last +> release. Fixed that. [[done]] --[[Joey]] diff --git a/doc/bugs/Update_dependency_on_certificate___62____61___1.3.3.mdwn b/doc/bugs/Update_dependency_on_certificate___62____61___1.3.3.mdwn new file mode 100644 index 0000000000..2a2a1469d4 --- /dev/null +++ b/doc/bugs/Update_dependency_on_certificate___62____61___1.3.3.mdwn @@ -0,0 +1,64 @@ +What steps will reproduce the problem? + +Run: + + cabal install git-annex + +What is the expected output? What do you see instead? + +The current output is the following: + + $ cabal install git-annex + Resolving dependencies... + Configuring certificate-1.3.2... + Building certificate-1.3.2... + Preprocessing library certificate-1.3.2... + [ 1 of 10] Compiling Data.Certificate.KeyDSA ( Data/Certificate/KeyDSA.hs, dist/build/Data/Certificate/KeyDSA.o ) + [ 2 of 10] Compiling Data.Certificate.KeyRSA ( Data/Certificate/KeyRSA.hs, dist/build/Data/Certificate/KeyRSA.o ) + + Data/Certificate/KeyRSA.hs:64:27: + `RSA.private_pub' is not a (visible) field of constructor `RSA.PrivateKey' + cabal: Error: some packages failed to install: + DAV-0.3 depends on certificate-1.3.2 which failed to install. + authenticate-1.3.2 depends on certificate-1.3.2 which failed to install. + certificate-1.3.2 failed during the building phase. The exception was: + ExitFailure 1 + git-annex-3.20130107 depends on certificate-1.3.2 which failed to install. + http-conduit-1.8.6.3 depends on certificate-1.3.2 which failed to install. + http-reverse-proxy-0.1.1.1 depends on certificate-1.3.2 which failed to install. + tls-1.0.3 depends on certificate-1.3.2 which failed to install. + tls-extra-0.5.1 depends on certificate-1.3.2 which failed to install. + yesod-1.1.7.2 depends on certificate-1.3.2 which failed to install. + yesod-auth-1.1.3 depends on certificate-1.3.2 which failed to install. + +I'd rather get a message stating how awesome the software I just installed is. :) + +What version of git-annex are you using? On what operating system? + + * Debian (testing) + * GHC 7.4.1 + * Cabal 1.14.0, cabal-install 0.14.0 + * cabal list git-annex says the installing version is: 3.20130107 + +Please provide any additional information below. + +The certificate package version 1.3.2 does not seem to install properly with +this version of GHC (I think). + +Version 1.3.3 solves the issue. I don't know if there is a way for me to +override the dependency tree to try to force the version update with +cabal-install, so maybe it's worth filing a bug. + +Thanks a lot for git-annex. + +> Welcome to cabal hell! This problem is why haskell's cabal system is not +> a sufficient way for users to install git-annex, and we have to provide +> prebuilt builds. +> +> No change to git-annex can fix this problem. The problem is that +> the old version of certificate got busted by some change to one of its +> dependencies, and several libraries that git-annex depends on have not +> yet been updated to use the new version of certificate. Once those +> libraries get updated, it'll fix itself. +> +> [[done]]; not git-annex bug. --[[Joey]] diff --git a/doc/bugs/Using_Github_as_remote_throws_proxy_errors.mdwn b/doc/bugs/Using_Github_as_remote_throws_proxy_errors.mdwn new file mode 100644 index 0000000000..34d9eafb4f --- /dev/null +++ b/doc/bugs/Using_Github_as_remote_throws_proxy_errors.mdwn @@ -0,0 +1,27 @@ +What steps will reproduce the problem? + +1. cd to an already existing git repository that uses Github as a remote, with the remote format similar to git@github.com:user/repo.git +2. git annex init +3. git annex status + +What is the expected output? What do you see instead? + + $ git annex status + supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: Invalid command: 'git-annex-shell 'configlist' '/~/dlo/objectifier.git'' + You appear to be using ssh to clone a git:// URL. + Make sure your core.gitProxy config option and the + GIT_PROXY_COMMAND environment variable are NOT set. + Command ssh ["-S","/Users/dan/Documents/Web/objectifier/.git/annex/ssh/git@github.com","-o","ControlMaster=auto","-o","ControlPersist=yes","git@github.com","git-annex-shell 'configlist' '/~/dlo/objectifier.git'"] failed; exit code 1 + 0 + # ... other stuff that isn't relevant + + +What version of git-annex are you using? On what operating system? + +git-annex-3.20120825 + +Max OS X 10.8.1 + +> [[done]]; see comments --[[Joey]] diff --git a/doc/bugs/Using_Github_as_remote_throws_proxy_errors/comment_1_10616b17c3fb8286fdc64c841023f8a1._comment b/doc/bugs/Using_Github_as_remote_throws_proxy_errors/comment_1_10616b17c3fb8286fdc64c841023f8a1._comment new file mode 100644 index 0000000000..77ebcaa1a2 --- /dev/null +++ b/doc/bugs/Using_Github_as_remote_throws_proxy_errors/comment_1_10616b17c3fb8286fdc64c841023f8a1._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.14.141" + subject="comment 1" + date="2012-09-24T17:24:47Z" + content=""" +The proxy message is sent from github, so I can't do anything about that. `git@github.com:user/repo.git` is a ssh url, so git-annex tries to use it as a full git-annex remote. If you use a git:// url, git-annex will +skip it. Or you can set `git config remote.origin.annex-ignore true` (replace origin with the name of the github remote). +"""]] diff --git a/doc/bugs/Using_Github_as_remote_throws_proxy_errors/comment_2_8a72887d33e492a041f8246d93d0c778._comment b/doc/bugs/Using_Github_as_remote_throws_proxy_errors/comment_2_8a72887d33e492a041f8246d93d0c778._comment new file mode 100644 index 0000000000..5fe4e6b5ae --- /dev/null +++ b/doc/bugs/Using_Github_as_remote_throws_proxy_errors/comment_2_8a72887d33e492a041f8246d93d0c778._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmG8L2pP-i6QATf6pK9WCSGpl0O9twwh8Q" + nickname="Dan" + subject="comment 2" + date="2012-09-24T18:00:17Z" + content=""" +Ah, so with the full ssh URL, git annex thinks it's a \"real\" server and can run commands on it. Makes sense. Thanks! +"""]] diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn new file mode 100644 index 0000000000..3c9374100c --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn @@ -0,0 +1,4 @@ +I have files with very long filenames on an xfs at home. On my laptop the annex should have been checked out on an encfs, but there filenames can't be as long as on the xfs. So perhaps it would be good to limit the keysize to a sane substring of the filename e.g. use only the first 120 characters. + +> Since there seems no strong argument for a WORM100, and better options +> exist, closing. [[done]] --[[Joey]] diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment new file mode 100644 index 0000000000..41d3afb3eb --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-08T17:14:25Z" + content=""" +Seems like you probably have files in git with nearly as long filenames as the key files. Course, you can rename those yourself. + +This couldn't be changed directly in WORM without some ugly transition, but it would be possible to implement it as a WORM100 or so. OTOH, if you're going to git annex migrate, you might as well use SHA1. +"""]] diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment new file mode 100644 index 0000000000..d00191f9d8 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-08T22:02:41Z" + content=""" +What if your files have the same prefix and it happens to be 100 chars long? This can not be solved within WORM, but as Joey pointed out, SHA* exists. +"""]] diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment new file mode 100644 index 0000000000..d9c291b178 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-09T20:11:59Z" + content=""" +I wouldn't say it's completly impossible for a WORM100 to work. It would just have the contract that the pair of mtime+100chars has to be unique for each unique piece of data. + +But, I have yet to be convinced there's any point, since SHA1 exists. +"""]] diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment new file mode 100644 index 0000000000..5c08cad6e0 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 4" + date="2011-04-09T23:45:28Z" + content=""" +mtime+100chars can still get collisions and a _lot_ easier than even SHA1. This introduces more problems that it solves, imo. +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn new file mode 100644 index 0000000000..f3833b3b3a --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn @@ -0,0 +1,31 @@ +After fixing a few things - see [[bugs/the tip at commit 6cecc26206c4a539999b04664136c6f785211a41 disables the watch command on OSX]], [[bugs/Missing dependancy in commit 6cecc26206c4a539999b04664136c6f785211a41]] and [[bugs/Fix for opening a browser on a mac (or xdg-open on linux/bsd?)]] I tried the watch command on my ~180gig annex of stuff. This might be yet again related to the issue of [[bugs/Issue on OSX with some system limits]] + +the watch command segfaults + +
+x00:annex jtang$ git annex watch --foreground -d
+watch . [2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","show-ref","git-annex"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","show-ref","--hash","refs/heads/git-annex"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..38d3f769ef004b96b6d640cfb59a45f7b4edf5f6","--oneline","-n1"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..ebabe9c92516c350a30126037173080648f5930b","--oneline","-n1"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..d36d8d88847decc2320f0be22892ad94a8abe594","--oneline","-n1"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..aaa62a8191b3c964fdf546077049f626e8561b22","--oneline","-n1"] 
+[2012-07-26 12:27:16 IST] chat: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","cat-file","--batch"] 
+(scanning...) error: git-annex died of signal 11
+
+ +The above was done on the usual OSX 10.7 system that I have. + +--- + +I'll try and bisect it and find out where the problem first appeared, does the tests currently test the watch command? (also my comments seem to get moderated whether i use my openid account with google or with the native ikiwiki account, so some comments might be hidden) + +> The test suite does not currently test the watch command, unfortunatly. +> +> Wow, I had not noticed the 30 pending moderated comments.. Let them all +> thru, and I guess I'll turn off comment spam filtering for now, since +> there has apparently been none. --[[Joey]] + +--- + +> Seems this segfault is fixed. [[done]] --[[Joey]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_10_6c872dff4fcc63c16bf69d1e96891c89._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_10_6c872dff4fcc63c16bf69d1e96891c89._comment new file mode 100644 index 0000000000..952ffadc4d --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_10_6c872dff4fcc63c16bf69d1e96891c89._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="jtang" + ip="79.97.135.214" + subject="comment 10" + date="2012-07-29T10:49:05Z" + content=""" +This is looking good, no more segfaulting. +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_1_5cad24007f819e4be193123dab0d511a._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_1_5cad24007f819e4be193123dab0d511a._comment new file mode 100644 index 0000000000..23a4165a40 --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_1_5cad24007f819e4be193123dab0d511a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.189" + subject="not good.." + date="2012-07-26T17:09:16Z" + content=""" +Do you see the segfault in a repo with fewer files? + + +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_2_d449bf656a59d424833f9ab5a7fb4e82._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_2_d449bf656a59d424833f9ab5a7fb4e82._comment new file mode 100644 index 0000000000..04b7ed1ae6 --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_2_d449bf656a59d424833f9ab5a7fb4e82._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.189" + subject="comment 2" + date="2012-07-26T17:09:59Z" + content=""" +Also, you might try bisecting to find whatever commit it first started segfaulting on. +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_3_ffb1ce41477ad60840abd7a89a133067._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_3_ffb1ce41477ad60840abd7a89a133067._comment new file mode 100644 index 0000000000..42f59d1bc4 --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_3_ffb1ce41477ad60840abd7a89a133067._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="jtang" + ip="79.97.135.214" + subject="comment 3" + date="2012-07-26T17:11:55Z" + content=""" +It fails on repos with either no files or smaller repos. +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_4_cebbc138c6861c086bb7937b54f5adbc._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_4_cebbc138c6861c086bb7937b54f5adbc._comment new file mode 100644 index 0000000000..71e810291e --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_4_cebbc138c6861c086bb7937b54f5adbc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="jtang" + ip="79.97.135.214" + subject="comment 4" + date="2012-07-26T17:50:42Z" + content=""" +It just occurred to me that bisecting won't help much, as the watch command was disabled accidentally in earlier commits and doing a script for bisecting is going to be as much work as just stepping through and debugging the issue with a debugger (i might need to fire up gdb on a mac (this wont be fun)) +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_5_5e27737a5bb0e9e46c98708700318e67._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_5_5e27737a5bb0e9e46c98708700318e67._comment new file mode 100644 index 0000000000..8f781ed97b --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_5_5e27737a5bb0e9e46c98708700318e67._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 5" + date="2012-07-27T12:20:07Z" + content=""" +After some debugging, I looked at the Utility/libkqueue.c and used it as a test, it seems to be hanging/segfaulting around the call to that library. Annoyingly I get segfaults from the library every so often on OSX, it's pretty a random event. +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_6_1f92da712232d050e085a4f39063d7a6._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_6_1f92da712232d050e085a4f39063d7a6._comment new file mode 100644 index 0000000000..2aa866410a --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_6_1f92da712232d050e085a4f39063d7a6._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.20" + subject="comment 6" + date="2012-07-27T16:34:45Z" + content=""" +Are you seeing libkqueue crash when it's called from a debugger or C program, rather than from Haskell? + +Are you building for 32 or 64 bit? You might try getting the 32 bit version of GCC (or The Haskell Platform) and see if it does better. There is a known GCC crashes on 64 bit OSX involving C libraries, +although this bug report doesn't seem to apply, since we're not using ghci . + +Are you building with cabal, or using the Makefile? + +You might try reverting git commit da4c506d61115236f3e43dd0bd17f30cd54df950 + +You might try disabling the -threaded option in the cabal file or Makefile. + +I ssh'd to the OSX box I have an account on, and confirmed that git-annex watch still works there as of the current head of the `assistant` branch. That's a 64 bit GHC system, FWIW. + +Do you see the crash when building from the `master` branch, or only `assistant`? Master has the watch command, but it's much out of date, so this will tell if the problem was introduced recently... and you might still have to bisect it since I can't reproduce it. :( +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_7_4153dc8029c545f8e86584a38bd536fb._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_7_4153dc8029c545f8e86584a38bd536fb._comment new file mode 100644 index 0000000000..ccb4593ec8 --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_7_4153dc8029c545f8e86584a38bd536fb._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 7" + date="2012-07-27T17:23:24Z" + content=""" +I'm using make and a 64bit version of haskell-platform, it's fine on the master branch. It's just crashing on the assistant branch, I'm just thinking out loud, but could I share the binaries that I have with you (I'd like to grab your binaries too) to see see if its just some silly problem with my build environment. + +I'm seeing the crash when I'm running git-annex (in haskell), when I run libkqueue in a debugger it behaves randomly, is mostly succeeds, but every so often it fails. A back trace reveals nothing so I am a bit at a loss. + +I've tried disabling the threaded option, but it still crashes, I will give it another try later when I get home. The problem seems to occur on my desktop mac in work and my home mac, however it is fine on my linux machines. + +Could I ask which version of OSX do you have access to? is it 10.6 or 10.7 ? + +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_8_f85b6eb5bfd28ffc6973fb4ab0fe4337._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_8_f85b6eb5bfd28ffc6973fb4ab0fe4337._comment new file mode 100644 index 0000000000..cbbd2933cf --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_8_f85b6eb5bfd28ffc6973fb4ab0fe4337._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.20" + subject="comment 8" + date="2012-07-27T18:10:06Z" + content=""" +I've reproduced a crash on OSX, involving not kqueue, but the WebApp's use of getaddrinfo. I've fixed that, but several things you've said in this bug report don't 100% add up to this being the same crash you've been seeing (for one thing, this can't affect `git annex watch`), so I'll wait for you to confirm. +"""]] diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_9_c747c488461c98cd285b51d3afc2c3eb._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_9_c747c488461c98cd285b51d3afc2c3eb._comment new file mode 100644 index 0000000000..b23ae0125c --- /dev/null +++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_9_c747c488461c98cd285b51d3afc2c3eb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="jtang" + ip="79.97.135.214" + subject="comment 9" + date="2012-07-27T18:17:45Z" + content=""" +Ah, it's alive, I'm testing on my home machine right now and it's functioning as expected. I've tested on my work machine as well and the watch command works as expected! (short of the existing system limits which fails for my bigger annexes). + +Apologies for sending you on the wrong path with the kqueue path. I reckon if I give this a good testing over the weekend is a good idea before closing this bug, I will report back in a few days on this after giving a thrashing of files! +"""]] diff --git a/doc/bugs/WebDAV_HandshakeFailed_.mdwn b/doc/bugs/WebDAV_HandshakeFailed_.mdwn new file mode 100644 index 0000000000..40c31b3cb5 --- /dev/null +++ b/doc/bugs/WebDAV_HandshakeFailed_.mdwn @@ -0,0 +1,5 @@ +When I attempt to add a Box.com special remote to my annex I get the following error: + + git-annex: HandshakeFailed (Error_Protocol ("certificate rejected: certificate is not allowed to sign another certificate",True,CertificateUnknown)) + +Running git-annex version: 3.20130207 diff --git a/doc/bugs/WebDAV_HandshakeFailed_/comment_1_40499110ea43bc99ad9dd9f642da434c._comment b/doc/bugs/WebDAV_HandshakeFailed_/comment_1_40499110ea43bc99ad9dd9f642da434c._comment new file mode 100644 index 0000000000..9bcc489ee5 --- /dev/null +++ b/doc/bugs/WebDAV_HandshakeFailed_/comment_1_40499110ea43bc99ad9dd9f642da434c._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlqOu7P4tb4D-Xo2pYrjln2NsAObtErliM" + nickname="Alexander" + subject="Same problem" + date="2013-02-24T16:51:52Z" + content=""" +Hello, + +same problem with box.com and german telekom (https://webdav.mediencenter.t-online.de). +I'm using git-annex (3.20130216) from debian/unstable on amd64. +git annex --debug is not helpful I think. Any other trace I can help with? + +Regards, +Alexander +"""]] diff --git a/doc/bugs/WebDAV_HandshakeFailed_/comment_2_506712e8cc5b47b9bd69edf67ae54da7._comment b/doc/bugs/WebDAV_HandshakeFailed_/comment_2_506712e8cc5b47b9bd69edf67ae54da7._comment new file mode 100644 index 0000000000..53642bce4c --- /dev/null +++ b/doc/bugs/WebDAV_HandshakeFailed_/comment_2_506712e8cc5b47b9bd69edf67ae54da7._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.210" + subject="comment 2" + date="2013-02-24T19:09:14Z" + content=""" +This is a reversion in version haskell-tls-extra-0.4.6.1. + +Opened a bug about this: +"""]] diff --git a/doc/bugs/Webapp_fails_to_resolve_ipv6_hostname.mdwn b/doc/bugs/Webapp_fails_to_resolve_ipv6_hostname.mdwn new file mode 100644 index 0000000000..da9bb46271 --- /dev/null +++ b/doc/bugs/Webapp_fails_to_resolve_ipv6_hostname.mdwn @@ -0,0 +1,15 @@ +What steps will reproduce the problem? + +From the webapp, go to Configuration > Manage repositories > Remote server. Enter a hostname that only has an IPv6 hostname (e.g. ipv6.google.com). Click Check this server. + +What is the expected output? What do you see instead? + +Expect the application to attempt to check the server via SSH. Instead, it results in error "cannot resolve host name". + +What version of git-annex are you using? On what operating system? + +git-annex 3.20120924 on Debian testing (amd64). + +Please provide any additional information below. + +> Thanks, [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace.mdwn b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace.mdwn new file mode 100644 index 0000000000..5942aa4917 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace.mdwn @@ -0,0 +1,47 @@ +What steps will reproduce the problem? +Created a repository (working directory), then an another one on the same computer, +in a different folder for backup repository. + +The initial folder, what I imported, contained: +741MB, 23381 files, most of the files are 30-150kB in sizes. + +After 18 hours of continous work +(I started 28 hours ago, but the laptop died one time, +and I needed to restart git-annex 4 times in total, and the laptop overheated once): +The initial directory (/home/user/down) contains at this point (still not finished): +8.1GB + +The target directory(/mnt/dat/annex2) contains: +975MB + + +What is the expected output? What do you see instead? +I expect maximum three times the usefull data size. So I can calculate with (rule of thumb). +(Ie. if I want to put 1GB data into annex, it would need maximum (at any point of time 3GB of data) + +Currently it uses 11x(!) times the original size, and still growing. + +I also expect to stop syncing, when it used up all the available disk space +(ie. it should leave at least 200MB on the original partition, +none of the other programs like to have 0 bytes left). +The machine just freeze, ie. takes almost 10 minutes to kill process, +and delete something to get back to life again. + +Also some kind of feedback how many files has been synchronized, +because currently the dashboard does not indicate any useful info. +Also the log page seems only growing, making firefox crashes. + +Self-controlled resource hogging. Ie. dont use more then 50% processor, +or dont make heavy disk usage for 3-4 hours, because the laptop can overheat. + +What version of git-annex are you using? On what operating system? +http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386.tar.gz +Version: 4.20130227 +Ubuntu 10.04, laptop Toshiba L300, Intel Core2Duo T5800@2.0Ghz, 3GB ram + +Please provide any additional information below. +The syncing is still not finished. +I hope it will finish within additional 24hours. + +Best, + Laszlo diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_0._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_0._comment new file mode 100644 index 0000000000..87bec85304 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_0._comment @@ -0,0 +1,34 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="Tried to add a bug, but website fails" + date="2013-03-01T14:40:18Z" + content=""" +@joeyh: I tried to add a comment to my bugreport: +http://git-annex.branchable.com/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace.Makefile/ + +I get this error: +Error: failed to create directory /home/b-git-annex/source/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace.Makefile/: File exists + +The comment I tried to add: +Seems like the logging is the culprit: + +user@usermachine:~/down/annex$ du -ak . | sort -nr | cut -f2 | xargs -d '\n' du -sh |head -n 30 - +8,2G . +7,5G ./.git +6,5G ./.git/annex +2,7G ./.git/annex/daemon.log +1,8G ./.git/annex/daemon.log.1 +1,5G ./.git/annex/daemon.log.5 +980M ./.git/objects +742M ./mydir +640M ./mydir/wp +616M ./mydir/wp/wd +314M ./mydir/wp/wd/2012 +278M ./.git/annex/daemon.log.4 +226M ./mydir/wp/wd/2011 +154M ./.git/annex/daemon.log.6 +109M ./.git/annex/objects + + +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_10_037a6dd6e15ef5f789a1f364f7507b53._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_10_037a6dd6e15ef5f789a1f364f7507b53._comment new file mode 100644 index 0000000000..f973238d83 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_10_037a6dd6e15ef5f789a1f364f7507b53._comment @@ -0,0 +1,45 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 10" + date="2013-03-03T18:31:24Z" + content=""" +The problem with restarting, that it seems, it puts everything one more time into .git. +(if we ignore the logging problem). + +Right now, here is the: + $ du -ak . | sort -nr | cut -f2 | xargs -d '\n' du -sh |head -n 30 - +output: + 8,7G . + 8,0G ./.git + 6,3G ./.git/annex + 3,5G ./.git/annex/daemon.log.2 + 2,5G ./.git/annex/daemon.log + 1,7G ./.git/objects + 742M ./mydir + 640M ./mydir/wp + 616M ./mydir/wp/wd + 314M ./mydir/wp/wd/2012 + 226M ./mydir/wp/wd/2011 + 227M ./.git/annex/daemon.log.1 + 109M ./.git/annex/objects + +If you observe ./.git/objects dir, is 1,7G, +while yesterday it was 742M (see my first comment). + +Other problem, that once it runs out of space, threads crashes: + Pusher crashed: user error (git [\"--git-dir=/home/user/down/annex/.git\",\"--work-tree=/home/user/down/annex\",\"update-index\",\"-z\",\"--index-info\"] exited 128) + NetWatcherFallback crashed: fd:28: hGetLine: end of file + DaemonStatus crashed: /home/user/down/annex/.git/annex/daemon.status.tmp7564: hClose: resource exhausted (No space left on device) + +Right now, I think I tested everything I could. + +Where is the autobuild? This one still is 2013 feb 27: +http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386.tar.gz + +Anyway, once the new version is out, I will retest this experiment. + +Best, + Laszlo + +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_11_614e4110188fc6474e7da50fc4281e13._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_11_614e4110188fc6474e7da50fc4281e13._comment new file mode 100644 index 0000000000..38d9bb6213 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_11_614e4110188fc6474e7da50fc4281e13._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 11" + date="2013-03-03T18:36:31Z" + content=""" +The autobuild is linked to on [[install/Linux_standalone]] + +I don't understand how you're \"restarting\" and not deleting the old .git repository. The right way would be to stop the assistant, run \"git annex uninit\", and then remove \".git\" +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_12_dcb74fb91e1c2f0db4efd68c8bcbc96c._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_12_dcb74fb91e1c2f0db4efd68c8bcbc96c._comment new file mode 100644 index 0000000000..8bf20fc18c --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_12_dcb74fb91e1c2f0db4efd68c8bcbc96c._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 12" + date="2013-03-03T21:24:18Z" + content=""" +restarting == Stop git-annex using the gui (Confirm shutdown daemon). Then check if there are running git process (ps -e |grep git), +if there are kill it, if (defunct) process present, then restart computer. Then launch git-annex using \"git-annex-webapp\" script). + +So its a normal program usage (start-stop cycle). + +The .git directory is growing. Without a single file moving/deleting/renaming/editing in the working directory. + +So the .git directory just grows, and I think it is a bug, and a bad one. (if we ignore the logging problem, the slow startup, the failing of disk full). + +I'll give it a spin to the autobuild (http://downloads.kitenet.net/git-annex/autobuild/i386/git-annex-standalone-i386.tar.gz), and report back. It will take at least 2 day, to repeat my whole experience. So expect update from me wednesday the soonest. + +Best, Laszlo + +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_1_d2c63723fa4bf828873770a42ffaab20._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_1_d2c63723fa4bf828873770a42ffaab20._comment new file mode 100644 index 0000000000..3096e596a0 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_1_d2c63723fa4bf828873770a42ffaab20._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-03-01T16:44:17Z" + content=""" +Can you please post a representative sample of the logs? +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_3_52f0db73dc38c3e3a73f6c7a420bf016._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_3_52f0db73dc38c3e3a73f6c7a420bf016._comment new file mode 100644 index 0000000000..b7b866dc6f --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_3_52f0db73dc38c3e3a73f6c7a420bf016._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 3" + date="2013-03-01T17:31:31Z" + content=""" +I have made the assistant check hourly if its logs are larger than one megabyte, and trim old logs. +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_4_93596b4d5a48ffcf4bc11ba9c83cf7ca._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_4_93596b4d5a48ffcf4bc11ba9c83cf7ca._comment new file mode 100644 index 0000000000..7efe773035 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_4_93596b4d5a48ffcf4bc11ba9c83cf7ca._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 4" + date="2013-03-01T19:52:13Z" + content=""" +Here is the last 300 lines of a 3.3GB daemon.log file: +http://pastebin.com/TF35H5gu +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_5_de94e80dde6d12485140bb079d74d775._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_5_de94e80dde6d12485140bb079d74d775._comment new file mode 100644 index 0000000000..16f8cf65a5 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_5_de94e80dde6d12485140bb079d74d775._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 5" + date="2013-03-01T20:20:21Z" + content=""" +Aha, so it's a direct mode repository and all this \"typechange\" stuff git outputs in such repositories is presumably the main culprit for log bloat. I have made it suppress that output. + +Excessive repacking may also explain a lot of the CPU usage and slowness. What does this say? + +`grep \"Auto packing the repository for optimum performance.\" .git/annex/*.log |wc -l` + +It may make sense for the assistant to tune `gc.auto` to avoid repacks. You might try disabling repacking altogether and see if it helps: `git config gc.auto 0` +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_6_5f34c3d449247b4bce4665b3ea4d054c._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_6_5f34c3d449247b4bce4665b3ea4d054c._comment new file mode 100644 index 0000000000..8da24d60f9 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_6_5f34c3d449247b4bce4665b3ea4d054c._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 6" + date="2013-03-02T10:05:38Z" + content=""" + $ ls -lah daemon.log + -rw-r--r-- 1 user user 3,3G 2013-03-02 10:54 daemon.log + + $ cat daemon.log |wc -l + 30746274 + + $ grep \"Auto packing the repository for optimum performance.\" daemon.log |wc -l + 568 + + $ grep \"typechange: \" daemon.log |wc -l + 30713158 + +I'm not sure where should I issue the git command. Inside the annex/.git directory? +The useful data is also in git repository locally. +I try to RTFM over the internet, thanks for the pointer! + + + +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_7_b43ae8aec23ba3acaf70edc0de058710._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_7_b43ae8aec23ba3acaf70edc0de058710._comment new file mode 100644 index 0000000000..4f53dedd1e --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_7_b43ae8aec23ba3acaf70edc0de058710._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 7" + date="2013-03-02T11:03:46Z" + content=""" + Ok, quick question: Should I issue git config gc.auto 0, in both repositories? + (it puts variable inside .git/config) + + With this gc.auto 0, should the \"typechange: \" logging change? (because it still in the log) + + I'm testing the whole experience, and report back, + also when new build is available i will switch to it, to report about the logging size. + +Thank you for your fast response! + +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_8_13b8e0a62f6b6d02960687e206a8b016._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_8_13b8e0a62f6b6d02960687e206a8b016._comment new file mode 100644 index 0000000000..4d3f11f365 --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_8_13b8e0a62f6b6d02960687e206a8b016._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawm5iosFbL2By7UFeViqkc6v-hoAtqILeDA" + nickname="Laszlo" + subject="comment 8" + date="2013-03-02T12:54:14Z" + content=""" +The startup time is somewhere between 4-5 hours +(dunno exactly, its only running about 3 hours) with this gc.auto 0. + +I will try to relaunch git-annex-assistant once it finishes startup, +but I can report back only tomorrow because it may take 5 hours/try. + +Laszlo + +"""]] diff --git a/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_9_818b94a74b01a210d1106dd35bc932d8._comment b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_9_818b94a74b01a210d1106dd35bc932d8._comment new file mode 100644 index 0000000000..11b3efec0b --- /dev/null +++ b/doc/bugs/When_syncing_two_repositories__44___git_annex_uses_9x_times_diskspace/comment_9_818b94a74b01a210d1106dd35bc932d8._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 9" + date="2013-03-03T17:45:34Z" + content=""" +Well, that sounds like an improvement for sure with gc.auto=0. I have made that be done by default when committing. + +The typechange fix is available in the autobuilds already. +"""]] diff --git a/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase.mdwn b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase.mdwn new file mode 100644 index 0000000000..e6f3524b2b --- /dev/null +++ b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase.mdwn @@ -0,0 +1,21 @@ +I'm using S3 special remote that I initialized like this: + + git annex initremote s3 type=S3 encryption=EEF2B390 datacenter="EU" storageclass="REDUCED_REDUNDANCY" + +Instead of asking me my key passphrase (I have gpg-agent running), it ask me a new passphase an then repeat it. So it's not my passphrase because if I kill gpg-agent, it will ask me three passphrase for each file: one mine and two for the new one. If i don't put anything there it says: + + copy GOPR1672.JPG (checking s3...) (to s3...) gpg: error creating passphrase: Invalid passphrase + gpg: symmetric encryption of `[stdin]' failed: Invalid passphrase + failed + git-annex: fd:13: hPutBuf: resource vanished (Broken pipe) + copy GOPR1673.JPG (checking s3...) (to s3...) + +So I create a new passphrase (two times) for each file I tried to upload. The problem is that I have more than 12000 files to upload! + +What's this new passphrase for? + +BTW: git-annex version: 3.20130102 same precompiled binary on Arch Linux. + +> I've reproduced this with gpg 2.0.19. It is a documented incompatability +> between gpg 1.x and 2.x; the latter needs --batch included in its +> parameters. I've put in a fix. [[done]] diff --git a/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_1_a4fc30bf7d39cae337286e9e815e6cba._comment b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_1_a4fc30bf7d39cae337286e9e815e6cba._comment new file mode 100644 index 0000000000..b006b3b442 --- /dev/null +++ b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_1_a4fc30bf7d39cae337286e9e815e6cba._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="joksnet" + ip="193.253.37.245" + subject="Working now" + date="2013-01-16T01:36:43Z" + content=""" +Ok. I don't what happend, but it's working now. I restart gpg-agent a few times with the script found [here](https://wiki.archlinux.org/index.php/GPG#gpg-agent). And now it doesn't ask for any passphrase. + +Par contre, I need to export the AWS_* variables if I reboot my pc. I think is because I didn't set embedcreds=yes when initremote. Can I change it now? O I have to remote rm and initremote again? + +Thank you, +Juan +"""]] diff --git a/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_2_e5d42b623017acedf6a3890ce15680a3._comment b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_2_e5d42b623017acedf6a3890ce15680a3._comment new file mode 100644 index 0000000000..d742bbe58d --- /dev/null +++ b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_2_e5d42b623017acedf6a3890ce15680a3._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 2" + date="2013-01-16T02:17:26Z" + content=""" +Someone else reported what sounds like the same bug at [[encryption_given_a_gpg_keyid_still_uses_symmetric_encryption]]. It sounds like this is somehow an agent bug. I cannot reproduce it. I can hypothesise that, if this bug is occurring, you'll be prompted for a passphrase when running this command.. which if it happens would certianly be a bug in gpg or its agent + +touch foo; echo foo| gpg --symmetric --passphrase-fd=0 foo + +(You can run `git annex initremote $yourremote embedcreds=yes` and it'll modify the existing configuration.) +"""]] diff --git a/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_3_e5150b65b514896e14b9ad3d951963f7._comment b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_3_e5150b65b514896e14b9ad3d951963f7._comment new file mode 100644 index 0000000000..4764dcd5d3 --- /dev/null +++ b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_3_e5150b65b514896e14b9ad3d951963f7._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joksnet" + ip="193.253.37.245" + subject="comment 3" + date="2013-01-16T17:23:26Z" + content=""" +It's a GPG/GPG-Agent (2.0.19) bug because the command you give me ask me for a new passphrase. + +Thank you for the answer. +"""]] diff --git a/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_4_47c2fc167b0c396edc40468fb7c7bfee._comment b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_4_47c2fc167b0c396edc40468fb7c7bfee._comment new file mode 100644 index 0000000000..20f6564c87 --- /dev/null +++ b/doc/bugs/With_S3__44___GPG_ask_for_a_new_passphrase/comment_4_47c2fc167b0c396edc40468fb7c7bfee._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 4" + date="2013-01-16T19:05:38Z" + content=""" +Does it ask for a passphrase if --batch is included in gpg's parameters? +"""]] diff --git a/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content.mdwn b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content.mdwn new file mode 100644 index 0000000000..bdf48760c1 --- /dev/null +++ b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content.mdwn @@ -0,0 +1,49 @@ +# What steps will reproduce the problem? + + echo "TEST CONTENT" > fileA + cp fileA fileB + git annex add file{A,B} + git annex drop fileA --force + cat fileB + +# What is the expected output? What do you see instead? + +## expected: + +--> TEST CONTENT + +## observed: + +--> cat: fileB: No such file or directory + + +# What version of git-annex are you using? On what operating system? + +git-annex version: 3.20121017 + +# Please provide any additional information below. + +I really like git annex's feature, to store the same content only once. But as this happens transparently (i.e. the user does not need to no, nor is he told, that contents are identical (which is very comfortable, of course)), the "git annex drop" function is broken. For it effectively deleting (seemingly) random files, WITHOUT notifying the user. + + +# Possible solution? + +One simple solution would be to use "git annex find" functionality to see who else uses the file and NOT deleting it. + +But this still leaves a problem: + +Consider the following variation of the above example and assume, that "drop" does not delete content that is still used (i.e. implementing the above solution). + + echo "TEST CONTENT" > fileA + cp fileA fileB + git annex add file{A,B} + git rm fileB + git annex drop fileA --force + git checkout --force + cat fileB +--> cat: fileB: No such file or directory + +Here again, the problem is, that the user would probably (correct me if I am wrong) expect that the fileB still exists, because removing a file and checking it out again is expected to not mess with the annex contents (?). He does not know, that the "annex frop fileA" actually drop fileB's contents, because there was no additional file linking to it. It effectively performed a "git annex dropunused". + +> We seem to have agreed this is reasonable behavior, and a doc change was done. +> Do feel free to suggest other doc changes.. [[done]] --[[Joey]] diff --git a/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_1_2eb20b65582fa7f271b1d0bb5560d08c._comment b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_1_2eb20b65582fa7f271b1d0bb5560d08c._comment new file mode 100644 index 0000000000..712657e82c --- /dev/null +++ b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_1_2eb20b65582fa7f271b1d0bb5560d08c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 1" + date="2012-10-28T17:13:32Z" + content=""" +You can avoid this by not using a deduplicating backend; for example you can use the WORM backend. + +However, the foot shooting actually occurs due to using drop --force, which is explicitly asking git-annex to be unsafe. If you want to be safe, simply don't use --force. If you want to safely delete a file, simply `git rm` it, and then `git annex unused` will come along and find content that can safely be removed. +"""]] diff --git a/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_2_b14e1d31dd6a8fb930fcc0bec798e194._comment b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_2_b14e1d31dd6a8fb930fcc0bec798e194._comment new file mode 100644 index 0000000000..005c45be26 --- /dev/null +++ b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_2_b14e1d31dd6a8fb930fcc0bec798e194._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlgyVag95OnpvSzQofjyX0WjW__MOMKsl0" + nickname="Sehr" + subject="comment 2" + date="2012-10-28T21:46:08Z" + content=""" +I onyl used \"--force\" for demonstration purposes. I could also set + + annex.numcopies = 0 + +which removes the need \"force\". While this setting can be totally reasonable in certain circumstancing it seems very dangerous, that completely unrelated files might unwillingly be deleted. + +I agree with you, that a possible solution could be to not use a deduplicating backend. But my point is, that this needs to be either changed or documented. Because even if the user can \"fix\" this by changing his behavior, he will probably only do so AFTER he lost something. + +Instead of changing the program (to include a check), I would at least suggest an addition to \"drop\"'s documentation: + + +\"drop\": keep in mind, that on dedcupliocating backends, you might end up deleting more than one file. to be perfectly safe, use git-rm and git-annex dropunused. +"""]] diff --git a/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_3_1892bcfbe3c462aa74552a241d65cad9._comment b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_3_1892bcfbe3c462aa74552a241d65cad9._comment new file mode 100644 index 0000000000..8dfc30a604 --- /dev/null +++ b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_3_1892bcfbe3c462aa74552a241d65cad9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 3" + date="2012-10-28T23:29:46Z" + content=""" +numcopies=0 is inherently unsafe, and unreasonable if you value your data at all. I've added some warnings about it to the man page. +"""]] diff --git a/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_4_dfa0e31996eaa14e2945c1d11670c4d9._comment b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_4_dfa0e31996eaa14e2945c1d11670c4d9._comment new file mode 100644 index 0000000000..d82a3026d4 --- /dev/null +++ b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_4_dfa0e31996eaa14e2945c1d11670c4d9._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlgyVag95OnpvSzQofjyX0WjW__MOMKsl0" + nickname="Sehr" + subject="comment 4" + date="2012-10-29T00:00:33Z" + content=""" +Thanks, that's cool. Admittedly, I cannot think of too many scenarios, where there are two identical files without the user's knowloedge. And an even smaller subset of scenarios, where one would want to issue a \"drop\" on (only) one of these due to storage shortages. + + + +By the way, I LOVE git-annex. + + +PS: I just realized, that the same applies to the \"move\" command. +"""]] diff --git a/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_5_e2a9336cf1080c158765d4adfe72f26b._comment b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_5_e2a9336cf1080c158765d4adfe72f26b._comment new file mode 100644 index 0000000000..cc93b0f49f --- /dev/null +++ b/doc/bugs/__34__drop__34___deletes_all_files_with_identical_content/comment_5_e2a9336cf1080c158765d4adfe72f26b._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="2001:4978:f:21a::2" + subject="comment 5" + date="2012-10-29T00:03:40Z" + content=""" +You're guaranteed to still have at least 1 copy of the file after move though, so you can get it back. + +"""]] diff --git a/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn b/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn new file mode 100644 index 0000000000..94c495735e --- /dev/null +++ b/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn @@ -0,0 +1,23 @@ +"git annex watch" will add the file generated by "git annex map", which is +probably not intended. Shouldn’t this file be created in /tmp or +.git/annex/ or somewhere else? + +> Indeed, so [[done]] --[[Joey]] + + /tmp $ cd test/ + /tmp/test $ git init + Initialized empty Git repository in /tmp/test/.git/ + /tmp/test $ git annex init + init ok + (Recording state in git...) + /tmp/test $ git annex watch + /tmp/test $ git annex map + map /tmp/test ok + + running: dot -Tx11 map.dot + + ok + /tmp/test $ ls -l + insgesamt 4 + lrwxrwxrwx 1 jojo jojo 180 Jul 15 23:36 map.dot -> .git/annex/objects/P3/76/SHA256-s208--44199582b5948512ff12cf03de0b86fa1bebf09785dba2827fe52afee0afbe3d/SHA256-s208--44199582b5948512ff12cf03de0b86fa1bebf09785dba2827fe52afee0afbe3d + diff --git a/doc/bugs/__34__make_test__34___fails_silently.mdwn b/doc/bugs/__34__make_test__34___fails_silently.mdwn new file mode 100644 index 0000000000..8632f03f51 --- /dev/null +++ b/doc/bugs/__34__make_test__34___fails_silently.mdwn @@ -0,0 +1,4 @@ +`make test` fails silently when the test program cannot be built. This happens, for example, when attempting to compile git-annex with `QuickCheck-2.4.2`. + +> I've made "make test" exit nonzero if the test suite cannot be built. +> [[done]] --[[Joey]] diff --git a/doc/bugs/__34__make_test__34___fails_silently/comment_1_f868e34f41d828d4571968d1ab07820a._comment b/doc/bugs/__34__make_test__34___fails_silently/comment_1_f868e34f41d828d4571968d1ab07820a._comment new file mode 100644 index 0000000000..00f67ad114 --- /dev/null +++ b/doc/bugs/__34__make_test__34___fails_silently/comment_1_f868e34f41d828d4571968d1ab07820a._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2012-01-03T16:54:19Z" + content=""" +The code is: + +
+@if ! $(GHCMAKE) -O0 test; then \
+                echo \"** not running test suite\" >&2; \
+else \
+
+ +The error message from the compiler, followed by the above error message does not seem \"silent\". It does exit 0 without running the test suite if it cannot be built. +"""]] diff --git a/doc/bugs/__34__make_test__34___fails_silently/comment_2_fb9e8e2716b0dea15b0d4807ae7cd114._comment b/doc/bugs/__34__make_test__34___fails_silently/comment_2_fb9e8e2716b0dea15b0d4807ae7cd114._comment new file mode 100644 index 0000000000..3b76e3f146 --- /dev/null +++ b/doc/bugs/__34__make_test__34___fails_silently/comment_2_fb9e8e2716b0dea15b0d4807ae7cd114._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://peter-simons.myopenid.com/" + ip="77.188.44.113" + subject="comment 2" + date="2012-01-03T18:09:38Z" + content=""" +When \"make test\" fails to run any tests at all, it should not return exit code 0. This behavior is quite misleading, and it means that automated build systems are not going to detect the fact that the test suit could not be run. +"""]] diff --git a/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn new file mode 100644 index 0000000000..f129abf623 --- /dev/null +++ b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn @@ -0,0 +1,15 @@ +The following commands show the failure: + +$ mkdir d && touch d/f + +$ mkdir g && cd g && git annex add ../d/f + +add ... ok + +error: Invalid path '.git/annex/objects/Jx/... + +... + +Then it seems it is enough to 'git add ../d/f' to complete the operation. + +> Thanks for reporting, [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content.mdwn b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content.mdwn new file mode 100644 index 0000000000..5f8cf2dd57 --- /dev/null +++ b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content.mdwn @@ -0,0 +1,22 @@ +What steps will reproduce the problem? + +running the assistant on my setup, which is a central 'client' repo, and two 'backup' repos on a USB drive and an ssh connection to another server. Both of those backup repos are non-bare, and I occasionally manually run 'git annex sync' on them to keep them up to date. I have 'numcopies' set to 2. + + +What is the expected output? What do you see instead? + +I expect everything to be copied to the 'backup' repos and content in the 'archive' subdirectories of the 'client' repo to be dropped. + +What happened instead: This morning, I started up the assistant without the USB drive present. I added one file to my client repo. The assistant began copying it to the SSH backup repo. I then plugged in the USB drive, and it began copying it to the USB repo; however, it *also* queued up large amounts of content in 'archive' subdirectories and started copying it down onto my client repo, which is unacceptable because there is not actually enough disk space on the client machine to hold everything in the 'archive' subdirectories! I stopped the assistant and started it up again. It started doing the same thing (queueing up archive content to transfer to client), and in addition started dropping content from my USB drive. This seems very similar to the bug in the previous version, where the globbing was broken. It's acting as if my usb remote preferred no content at all, and it transfers everything away from it and drops it -- and at the same time, brings in content which is not supposed to be there on my client machine. Strangely it's not doing the same thing on the ssh remote -- the ssh remote is fine. No anomalies there -- it has not been trying to drop content from there. + +This is just happening this morning -- I compiled 4.20130227 last night and ran the assistant and it behaved correctly, with no bugs at all. Now when I run it, it's going crazy. + +What version of git-annex are you using? On what operating system? + +4.20130227 on OS X. + + +Please provide any additional information below. + +One thing that would be incredibly helpful with the kinds of bugs I've seen in the assistant is if the daemon.log contained info about *why* it is dropping or transferring content. "transferring content because there was only 1 copy and we need to fulfill numcopies" "transferring content because it matches such-and-such a preferred content expresson" "dropping content because it does not match this preferred content expression" -- that would remove a lot of mystery about the way the assistant is working. + diff --git a/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_1_56f9cd5cc2e089b32cb076dc2e2a8ca5._comment b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_1_56f9cd5cc2e089b32cb076dc2e2a8ca5._comment new file mode 100644 index 0000000000..cf309b8b34 --- /dev/null +++ b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_1_56f9cd5cc2e089b32cb076dc2e2a8ca5._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-03-01T19:59:23Z" + content=""" +I've added some debug info to the log for transfers and drops. You have to start the assistant with the --debug flag, then you'll get stuff like: + +
+[2013-03-01 15:23:10 JEST] Committer: queued Upload UUID \"a0cd7cb6-57ee-4551-955f-aa77738135cc\" myfile : newly added file
+[2013-03-01 15:25:00 JEST] Watcher: queued Download UUID \"a0cd7cb6-57ee-4551-955f-aa77738135cc\" ./foo : new or renamed file wanted
+[2013-03-01 15:54:00 JEST] Watcher: dropped ./archive/foo (from: here) (copies now: 1): file renamed
+
+ +It would take more work to show the preferred content expressions that caused the transfer or drop, but this is simple to determine: + +* for Uploads it should always be the preferred content expression of the remote being uploaded to +* for Downloads it is always the preferred content expression of the local repository +* for drops it is always the preferred content expression of the repository it is dropped from + +Have not managed to reproduce problem yet. +"""]] diff --git a/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_2_21c0f7f328cb51080fbd97e086c47a30._comment b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_2_21c0f7f328cb51080fbd97e086c47a30._comment new file mode 100644 index 0000000000..5eb511b655 --- /dev/null +++ b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_2_21c0f7f328cb51080fbd97e086c47a30._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 2" + date="2013-03-03T18:44:35Z" + content=""" +I rebuilt and ran with --debug, and started up git annex. It behaved itself. + +I tried restarting it, unplugging and plugging back in my USB drive to see if it provoked any problems -- none. + +I decided to try adding something. I added a music video to my Movies/Music directory. It uploaded it to my ssh remote but not to the USB drive and, bizarrely, when I use \"git annex whereis\" it doesn't show it as existing *anywhere*: + + annex$ git annex whereis Movies/Music/Tanlines\ -\ All\ Of\ Me.flv + annex$ + +It does in fact exist in all three repos! The link is there: + +annex$ ls -l Movies/Music/Tanlines\ -\ All\ Of\ Me.flv +lrwxr-xr-x 1 ed staff 206 Mar 3 13:18 Movies/Music/Tanlines - All Of Me.flv -> ../../.git/annex/objects/0k/j6/SHA256E-s37822147--c9df1d6c9f6d2d72e039de9705ea4673160da32eb0cc9ea87e65003506d9297d.flv/SHA256E-s37822147 + +the object is there! + +annex$ ls -l .git/annex/objects/0k/j6/SHA256E-s37822147--c9df1d6c9f6d2d72e039de9705ea4673160da32eb0cc9ea87e65003506d9297d.flv/SHA256E-s37822147--c9df1d6c9f6d2d72e039de9705ea4673160da32eb0cc9ea87e65003506d9297d.flv +-r--r--r-- 1 ed staff 37822147 Mar 3 13:17 .git/annex/objects/0k/j6/SHA256E-s37822147--c9df1d6c9f6d2d72e039de9705ea4673160da32eb0cc9ea87e65003506d9297d.flv/SHA256E-s37822147--c9df1d6c9f6d2d72e039de9705ea4673160da32eb0cc9ea87e65003506d9297d.flv + +The corresponding object also exists in the .git/annex/objects directory of the USB remote and the ssh remote. + +here's a pastebin of the daemon.log: http://pastebin.com/BGRBQ6Rx + +Shut down the daemon to see if that changed anything; nope. \"Whereis\" still comes up blank. Tried a git annex fsck --fast to see if that changed anything; nope. \"Whereis\" still comes up blank. + +Any ideas? Is the log revealing at all? + + + + +"""]] diff --git a/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_3_3287b2f25f3b5ae4c27f4748694563ee._comment b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_3_3287b2f25f3b5ae4c27f4748694563ee._comment new file mode 100644 index 0000000000..e108052d77 --- /dev/null +++ b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_3_3287b2f25f3b5ae4c27f4748694563ee._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 3" + date="2013-03-03T18:53:23Z" + content=""" +Are you sure the file in question is checked into git? Because git-annex ignores symlinks that are not checked into git. +"""]] diff --git a/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_4_e515eca68a70d40c522805d7e0d7c0e6._comment b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_4_e515eca68a70d40c522805d7e0d7c0e6._comment new file mode 100644 index 0000000000..4a533fc92a --- /dev/null +++ b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_4_e515eca68a70d40c522805d7e0d7c0e6._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 4" + date="2013-03-03T18:58:42Z" + content=""" +Just to see what happened, I re-added that same file in another directory manually (not with the assistant). + +git annex whereis instantly showed that it was in all the other repositories. + +I finally thought to try a \"git status\" -- it showed that with for the original copy of this I added, the actual symlink was never checked into git. + + annex$ git status + # On branch master + # Untracked files: + # (use \"git add ...\" to include in what will be committed) + # + # Movies/Music/Tanlines - All Of Me.flv + nothing added to commit but untracked files present (use \"git add\" to track) + annex$ + +This is probably all unrelated to the original topic of the bug -- I am not seeing the \"client repo starts pulling in archive content, dropping it from backup remote\" issue anymore. + +"""]] diff --git a/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_5_b27f4c103dda050b6e9cf03ea3157abc._comment b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_5_b27f4c103dda050b6e9cf03ea3157abc._comment new file mode 100644 index 0000000000..434bcb1ddf --- /dev/null +++ b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_5_b27f4c103dda050b6e9cf03ea3157abc._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 5" + date="2013-03-03T19:42:19Z" + content=""" +A symlink not added to git could happen with the assistant for various reasons. For example, if the assistant was shut down before it got a chance to stage a newly added file. The assistant detects such symlinks when started up and will add them then. + +Back to the original bug report.. +"""]] diff --git a/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_6_2cc7083dab944705bf91fc00319b75e6._comment b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_6_2cc7083dab944705bf91fc00319b75e6._comment new file mode 100644 index 0000000000..9f110d30d1 --- /dev/null +++ b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_6_2cc7083dab944705bf91fc00319b75e6._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 6" + date="2013-03-04T00:20:25Z" + content=""" +Since I can't reproduce the \"copy archived content to the client repo then drop it from the backup repo\" anymore, I guess we can close. If I notice anything like that happening again I have the --debug flag available to investigate and open a new bug report. + +"""]] diff --git a/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_7_1175f9be789d4c1907f0be98e435bd2f._comment b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_7_1175f9be789d4c1907f0be98e435bd2f._comment new file mode 100644 index 0000000000..1f11f8bedc --- /dev/null +++ b/doc/bugs/__39__client__39___repo_starts_pulling_in___39__archive__39___content/comment_7_1175f9be789d4c1907f0be98e435bd2f._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 7" + date="2013-03-04T00:57:38Z" + content=""" +With the new Preferences page I added to the webapp today you can also enable debugging while the assistant is running if it starts to misbehave. + +However, I'm in no hurry to close this bug just yet. +"""]] diff --git a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn new file mode 100644 index 0000000000..a6e2423a82 --- /dev/null +++ b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn @@ -0,0 +1,40 @@ +Hi, + +I just wanted to install git-annex via cabal, as described in the install document. More specifically, I did this on my Ubuntu Lucid box: + + andreas@antares:~$ sudo aptitude install cabal-install + [...] + andreas@antares:~$ cabal update + andreas@antares:~$ cabal install quickcheck --bindir=$HOME/bin + andreas@antares:~$ cabal install git-annex -v --bindir=$HOME/bin + +However, I got this error: + + /usr/bin/ghc --numeric-version + looking for package tool: ghc-pkg near compiler in /usr/bin + found package tool in /usr/bin/ghc-pkg + /usr/bin/ghc-pkg --version + /usr/bin/ghc --supported-languages + Reading installed packages... + /usr/bin/ghc-pkg dump --global + /usr/bin/ghc-pkg dump --user + Reading available packages... + Resolving dependencies... + selecting + cabal: cannot configure git-annex-3.20120113. It requires quickcheck >=2.1 + There is no available version of quickcheck that satisfies >=2.1 + +which is really strange, because quickcheck 2.4.2 is installed: + + andreas@antares:~$ ls -a .cabal/lib/ + . .. QuickCheck-2.4.2 + +Any help is greatly appreciated :) +Andreas. + +> QuickCheck has to be spelled in mixed case. --[[Joey]] + +Sorry to disagree, this doesn't fix my problem. cabal still complains that no version >= 2.1 is available, even though 2.4.2 is installed. This problem already occurred before I explicitly installed QuickCheck. According to [[install]], the `cabal install git-annex -v --bindir=$HOME/bin` should already take care of the dependencies. + +>> You need to `cabal update` to get the fixed version of git-annex which +>> spells QuickCheck correctly. [[done]] --[[Joey]] diff --git a/doc/bugs/__91__webapp__93___pause_syncing_with_specific_repository.mdwn b/doc/bugs/__91__webapp__93___pause_syncing_with_specific_repository.mdwn new file mode 100644 index 0000000000..dad961d9fa --- /dev/null +++ b/doc/bugs/__91__webapp__93___pause_syncing_with_specific_repository.mdwn @@ -0,0 +1,5 @@ +[Due to some stupid issue on my and AT&T's part] one of my remote repositories is currently unreachable. I would like to tell the webapp/assistant to not attempt to sync with it, or, at least, modify this error message to be more specific (by telling me which repository is unreachable). + +In a red bubble it says: "Synced with rose 60justin" + +That verbage is the same if they all succeed. The only difference is the red instead of green. Would be nice to know exactly which machine to kick (if I didn't already know, eg I was syncing only with repositories not under my control). diff --git a/doc/bugs/acl_not_honoured_in_rsync_remote.mdwn b/doc/bugs/acl_not_honoured_in_rsync_remote.mdwn new file mode 100644 index 0000000000..b5f92a9fb9 --- /dev/null +++ b/doc/bugs/acl_not_honoured_in_rsync_remote.mdwn @@ -0,0 +1,57 @@ +in a setup where an rsync(+gnupg) remote is shared among different users of the same git-annex repository (ie, the people copying to there use different accounts on the rsync server), acls are not honored under some circumstances. + +the error message reads as follows: + + copy …filename… (to prometheus...) Reading passphrase from file descriptor 11 + + sending incremental file list + rsync: recv_generator: mkdir "/home/shared/photos/encrypted_storage/9a6/0ff" failed: Permission denied (13) + *** Skipping any contents from this failed directory *** + 9a6/0ff/ + rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1070) [sender=3.0.8] + + sent 185 bytes received 18 bytes 135.33 bytes/sec + total size is 2119419 speedup is 10440.49 + + This could have failed because --fast is enabled. + failed + +the acl used in my particular case is: + + # file: . + # owner: chrysn + # group: chrysn + user::rwx + group::rwx + group:family:rwx + mask::rwx + other::r-x + default:user::rwx + default:group::rwx + default:group:family:rwx + default:mask::rwx + default:other::r-x + +sub-directories are observed to have diverging permissions, though: + + # file: 794 + # owner: chrysn + # group: chrysn + user::rwx + group::rwx #effective:r-x + group:family:rwx #effective:r-x + mask::r-x + other::r-x + default:user::rwx + default:group::rwx + default:group:family:rwx + default:mask::rwx + default:other::r-x + +something seems to apply the umask (default 022) and revoke group write access from the files, overruling the acl. this is not what a umask is normally used for, and smells of [coreutils slavishly observing posix specs that don't consider all features](http://savannah.gnu.org/bugs/?19546) -- the observed effect is exactly what's described there. + +the git annex version used is 3.20121017 as in debian, the receiving site uses rsync 3.0.7; the affected directories come from a time when these very versions are known to have been used. + +this is probably not a bug of git-annex alone, but affects its operation and might be solvable by invoking rsync differently. + +(this is kind of a follow-up on [[forum/__34__permission_denied__34___in_fsck_on_shared_repo]]) diff --git a/doc/bugs/acl_not_honoured_in_rsync_remote/comment_1_aa6fe1d7b029eae7ee71c97e0f0937a6._comment b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_1_aa6fe1d7b029eae7ee71c97e0f0937a6._comment new file mode 100644 index 0000000000..c24b021572 --- /dev/null +++ b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_1_aa6fe1d7b029eae7ee71c97e0f0937a6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.27" + subject="comment 1" + date="2012-11-06T20:49:47Z" + content=""" +I don't know much about ACLs, but you might try setting `remote.prometheus.annex-rsync-options` with rsync options such as --acls or --no-acls +"""]] diff --git a/doc/bugs/acl_not_honoured_in_rsync_remote/comment_2_ffb9424e966ee10a4fe2d446b3042cb2._comment b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_2_ffb9424e966ee10a4fe2d446b3042cb2._comment new file mode 100644 index 0000000000..96dbc6a2e2 --- /dev/null +++ b/doc/bugs/acl_not_honoured_in_rsync_remote/comment_2_ffb9424e966ee10a4fe2d446b3042cb2._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="further tracking" + date="2012-11-11T01:15:41Z" + content=""" +the behavior is rooted in rsync, which behaves similar to `mkdir -p`. it can probably be worked around by configuring the rsync option `--chmod=775`, but that would add executability to files. + +i've opened a bug in rsync's bug tracker at , let's see what the developer says. +"""]] diff --git a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn new file mode 100644 index 0000000000..471a698a0e --- /dev/null +++ b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn @@ -0,0 +1,21 @@ +The command `git annex dropunused` currently takes a number, as referenced in output of last `git annex unused` command. + +When you want to drop all, or a range, this may be annoying, as you have to specify each number on the command line. + +A range argument, such as `1-1845`, possibly combined with other argument types (Cf. many print dialogues: `1,3,5-7,9`) would be great. + +I work around this lack as I want to drop all unused files anyway by something like this: + + git annex unused | grep -o -P "^ [0-9]+" | xargs git annex dropunused + +> It's designed to be used with `seq`. There's an example in the +> [[walkthrough|walkthrough/unused_data]], and of course multiple seq calls can be used to +> specifiy multiple ranges. So: + + git annex dropunused `seq 1 9` `seq 11 1845` + +> I don't see adding my own range operations to be an improvement worth +> making; it'd arguably only be a complication. --[[Joey]] [[done]] + +>> Actually, this did get implemented, since using seq could fall afoul +>> of command-line length limits in extreme cases. diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn new file mode 100644 index 0000000000..7d7bdfc51a --- /dev/null +++ b/doc/bugs/add_script-friendly_output_options.mdwn @@ -0,0 +1,19 @@ +I have a need to use git-annex from a larger program. It'd be great if the information output by some of the commands that is descriptive (for example, whereis) could be sent to stdout in a machine-readable format like (preferably) JSON, or XML. That way I can simply read in the output of the command and use the data directly instead of having to parse it via regexes or other such string manipulation. + +This could perhaps be triggered by a --json or --xml flag to the relevant commands. + +> This is [[done]], --json is supported by all commands, more or less. +> +> Caveats: +> +> * the version, status, and find commands produce custom output and so +> no json. This could change for version and status; find needs to just +> be a simple list of files, I think +> * The "note" fields may repeat multiple times per object with different +> notes and are of course not machine readable, and subject to change. +> * Output of helper commands like rsync is not diverted away, and +> could clutter up the json output badly. Should only affect commands +> that transfer data. And AFAICS, wget and rsync both output their +> progress displays to stderr, so shouldn't be a problem. +> +> --[[Joey]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow.mdwn b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow.mdwn new file mode 100644 index 0000000000..46024e5007 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow.mdwn @@ -0,0 +1,81 @@ +Creating additional branches in history seems to slow down the 'git annex unused' command quadratically, even if the location of the branches should be irrelevant as far as unused data goes. + +This was tested on: + + $ git annex version + git-annex version: 3.20130216 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +What steps will reproduce the problem? + + $ mkdir a + $ cd a + $ git init + $ git annex init + $ i=0 ; while test $i -lt 1000; do dd if=/dev/urandom of=$i.img bs=1M count=1; i=$(($i+1)); done + $ git annex add . + $ git commit -m"foo" + $ git rm 1* + $ git commit -m"bar" + $ git log --oneline --decorate + ffcca3a (HEAD, master) bar + 3e7793d foo + $ time -p git annex unused + unused . (checking for unused data...) (checking master...) + (...) + real 0.76 + user 0.40 + sys 0.06 + git commit --allow-empty -m"baz" + $ git log --oneline --decorate + 4390c32 (HEAD, master) baz + ffcca3a bar + 3e7793d foo + $ time -p git annex unused + unused . (checking for unused data...) (checking master...) + (...) + real 0.75 + user 0.38 + sys 0.07 + $ git branch boo HEAD^ + $ time -p git annex unused + unused . (checking for unused data...) (checking boo...) (checking master...) + (...) + real 1.29 + user 0.62 + sys 0.08 + arand@mas:~/tmp/more/a(master)$ git branch beeboo HEAD^ + 4390c32 (HEAD, master) baz + ffcca3a (boo, beeboo) bar + 3e7793d foo + arand@mas:~/tmp/more/a(master)$ time -p git annex unused + unused . (checking for unused data...) (checking beeboo...) (checking master...) + (...) + real 2.50 + user 1.12 + sys 0.14 + $ git branch -d boo beeboo + $ git log --oneline --decorate + 4390c32 (HEAD, master) baz + ffcca3a bar + 3e7793d foo + $ time -p git annex unused + unused . (checking for unused data...) (checking master...) + (...) + real 0.77 + user 0.42 + sys 0.04 + +What is the expected output? What do you see instead? + +I would expect the time to be the same in all the above cases. + +What version of git-annex are you using? On what operating system? + + $ git annex version + git-annex version: 3.20130216 + +On current Debian sid/experimental diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_1_d350c39c67031c500e3224e92c0029ea._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_1_d350c39c67031c500e3224e92c0029ea._comment new file mode 100644 index 0000000000..f8b7e789b5 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_1_d350c39c67031c500e3224e92c0029ea._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.210" + subject="comment 1" + date="2013-02-26T20:29:05Z" + content=""" +`git annex unused` finds content that is not used by the working tree or by *any* branch is unused. For the working tree, it can look at the symlinks on disk, which is the fastest option. + +For branches, it has to first use `git ls-tree` to find the files on the branch, and then use `git cat-file` to look up the key used by each file. It does this as fast as it can (eg, it runs a single `git cat-file --batch`, rather than one process per file). Still, this is pulling potentially a lot of data out of git, and it gets pretty slow. + +I've spent a lot of time optimising this as much as is possible with these constraints. One nice one is that, if it finds no unused keys after checking the working tree, it can stop, rather than checking any branches. Your example avoids this optimisation. + +Another optimisation is to only check each git ref once, even if multiple branches refer to it. You can see this optmisation firing in your transcript, when it only shows it's checking one branch of the two identical branches you've made. + +Indeed, if you go on and add 100 identical branches, you'll find it runs in just about the same time it ran with 2 branches. (There's a little overhead in getting the list of branches and throwing out the duplicates, but that's all.) + +What then explains your numbers? Well, I have no idea. I cannot replicate them; I tend to see about the same amount of time taken with two duplicate branches as with one branch. I suspect you just didn't get statistically valid results, which playing around with `time` at the command line often doesn't, +due to caching, other active processes, etc. +"""]] diff --git a/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_2_b2d2b1caa51ffec3d87c36b373cb8d4a._comment b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_2_b2d2b1caa51ffec3d87c36b373cb8d4a._comment new file mode 100644 index 0000000000..0954e89880 --- /dev/null +++ b/doc/bugs/added_branches_makes___39__git_annex_unused__39___slow/comment_2_b2d2b1caa51ffec3d87c36b373cb8d4a._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://launchpad.net/~arand" + nickname="arand" + subject="comment 2" + date="2013-02-27T15:53:37Z" + content=""" +Hmm, indeed, after further testing it seems like the increased time due to the duplicate branch seems to have been a random quirk, bleh :( + +But shouldn't it theoretically be possible to optimize out much of the overhead of multiple very-similar (though not identical) branches though? + +I've experimented around with ls-tree and cat-file in a bash script[1], and in this primitive implementation the running time seems to be considerably lower (~0.3s vs ~4s), with much less overhead for extra very-similar branches (~0.7s vs ~37s) + +Am I missing some key element that's the reason for the time taken by git annex unused? + + +[1] primitive annex unused script: [https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-funused](https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-funused) + +timing script: [https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-testunused](https://gitorious.org/arand-scripts/arand-scripts/blobs/master/annex-testunused) + +"""]] diff --git a/doc/bugs/annex-rsync-options_shell-split_carelessly.mdwn b/doc/bugs/annex-rsync-options_shell-split_carelessly.mdwn new file mode 100644 index 0000000000..c8eeb7160d --- /dev/null +++ b/doc/bugs/annex-rsync-options_shell-split_carelessly.mdwn @@ -0,0 +1,16 @@ +with rsync, it is sometimes the case that one needs to specify ssh options -- typical examples from the rsync man page are `rsync -e 'ssh -p 2234'`. as git-annex does the shell splitting of the arguments in `annex-rsync-options` (see [[special remotes/rsync]]) itself by looking for whitespace, these options can't be passed directly. (`annex-rsync-options = -e 'ssh -p 2234'` gets split to `["rsync", "-e", "'ssh", "-p", "2234'"]` instead of `["rsync", "-e", "ssh -p 2234"]`). + +git-annex should respect shell splitting rules when looking at annex-rsync-options. (i suppose there is a haskell library or module for that; in python, we have the `shlex` module for that). + +## workaround + +put this in .git/ssh and mark it as executable: + + #!/bin/sh + exec ssh -p 2234 $@ + +put this in your git annex config in the particular remote's section: + + annex-rsync-options = -e /local/path/to/your/repo/.git/ssh + +(typical bug report information: observed with git-annex 3.20121127 on debian) diff --git a/doc/bugs/annex-rsync-options_shell-split_carelessly/comment_1_2636e0d224317f2e6db94658d8a094c4._comment b/doc/bugs/annex-rsync-options_shell-split_carelessly/comment_1_2636e0d224317f2e6db94658d8a094c4._comment new file mode 100644 index 0000000000..42b92ce281 --- /dev/null +++ b/doc/bugs/annex-rsync-options_shell-split_carelessly/comment_1_2636e0d224317f2e6db94658d8a094c4._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.8.117" + subject="comment 1" + date="2012-12-13T17:03:08Z" + content=""" +Due to the way git-annex runs rsync, which involves a specific -e parameter it constructs that, you cannot pass -e in annex-rsync-options anyway; or if you do you'll bypass use of git-annex-shell, which is not desirable. I have not checked which, but would not recommend use of it. + +There is no need for ugly workarounds. Just use ~/.ssh/config to configure the hostname to use the nonstandard port it needs. For example: + +
+Host example.com
+Port 2234
+
+ +Or, to make a separate example.com-2234 host that can be used to use the nonstandard port: + +
+Host example.com-2234
+Hostname example.com
+Port 2234
+
+"""]] diff --git a/doc/bugs/annex_add_in_annex.mdwn b/doc/bugs/annex_add_in_annex.mdwn new file mode 100644 index 0000000000..e12826f00e --- /dev/null +++ b/doc/bugs/annex_add_in_annex.mdwn @@ -0,0 +1,6 @@ +I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories. + +> There is a guard against `git annex add .git-annex/foo`, but it doesn't +> notice `cd .git-annex; git annex add foo`. --[[Joey]] + +> Now fixed, by removing the .git-annex directory. [[done]] --[[Joey]] diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn new file mode 100644 index 0000000000..e830f11564 --- /dev/null +++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn @@ -0,0 +1,20 @@ +Just starting using v3, even more awesome, thanks! + +With git-annex, I take the habit to do copies of files without restriction, as they end up into (cheap) symlink copies. +However, if 2 copies are unannexed, only one is restored, the other becomes a broken symlink, so I kind of loose some information +(my use case: I have a repo on which I recently started using annex, but most of the files, which i would want to be annexed, are only in git, +so my plan is to unninit this repo, delete the .git dir, and then annex everything, as I don't mind the history). + +Rafaël + +> The only way for git-annex to support this in its current state would be +> for the unannex command to copy the file content from the annex, rather +> than moving it out. Then multiple links to the same content could be +> unannexed. +> +> But, this would be slower, and would depend on a later `unused` and +> `dropunused` to actually remove the content. While doable, by use case +> for unannex is more to quickly undo a mistaken add, and it's unlikely there +> are multiple symlinks to the same content in this situation. --[[Joey]] + +[[!tag done]] diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment new file mode 100644 index 0000000000..839992477a --- /dev/null +++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w" + nickname="Rafaël" + subject="comment 1" + date="2011-07-04T16:57:25Z" + content=""" +You convince me for unannex, but isn't the goal of uninit to revert all annex operations? In the current state, a clean revert is not possible (because of the broken symlinks after uninit). Instead of copying, using hard links is out of question? + +For my needs, is the command \"git annex unlock .\" (from the root of the repo) a correct workaround? +"""]] diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment new file mode 100644 index 0000000000..21c0c449b0 --- /dev/null +++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY" + nickname="Joey" + subject="comment 2" + date="2011-07-04T20:25:38Z" + content=""" +Indeed, uninit needed to be improved. I've done so. Also, unannex --fast can be used to make hard links to content left in the annex. +"""]] diff --git a/doc/bugs/another_build_error_in_assistant.mdwn b/doc/bugs/another_build_error_in_assistant.mdwn new file mode 100644 index 0000000000..c21f1ac4ed --- /dev/null +++ b/doc/bugs/another_build_error_in_assistant.mdwn @@ -0,0 +1,79 @@ +What steps will reproduce the problem? +Just trying to install git-annex last release (20121120) from cabal or from bundled sources + +What is the expected output? What do you see instead? +Build stop like this : +(doing cabal install or build from the bundle) +... +[161 of 284] Compiling Assistant.Alert ( Assistant/Alert.hs, dist/build/git-annex/git-annex-tmp/Assistant/Alert.o ) +[162 of 284] Compiling Assistant.Types.DaemonStatus ( Assistant/Types/DaemonStatus.hs, dist/build/git-annex/git-annex-tmp/Assistant/Types/DaemonStatus.o ) +[163 of 284] Compiling Assistant.Monad ( Assistant/Monad.hs, dist/build/git-annex/git-annex-tmp/Assistant/Monad.o ) + +Assistant/Monad.hs:86:16: + Couldn't match expected type `Assistant a' + with actual type `Reader AssistantData a' + Expected type: (AssistantData -> a) -> Assistant a + Actual type: (AssistantData -> a) -> Reader AssistantData a + In the expression: reader + In an equation for `getAssistant': getAssistant = reader + +Assistant/Monad.hs:93:15: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: st <- reader threadState + In the expression: + do { st <- reader threadState; + liftIO $ runThreadState st a } + +Assistant/Monad.hs:99:14: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: d <- reader id + In the expression: + do { d <- reader id; + liftIO $ io $ runAssistant d a } + +Assistant/Monad.hs:105:14: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: d <- reader id + In the expression: + do { d <- reader id; + return $ runAssistant d a } + +Assistant/Monad.hs:110:14: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: d <- reader id + In the expression: + do { d <- reader id; + return $ \ v -> runAssistant d $ a v } + +Assistant/Monad.hs:115:14: + Couldn't match expected type `Assistant t0' + with actual type `Reader r0 a0' + In the return type of a call of `reader' + In a stmt of a 'do' block: d <- reader id + In the expression: + do { d <- reader id; + return $ \ v1 v2 -> runAssistant d (a v1 v2) } + +Assistant/Monad.hs:120:12: + Couldn't match expected type `Assistant a0' + with actual type `Reader r0 a1' + In the return type of a call of `reader' + In the first argument of `(>>=)', namely `reader v' + In the expression: reader v >>= liftIO . io + + + + +What version of git-annex are you using? On what operating system? +- version 3.20121112 +- Ubuntu 12.04 LTS, 64 bits + +> Dup of [[3.20121112_build_fails_on_Ubuntu_12.04]]. --[[Joey]] [[done]] diff --git a/doc/bugs/archiving_git_repositories.mdwn b/doc/bugs/archiving_git_repositories.mdwn new file mode 100644 index 0000000000..1753c10ae3 --- /dev/null +++ b/doc/bugs/archiving_git_repositories.mdwn @@ -0,0 +1 @@ +In a true dropbox-like fashion, I tried to import my entire homefolder into the git-annex assistant. However, it seems that git-annex breaks on the several git repositories I've got checked out in my "Projects" folder. Is this a possible use case, or should I look at other tools to perform this with? diff --git a/doc/bugs/archiving_git_repositories/comment_1_51f546a571303118446a9e0b3e6482c9._comment b/doc/bugs/archiving_git_repositories/comment_1_51f546a571303118446a9e0b3e6482c9._comment new file mode 100644 index 0000000000..cec1398fc4 --- /dev/null +++ b/doc/bugs/archiving_git_repositories/comment_1_51f546a571303118446a9e0b3e6482c9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.23" + subject="comment 1" + date="2012-10-19T21:01:44Z" + content=""" +The assistant is not ready to manage whole home directories, and the webapp refuses to allow you to do that. + +git doesn't allow checking a git repository into a git repository either. I archive old git repos inside one of my annexes by tarring them up. +"""]] diff --git a/doc/bugs/assistant___40__OS_X_Lion__41___-___34__too_many_open_files__34___error.mdwn b/doc/bugs/assistant___40__OS_X_Lion__41___-___34__too_many_open_files__34___error.mdwn new file mode 100644 index 0000000000..1b3879e639 --- /dev/null +++ b/doc/bugs/assistant___40__OS_X_Lion__41___-___34__too_many_open_files__34___error.mdwn @@ -0,0 +1,32 @@ +What steps will reproduce the problem? + +I downloaded the most recent OS X Lion build of the Git Annex application bundle and ran it. + +I wanted to test the direct mode, as my existing annex is in the old style. +So I set up a new (direct) repository, then added an SSH remote, then dropped a large number of files (170) in there. + + +What is the expected output? What do you see instead? + +Expected: the files will be added to the repository in direct mode and stored on the ssh remote. +Instead: the following error appeared in the webapp: + +Committer crashed: /Users/ed/directannex/.git/annex/tmp/: openTempFile: resource exhausted (Too many open files) + +nothing seems to have been synced to the remote -- I gather this from the fact that the annex directory in the remote git repo is only 70K instead of several hundred megs. + + +What version of git-annex are you using? On what operating system? + + Version: 3.20130102 on OS X lion, using the version bundled as an application bundle. + + +Please provide any additional information below. + +I imagine I could avoid this error by using the "ulimit -n" command to increase number of files in a shell session and then running assistant manually, so this is really only a bug report about the bundled application. + +> This seems to be caused by Command.Add.lockdown not closing the +> temporary file handle, so when called in a mapM by the committer +> thread when there are a lot of files, it could build up a lot of +> open handles before later GC closes them. Added a manual close, +> so I think this is [[done]]. --[[Joey]] diff --git a/doc/bugs/assistant___40__OS_X_Lion__41___-___34__too_many_open_files__34___error/comment_1_9904c30a4c24a699d71e90ce5e9b89cf._comment b/doc/bugs/assistant___40__OS_X_Lion__41___-___34__too_many_open_files__34___error/comment_1_9904c30a4c24a699d71e90ce5e9b89cf._comment new file mode 100644 index 0000000000..4db7bf9a8b --- /dev/null +++ b/doc/bugs/assistant___40__OS_X_Lion__41___-___34__too_many_open_files__34___error/comment_1_9904c30a4c24a699d71e90ce5e9b89cf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 1" + date="2013-01-05T00:32:44Z" + content=""" +After I painstakingly killed off the assistant's processes (I didn't see a way to stop it in the web interface, and issuing \"git annex assistant --stop\" did not stop it), and restarted the assistant, it synced successfully, so the error seems to have been transient. +"""]] diff --git a/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address.mdwn b/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address.mdwn new file mode 100644 index 0000000000..3834107048 --- /dev/null +++ b/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address.mdwn @@ -0,0 +1,46 @@ +What steps will reproduce the problem? + +Using the assistant, create an SSH remote. Try to use an alias as the name +of the remote (e.g. I have a server which I have aliased to "homeworld" in +my .ssh/config. When I'm at home, that is an alias for 192.168.1.253. +When I'm not at home, I edit .ssh/config so that "homeworld" becomes an +alias for a hostname at no-ip.com.) Despite the fact that "homeworld" is a +viable ssh target because of the alias, the assistant doesn't recognize it +as a valid host to ssh to. + +I had trouble with an ip address the first time I tried it but just tried it again and it worked fine, so please disregard that part of the title of this bug report. + + +What is the expected output? What do you see instead? + +expected output = move to the "create a repository -- rsync or regular" page. +observed output = "cannot resolve host name" + + +What version of git-annex are you using? On what operating system? + + Version: 3.20130102 OS X Lion + + +Please provide any additional information below. + +I realize this is kind of a power user whine. Using an ssh alias which +does not correspond to an actual resolvable hostname (and cannot, because +it's supposed to be a layer of indirection over the hostname) is not an +everyday problem for an average user. + +> The assistant tries to resolve the hostname explicitly +> to catch user's typos, and also expands it to a FQDN, to make +> it more likely to be able to reach the host when roaming to other +> networks. +> +> Also, the assistant sets up it *own* .ssh/config hostname alias, +> in order to make it use the special ssh key that it generates for the host. +> So that is not compatable with using a ssh host alias you've set up. +> Even if it knew about your alias, it would set up a new hostname alias, and +> whatever machinery you have to update the alias would not work. +> +> You can, of course, add git remotes using any ssh alias you like, by +> hand, and restart the assistant and it will use them. --[[Joey]] + +[[!tag /design/assistant]] diff --git a/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_1_1650539846521ae11837e4ac73348af6._comment b/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_1_1650539846521ae11837e4ac73348af6._comment new file mode 100644 index 0000000000..859810a444 --- /dev/null +++ b/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_1_1650539846521ae11837e4ac73348af6._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 1" + date="2013-01-06T05:23:25Z" + content=""" +Fair enough. :) Power user solution to power user whine! + +"""]] diff --git a/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_2_b91415e4ee74eb12bc6e6faddd00af6e._comment b/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_2_b91415e4ee74eb12bc6e6faddd00af6e._comment new file mode 100644 index 0000000000..d910e016e8 --- /dev/null +++ b/doc/bugs/assistant_cannot_set_up_remote_repo_via_an_ssh_alias_or_an_ip_address/comment_2_b91415e4ee74eb12bc6e6faddd00af6e._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmcYryijvlF8bJvM_eZNSrUPEkMlxMDGTQ" + nickname="Thiago" + subject="Also relevant for machines behind pagekite.net" + date="2013-02-19T20:00:30Z" + content=""" +I'm running into this problem too. I need to use an ssh alias because machines behind pagekite have to be accessed using an HTTP connect proxy, which is done with the ProxyCommand option in the ssh config file. I'm mentioning this because IMHO it's much less of a power user problem than you seem to think. + +In fact, I'd argue that a power user is more likely to have access to a machine with a fixed IP address or FQDN. It is people which fall more into the \"mere mortal\" side of the spectrum that need to resort to ssh aliases to access machines behind NAT or dynamic IPs. :-) + +All that to say that it would be nice if this usecase was explicitly supported in the UI (maybe a checkbox saying \"this is an ssh alias\", or a textfield to enter a proxy through which the connection has to be routed). +"""]] diff --git a/doc/bugs/assistant_does_not_always_use_repo_cost_info_when_queueing_downloads.mdwn b/doc/bugs/assistant_does_not_always_use_repo_cost_info_when_queueing_downloads.mdwn new file mode 100644 index 0000000000..c00c3448d7 --- /dev/null +++ b/doc/bugs/assistant_does_not_always_use_repo_cost_info_when_queueing_downloads.mdwn @@ -0,0 +1,16 @@ +git-annex has nice repository cost tracking, but the assistant doesn't +always use this when queuing downloads. + +For example, if a network comes up, it'll queue a lot of downloads from a +remote, perhaps expensive server. If a removable drive is then attached, +which has a lower cost, it'll queue downloads from there, but +only once it works through all the ones it's already queued from the +network. + +There is a workaround: Since the webapp allows disabling syncing to a +remote, the user can go in and disable syncing to the network remote +temporarily. + +[[!tag /design/assistant]] + +--[[Joey]] diff --git a/doc/bugs/assistant_does_not_list_remote___39__origin__39__.mdwn b/doc/bugs/assistant_does_not_list_remote___39__origin__39__.mdwn new file mode 100644 index 0000000000..3607cbcaf6 --- /dev/null +++ b/doc/bugs/assistant_does_not_list_remote___39__origin__39__.mdwn @@ -0,0 +1,22 @@ +What steps will reproduce the problem? + +1. create a git annex repo on a server +2. clone it on workstation +3. open the webapp on the workstation + + +What is the expected output? What do you see instead? + +The webapp should show the 'origin' remote and the assistant should ensure syncing. +Instead the remote does not show up in the webapp. +I checked with `git annex status` and the remote is there. + +What version of git-annex are you using? On what operating system? + +3.20130207 on latest Ubuntu + +Please provide any additional information below. + +I tried both with direct and indirect mode for the local annex repo. + +I am sorry if I am missing the point. I checked the docs, however without much success. diff --git a/doc/bugs/assistant_does_not_list_remote___39__origin__39__/comment_1_ffa008240c61b50396aa92f467731db6._comment b/doc/bugs/assistant_does_not_list_remote___39__origin__39__/comment_1_ffa008240c61b50396aa92f467731db6._comment new file mode 100644 index 0000000000..8f6cb53175 --- /dev/null +++ b/doc/bugs/assistant_does_not_list_remote___39__origin__39__/comment_1_ffa008240c61b50396aa92f467731db6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmaMxYm33s0H-nxBo5uzYUzdIECoyR8Ug8" + nickname="Stefan" + subject="an update" + date="2013-02-13T20:33:22Z" + content=""" +The remotes were not listed because git-annex-shell was not working (found in the path, however necessary *.so files were not found). + +Now all remotes are listed. `.git/config` is ok, with the uuid listed. `git annex status` returns all. + +However sync is not working. The webapp on the workstation states that it has synced with the server, however `ls` on the server returns empty. The server is set as archive, the workstation as client. +"""]] diff --git a/doc/bugs/assistant_does_not_list_remote___39__origin__39__/comment_2_a53f80090bc2a0f32b8d8307cb24b563._comment b/doc/bugs/assistant_does_not_list_remote___39__origin__39__/comment_2_a53f80090bc2a0f32b8d8307cb24b563._comment new file mode 100644 index 0000000000..5915fd8445 --- /dev/null +++ b/doc/bugs/assistant_does_not_list_remote___39__origin__39__/comment_2_a53f80090bc2a0f32b8d8307cb24b563._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.210" + subject="comment 2" + date="2013-02-26T19:51:47Z" + content=""" +You are probably not running the assistant on the server. This is fine, but it means that git pushes made to the server when files are added to the client are not merged into its working copy. Which is why you don't see the files. Running `git annex sync` on the server should make all the files that have been sent to it show up. +"""]] diff --git a/doc/bugs/assistant_ignore_.gitignore.mdwn b/doc/bugs/assistant_ignore_.gitignore.mdwn new file mode 100644 index 0000000000..a32fd457a8 --- /dev/null +++ b/doc/bugs/assistant_ignore_.gitignore.mdwn @@ -0,0 +1,27 @@ +What steps will reproduce the problem? + +1. have an existing directory with a bunch of files +2. create a `.gitignore` file that matches some files (*.log *.aux *~ etc.) +3. `git init .` +4. `git annex init work` +5. `git remote add server server:Blabla` +6. `ssh server` +7. `@server $ mkdir Blabla` +8. `@server $ cd Blabla` +9. `@server $ git init .` +10. `@server $ git annex init server` +11. `@server $ exit` +12. `git annex webapp` + +What is the expected output? What do you see instead? + +I expect that ingored files stay ignored, +I see instead that all the files (including the ignored) are transfered to the server + +What version of git-annex are you using? On what operating system? + +3.20130124, debian sid (on both machines) + +> As noted in [[design/assistant/inotify]]'s TODO list, this +> needs an efficient gitignore query interface in git (DNE) +> or a gitignore parser. --[[Joey]] diff --git a/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts.mdwn b/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts.mdwn new file mode 100644 index 0000000000..90034f7f57 --- /dev/null +++ b/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts.mdwn @@ -0,0 +1,30 @@ +What steps will reproduce the problem? + +Using the Linux tarball (i386) with configured IPv6: + +* git-annex.linux/git-annex-webapp + +A browser is then started, pointing to file:///tmp/webapp313.html which in turn points to http://localhost:$port/$blah . + +On my box localhost resolves to ::1, but the webapp is only listening on 127.0.0.1 so . While I can work around this by specifying 127.0.0.1 as the hostname, the next page that is loaded goes back to localhost. + +What is the expected output? What do you see instead? + +I would expect that the webapp would bind to ::1 if possible. + +What version of git-annex are you using? On what operating system? + +3.20130102 fromt the Linux tarball release on Debian Squeeze. + +Please provide any additional information below. + +I've tested this with: + +* epiphany +* iceweasel +* chromium + +Iceweasel is the only one which correctly fell back to IPv4 and worked. + +> Ok, I've made it use the IP address in the URL. Ugly, but avoids +> whatever mess results in this behavior. [[done]] --[[Joey]] diff --git a/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_1_91a62a2ce14a1027d2ac8b8e88df5f0c._comment b/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_1_91a62a2ce14a1027d2ac8b8e88df5f0c._comment new file mode 100644 index 0000000000..540e06ee3c --- /dev/null +++ b/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_1_91a62a2ce14a1027d2ac8b8e88df5f0c._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.1.42" + subject="comment 1" + date="2013-01-09T23:21:58Z" + content=""" +I have a ipv6 enabled computer here, where the webapp works fine. I think you must have something odd in your network configuration if web browsers, when sent to \"http://localhost:port/ don't manage to connect to a web server running on ipv4. + +The webapp only uses 127.0.0.1 if \"localhost\" resolves to that address. If \"localhost\" resolves to both ipv6 and ipv4 addresses, the webapp uses the ipv4 and not the ipv6 address. If \"localhost\" resolves to only ipv6, the webapp uses ipv6. + +AFAIK it's not possible to bind to both an ipv4 and an ipv6 address, and get the same port number for both, and the webapp needs a port number to use to launch the web browser. Therefore, it has to choose one address. It choses ipv4 in preference to ipv6 because there are some things that still don't support ipv6. [This commit](http://source.git-annex.branchable.com/?p=source.git;a=commit;h=467844d7d3f703f99fcde1f951f33efda5e90074) is relevant. +"""]] diff --git a/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_2_4982cd373eaaeee180be03c6e9fda7b1._comment b/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_2_4982cd373eaaeee180be03c6e9fda7b1._comment new file mode 100644 index 0000000000..849bde6e47 --- /dev/null +++ b/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_2_4982cd373eaaeee180be03c6e9fda7b1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkF8_uQjLYm5Mf5F_JuVW-BxlvzpWjvR_o" + nickname="Andrew" + subject="comment 2" + date="2013-01-09T23:51:18Z" + content=""" +Watching tcpdump on loopback I only see a request from Epiphany to ::1, after that it gives up. So not terribly helpful. + +It looks like it might be a bug in the browsers. :( +"""]] diff --git a/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_3_85d264e311acaa91dac0597ee8deda82._comment b/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_3_85d264e311acaa91dac0597ee8deda82._comment new file mode 100644 index 0000000000..91da74efc1 --- /dev/null +++ b/doc/bugs/assistant_listens_on_127.0.0.1_not_::1_which_breaks_IPv6_enabled_hosts/comment_3_85d264e311acaa91dac0597ee8deda82._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkF8_uQjLYm5Mf5F_JuVW-BxlvzpWjvR_o" + nickname="Andrew" + subject="comment 3" + date="2013-01-10T22:10:00Z" + content=""" +Cool, thanks Joey. Let's hope we can remove the ugly hack one day. ;) +"""]] diff --git a/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files.mdwn b/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files.mdwn new file mode 100644 index 0000000000..d009b55cd7 --- /dev/null +++ b/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files.mdwn @@ -0,0 +1,68 @@ +What steps will reproduce the problem? + +In one terminal, I created a new annex and started the assistant watching it. + +In another, I added a file file1; assistant noticed it and added it to the annex. + +I moved file1 to a directory directory1; the link broke and assistant did not fix it. + +I created a file called file2 inside directory 2; assistant noticed it and added it to the annex. + +I moved file2 back up to the annex root directory; the link broke and assistant did not fix it. + +I created a file file3 in the annex root directory; assistant noticed it and added it to the annex. + +Here is the content of the first terminal, where I created the annex and ran assistant: + + + ~$ mkdir testannex + ~$ cd testannex/ + testannex$ git init . + Initialized empty Git repository in /Users/ed/testannex/.git/ + testannex$ git annex init "test annex" + init test annex ok + (Recording state in git...) + testannex$ git annex assistant --foreground + assistant . (scanning...) (started...) add file1 (checksum...) ok + (Recording state in git...) + (Recording state in git...) + add directory1/file2 (checksum...) ok + (Recording state in git...) + (Recording state in git...) + add file3 (checksum...) + +here is the content of the second terminal, where I created and moved files: + + ~$ cd testannex + testannex$ echo "file1 content" > file1 + testannex$ mkdir directory1 + testannex$ ls -l file1 + lrwxr-xr-x 1 ed staff 180 Dec 13 15:40 file1 -> .git/annex/objects/FX/51/SHA256E-s14--edac79763e630b1b77aefb6c284bcb0362dea71c0548be0e793ffa8fd5907b80/SHA256E-s14--edac79763e630b1b77aefb6c284bcb0362dea71c0548be0e793ffa8fd5907b80 + testannex$ mv file1 directory1/ + testannex$ cd directory1/ + directory1$ cat file1 + cat: file1: No such file or directory + directory1$ echo "file2 content" > file2 + directory1$ cat file2 + file2 content + directory1$ cat file1 + cat: file1: No such file or directory + directory1$ mv file2 ../ + directory1$ cd .. + testannex$ echo "file3 content" > file3 + testannex$ + + +What is the expected output? What do you see instead? + +The links do not break when moved to another directory. + +What version of git-annex are you using? On what operating system? + +One compiled using cabal from checkout 739c937 + +Please provide any additional information below. + +> [[fixed|done]]; this turned out to be an kqueue specific bug, +> the kqueue code statted new files, but that files for a broken symlink. +> Using lstat instead fixed this. --[[Joey]] diff --git a/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files/comment_1_e0dafc410ffd617d445bb9403c7bfafe._comment b/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files/comment_1_e0dafc410ffd617d445bb9403c7bfafe._comment new file mode 100644 index 0000000000..fef95243b9 --- /dev/null +++ b/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files/comment_1_e0dafc410ffd617d445bb9403c7bfafe._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 1" + date="2012-12-13T20:50:18Z" + content=""" +(Operating system is OS X Lion) + +"""]] diff --git a/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files/comment_2_2af247c8a1fcbde10795a990ef3303e9._comment b/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files/comment_2_2af247c8a1fcbde10795a990ef3303e9._comment new file mode 100644 index 0000000000..6d93d7e619 --- /dev/null +++ b/doc/bugs/assistant_not_noticing_file_renames__44___not_fixing_files/comment_2_2af247c8a1fcbde10795a990ef3303e9._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 2" + date="2012-12-15T05:47:25Z" + content=""" +thanks, fix worked for me! Now the links are fixed virtually instantaneously when the file is moved. + +"""]] diff --git a/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote.mdwn b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote.mdwn new file mode 100644 index 0000000000..657a4dd3d8 --- /dev/null +++ b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote.mdwn @@ -0,0 +1,5 @@ +I am running 3.20121112 and this bug appeared when I upgraded to that from 3.20121017. + +After performing the startup scan after I login, git-annex gets GPG to sends me a pinentry pop-up asking for my GPG passphrase. However, I know that there is nothing to be dropped or copied to the encrypted SSH remote that I am being asked to provide access to. So I can't see any good reason why I would be asked for the passphrase. + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_1_10a9570a5d762ba2da271b38dc63edb6._comment b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_1_10a9570a5d762ba2da271b38dc63edb6._comment new file mode 100644 index 0000000000..548db2386f --- /dev/null +++ b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_1_10a9570a5d762ba2da271b38dc63edb6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 1" + date="2012-11-29T20:13:53Z" + content=""" +Hmm, is this an encrypted rsync remote? + +I have not been able to reproduce the problem so fat. + +If you start the assistant with --debug, and look in .git/annex/daemon.log, you should find where it runs gpg, and the lines showing what it's doing just before/after that should provide a hint what it's doing. +"""]] diff --git a/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_2_57d50955b038c2e2405068536c7e83f3._comment b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_2_57d50955b038c2e2405068536c7e83f3._comment new file mode 100644 index 0000000000..ce57194c9c --- /dev/null +++ b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_2_57d50955b038c2e2405068536c7e83f3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="spwhitton" + ip="163.1.167.50" + subject="comment 2" + date="2012-12-01T18:34:10Z" + content=""" +Thanks for the log advice. I looked and it wants to drop a file `ttmik/TTMIK-Lesson-L1L1.mp3` from the encrypted rsync remote ma. So I did `git annex whereis ttmik/TTMIK-Lesson-L1L1.mp3` and learnt that the file is not on ma. I tried `git annex drop --from ma ttmik` to be sure, and the command was successful, but it still tries to drop the file from ma on startup. Presumably it would try all the other files in the ttmik directory if I gave it the chance to try to drop this first one. The only thing special about the ttmik directory is that every file in there was added using addurl, so I guess the problem has something to do with that. +"""]] diff --git a/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_3_a66f34daaba421c87eb404ef933e5191._comment b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_3_a66f34daaba421c87eb404ef933e5191._comment new file mode 100644 index 0000000000..9bfade666d --- /dev/null +++ b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_3_a66f34daaba421c87eb404ef933e5191._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 3" + date="2012-12-01T18:59:45Z" + content=""" +How is this rsync remote configured? Is it configured as a transfer remote? If not, the assistant's default behavior is to sync all files to it, and since you say the file is not there, it'd make sense it would start transferring it to there on startup. + +OTOH, this could be a complete red herring. You haven't shown me the log file. Perhaps the drop you're seeing in the log is before the operation that is asking for the GPG passphrase. I can't tell until you show me the log. +"""]] diff --git a/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_4_094a3272eca1c6d2b4d264911ffe96e5._comment b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_4_094a3272eca1c6d2b4d264911ffe96e5._comment new file mode 100644 index 0000000000..8f9f546103 --- /dev/null +++ b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_4_094a3272eca1c6d2b4d264911ffe96e5._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="spwhitton" + ip="163.1.167.50" + subject="comment 4" + date="2012-12-02T18:27:09Z" + content=""" +The rsync remote is in the archive group but it has the preferred content string `exclude=video/* and exclude=ttmik/*`. So it ought to be dropping the files, but not over and over again. + +Here is the log file, showing me hitting Escape on the pinentry dialog: + + (scanning...) Already up-to-date. + Already up-to-date. + Already up-to-date. + drop ma ttmik/TTMIK-Lesson-L1L1.mp3 (gpg) gpg: cancelled by user + gpg: decryption failed: secret key not available + TransferScanner crashed: user error (gpg [\"--batch\",\"--no-tty\",\"--use-agent\",\"-- + quiet\",\"--trust-model\",\"always\",\"--decrypt\"] exited 2) + (started...) +"""]] diff --git a/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_5_0161410d042a3421addd4a1fc7c1cd01._comment b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_5_0161410d042a3421addd4a1fc7c1cd01._comment new file mode 100644 index 0000000000..89ef87c2a8 --- /dev/null +++ b/doc/bugs/assistant_wants_my_gpg_passphrase_when_it_has_nothing_to_drop_or_copy_to_that_remote/comment_5_0161410d042a3421addd4a1fc7c1cd01._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.72" + subject="comment 5" + date="2012-12-05T16:34:24Z" + content=""" +Aha, I see the bug now. It is indeed trying to drop from every remote, even ones without the data. + +(Also, the thread should not crash here if the drop fails.. I've fixed that.) +"""]] diff --git a/doc/bugs/automatic_merge_of_synced_branches_upon___34__git_annex_sync__34__.mdwn b/doc/bugs/automatic_merge_of_synced_branches_upon___34__git_annex_sync__34__.mdwn new file mode 100644 index 0000000000..361585a782 --- /dev/null +++ b/doc/bugs/automatic_merge_of_synced_branches_upon___34__git_annex_sync__34__.mdwn @@ -0,0 +1,16 @@ +When maintaining several replica of the same git-annex repo "git annex sync" is quite handy. +But it would be even handier if "git annex sync" would also perform automatic "git merge synced/*" actions on all remotes. + +Clearly, this is beneficial when the user wants to keep all working copies synchronized. +This is likely the case in git annex assistant like scenarios. And it's always the case in my day to day scenarios :-) +I'm not sure about other use cases that I've hard time imagining... + +As just discussed on IRC (#vcs-home/OFTC), this could be implemented in various ways: + +1) By doing ssh on each remote and running the appropriate "git merge ..." commands there. + The drawback of this is that quite often it won't be permitted to ssh on the remote and run arbitrary commands there. + +2) Having a default post-receive hook, created at the time of "git annex init" that automatically does the merges when contacted by other remotes as a consequence of "git annex sync". + + +Thanks for git-annex! diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn new file mode 100644 index 0000000000..122224a8f3 --- /dev/null +++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn @@ -0,0 +1,72 @@ +foo is a local repo, bar is a bare remote. + +I upgraded foo's git-annex to 0.20110325 and upgraded a local repo backend +to version 2. I then ran `git annex copy . --to bar` and checked the +remote. This created WORM:SHA512--123123 files in annex/objects. +Understandable but unwanted. So I upgraded git-annex on bar's machine, as +well. + + % git annex copy . --to bar + copy quux (checking bar) git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade (to bar) + git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade + rsync: connection unexpectedly closed (0 bytes received so far) [sender] + rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.7] + + rsync failed -- run git annex again to resume file transfer + failed + +Running `git annex upgrade` on bar's machine I get: + + % git annex upgrade + upgrade (v1 to v2) (moving content...) git-annex: Prelude.read: no parse + +Again, bar is a bare repo. +Running the copy job again, I am still getting the same error as above (as expected). Partial contents of annex/objects on bar: + + [...] + SHA512:123 + WORM:SHA512--234 + [...] + + +-- RichiH + +> Upgrading bare repos to v2 generally works fine, so I actually need +> to see the full content of annex/, not a fragment, in order to debug this. +> (Filename contents I don't need to see.) Feel free to email me the details at +> joey@kitenet.net if you don't want to post them here. --[[Joey]] + +>> Sent. -- RichiH + +>>> Ok, I'm going to go work on my reading comprehension. I see now +>>> that you +>>> explained the problem pretty well. The problem is caused by these +>>> few weird v1 mixed with v2 keys in the annex. +>>> Ones like "annex/objects/WORM:SHA512--$sha512". +>>> +>>> That's a v1 key, but a corrupt form of the key; it's missing the +>>> size and mtime fields that all WORM keys have in v1. And +>>> the filename is itself a key, a v2 SHA512 key. These were +>>> created when you did the `git annex copy to the v1 bare repo. +>>> In v2, git-annex-shell takes a full key object, while in v1, +>>> it takes a key name and a backend name. This incompatability +>>> leads to the weird behavior seen. +>>> +>>> I had suggested you delete data.. don't. On second thought, +>>> you shouldn't delete anything. I'll simply make the v2 upgrade +>>> detect and work around this bug. +>>> --[[Joey]] + +>>>> This should be fixed in current git. The scambled keys will be +>>>> fixed up on upgrade. Thanks for your patience! [[done]] --[[Joey]] + +>>>>> I should stop reading your answers via git; by the time I got to +>>>>> "second thoughts", I had already deleted the files & directories +>>>>> in question, upgraded the bare repo and was busy uploading from my +>>>>> local repo. I agree that taking care of this in the upgrade code +>>>>> is the cleanest approach, by the way. +>>>>> No need to thank me for my patience; thank you for your quickness! +>>>>> RichiH +>>>>> +>>>>> PS: If I get a handle on the mtime issue in the SHA backend, git +>>>>> annex will be pretty much perfect :) diff --git a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn new file mode 100644 index 0000000000..530a8da5d5 --- /dev/null +++ b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn @@ -0,0 +1,5 @@ +Found this out the hard way. See the comment in the below post for what happens. + +[[/forum/git_annex_add_crash_and_subsequent_recovery/]] + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment new file mode 100644 index 0000000000..7ff8f8e3d9 --- /dev/null +++ b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-12-06T16:49:32Z" + content=""" +This only happens with the WORM backend (or possibly with SHA1E if the file's extension has a newline). + +The problem is not the newline in the file, but the newline in the key generated for the file. It's probably best to just disallow such keys being created. +"""]] diff --git a/doc/bugs/bad_comment_in_ssh_public_key_ssh-rsa.mdwn b/doc/bugs/bad_comment_in_ssh_public_key_ssh-rsa.mdwn new file mode 100644 index 0000000000..87e937d42d --- /dev/null +++ b/doc/bugs/bad_comment_in_ssh_public_key_ssh-rsa.mdwn @@ -0,0 +1,22 @@ +What steps will reproduce the problem? +I was trying to pair two repositories on 2 different computers, +but my public key had this email address at the end: +bachir@Bachirs-iMac.local + +What is the expected output? What do you see instead? +I was expecting successful pairing. +I got: +bad comment in ssh public key ssh-rsa +AAB3....SAK +bachir@Bachirs-iMac.local + +What version of git-annex are you using? On what operating system? +I am using the package git-annex Version: 3.20120925 +on MacOSX Lion + +Please provide any additional information below. +I've checked your code, seems to complain about the dash '-' in the email address +bachir@Bachirs-iMac.local + +> This bug is already fixed, the fix is in the +> 3.20121001 release. [[done]] --[[Joey]] diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn new file mode 100644 index 0000000000..5e9100acfe --- /dev/null +++ b/doc/bugs/bare_git_repos.mdwn @@ -0,0 +1,29 @@ +It would be nice if git-annex could be used in bare git repos. +However, that is not currently supported. Problems include: + +* git-annex often does not read a git repo's config before touching it, + so it doesn't know if the repo is bare or not + (reading the config when operating on ssh repos would be a pain and SLOW; + I had some of that code in as of 1aa19422ac8748eeff219ac4f46df166dae783c5, + but ripped it all out) +* .. which results in creating `.git/annex` in a bare repo, which mightily + confuses git (so it will complain that the bare repo is not + a git repo at all!) +* `.git-annex/` needs to have state recorded to it and committed, and that + is not possible with a bare repo. (If [[todo/branching]] were done, + that might be fixed.) (now fixed) + +---- + +Update: Now that git-annex-shell is used for accessing remote repos, +it would be possible to add smarts about bare repos there, and avoid +some of the above problems. Probably only the state recording problem +remains. + +A possible other approach to the state recording repo is to not +record state changes on the remote in that case. Git-annex already +records remote state changes locally whenever it modifies the state of a +remote. --[[Joey]] + +> And... [[done]]! See [[/bare_repositories]] for current status +> and gotchas. --[[Joey]] diff --git a/doc/bugs/bug_in_download_prebuilt_linux_tarball__44___and_constraints_issues_with_3.20121112.mdwn b/doc/bugs/bug_in_download_prebuilt_linux_tarball__44___and_constraints_issues_with_3.20121112.mdwn new file mode 100644 index 0000000000..4a942ec9ee --- /dev/null +++ b/doc/bugs/bug_in_download_prebuilt_linux_tarball__44___and_constraints_issues_with_3.20121112.mdwn @@ -0,0 +1,45 @@ +What steps will reproduce the problem? + +First issue: +* The prebuilt tarball doesn't synchronize properly. + +Reproduce: +* Download and untar tarball +* Start gitannex webapp on multiple computers(i had three in use) +* Synchronize with xmpp and an ssh server backend. +* The tarball versions don't push/get from ssh server backend. + +Second issue: +* I can't install git-annex on ubuntu 12.10(Works fine in debian unstable) +* http://hpaste.org/77684 + +Reproduce: +* cabal update +* cabal install --only-dependencies +* cabal configure +* cabal build +* cabal install --bindir=$HOME/bin + +With these constraints the cabal install can work: +* cabal install --only-dependencies ./ --constraint=certificate==1.2.2 --constraint=crypto-pubkey-types==0.1.1 + +What is the expected output? What do you see instead? +* Tarball version doesn't push to ssh backend. +* cabal install git-annex gives http://hpaste.org/77684 + +What version of git-annex are you using? On what operating system? +* git-annex 3.20121112 on debian unstale (working) +* ubuntu 12.10(failing) + +Please provide any additional information below. + +With these constraints the cabal install can work: +* cabal install --only-dependencies ./ --constraint=certificate==1.2.2 --constraint=crypto-pubkey-types==0.1.1 + +NOTE: +I couldn't get the markdown to cooperate, so using pl pagetype. + +> I suspect this is [[done]].. +> +> I fixed some bugs in the prebuilt tarball in the past 2 days that prevented it +> from transferring files. diff --git a/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn b/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn new file mode 100644 index 0000000000..d0551c151b --- /dev/null +++ b/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn @@ -0,0 +1,25 @@ +Here goes... + +
+laplace:git-annex jtang$ make
+ghc -O2 -threaded -Wall -ignore-package monads-fd -ignore-package monads-tf -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_KQUEUE -DOSX --make git-annex Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
+
+Assistant/Threads/MountWatcher.hs:40:0:
+     warning: #warning Building without dbus support; will use mtab polling
+[165 of 208] Compiling Assistant.Alert  ( Assistant/Alert.hs, tmp/Assistant/Alert.o )
+[173 of 208] Compiling Assistant.DaemonStatus ( Assistant/DaemonStatus.hs, tmp/Assistant/DaemonStatus.o )
+[174 of 208] Compiling Assistant.TransferQueue ( Assistant/TransferQueue.hs, tmp/Assistant/TransferQueue.o )
+[175 of 208] Compiling Assistant.Threads.Watcher ( Assistant/Threads/Watcher.hs, tmp/Assistant/Threads/Watcher.o )
+
+Assistant/Threads/Watcher.hs:61:43:
+    Couldn't match expected type `Utility.Kqueue.Kqueue'
+                with actual type `()'
+    Expected type: IO Utility.Kqueue.Kqueue -> IO Utility.Kqueue.Kqueue
+      Actual type: IO Utility.Kqueue.Kqueue -> IO ()
+    In the fourth argument of `watchDir', namely `startup'
+    In the second argument of `($)', namely
+      `watchDir "." ignored hooks startup'
+make: *** [git-annex] Error 1
+
+ +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/build_is_broken_at_commit_cc0e5b7.mdwn b/doc/bugs/build_is_broken_at_commit_cc0e5b7.mdwn new file mode 100644 index 0000000000..43074915f0 --- /dev/null +++ b/doc/bugs/build_is_broken_at_commit_cc0e5b7.mdwn @@ -0,0 +1,13 @@ +the build is currently borked with + +
+ghc -O2 -threaded -Wall -ignore-package monads-fd -ignore-package monads-tf -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING -DWITH_KQUEUE -DOSX --make git-annex -o tmp/git-annex Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
+Assistant/Install.hs:19:8:
+Could not find module `Utility'
+Use -v to see a list of the files searched for.
+make: *** [git-annex] Error 1
+
+ +This was first introduced at commit e88e3ba + +> oops, -DOSX is not a good idea. [[done]] --[[Joey]] diff --git a/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn b/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn new file mode 100644 index 0000000000..34c2eef5f0 --- /dev/null +++ b/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn @@ -0,0 +1,43 @@ +Building commit 8baff14054e65ecbe801eb66786a55fa5245cb30 yields this... + + +
+[164 of 189] Compiling Command.Sync ( Command/Sync.hs, tmp/Command/Sync.o )
+Command/Sync.hs:268:34:
+Not in scope: `vermarker'
+Perhaps you meant `varmarker' (line 267)
+make: *** [git-annex] Error 1
+
+ +Supplied fix... + +
+
+From a23a1af99c7a95c316a87f9c6f5f67a6f8ff6937 Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Wed, 27 Jun 2012 21:55:22 +0100
+Subject: [PATCH 14/14] fix build issue introduced in
+ 8baff14054e65ecbe801eb66786a55fa5245cb30
+
+---
+ Command/Sync.hs | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Command/Sync.hs b/Command/Sync.hs
+index b2bf24d..dfaed59 100644
+--- a/Command/Sync.hs
++++ b/Command/Sync.hs
+@@ -265,7 +265,7 @@ mergeFile file key
+        | otherwise = go $ shortHash $ show key
+        where
+                varmarker = ".variant-"
+-               doubleconflict = vermarker `isSuffixOf` (dropExtension file)
++               doubleconflict = varmarker `isSuffixOf` (dropExtension file)
+                go v = takeDirectory file
+                         dropExtension (takeFileName file)
+                        ++ varmarker ++ v
+-- 
+1.7.11.1
+
+ +[[fixed|done]] diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn new file mode 100644 index 0000000000..a7bae50b8b --- /dev/null +++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn @@ -0,0 +1,14 @@ +A recent checkout of git-annex fails to build for me (I've installed the new dependancies as well) + +
+[70 of 81] Compiling Command.DropUnused ( Command/DropUnused.hs, Command/DropUnused.o )
+[71 of 81] Compiling Command.Status   ( Command/Status.hs, Command/Status.o )
+
+Command/Status.hs:133:37: Not in scope: `swap'
+make: *** [git-annex] Error 1
+
+ +it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install. I ran a bisect and found that commit 75a3f5027f74565d909fb940893636d081d9872a seems to have broken git-annex for me, reverting the commit allows me to build git-annex, I have not run the tests to verify everything is working correctly though. + +> Probably `swap` appeared only in a newer GHC. I've reverted to avoid a +> versioned build dependency. [[done]] --[[Joey]] diff --git a/doc/bugs/build_problem_on_OSX.mdwn b/doc/bugs/build_problem_on_OSX.mdwn new file mode 100644 index 0000000000..e859a11501 --- /dev/null +++ b/doc/bugs/build_problem_on_OSX.mdwn @@ -0,0 +1,18 @@ +I just squelched a bunch of build issues (to do with dependancies) on my autobuilder for OSX, this is currently happening + +
+install -d tmp
+ghc -O2 -Wall -outputdir tmp -IUtility  -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_DNS -DWITH_KQUEUE -threaded --make git-annex -o tmp/git-annex Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
+
+Assistant/Threads/NetWatcher.hs:29:0:
+     warning: #warning Building without dbus support; will poll for network connection changes
+
+Assistant/Threads/MountWatcher.hs:36:0:
+     warning: #warning Building without dbus support; will use mtab polling
+[ 29 of 259] Compiling Utility.OSX      ( Utility/OSX.hs, tmp/Utility/OSX.o )
+
+Utility/OSX.hs:22:17: Not in scope: `myHomeDir'
+make: *** [git-annex] Error 1
+
+ +> Someone else reported that too; I fixed it. [[done]] --[[Joey]] diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn new file mode 100644 index 0000000000..48386bde47 --- /dev/null +++ b/doc/bugs/building_on_lenny.mdwn @@ -0,0 +1,80 @@ +hi, + +I am trying to build git annex on lenny. + +I checked out the latest from git c88d4939453845efee04da811d64aa41046f9c11, +installed all the packages (some from backports) as required by dpkg-buildpackage + +Then I get this: + + ... + mkdir -p build + ghc -odir build -hidir build --make git-annex + [ 1 of 19] Compiling Utility ( Utility.hs, build/Utility.o ) + [ 2 of 19] Compiling GitRepo ( GitRepo.hs, build/GitRepo.o ) + [ 3 of 19] Compiling GitQueue ( GitQueue.hs, build/GitQueue.o ) + [ 4 of 19] Compiling TypeInternals ( TypeInternals.hs, build/TypeInternals.o ) + [ 5 of 19] Compiling Types ( Types.hs, build/Types.o ) + [ 6 of 19] Compiling Annex ( Annex.hs, build/Annex.o ) + [ 7 of 19] Compiling Locations ( Locations.hs, build/Locations.o ) + [ 8 of 19] Compiling UUID ( UUID.hs, build/UUID.o ) + [ 9 of 19] Compiling LocationLog ( LocationLog.hs, build/LocationLog.o ) + [10 of 19] Compiling Core ( Core.hs, build/Core.o ) + [11 of 19] Compiling Backend.URL ( Backend/URL.hs, build/Backend/URL.o ) + [12 of 19] Compiling Backend ( Backend.hs, build/Backend.o ) + + Backend.hs:114:50: + Not in scope: type constructor or class `SomeException' + make[1]: *** [git-annex] Error 1 + make[1]: Leaving directory `/home/cstamas/tmp/git-annex' + dh_auto_build: make -j1 returned exit code 2 + make: *** [build] Error 2 + dpkg-buildpackage: failure: debian/rules build gave error exit status 2 + +I will try to check the mentioned file for error, but I do not know how to program in haskell. + +Thanks for your help! --[[cstamas]] + +> Newer versions of ghc changed their exception handling types, and +> I coded git-annex to use the new style and not the old. gch6 6.12 will +> work. I do not think there is a backport available though. --[[Joey]] +> +> Ok, found and deployed a workaround. It is not tested. Let me know how it +> works for you. --[[Joey]] + +>> I did a git pull and now I get: + + mkdir -p build + ghc -cpp -odir build -hidir build --make git-annex + [ 1 of 20] Compiling Portability ( Portability.hs, build/Portability.o ) + + Portability.hs:13:21: + Not in scope: type constructor or class `Exception' + make[1]: *** [git-annex] Error 1 + make[1]: Leaving directory `/home/cstamas/tmp/git-annex' + dh_auto_build: make -j1 returned exit code 2 + make: *** [build] Error 2 + dpkg-buildpackage: failure: debian/rules build gave error exit status 2 + +>> --[[cstamas]] + +>>> Ok well, I'm not going to try to reimplement all of +>>> Control.Exception.Extensible so I've made it use it. You will have to +>>> figure out how to install that library yourself though, I don't know +>>> how to use cabal with such an old ghc. Library is here: +>>> +>>> and I asked how to get it on stable here: +>>> --[[Joey]] + +>>>> I made some effort with cabal on lenny. I can install (and I did it) cabal +>>>> from squeeze as dependencies are ok. Then I installed extensible +>>>> exceptions, but it places it in some local dir that git-annex's installer +>>>> (or ghc itself) does not know about. +>>>> +>>>> Later I realized that *only* for the compilation ghc6 and its friends are +>>>> needed. So I built the package on my other machine running squeeze. Then +>>>> resulting deb packages cleanly installs on lenny +>>>> +>>>> For me this is OK. Thanks! --[[cstamas]] + +[[done]] diff --git a/doc/bugs/cabal_build:___34__Could_not_find_module___96__Data.AssocList__39____34__.mdwn b/doc/bugs/cabal_build:___34__Could_not_find_module___96__Data.AssocList__39____34__.mdwn new file mode 100644 index 0000000000..a23930bd75 --- /dev/null +++ b/doc/bugs/cabal_build:___34__Could_not_find_module___96__Data.AssocList__39____34__.mdwn @@ -0,0 +1,23 @@ +What steps will reproduce the problem? + +Download current git head (fa5100d) and run cabal update; cabal install --only-dependencies; cabal configure; cabal build + +What is the expected output? What do you see instead? + +Expect succcessful build, get: + +Assistant/Install.hs:24:8: + Could not find module `Data.AssocList' + It is a member of the hidden package `hxt-9.3.1.1'. + Perhaps you need to add `hxt' to the build-depends in your .cabal file. + Use -v to see a list of the files searched for. + + +What version of git-annex are you using? On what operating system? + +building using cabal from clone of git clone git://git-annex.branchable.com/, commit fa5100d (same problem happens with last release, 3.20121127). On OS X, "Lion". + + +Please provide any additional information below. + +> [[done]], using Data.List instead now. diff --git a/doc/bugs/cabal_build:___34__Could_not_find_module___96__Data.AssocList__39____34__/comment_1_0da9fd67c3cc01b316f95a1df4eb62ae._comment b/doc/bugs/cabal_build:___34__Could_not_find_module___96__Data.AssocList__39____34__/comment_1_0da9fd67c3cc01b316f95a1df4eb62ae._comment new file mode 100644 index 0000000000..967e203c1b --- /dev/null +++ b/doc/bugs/cabal_build:___34__Could_not_find_module___96__Data.AssocList__39____34__/comment_1_0da9fd67c3cc01b316f95a1df4eb62ae._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="Could not find module `Data.AssocList' confirmed fixed." + date="2012-11-28T03:38:19Z" + content=""" +builds fine now, thank you! +"""]] diff --git a/doc/bugs/cabal_configure_is_broken_on_OSX_builds.mdwn b/doc/bugs/cabal_configure_is_broken_on_OSX_builds.mdwn new file mode 100644 index 0000000000..ddbbcb8746 --- /dev/null +++ b/doc/bugs/cabal_configure_is_broken_on_OSX_builds.mdwn @@ -0,0 +1,14 @@ +Seems the last few commits have borked 'cabal configure' on OSX with the following error message + +
+[jtang@laplace git-annex (master)]$ cabal configure 
+Resolving dependencies...
+
+Build/InstallDesktopFile.hs:19:8:
+    Could not find module `Assistant.OSX'
+    Use -v to see a list of the files searched for.
+
+ +Looks like a missing module. + +> Was broken everywhere really, so I fixed it. [[done]] --[[Joey]] diff --git a/doc/bugs/cabal_install_on_Ubuntu_12.04_fails_with_complaint_about_regex-base.mdwn b/doc/bugs/cabal_install_on_Ubuntu_12.04_fails_with_complaint_about_regex-base.mdwn new file mode 100644 index 0000000000..b84d930b47 --- /dev/null +++ b/doc/bugs/cabal_install_on_Ubuntu_12.04_fails_with_complaint_about_regex-base.mdwn @@ -0,0 +1,34 @@ +## What steps will reproduce the problem? + +I attempt + + cabal install git-annex + +(already having installed `c2hs`) + +## What is the expected output? What do you see instead? + +I get + + $ cabal install git-annex + Resolving dependencies... + cabal: Could not resolve dependencies: + trying: git-annex-3.20130216.1 (user goal) + trying: git-annex-3.20130216.1:+webapp + trying: git-annex-3.20130216.1:+assistant + trying: yesod-1.1.9 (dependency of git-annex-3.20130216.1:+assistant) + trying: http-conduit-1.9.0 (dependency of yesod-1.1.9) + trying: regex-compat-0.95.1/installed-851... (dependency of + http-conduit-1.9.0) + next goal: regex-base (dependency of regex-compat-0.95.1/installed-851...) + rejecting: regex-base-0.93.2/installed-999... (conflict: regex-base => + mtl==2.0.1.0/installed-db1..., git-annex => mtl(>=2.1.1)) + rejecting: regex-base-0.93.2, 0.93.1, 0.93, 0.92, 0.91, 0.90, 0.83, 0.72.0.2, + 0.72.0.1, 0.71 (conflict: regex-compat => regex-base==0.93.2/installed-999...) + +> This is a transient version skew in cabal, generally +> known as "cabal hell". It will be fixed when the libraries +> involved fix their versioned dependencies, which may have +> already happened. Precompiled builds of git-annex are provided +> so I do not have to chase every transient version skew +> that occurs for every use who runs into it. [[done]] --[[Joey]] diff --git a/doc/bugs/can__39__t_run_the_assistant_from_the_command_line_anymore__63__.mdwn b/doc/bugs/can__39__t_run_the_assistant_from_the_command_line_anymore__63__.mdwn new file mode 100644 index 0000000000..58828e8ca8 --- /dev/null +++ b/doc/bugs/can__39__t_run_the_assistant_from_the_command_line_anymore__63__.mdwn @@ -0,0 +1,31 @@ +What steps will reproduce the problem? + +just compiled the latest version of git annex from git, checkout hash was c9c0042 + +Tried running the assistant from the command line, got unexpected error + +What is the expected output? What do you see instead? + +Expected assistant to start, instead got: + + annex$ git annex assistant + git-annex: watch mode is not available on this system + + +What version of git-annex are you using? On what operating system? + +annex$ git annex version +git-annex version: 3.20130103 +local repository version: 3 +default repository version: 3 +supported repository versions: 3 +upgrade supported from repository versions: 0 1 2 +annex$ + +OS X Lion. + + +Please provide any additional information below. + +> The cabal file had a typo that prevented it from using hfsevents. I'm +> uploading a fix now. [[done]] --[[Joey]] diff --git a/doc/bugs/can_not_add_ssh_remote_to_assistant_with___34__host:port__34___syntax.mdwn b/doc/bugs/can_not_add_ssh_remote_to_assistant_with___34__host:port__34___syntax.mdwn new file mode 100644 index 0000000000..dfe2738ef5 --- /dev/null +++ b/doc/bugs/can_not_add_ssh_remote_to_assistant_with___34__host:port__34___syntax.mdwn @@ -0,0 +1,30 @@ +What steps will reproduce the problem? + +* Start the assistant +* try to add a remote server (ssh) which is not on port 22. E.g. using myhost:1234 +* after a rather long time the connection fails + + +What is the expected output? What do you see instead? + +it would be nice if this syntax was supported, or if an (optional) port field was provided. +second best solution: inform the user that "myhost:1234" is not the expected format. +third best solution (already in place) fail with "some error message". + + + + +What version of git-annex are you using? On what operating system? + +3.20121016 on Ubuntu 12.04 (in future maybe also on home nas with wheezy) + + +Please provide any additional information below. + +Thanks for a nice program and all your work on debian! +this is not really a bug more of a wishlist feature. + + +[[!tag /design/assistant]] + +> Ok, it has a port field now. [[done]] --[[Joey]] diff --git a/doc/bugs/can_not_add_ssh_remote_to_assistant_with___34__host:port__34___syntax/comment_1_397eb359c3f8ef30460a9556b6f55848._comment b/doc/bugs/can_not_add_ssh_remote_to_assistant_with___34__host:port__34___syntax/comment_1_397eb359c3f8ef30460a9556b6f55848._comment new file mode 100644 index 0000000000..991a7483d9 --- /dev/null +++ b/doc/bugs/can_not_add_ssh_remote_to_assistant_with___34__host:port__34___syntax/comment_1_397eb359c3f8ef30460a9556b6f55848._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 1" + date="2012-11-09T14:04:57Z" + content=""" +edit ~/.ssh/config. add + + Host myhost + Port 1234 + + +now you never have to specify the port again(for ssh,scp,rsync,etc) +"""]] diff --git a/doc/bugs/case-insensitive.mdwn b/doc/bugs/case-insensitive.mdwn new file mode 100644 index 0000000000..a917f64c28 --- /dev/null +++ b/doc/bugs/case-insensitive.mdwn @@ -0,0 +1,20 @@ +What steps will reproduce the problem? + +> Building git-annex on the ghc7.0 branch on a Mac with the default case-insensitive file system + +What is the expected output? What do you see instead? + +> Expected: build successfully; instead: + + ld: duplicate symbol _UtilityziDiskFree_zdwa_info in dist/build/git-annex/git-annex-tmp/Utility/diskfree.o and dist/build/git-annex/git-annex-tmp/Utility/DiskFree.o for architecture x86_64 + +What version of git-annex are you using? On what operating system? + +> commit `0bd5c90ef0518f75d52f0c5889422d8233df847d` on a Mac OS 10.6 and 10.7, using the Haskell Platform 2012.04 + +Please provide any additional information below. + +> The problem is that since `DiskFree.hs` generates `DiskFree.o` and `diskfree.c` generates `diskfree.o`, a case-insensitive file system overwrites one object file with the other. Renaming `diskfree.c` to `diskfreec.c` and changing the corresponding filenames in `git-annex.cabal` fixes the problem. + +>> Man, not this again. The 80's called, they want their +>> unix portability wars back. [[fixed|done]]. --[[Joey]] diff --git a/doc/bugs/case_sensitivity_on_FAT.mdwn b/doc/bugs/case_sensitivity_on_FAT.mdwn new file mode 100644 index 0000000000..682acc71d7 --- /dev/null +++ b/doc/bugs/case_sensitivity_on_FAT.mdwn @@ -0,0 +1,49 @@ +I was copying files to a directory remote with `git annex copy`. Out of 114 files, 9 of them failed with no message, just: + + copy data/foo.dat (to usbdrive...) failed + copy data/bar.dat (to usbdrive...) failed + +According to strace: + + 31338 mkdir("/media/annex/Zp/9v/SHA256-s1362999320--d650297c8cf8c2dc0575110a52d0c5cc0ff266f294a0599f85796a6b44b23492", 0777) = -1 ENOENT (No such file or directory) + 31338 mkdir("/media/annex/Zp/9v", 0777) = -1 ENOENT (No such file or directory) + 31338 mkdir("/media/annex/Zp", 0777) = -1 EEXIST (File exists) + 31338 stat("/media/annex/Zp", 0x7f8449f170d0) = -1 ENOENT (No such file or directory) + +The filesystem is FAT32 and has weird case semantics. This was mounted by udisks with its default options: + + /dev/sdb1 on /media/annex type vfat (rw,nosuid,nodev,uhelper=udisks,uid=1000,gid=1000,shortname=mixed,dmask=0077,utf8=1,showexec) + +I wonder if the directory remote should use hashDirLower instead of hashDirMixed? + +> git-annex intentionally uses the same layout for directory and rsync +> special remotes as it does for the .git/annex directory. As far +> as I know it works ok on (truely) case-insensative filesystems. +> +> Based on your strace, if you `ls /media/annex/Zp`, you will see +> "No such file or directory", but if you `mkdir /media/annex/Zp` it will +> fail with "File exists". Doesn't make much sense to me. +> +> The (default) VFAT mount option shortname=mixed causes this behavior. +> With shortname=lower it works ok. --[[Joey]] +> +>> So, the options for fixing this bug seem to be to fix Linux (which would +>> be a good idea IMHO but I don't really want to go there), or generally +>> convert git-annex to using lowercase for its hashing (which would be a +>> large amount of pain to rewrite all the symlinks in every git repo), +>> or some special hack around this problem. +>> +>> I've put in a workaround for the problem in the directory special +>> remote; it will use mixed case but fall-back to lowercase as necessary. +>> +>> That does leave the case of a bare git repository with annexed content +>> stored on VFAT. More special casing could fix it, but that is, I +>> think, an unusual configuration. Leaving the bug open for that case, +>> and for the even more unlikely configuration of a rsync special remote +>> stored on VFAT. --[[Joey]] + +>>> Bare repositories now use lowercase. rsync is the only remaining +>>> unsupported possibility. --[[Joey]] +>>>> Everything now uses lowercase, with the exception of non-bare +>>>> repos, which cannot be on FAT anyway due to using symlinks. [[done]] +>>>> --[[Joey]] diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn new file mode 100644 index 0000000000..a880392bf7 --- /dev/null +++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn @@ -0,0 +1,92 @@ +[[!meta title="arbitrary/configurable backends"]] + +(Retitling as this has drifted..) + +--- + +I thought this might be useful, since curl is being used for the URL backend, it might be worth checking for it's existence. + +
+diff --git a/configure.hs b/configure.hs
+index 772ba54..1a563e0 100644
+--- a/configure.hs
++++ b/configure.hs
+@@ -13,6 +13,7 @@ tests = [
+        , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"]
+        , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null"
++       , TestCase "curl" $ requireCmd "curl" "curl --version >/dev/null"
+        , TestCase "unicode FilePath support" $ unicodeFilePath
+        ] ++ shaTestCases [1, 256, 512, 224, 384]
+
+ +> Well, curl is an optional extra, so requireCmd is too strong. Changed +> to testCmd and applied, thank you! +> +> I thought about actually *using* the resulting SysConfig.curl +> to disable the URL backend if False.. but probably it's better +> to just let it fail if curl is not available. Although, if we wanted +> to add a check for wget or something and use it when curl was not +> available, that might be worth doing. --[[Joey]] + +>> I was thinking that is it worth doing a generic "stat", "delete", "get" +>> and "put" options, I do like the idea of having the possibility of +>> being about to use completely arbitrary storage systems or arbitrary +>> transfer systems. If there was the capability of doing so it would be +>> interesting to see possibilities of using aria2 for using something +>> like bittorrent as backend, or using something like irods or some +>> grid storage system as the storage archive. It's just an idea as +>> I have seen it implemented quite well in irods. + +>>> I'm unsure about the idea of having a backend where that is +>>> parameterized. It would mean that one annex's GENERIC-foo key +>>> might be entirely different from another's key with the same backend +>>> and details. And a misconfiguration could get data the wrong +>>> way and get the wrong data, etc. +>>> +>>> I mostly look at the URL backend as an example that can be modified to +>>> make this kind of custom backend. You already probably know enough to +>>> make a TORRENT backend where keys are the urls to torrents to download +>>> with `aria2c --follow-torrent=mem`. +>>> +>>> I am also interested in doing backends that use eg, cloud storage. +>>> A S3 backend that could upload files to S3 in addition to downloading +>>> them, for example, would be handy. --[[Joey]] + +>>>> So, rather than use backends to do this, it instead made more sense +>>>> to make them [[special_remotes]]. The URL backend remains a bit +>>>> of a special case, and a bittorrent backend that downloaded a file +>>>> from a bittorrent url would still be a good use of backend, but for +>>>> storing files in external data stores like S3, making it a remote +>>>> makes better sense. I think I can close this bug now, [[done]] +>>>> --[[Joey]] + +also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not) + +> It's correct, typewise, but I don't see any real reason to bother +> with the change. But I do appreciate patches, which have been rare +> so far, probaby because of Haskell.. :) --[[Joey]] + +>> heh agreed + +
+diff --git a/Backend/URL.hs b/Backend/URL.hs
+index 29dc8fe..4afcf86 100644
+--- a/Backend/URL.hs
++++ b/Backend/URL.hs
+@@ -50,10 +50,13 @@ dummyFsck _ _ _ = return True
+ dummyOk :: Key -> Annex Bool
+ dummyOk _ = return True
+ 
++curl :: [CommandParam] -> IO Bool
++curl = boolSystem "curl"
++
+ downloadUrl :: Key -> FilePath -> Annex Bool
+ downloadUrl key file = do
+        showNote "downloading"
+        showProgress -- make way for curl progress bar
+-       liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
++       liftIO $ curl [Params "-# -o", File file, File url]
+        where
+                url = join ":" $ drop 1 $ split ":" $ show key 
+
diff --git a/doc/bugs/com.branchable.git-annex.assistant.plist_is_invalid.mdwn b/doc/bugs/com.branchable.git-annex.assistant.plist_is_invalid.mdwn new file mode 100644 index 0000000000..d67cdcd404 --- /dev/null +++ b/doc/bugs/com.branchable.git-annex.assistant.plist_is_invalid.mdwn @@ -0,0 +1,15 @@ +What steps will reproduce the problem? +`cat com.branchable.git-annex.assistant` + +What version of git-annex are you using? On what operating system? + git-annex version: 3.20121112 on OS X Mountain Lion + +Please provide any additional information below. +The '`RunAtLoad`' key is missing a value. + +It should say: + +`RunAtLoad`
+`` + +> Fixed in git. [[done]] --[[Joey]] diff --git a/doc/bugs/commitBuffer:_invalid_argument___40__invalid_character__41__.mdwn b/doc/bugs/commitBuffer:_invalid_argument___40__invalid_character__41__.mdwn new file mode 100644 index 0000000000..027d22431b --- /dev/null +++ b/doc/bugs/commitBuffer:_invalid_argument___40__invalid_character__41__.mdwn @@ -0,0 +1,228 @@ +What steps will reproduce the problem? + + $ git init a.git + Initialized empty Git repository in /var/tmp/git-annex-bug/a.git/.git/ + $ cd a.git + $ git annex init a + init a ok + (Recording state in git...) + $ touch Ren$'\351' + $ git annex add Ren$'\351' + add René (checksum...) ok + (Recording state in git...) + $ git ci -m "Added Rene'." + [master (root-commit) a61b796] Added Rene'. + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 "Ren\351" + $ cd .. + $ git clone -o a a.git b.git + Cloning into b.git... + remote: Counting objects: 13, done. + remote: Compressing objects: 100% (9/9), done. + remote: Total 13 (delta 1), reused 0 (delta 0) + Receiving objects: 100% (13/13), done. + Resolving deltas: 100% (1/1), done. + $ cd b.git + $ git annex copy --from=a --fast -v + (merging a/git-annex into git-annex...) + copy René + git-annex: /var/tmp/git-annex-bug/b.git/.git/annex/transfer/download/7c5ee764-e8c6-11e1-b0c5-67c6ec1236d6/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855: commitBuffer: invalid argument (invalid character) + failed + (Recording state in git...) + git-annex: copy: 1 failed + +What is the expected output? What do you see instead? + +Expect that files will be copied, but they are not. + +What version of git-annex are you using? On what operating system? + + $ echo $LANG + en_US.UTF-8 + $ lsb_release -a + No LSB modules are available. + Distributor ID: Ubuntu + Description: Ubuntu 11.10 + Release: 11.10 + Codename: oneiric + $ uname -a + Linux pdx-desktop 3.0.0-23-generic #39-Ubuntu SMP Thu Jul 19 19:18:53 UTC 2012 i686 i686 i386 GNU/Linux + $ bash --version + GNU bash, version 4.2.10(1)-release (i686-pc-linux-gnu) + Copyright (C) 2011 Free Software Foundation, Inc. + License GPLv3+: GNU GPL version 3 or later + + This is free software; you are free to change and redistribute it. + There is NO WARRANTY, to the extent permitted by law. + $ ghc --version + The Glorious Glasgow Haskell Compilation System, version 7.4.2 + $ git annex version + git-annex version: 3.20120807 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +Please provide any additional information below. + +The problem is related to weird characters in file names. In the +above example, the "weird character" is an accented 'e' (entered with +$'\351' in bash and zsh). I am able to add the files with weird +characters in their name to one annex, but I cannot copy them to other +annexes using `git annex copy`. + +The above example is a simplification of a real problem I am +experiencing. In my real scenario, the file is not empty. See the +attachment for some variations, including with non-empty +files. UPDATE: I'm not allowed to add attachments. See below. + +May be related to these (long-ago fixed) bugs: +http://git-annex.branchable.com/todo/support-non-utf8-locales/ + + +"Attachment": Here are my notes, including more examples: + + Programs I'm using: + + $ echo $LANG + en_US.UTF-8 + $ lsb_release -a + No LSB modules are available. + Distributor ID: Ubuntu + Description: Ubuntu 11.10 + Release: 11.10 + Codename: oneiric + $ uname -a + Linux pdx-desktop 3.0.0-23-generic #39-Ubuntu SMP Thu Jul 19 19:18:53 UTC 2012 i686 i686 i386 GNU/Linux + $ bash --version + GNU bash, version 4.2.10(1)-release (i686-pc-linux-gnu) + Copyright (C) 2011 Free Software Foundation, Inc. + License GPLv3+: GNU GPL version 3 or later + + This is free software; you are free to change and redistribute it. + There is NO WARRANTY, to the extent permitted by law. + $ ghc --version + The Glorious Glasgow Haskell Compilation System, version 7.4.2 + $ git annex version + git-annex version: 3.20120807 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + + + Simplest way to see problem: one empty file with weird character + (accented e: $'\351') in name: + + $ git init a.git + Initialized empty Git repository in /var/tmp/git-annex-bug/a.git/.git/ + $ cd a.git + $ git annex init a + init a ok + (Recording state in git...) + $ touch Ren$'\351' + $ git annex add Ren$'\351' + add René (checksum...) ok + (Recording state in git...) + $ git ci -m "Added Rene'." + [master (root-commit) a61b796] Added Rene'. + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 "Ren\351" + $ cd .. + $ git clone -o a a.git b.git + Cloning into b.git... + remote: Counting objects: 13, done. + remote: Compressing objects: 100% (9/9), done. + remote: Total 13 (delta 1), reused 0 (delta 0) + Receiving objects: 100% (13/13), done. + Resolving deltas: 100% (1/1), done. + $ cd b.git + $ git annex copy --from=a --fast -v + (merging a/git-annex into git-annex...) + copy René + git-annex: /var/tmp/git-annex-bug/b.git/.git/annex/transfer/download/7c5ee764-e8c6-11e1-b0c5-67c6ec1236d6/SHA256-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855: commitBuffer: invalid argument (invalid character) + failed + (Recording state in git...) + git-annex: copy: 1 failed + + + Problem disappears with two empty files: + + $ cd .. + $ git init a2.git + Initialized empty Git repository in /var/tmp/git-annex-bug/a2.git/.git/ + $ cd a2.git + $ git annex init a2 + init a2 ok + (Recording state in git...) + $ touch Ren$'\351' + $ git annex add Ren$'\351' + add René (checksum...) ok + (Recording state in git...) + $ git ci -m "Add Ren$'\351'." + [master (root-commit) 62ac1c8] Add Ren$'\351'. + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 "Ren\351" + $ touch Rene + $ git annex add Rene + add Rene (checksum...) ok + (Recording state in git...) + $ git ci -m "Add Rene." + [master c455523] Add Rene. + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 Rene + $ cd .. + $ git clone -o a2 a2.git b2.git + Cloning into b2.git... + done. + $ cd b2.git + $ git annex copy --from=a2 --fast -v + (merging a2/git-annex into git-annex...) + copy Rene (from a2...) ok + (Recording state in git...) + + + Problem returns with two non-empty files: + + $ cd .. + $ git init a4.git + Initialized empty Git repository in /var/tmp/git-annex-bug/a4.git/.git/ + $ cd a4.git + $ git annex init a4 + init a4 ok + (Recording state in git...) + $ touch Ren$'\351' + $ rm Ren$'\351' + $ ls + $ echo "some content" > Ren$'\351' + $ git annex add Ren$'\351' + add René (checksum...) ok + (Recording state in git...) + $ git ci -m "Add Ren$'\351'." + [master (root-commit) f090d90] Add Ren$'\351'. + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 "Ren\351" + $ echo "some other content" > Rene + $ git annex add Rene + add Rene (checksum...) ok + (Recording state in git...) + $ git ci -m "Add Rene." + [master 97c20cd] Add Rene. + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 Rene + $ cd .. + $ git clone -o a4 a4.git b4.git + Cloning into b4.git... + done. + $ cd b4.git + $ git annex copy --from=a4 --fast -v + (merging a4/git-annex into git-annex...) + copy Rene (from a4...) ok + copy René + git-annex: /var/tmp/git-annex-bug/b4.git/.git/annex/transfer/download/a5fcd0d4-e8c8-11e1-bb41-43ce1cb9a9f1/SHA256-s13--1c87b6727f523662df714f06a94ea27fa4d9050c38f4f7712bd4663ffbfdfa01: commitBuffer: invalid argument (invalid character) + failed + (Recording state in git...) + git-annex: copy: 1 failed + +> [[Fixed|done]]. Sorry this took so long, I was at a very busy point when +> you filed this and am only just getting caught up. --[[Joey]] diff --git a/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn b/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn new file mode 100644 index 0000000000..dfd3ffb829 --- /dev/null +++ b/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn @@ -0,0 +1,8 @@ +commit f20a40f breaks on OSX as mntent.h doesn't exist, the closet thing available to what mntent.h provides is getmntinfo(), it looks yet another bunch of ifdef's might be needed to work around OSX. This problem maybe similarly true with FreeBSD, libfam seems to have worked around the issue - + +hope the above report helps. + +> Thanks, that was a very useful pointer. I couldn't figure out how to +> use Haskell's FFI to loop over the list of statfs structs returned by +> getmntinfo, so I incorporated that C code into a little library, +> and it seems to work ok. [[done]] --[[Joey]] diff --git a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn new file mode 100644 index 0000000000..2485e7b19e --- /dev/null +++ b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn @@ -0,0 +1,53 @@ +When two git-annex processes are running and both modifying the git-annex +branch, it's possible one will fail due to git's locking. When this +happens, git-annex has already recorded its state in the journal (so no +data is lost), but git-annex does crash, which can be surprising. + +I feel that, in general, multiple git-annex processes should be able to run +concurrently. A big lock around all commands, or even all +repository-modifying commands is a bad idea. Also, it's probably best to +only worry about locking conflicts editing the git-annex branch. While `git +annex add` and a few other commands make changes to the main git repo, +and can have similar locking issues, so can any git commands that stage +changes (I think.. check). + +Probably should KISS. Just add a lock file that is taken before changes to +the git-annex branch, and if it's locked, wait. Changes to the git-annex +branch tend to happen quickly (unless it's committing an enormous set of +changes, and even that is relatively fast), so waiting seems ok. --[[Joey]] + +---- + +Commit 7981eb4cb512fbe3c49a3dd165c31be14ae4bc49 is more pessimistic, +describes some other potential issues. + +* The journal needs to be emptied (done) and kept locked (not done) during + a merge, since a merge operates at a level below the journal, and any + changes that are journaled during a merge can overwrite changes merged + in from another branch. + +* Two git-annex processes can be doing conflicting things and inconsistent + information be written to the journal. + + - One example would be concurrent get and drop of the same key. + But could this really race? If the key was already present, the get + would do nothing, so record no changes. If the key was not yet present, + the drop would do nothing, and record no changes. + + - Instead, consider two copys of a key to different locations. If the + slower copy starts first and ends last, it could cache the location + info, add the new location, and lose the other location it was copied to. + Tested it and the location is not cached during the whole copy (logChange + reads the current log immediatly before writing), so this + race's window is very small -- but does exist. + +---- + +## Updated plan + +Make Branch.change transactional, so it takes a lock, reads a file, +applies a function to it, and writes the changed file. + +Make Branch.update hold the same lock. + +> [[Done]]. diff --git a/doc/bugs/configurable_path_to_git-annex-shell.mdwn b/doc/bugs/configurable_path_to_git-annex-shell.mdwn new file mode 100644 index 0000000000..af2474451c --- /dev/null +++ b/doc/bugs/configurable_path_to_git-annex-shell.mdwn @@ -0,0 +1,7 @@ +On one of my ssh remotes I had to install git-annex using cabal. No system-wide installation possible. Hence `git-annex` and `git-annex-shell` are not in the default `$PATH` but in `$HOME/.cabal/bin`. + +Right now the command run by git-annex when ssh'ing to a remote is hardcoded to "`git-annex-shell`", which doesn't work for me. It would be nice to be able to change this per remote, for example with an option named `annex..annex-shell-command`. Changing "`git-annex-shell`" in `Remote/Helper/Ssh.hs` to "`~/.cabal/bin/git-annex-shell`" worked for me, but it's obviously very ugly :) + +Could you do that please? I'll try to hack it myself and send you a patch in the next few days, but I'm pretty new to Haskell so it may take me a while... Thanks! + +> [[closing|done]], see comments --[[Joey]] diff --git a/doc/bugs/configurable_path_to_git-annex-shell/comment_1_fb6771f902b57f2b690e7cc46fdac47e._comment b/doc/bugs/configurable_path_to_git-annex-shell/comment_1_fb6771f902b57f2b690e7cc46fdac47e._comment new file mode 100644 index 0000000000..50349363b1 --- /dev/null +++ b/doc/bugs/configurable_path_to_git-annex-shell/comment_1_fb6771f902b57f2b690e7cc46fdac47e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2012-03-22T22:26:02Z" + content=""" +It doesn't need to be installed into the system PATH; just the user PATH. Which you should be able to control. + +Exactly how to do this surely varies, but here I have a `~/.bashrc` containing `PATH=$HOME/bin:$PATH; export PATH` and I keep git-annex-shell in bin and it's available to eg \"ssh mybox git-annex-shell\" +"""]] diff --git a/doc/bugs/configurable_path_to_git-annex-shell/comment_2_2b856f4f0b65c2331be7d565f0e4e8a8._comment b/doc/bugs/configurable_path_to_git-annex-shell/comment_2_2b856f4f0b65c2331be7d565f0e4e8a8._comment new file mode 100644 index 0000000000..adea95ecf2 --- /dev/null +++ b/doc/bugs/configurable_path_to_git-annex-shell/comment_2_2b856f4f0b65c2331be7d565f0e4e8a8._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://schnouki.net/" + nickname="Schnouki" + subject="comment 2" + date="2012-03-23T13:27:12Z" + content=""" +Hmm, ok, solved. I'm using zsh, which is a little different: `.zshrc` is only read for interactive shells, so `ssh mybox 'echo $PATH'` displayed `/usr/bin:/bin:/usr/sbin:/sbin`. Using `.zshenv`, which is used even for non-interactive shells, did the trick. Thanks! +"""]] diff --git a/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn b/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn new file mode 100644 index 0000000000..b71f39e5a7 --- /dev/null +++ b/doc/bugs/configure_mistakes_hashalot_bins_for_sha__63____63____63__sum_and_builds_a_broken_git-annex_executable.mdwn @@ -0,0 +1,57 @@ +git-annex's configure step finds hashalot's /usr/sbin/sha256, /usr/sbin/sha384, and /usr/sbin/sha512 executables and mistakes them for sha256sum, sha384sum, and sha512sum and prefers them over the correct executables. Hashalot is not compatible, but the build does not fail producing a broken git-annex executable which tries to use hashalot's programs instead of the appropriate shaXXXsum program and is non-functional. + +Hashalot can be found at: + + +What steps will reproduce the problem? + +Compile with hashalot's programs in the path. + + +What is the expected output? What do you see instead? + +Expect to see configure output: + +[...] +
+  checking sha1... sha1sum
+  checking sha512... sha512sum
+  checking sha224... sha224sum
+  checking sha384... sha384sum
+  checking sha256... sha256sum
+
+[...] + + +Instead I see configure output: + +[...] +
+  checking sha1... sha1sum
+  checking sha512... sha512
+  checking sha224... sha224sum
+  checking sha384... sha384
+  checking sha256... sha256
+
+[...] + + +What version of git-annex are you using? On what operating system? + +I am using 3.20120605, but have checked out the latest GIT and confirmed the bug is still there. + + +Please provide any additional information below. + +This is not a runtime bug, only compile time. Uninstalling Hashalot or simply removing it from the PATH is enough to work around this bug. The bug is, however, frustrating because at first glance there appears to be no problem. However any functions of git-annex which use the affected SHA hash functions will fail with the resulting executable and the failure gives no clear indication of why. + +I found this bug on Gentoo when I installed git-annex on a system which already had hashalot installed. In the case of Gentoo, git-annex is compiled with hashalot's executables in the path, but normal users don't have /usr/sbin/ in their path so git-annex just fails to find the executable. If you put hashalot in the path, then git annex still fails to work as hashalot is not a replacement for sha1sum and friends. + +It may be enough to just prefer sha???sum over sha??? if they both exist. + +> Grr. There is no consistency across unixes as to the names of these +> programs and now something is installing shaN commands that don't +> generate a checksum?! +> +> Ok, fine, configure now checks that the program it finds outputs a known +> good checksum. [[done]] --[[Joey]] diff --git a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn new file mode 100644 index 0000000000..2b9c773678 --- /dev/null +++ b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn @@ -0,0 +1,6 @@ +On RHEL5 (and clones) systems uuidgen is available as an alternative to +uuid, the configure script fails, it should probably detect either uuid or +uuidgen, or let the user decide? - also uuidgen behaves differently from +uuid on debian. + +> uuidgen is now supported. --[[Joey]] [[done]] diff --git a/doc/bugs/conflicting_haskell_packages.mdwn b/doc/bugs/conflicting_haskell_packages.mdwn new file mode 100644 index 0000000000..5528fad824 --- /dev/null +++ b/doc/bugs/conflicting_haskell_packages.mdwn @@ -0,0 +1,17 @@ +The compilation command should states which packages are used and avoid the default mechnasim that automatically search for them. + +This can be done by the flags -hide-packages and then -package foo + +> My ghc does not have a `--hide-packages` option. +> +> Could you just show the build problem that you are suggesting I work +> around? --[[Joey]] + + +> Thanks npouillard, I see the problem now. +> +> +> I've added "-ignore-package monads-fd" to GHCFLAGS. I hope I don't +> really have to hide all packages and individually turn them back on; +> surely this monads-fd/mtl conflict is an exception, and Haskell's module +> system is not a mess of conflicting modules? --[[Joey]] [[done]] diff --git a/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment b/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment new file mode 100644 index 0000000000..42f44bf9cf --- /dev/null +++ b/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="http://ertai.myopenid.com/" + nickname="npouillard" + subject="how to reproduce the package conflict issue" + date="2011-02-07T14:12:43Z" + content=""" +If you install the monads-fd package (with cabal install for instance), then you can no longer build git-annex: + +
+./configure
+  checking cp -a... yes
+  checking cp -p... yes
+  checking cp --reflink=auto... yes
+  checking uuid generator... uuid
+  checking xargs -0... yes
+  checking rsync... yes
+ghc -O2 -Wall --make git-annex
+
+Annex.hs:22:7:
+    Ambiguous module name `Control.Monad.State':
+      it was found in multiple packages: monads-fd-0.2.0.0 mtl-2.0.1.0
+make: *** [git-annex] Error 1
+
+"""]] diff --git a/doc/bugs/conq:_invalid_command_syntax.mdwn b/doc/bugs/conq:_invalid_command_syntax.mdwn new file mode 100644 index 0000000000..82cd51d8da --- /dev/null +++ b/doc/bugs/conq:_invalid_command_syntax.mdwn @@ -0,0 +1,30 @@ +I've been getting an occasional error from git-annex. + +The error is: 'conq: invalid command syntax.' + +For example, the last two commands I ran are: + + $ git annex unused + unused . (checking for unused data...) (checking master...) (checking origin/master...) + Some annexed data is no longer used by any files: + NUMBER KEY + 1 SHA256-s..... + (To see where data was previously used, try: git log --stat -S'KEY') + + To remove unwanted data: git-annex dropunused NUMBER + + ok + + $ git annex dropunused 1 + dropunused 1 conq: invalid command syntax. + ok + + + +*OS:* OSX + port installs of the GNU tools + +*git-annex-version:* 3.20111211 + +*git-version:* 1.7.7.4 + +> [[done]], apparently not a git-annex bug --[[Joey]] diff --git a/doc/bugs/conq:_invalid_command_syntax/comment_1_f33b83025ce974e496f83f248275a66a._comment b/doc/bugs/conq:_invalid_command_syntax/comment_1_f33b83025ce974e496f83f248275a66a._comment new file mode 100644 index 0000000000..62881bfc7a --- /dev/null +++ b/doc/bugs/conq:_invalid_command_syntax/comment_1_f33b83025ce974e496f83f248275a66a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2012-01-03T00:34:59Z" + content=""" +AFAICs, you probably just have a \"conq\" program that is running in the background and emitted this error. + +The error message is not part of git-annex; it does not run any \"conq\" thing itself. Although you could try passing the --debug parameter to check the commands it does run to see if one of them somehow causes this conq thing. +"""]] diff --git a/doc/bugs/conq:_invalid_command_syntax/comment_2_195106ca8dedad5f4d755f625e38e8af._comment b/doc/bugs/conq:_invalid_command_syntax/comment_2_195106ca8dedad5f4d755f625e38e8af._comment new file mode 100644 index 0000000000..39ece1eca2 --- /dev/null +++ b/doc/bugs/conq:_invalid_command_syntax/comment_2_195106ca8dedad5f4d755f625e38e8af._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2012-01-03T00:41:08Z" + content=""" +A google search +finds other examples of this error message related to ssh, mercurial, and bitbucket. What that has to do with git is anyone's guess, but I'm pretty sure git-annex is not related to it at all. +"""]] diff --git a/doc/bugs/conq:_invalid_command_syntax/comment_3_55af43e2f43a4c373f7a0a33678d0b1c._comment b/doc/bugs/conq:_invalid_command_syntax/comment_3_55af43e2f43a4c373f7a0a33678d0b1c._comment new file mode 100644 index 0000000000..75132c1d6b --- /dev/null +++ b/doc/bugs/conq:_invalid_command_syntax/comment_3_55af43e2f43a4c373f7a0a33678d0b1c._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://a-or-b.myopenid.com/" + ip="203.45.2.230" + subject="comment 3" + date="2012-01-03T00:49:41Z" + content=""" +Yeah, I saw those google links myself, but couldn't see why the bitbucket/ssh would be relevant. + +The strange thing is that I *only* get this message when running git-annex. + +I also don't have a conq in my path so I don't know where it is running from. + + +Oh well, if I ever sort it out I'll post back here. +"""]] diff --git a/doc/bugs/copy_doesn__39__t_scale.mdwn b/doc/bugs/copy_doesn__39__t_scale.mdwn new file mode 100644 index 0000000000..a5ca9d9ee6 --- /dev/null +++ b/doc/bugs/copy_doesn__39__t_scale.mdwn @@ -0,0 +1,38 @@ +It seems that git-annex copies every individual file in a separate +transaction. This is quite costly for mass transfers: each file involves a +separate rsync invocation and the creation of a new commit. Even with a +meager thousand files or so in the annex, I have to wait for fifteen +minutes to copy the contents to another disk, simply because every +individual file involves some disk thrashing. Also, it seems suspicious +that the git-annex branch would get a thousands commits of history from the +simple procedure of copying everything to a new repository. Surely it would +be better to first copy everything and then create only a single commit +that registers the changes to the files' availability? + +> git-annex is very careful to commit as infrequently as possible, +> and the current version makes *1* commit after all the copies are +> complete, even if it transferred a billion files. The only overhead +> incurred for each file is writing a journal file. +> You must have an old version. +> --[[Joey]] + +(I'm also not quite clear on why rsync is being used when both repositories +are local. It seems to be just overhead.) + +> Even when copying to another disk it's often on +> some slow bus, and the file is by definition large. So it's +> nice to support resumes of interrupted transfers of files. +> Also because rsync has a handy progress display that is hard to get with cp. +> +> (However, if the copy is to another directory in the same disk, it does +> use cp, and even supports really fast copies on COW filesystems.) +> --[[Joey]] + +--- + +Oneshot mode is now implemented, making git-annex-shell and other +short lifetime processes not bother with committing changes. +[[done]] --[[Joey]] + +Update: Now it makes one commit at the very end of such a mass transfer. +--[[Joey]] diff --git a/doc/bugs/copy_doesn__39__t_scale/comment_1_7c12499c9ac28a9883c029f8c659eb57._comment b/doc/bugs/copy_doesn__39__t_scale/comment_1_7c12499c9ac28a9883c029f8c659eb57._comment new file mode 100644 index 0000000000..749b3ba108 --- /dev/null +++ b/doc/bugs/copy_doesn__39__t_scale/comment_1_7c12499c9ac28a9883c029f8c659eb57._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4" + nickname="Lauri" + subject="comment 1" + date="2012-01-28T00:17:37Z" + content=""" +To me it very much seems that a commit per file is indeed created at the remote end, although not at the local end. See the following transcript: . + + +"""]] diff --git a/doc/bugs/copy_doesn__39__t_scale/comment_2_f85d8023cdbc203bb439644cf7245d4e._comment b/doc/bugs/copy_doesn__39__t_scale/comment_2_f85d8023cdbc203bb439644cf7245d4e._comment new file mode 100644 index 0000000000..9a2bd92fa3 --- /dev/null +++ b/doc/bugs/copy_doesn__39__t_scale/comment_2_f85d8023cdbc203bb439644cf7245d4e._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2012-01-28T19:32:36Z" + content=""" +Ah, I see, I was not thinking about the location log update that's done on the remote side. + +For transfers over ssh, that's a separate git-annex-shell invoked per change. For local-local transfers, it's all done in a single process but it spins up a state to handle the remote and then immediately shuts it down, also generating a commit. + +In either case, I think there is a nice fix. Since git-annex *does* have a journal nowadays, and goes to all the bother to +support recovery if a process was interrupted and journalled changes that did not get committed, there's really no reason in either of these cases for the remote end to do anything more than journal the change. The next time git-annex is actually run on the remote, and needs to look up location information, it will merge the journalled changes into the branch, in a single commit. + +My only real concern is that some remotes might *never* have git-annex run in them directly, and would just continue to accumulate journal files forever. Although due to the way the journal is structured, it can have, at a maximum, the number of files in the git-annex branch. However, the number of files in it is expected to be relatively smal and it might get a trifle innefficient, as it lacks directory hashing. These performance problems could certainly be dealt with if they do turn out to be a problem. +"""]] diff --git a/doc/bugs/copy_doesn__39__t_scale/comment_3_4592765c3d77bb5664b8d16867e9d79c._comment b/doc/bugs/copy_doesn__39__t_scale/comment_3_4592765c3d77bb5664b8d16867e9d79c._comment new file mode 100644 index 0000000000..aa9bf1e45a --- /dev/null +++ b/doc/bugs/copy_doesn__39__t_scale/comment_3_4592765c3d77bb5664b8d16867e9d79c._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4" + nickname="Lauri" + subject="comment 3" + date="2012-01-29T01:51:35Z" + content=""" +That sounds just fine, but indeed my use case was a bare backup/transfer repository that is meant to always be only at the remote end of git-annex operations. So why not as well do a single commit after everything has been copied and journaled? That's what's done at the other end too, after all. Or, if commits are to be minimized, just stage the journal into the index before finishing, but don't commit it yet? + +(I would actually prefer this mode of usage for other git-annex operations, too. In git you can add stuff little by little and commit them all in one go. In git-annex the add immediately creates a commit, which is unexpected and a bit annoying.) + +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn new file mode 100644 index 0000000000..69fbc816f0 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn @@ -0,0 +1,6 @@ +Conversation moved from [[tips/recover_data_from_lost+found]] +to a proper bug. --[[Joey]] + +(Unfortunatly that scrambled the comment creation times and thus order.) + +> Added a message [[done]] --[[Joey]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment new file mode 100644 index 0000000000..ec24c478d8 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 10" + date="2011-05-15T16:47:53Z" + content=""" +The key is the basename of the symlink target. +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment new file mode 100644 index 0000000000..7bc54573ed --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 11" + date="2011-05-15T18:53:26Z" + content=""" +It seems the objects are in the remote after all, but the remote is unaware of this fact. No idea where/why the remote lost that info, but.. Anyway, with the SHA backends, wouldn't it make sense to simply return \"OK\" and update the annex logs accordingly, no? + +Local: + + % ls -l foo + lrwxrwxrwx 1 richih richih 312 Apr 3 01:18 foo -> .git/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 + % + +Remote: + + % git-annex-shell recvkey SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 + git-annex-shell: key is already present in annex + % strace git-annex-shell recvkey /base/git-annex/fun SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 2>&1 | grep SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 + stat64(\"/base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491\", {st_mode=S_IFREG|0444, st_size=80781, ...}) = 0 + % ls -l /base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 + -r--r--r-- 1 richih richih 80781 2011-04-01 12:44 /base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 + % +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment new file mode 100644 index 0000000000..b458a37b69 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 12" + date="2011-05-15T19:40:47Z" + content=""" +So, it appears that you're using git annex copy --fast. As documented that assumes the location log is correct. So it avoids directly checking if the bare repo contains the file, and tries to upload it, and the bare repo is all like \"but I've already got this file!\". The only way to improve that behavior might be to let rsync go ahead and retransfer the file, which, with recovery, should require sending little data etc. But I can't say I like the idea much, as the repo already has the content, so unlocking it and letting rsync mess with it is an unnecessary risk. I think it's ok for --force to blow up +if its assumptions turn out to be wrong. + +If you use git annex copy without --fast in this situation, it will do the right thing. +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment new file mode 100644 index 0000000000..d92ecbba03 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 13" + date="2011-05-15T20:25:25Z" + content=""" +Yes, makes sense. I am so used to using --fast, I forgot a non-fast mode existed. I still think it would be a good idea to fall back to non-fast mode if --fast runs into an error from the remote, but as that is well without my abilities how about this patch? + + + From 4855510c7a84eb5d28fdada429580a8a42b7112a Mon Sep 17 00:00:00 2001 + From: Richard Hartmann + Date: Sun, 15 May 2011 22:20:42 +0200 + Subject: [PATCH] Make error in RecvKey.hs suggest possible solution + + --- + Command/RecvKey.hs | 2 +- + 1 files changed, 1 insertions(+), 1 deletions(-) + + diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs + index 126608f..b917a1c 100644 + --- a/Command/RecvKey.hs + +++ b/Command/RecvKey.hs + @@ -27,7 +27,7 @@ start :: CommandStartKey + start key = do + present <- inAnnex key + when present $ + - error \"key is already present in annex\" + + error \"key is already present in annex. If you are running copy, try without '--fast'\" + + ok <- getViaTmp key (liftIO . rsyncServerReceive) + if ok + -- + 1.7.4.4 + +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment new file mode 100644 index 0000000000..f45bd70468 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 14" + date="2011-05-15T20:50:26Z" + content=""" +Or, even better, wouldn't it make sense to have SHA backends always default to --fast and only use non-fast when any snags are hit, use non-fast mode for that file. + +Though if we continue here, we should probably move this to its own page. +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment new file mode 100644 index 0000000000..b4a00bd7e1 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 15" + date="2011-05-15T21:38:47Z" + content=""" +PS: Just to make this clear, I am using a custom alias for all my copying needs and thus didn't even see that I used --fast. :p +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment new file mode 100644 index 0000000000..6d3dabb92b --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 16" + date="2011-05-16T20:01:28Z" + content=""" +Thanks. +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment new file mode 100644 index 0000000000..59c30de534 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-05-12T00:07:29Z" + content=""" +I followed this to re-inject files which git annex fsck listed as missing. + +For everyone of those files, I get + + git-annex-shell: key is already present in annex + rsync: connection unexpectedly closed (0 bytes received so far) [sender] + rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.8] + +when trying to copy the files to the remote. + +-- Richard +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment new file mode 100644 index 0000000000..44aab3baa0 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-05-12T01:01:34Z" + content=""" +Sounds like you probably didn't commit after the fsck, or didn't push so the other repository did not know the first had the content again -- but I'm not 100% sure. +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment new file mode 100644 index 0000000000..4744db995c --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 3" + date="2011-05-14T09:06:54Z" + content=""" +As my comment from work is stuck in moderation: + +I ran this twice: + + git pull && git annex add . && git annex copy . --to --fast --quiet && git commit -a -m \"$HOST $(date +%F--%H-%M-%S-%Z)\" && git push + +but nothing changed +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment new file mode 100644 index 0000000000..1fb19ab192 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-05-14T16:13:58Z" + content=""" +Hmm. Old versions may have forgotten to git add a .git-annex location log file when recovering content with fsck. That could be another reason things are out of sync. + +But I'm not clear on which repo is trying to copy files to which. + +(NB: If the files were recovered on a bare git repo, fsck cannot update the location log there, which could also explain this.) +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment new file mode 100644 index 0000000000..0a546bd88c --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 5" + date="2011-05-14T19:03:43Z" + content=""" +Version: 0.20110503 + +My local non-bare repo is copying to a remote bare repo. + +I have been recovering in a non-bare repo. + +If there is anything I can send you to help... If I removed said files and went through http://git-annex.branchable.com/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/ -- would that help? +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment new file mode 100644 index 0000000000..1e3f325319 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 6" + date="2011-05-14T19:23:45Z" + content=""" +Well, focus on a specific file that exhibits the problem. What does `git annex whereis` say about it? Is the content actually present in annex/objects/ on the bare repository? Does that contradict whereis? +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment new file mode 100644 index 0000000000..f7dfad68ca --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 7" + date="2011-05-14T23:13:15Z" + content=""" +It exists locally, whereis tells me it exists locally and locally, only. + +The object is _not_ in the bare repo. + +The file _might_ have gone missing before I upgraded my annex backend version to 2. Could this be a factor? +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment new file mode 100644 index 0000000000..01248914c3 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 8" + date="2011-05-15T00:09:34Z" + content=""" +What you're describing should be impossible; the error message shown can only occur if the object is present in the annex where `git-annex-shell recvkey` is run. So something strange is going on. + +Try reproducing it by running on the remote system, `git-annex-shell recvkey /remote/repo.git $key` .. if you can reproduce it, I guess the next thing to do will be to strace the command and see why it's thinking the object is there. +"""]] diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment new file mode 100644 index 0000000000..2755cf3317 --- /dev/null +++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 9" + date="2011-05-15T09:16:49Z" + content=""" +Just to make sure: How do I get $key? What I did was look at the path in the object store of the local repo and see if that exact same path & file existed in the remote. +"""]] diff --git a/doc/bugs/creating_a_remote_server_repository.mdwn b/doc/bugs/creating_a_remote_server_repository.mdwn new file mode 100644 index 0000000000..90e85bc8b7 --- /dev/null +++ b/doc/bugs/creating_a_remote_server_repository.mdwn @@ -0,0 +1,24 @@ +What steps will reproduce the problem? + +I was trying to add a remote server repository. Unfortunately, this didn't work. Enter Host name, user name, directory, port +(left most of them at their default), but there is no way, to specify a password.) Clicking check this server failed: + + Failed to ssh to the server. Transcript: Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,password). + +(Problem was, I could never enter a password. Interestingly, on the konsole, I get a prompt for a password, but I can't enter anything there). + + +What is the expected output? What do you see instead? + +Successfully create a connection and use the remote server. + + +What version of git-annex are you using? On what operating system? +Version: 3.20130124 + + + +Please provide any additional information below. + +[[!tag /design/assistant]] +[[!meta title="ssh password prompting"]] diff --git a/doc/bugs/creating_a_remote_server_repository/comment_1_de1a370347428245bcfca60eaca96779._comment b/doc/bugs/creating_a_remote_server_repository/comment_1_de1a370347428245bcfca60eaca96779._comment new file mode 100644 index 0000000000..aad008fd01 --- /dev/null +++ b/doc/bugs/creating_a_remote_server_repository/comment_1_de1a370347428245bcfca60eaca96779._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.3.125" + subject="comment 1" + date="2013-02-05T19:22:04Z" + content=""" +You're not really intended to start the git-annex assistant from a console. If you do, ssh will go ahead and prompt for passwords using that controlling console. + +When the assistant is not started from a console, ssh should use the `ssh-askpass` to prompt for the password. Assuming your system has that installed. +"""]] diff --git a/doc/bugs/creds_directory_not_automatically_created.mdwn b/doc/bugs/creds_directory_not_automatically_created.mdwn new file mode 100644 index 0000000000..d4e436da42 --- /dev/null +++ b/doc/bugs/creds_directory_not_automatically_created.mdwn @@ -0,0 +1,3 @@ +I just compiled ff7810eb83d8372e6206d487c63482d678e0b3d4 and created a new git-annex repository through the setup steps of "git-annex webapp". Then I tried configuring a Jabber account from the webapp. It then failed to create $REPO/.git/annex/creds/xmpp with a "No such file or directory" message because $REPO/.git/annex/creds did not get created. After doing a manual mkdir the Jabber setup went through fine. + +> [[Fixed|done]], thanks. --[[Joey]] diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn new file mode 100644 index 0000000000..296d61aac5 --- /dev/null +++ b/doc/bugs/cyclic_drop.mdwn @@ -0,0 +1,104 @@ +drop's verification that a remote still has content can fail +if the remote is also dropping the content at the same time. Each +repository checks that the other still has the content, and then both +drop it. Could also happen with larger cycles of repositories. + +> Confirmed fixed now. All cases tested. [[done]] + +--- + +Fixing this requires locking. (Well, there are other ways, like moving the +content to a holding area when checking if it's safe to drop, but they +seem complicated, and would be hard to implement for move --from.) + +Add per-content lock files. An exclusive lock is held on content when +it's in the process of being dropped, or moved. The lock is taken +nonblocking; if it cannot be obtained, something else is acting on the +content and git-annex should refuse to do anything. + +Then when checking inannex, try to take a shared lock. Note that to avoid +deadlock, this must be a nonblocking lock. (Actually, with fcntl locking, +can just check if there is a lock, without needing to take one.) +If it fails, the status of the content is unknown, so inannex should fail. +Note that this failure needs to be distinguishable from "not in annex". + +> Thinking about these lock files, this would be a lot more files, +> and would possibly break some assumptions that everything in +> `.git/annex/objects` is a key's content. (Or would need lots more +> directories to put the lock files elsewhere.) There would be more +> overhead to manage these and have them on disk. +> +> What if it just locked the actual content file? The obvious limitation +> is only content that was already inannex could be locked, but that +> happens to be exactly what's needed here; if content is not present, +> it's not going to get dropped or moved. +> +> Of course, if some consumer of a file locked it, then it could prevent it +> from being dropped or moved. This could be considered a bug, or a feature. :) +> +> However, this would mean that such a hypothetical consumer could also +> make inannex checks fail. +> +> The other downside is that, for fcntl exclusive locking, the file has to +> be opened for write. Normally the modes of content files are locked down +> to prevent modifcation. Dealt with, but oh so nasty. Almost makes flock +> locking seem worth using. + +--- + +drop --from could also cycle. Locking should fix. + +> Confirmed fixed now. + +--- + +move --to can also be included in the cycle, since it can drop data. + +Consider move to a remote that already has the content and +is at the same time doing a drop (or a move). The remote +verifies the content is present on the movee, and removes its copy. +The movee removes its copy. + +So move --to needs to take the content lock on start. Then the inannex +will fail. + +This is why it's important for inannex to fail in a way that is +distinguishable from "not in annex". Otherwise, move --to +would see the cycle as the remote not having content, and try to +redundantly send it, drop it locally, and still race. + +> Confirmed fixed now. + +-- + +move --from is similar. Consider a case where both the local and the remote +are doing a move --from. Both have the content, and confirm the other does, +via inannex checks. Then both run git-annex-shell dropkey, removing both +copies. + +So move --from needs to take the content lock on start, so the inannex will +fail. NB: If the content is not locally present, don't take the lock. + +> Confirmed fixed now. + +--- + +Another cycle might be running move --to and move --from on the same file, +locally. The exclusivity of the content lock solves this, as only one can +run at a time. + +Would it work with a shared lock? The --to would run git-annex-shell +inannex. The --from would also be running, and would run git-annex-shell +dropkey. So inannex and dropkey would end up running on the remote +at the same time. Dropkey takes the content lock, and inannex checks it, +but what if inannex runs first? Then it returns true, and then the content +is removed, so both the --to and --from see success and the --to proceeds +to remove the local content that the --from already caused to be removed +from the remote. So, no, the nasty exclusive lock is needed. + +> Confirmed fixed now. + +--- + +Another cycle might involve move --from and drop, both run on the same +file, locally. Again, the exclusive lock solves this. diff --git a/doc/bugs/direct_mode_renames.mdwn b/doc/bugs/direct_mode_renames.mdwn new file mode 100644 index 0000000000..60f4493005 --- /dev/null +++ b/doc/bugs/direct_mode_renames.mdwn @@ -0,0 +1,15 @@ +When in direct mode, renaming a file with `git mv` does not update the +direct mode mapping to use the new filename. --[[Joey]] + +Consistency checks now prevent anything bad happening when the mapping file +contains old filenames. Still, missing the new filename will prevent that +file working properly in direct mode. + +Perhaps the pre-commit hook needs to update the mapping for files that were +deleted or added. + +This also affects moves of files when the assistant is being used. +In this case, the assistant updates the mapping to add the new name, +but does not delete the old name from the mapping. + +> [[done]]; the pre-commit hook now updates the mappings. --[[Joey]] diff --git a/doc/bugs/direct_mode_renames/comment_1_f18c335e0d6f4259d2470935ef391cb8._comment b/doc/bugs/direct_mode_renames/comment_1_f18c335e0d6f4259d2470935ef391cb8._comment new file mode 100644 index 0000000000..aaf43a24ee --- /dev/null +++ b/doc/bugs/direct_mode_renames/comment_1_f18c335e0d6f4259d2470935ef391cb8._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkZktNHFhxC1kYA9KKdKpYJO4clq9WDsjE" + nickname="Jason" + subject="comment 1" + date="2013-02-06T06:35:21Z" + content=""" +I'm not sure if this matters, but in my case I wasn't even using `git mv`, I was just using `mv`. +"""]] diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment new file mode 100644 index 0000000000..be8b8b0a72 --- /dev/null +++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment @@ -0,0 +1,79 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck" + nickname="Jim" + subject="Case sensitivity" + date="2011-11-22T18:51:03Z" + content=""" +I agree, it's weird, but that's what I'm seeing: + + #!/bin/sh + + if [ $UID != 0 ] ; then echo \"need root\" ; exit 1 ; fi + + set -x + + # make image + cd /tmp + dd if=/dev/zero of=diskimage bs=1M count=40 + DEV=$(losetup --find --show diskimage) + + # make FAT32 fs + mkfs.vfat -F 32 $DEV + + # mount it + mkdir annex + mount -o shortname=mixed,utf8=1 $DEV annex + + # show bug + ( + cd annex + mkdir zP + mkdir Zp + ls Zp + ls + touch zP + touch Zp + ) + + # cleanup + umount annex + rm -r annex + losetup -d $DEV + rm diskimage + + # info + uname -a + +Output: + + + cd /tmp + + dd if=/dev/zero of=diskimage bs=1M count=40 + 40+0 records in + 40+0 records out + 41943040 bytes (42 MB) copied, 0.0847729 s, 495 MB/s + ++ losetup --find --show diskimage + + DEV=/dev/loop0 + + mkfs.vfat -F 32 /dev/loop0 + mkfs.vfat 3.0.9 (31 Jan 2010) + Loop device does not match a floppy size, using default hd params + + mkdir annex + + mount -o shortname=mixed,utf8=1 /dev/loop0 annex + + cd annex + + mkdir zP + + mkdir Zp + mkdir: cannot create directory `Zp': File exists + + ls Zp + ls: cannot access Zp: No such file or directory + + ls + zP + + touch zP + + touch Zp + touch: cannot touch `Zp': File exists + + umount annex + + rm -r annex + + losetup -d /dev/loop0 + + rm diskimage + + uname -a + Linux pilot 3.0.3+ #1 SMP Mon Aug 29 15:21:18 EDT 2011 x86_64 GNU/Linux + +"""]] diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment new file mode 100644 index 0000000000..5040b3120f --- /dev/null +++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-11-22T19:56:55Z" + content=""" +All right, I see the same thing with linux 3.1.0. It seems this behavior has changed since linux 3.0.0. Mounting with shortname=lower avoids the problem. + +I feel a good case could be made that this new behavior is a linux bug. Your example with touch particularly shows how weird it is. + +
+$ touch Foo
+$ echo hi > foo
+sh: cannot create foo: File exists
+$ rm foo
+rm: cannot remove `foo': No such file or directory
+
+"""]] diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment new file mode 100644 index 0000000000..54d6ff50ab --- /dev/null +++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck" + nickname="Jim" + subject="comment 3" + date="2011-11-22T20:35:01Z" + content=""" +I see the same results (\"`touch: cannot touch 'Zp': File exists`\") on these Debian systems: + + Linux pilot 3.0.3+ #1 SMP Mon Aug 29 15:21:18 EDT 2011 x86_64 GNU/Linux + Linux neurosis 3.0.0-1-amd64 #1 SMP Sun Jul 24 02:24:44 UTC 2011 x86_64 GNU/Linux + Linux bucket 2.6.39-2-amd64 #1 SMP Tue Jul 5 02:51:22 UTC 2011 x86_64 GNU/Linux + Linux psychosis 2.6.37-trunk-amd64 #1 SMP Thu Jan 6 14:13:28 UTC 2011 x86_64 GNU/Linux + Linux bacon 2.6.32-5-amd64 #1 SMP Thu Aug 12 13:01:50 UTC 2010 x86_64 GNU/Linux + +It does NOT happen on this Ubuntu system: + + Linux esensor 3.0.0-12-generic #20-Ubuntu SMP Fri Oct 7 14:56:25 UTC 2011 x86_64 x86_64 x86_64 GNU/Linux + +So really it seems like only the Ubuntu kernel is the outlier here? Maybe it has something to do with charsets or something; I think FAT is a mess in that regard and even long versus short filenames can behave differently. +"""]] diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment new file mode 100644 index 0000000000..406a6b18ee --- /dev/null +++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-11-22T20:59:55Z" + content=""" +Your ubuntu system has 3.0.0 which as noted does not have the problem. +"""]] diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment new file mode 100644 index 0000000000..1656ff2075 --- /dev/null +++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 5" + date="2011-11-22T21:01:14Z" + content=""" +I am surprised if it happens on 2.6.x though. Debian 3.0.0 seemed to not have the problem but perhaps my test was bad. +"""]] diff --git a/doc/bugs/done.mdwn b/doc/bugs/done.mdwn new file mode 100644 index 0000000000..a35d427198 --- /dev/null +++ b/doc/bugs/done.mdwn @@ -0,0 +1,4 @@ +recently fixed [[bugs]] + +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10 +archive=yes]] diff --git a/doc/bugs/dotdot_problem.mdwn b/doc/bugs/dotdot_problem.mdwn new file mode 100644 index 0000000000..cbefd5dae5 --- /dev/null +++ b/doc/bugs/dotdot_problem.mdwn @@ -0,0 +1,4 @@ +cannot "git annex ../foo" (GitRepo.relative is buggy and +git-ls-files also refuses w/o --full-name, which would need other changes) + +[[done]] diff --git a/doc/bugs/drop_fails_to_see_copies_that_whereis_sees.mdwn b/doc/bugs/drop_fails_to_see_copies_that_whereis_sees.mdwn new file mode 100644 index 0000000000..59b76c92b8 --- /dev/null +++ b/doc/bugs/drop_fails_to_see_copies_that_whereis_sees.mdwn @@ -0,0 +1,69 @@ +What steps will reproduce the problem? +------------------------------------------------------ + +I haven't tried doing this again, but here's what I did: + +I was copying files to my usb drive (hugex) with probably this command: + + git annex copy --to=hugex . + +While doing that, I did something stupid in another window, and filled my local hard drive (the one I was copying from) and git annex spit out some errors about not being able to write to log files (because my local drive was full.) + +I suspended (^Z) git annex, made some hard drive space, then resumed (fg). + +At first, git annex whereis didn't see the copies of some of the files in hugex (apparently the ones where git annex had trouble writing the local log files). After a "git remote update hugex" and I think a ``git annex merge``, whereis was able to see both copies. + +But git drop can't see both copies, and won't let me drop my local copy. I ran ``git annex fsck .`` on the directory with the now files I was copying above, and that didn't seem to change anything (or report errors.) + +Here's a terminal session where I'm showing the problem and some hopefully useful text: + + compy compy compy ~/video/tv/keen-eddie> git annex whereis 02-horse-heir.avi + whereis 02-horse-heir.avi (2 copies) + 5bfe091c-ed07-11df-842e-eb791a485889 -- here (compy) + e15161ec-f1bf-11df-a7b5-eb1f0e6921ee -- hugex + ok + compy compy compy ~/video/tv/keen-eddie> git annex drop 02-horse-heir.avi + drop 02-horse-heir.avi (unsafe) + Could only verify the existence of 1 out of 2 necessary copies + + No other repository is known to contain the file. + + (Use --force to override this check, or adjust annex.numcopies.) + failed + git-annex: drop: 1 failed + zsh: exit 1 git annex drop 02-horse-heir.avi + compy compy compy ~/video/tv/keen-eddie> ls -lh 02-horse-heir.avi + lrwxrwxrwx 1 jasonwoof jasonwoof 149 Nov 15 04:19 02-horse-heir.avi -> ../../../.git/annex/objects/KV/8G/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7 + compy compy compy ~/video/tv/keen-eddie> ls --dereference -lh 02-horse-heir.avi + -r--r--r-- 1 jasonwoof jasonwoof 342M Nov 15 04:19 02-horse-heir.avi + compy compy compy ~/video/tv/keen-eddie> ls -lh ../../../.git/annex/objects/KV/8G/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7 + -r--r--r-- 1 jasonwoof jasonwoof 342M Nov 15 04:19 ../../../.git/annex/objects/KV/8G/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7 + compy compy compy ~/video/tv/keen-eddie> ls -lh /media/hugex/jason/home.git/annex/objects/5bd/6a1/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7 + -r--r--r-- 1 jasonwoof jasonwoof 342M Jan 7 16:27 /media/hugex/jason/home.git/annex/objects/5bd/6a1/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7/SHA1-s358316330--7fcbf33b711e41def269f042842212d0bf3736a7 + +That last command is showing that the file is indeed in hugex's annex/objects + + +Oh, and another wrinkle. hugex is a bare repo. I tried to fsck hugex's video/tv directory, but it said that directory doesn't exist (I assume because there's no working copy.) + + +What is the expected output? What do you see instead? +------------------------------------------------------------------------------ + +I'd like ``git annex drop`` to have the same kind of confidence in the copies of these files on hugex. + +How can I resolve this. I tried ``git annex copy --to=hugex .`` again, but that command knows that these files are already copied to hugex. + +What version of git-annex are you using? On what operating system? + +3.20130102 on debian unstable (thanks for packaging!) + +Please provide any additional information below. +------------------------------------------------------------------ + +You can also reach me at ``jason@jasonwoof.com`` + +> [[done]]; the confusing message has been improved. +> +> BTW, you can use `git annex move` to ensure a file is on another repo and +> drop it locally. --[[Joey]] diff --git a/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_1_f5a9d99d90daf5eba4773d361fa1807a._comment b/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_1_f5a9d99d90daf5eba4773d361fa1807a._comment new file mode 100644 index 0000000000..647c6db3a8 --- /dev/null +++ b/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_1_f5a9d99d90daf5eba4773d361fa1807a._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.211" + subject="comment 1" + date="2013-01-08T18:43:33Z" + content=""" +So, the first part of this is git-annex behaving as designed. When there's a fatal disk problem and it cannot update its local location log, only the remote's location log gets updated. After a `git annex sync` (or your manual equivilant), the location log changes get propigated from the remote, and the local repository returns to a consistent state. + +As to the drop problem, look at it again. You have numcopies configured to 2, and there are currently 2 copies of the file. So dropping 1 would leave the numcopies constraint unsatisfied, and so it doesn't. You can override --numcopies 1 or --force the drop, but I don't see a bug here. + +(Perhaps the \"No other repository is known to contain the file.\" message is confusing in this context? It doesn't mean that no other repo has the file at all, but that there are no other repositories, that it was not able to check, that might have the file. If you had another removable drive with the file, and the drive was detached, this message would instead say \"Try making some of these repositories available: otherdrive\") + +(Re fscking in a bare repo, in a bare repository, you cannot fsck specific files/directories; fsck just checks every single key that git-annex knows about.) +"""]] diff --git a/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_2_040aa454cd8acd2857ef36884465576f._comment b/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_2_040aa454cd8acd2857ef36884465576f._comment new file mode 100644 index 0000000000..58f69646b5 --- /dev/null +++ b/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_2_040aa454cd8acd2857ef36884465576f._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://jasonwoof.com/" + nickname="JasonWoof" + subject="Thank you." + date="2013-01-08T23:02:33Z" + content=""" +First off, awesome that git annex auto-fixes itself after I make hard drive space and merge. + +The problem was that I didn't realize that my .gitattributes that were supposed to set numcopies to 1 didn't work. (Because I didn't realize that they don't apply recursively.) I fixed my .gitattributes, and then was able to drop those files. + +And you are right that the text \"No other repository is known to contain the file\" threw me off. Perhaps when numcopies > 1 you could change that message to \"Not enough other repositories are known to contain the file\". + +Thank you. + +-- Jason +"""]] diff --git a/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_3_f5d8faab325ee26800ecad5aba49b54b._comment b/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_3_f5d8faab325ee26800ecad5aba49b54b._comment new file mode 100644 index 0000000000..ca6287dd6f --- /dev/null +++ b/doc/bugs/drop_fails_to_see_copies_that_whereis_sees/comment_3_f5d8faab325ee26800ecad5aba49b54b._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://jasonwoof.com/" + nickname="JasonWoof" + subject="language suggestions" + date="2013-01-09T04:31:40Z" + content=""" +Maybe it could read \"Could not verify that enough copies of the file exist to safely drop it\". Or \"Could not verify that enough other repositories contain this file to safely drop it.\" + +It might be cool (if this isn't supported already) if there was a command to drop a file that would automatically copy it to another repo first if needed to satisfy numcopies. +"""]] diff --git a/doc/bugs/dropping_and_re-adding_from_web_remotes_doesn__39__t_work.mdwn b/doc/bugs/dropping_and_re-adding_from_web_remotes_doesn__39__t_work.mdwn new file mode 100644 index 0000000000..30327ba1fc --- /dev/null +++ b/doc/bugs/dropping_and_re-adding_from_web_remotes_doesn__39__t_work.mdwn @@ -0,0 +1,139 @@ +In experimenting with the web remote, I found that dropping a URL gave an error, but still succeeded, while re-adding the same URL fails to work correctly. + +#What steps will reproduce the problem? + +
+/tmp $ dd if=/dev/zero of=/tmp/file.bin count=1024
+1024+0 records in
+1024+0 records out
+524288 bytes (524 kB) copied, 0.0135652 s, 38.6 MB/s
+/tmp $ mkdir /tmp/repo
+/tmp $ cd /tmp/repo
+/tmp/repo $ git init
+Initialized empty Git repository in /tmp/repo/.git/
+/tmp/repo $ git annex init "test"
+init test ok
+(Recording state in git...)
+/tmp/repo $ git annex addurl file:///tmp/file.bin --file annexed.bin
+######################################################################## 100.0%
+(checksum...) ok
+(Recording state in git...)
+/tmp/repo $ git annex drop annexed.bin 
+drop annexed.bin (checking file:///tmp/file.bin...) ok
+(Recording state in git...)
+/tmp/repo $ mv /tmp/file.bin /tmp/file2.bin 
+/tmp/repo $ git annex get annexed.bin 
+get annexed.bin (from web...) 
+curl: (37) Couldn't open file /tmp/file.bin
+
+  Unable to access these remotes: web
+
+  Try making some of these repositories available:
+        00000000-0000-0000-0000-000000000001 -- web
+failed
+git-annex: get: 1 failed
+/tmp/repo $ git annex drop --from web annexed.bin --force
+drop web annexed.bin
+  removal from web not supported
+failed
+(Recording state in git...)
+git-annex: drop: 1 failed
+/tmp/repo $ git annex get annexed.bin 
+get annexed.bin (not available) 
+  No other repository is known to contain the file.
+failed
+git-annex: get: 1 failed
+/tmp/repo $ mv /tmp/file2.bin /tmp/file.bin 
+/tmp/repo $ git annex addurl file:///tmp/file.bin --file annexed.bin
+addurl annexed.bin ok
+/tmp/repo $ git annex whereis annexed.bin
+whereis annexed.bin (0 copies) failed
+git-annex: whereis: 1 failed
+/tmp/repo $ git annex addurl file:///tmp/file.bin --file annexed2.bin
+######################################################################## 100.0%
+(checksum...) ok
+(Recording state in git...)
+/tmp/repo $ git annex whereis annexed.bin
+whereis annexed.bin (1 copy)
+        e2418e81-ec04-4091-aabe-ed75d65f93fa -- here (test)
+ok
+/tmp/repo $ git annex whereis annexed2.bin
+whereis annexed2.bin (1 copy)
+        e2418e81-ec04-4091-aabe-ed75d65f93fa -- here (test)
+ok
+/tmp/repo $ git annex drop annexed.bin
+drop annexed.bin (unsafe)
+  Could only verify the existence of 0 out of 1 necessary copies
+
+  No other repository is known to contain the file.
+
+  (Use --force to override this check, or adjust annex.numcopies.)
+failed
+git-annex: drop: 1 failed
+/tmp/repo $ git annex drop annexed2.bin
+drop annexed2.bin (unsafe)
+  Could only verify the existence of 0 out of 1 necessary copies
+
+  No other repository is known to contain the file.
+
+  (Use --force to override this check, or adjust annex.numcopies.)
+failed
+git-annex: drop: 1 failed
+/tmp/repo $ mv /tmp/file.bin /tmp/file2.bin 
+/tmp/repo $ git annex addurl file:///tmp/file2.bin --file annexed.bin
+addurl annexed.bin ok
+(Recording state in git...)
+/tmp/repo $ git annex whereis annexed2.bin 
+whereis annexed2.bin (2 copies) 
+        00000000-0000-0000-0000-000000000001 -- web
+        e2418e81-ec04-4091-aabe-ed75d65f93fa -- here (test)
+
+  web: file:///tmp/file.bin
+  web: file:///tmp/file2.bin
+ok
+/tmp/repo $ mv /tmp/file2.bin /tmp/file.bin 
+/tmp/repo $ git annex drop annexed.bin 
+drop annexed.bin (checking file:///tmp/file.bin...) ok
+(Recording state in git...)
+/tmp/repo $ git annex get annexed.bin 
+get annexed.bin (from web...) 
+######################################################################## 100.0%
+ok
+(Recording state in git...)
+/tmp/repo $ git annex drop annexed.bin 
+drop annexed.bin (checking file:///tmp/file.bin...) ok
+(Recording state in git...)
+/tmp/repo $ mv /tmp/file.bin /tmp/file2.bin 
+/tmp/repo $ git annex get annexed.bin 
+get annexed.bin (from web...) 
+curl: (37) Couldn't open file /tmp/file.bin
+######################################################################## 100.0%
+ok
+(Recording state in git...)
+
+ +#What is the expected output? What do you see inst + + +When dropping one file and I see "git-annex: drop: 1 failed" I would expect the file to still be in the remote as far as git-annex is concerned. + +When re-adding the URL, I expect the file to be re-added to the web remote. (note that it re-appears after adding the same file via a different URL) + + +#What version of git-annex are you using? On what operating system? + + +3.20121112 on Gentoo Linux + + +#Please provide any additional information below. + +This seems to be a corner case, and would probably have minimal impact on most people. + +> Yeah, dropping from the web is pretty weird. +> +> I guess it makes sense to do if a website stops having a file and you don't +> want git-annex to try to download from it anymore. So, I've made dropping +> from the web remove all urls associated with the file, rather than failing. +> +> [[done]] --[[Joey]] diff --git a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn new file mode 100644 index 0000000000..c6ef13f844 --- /dev/null +++ b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn @@ -0,0 +1,13 @@ +I was trying out the example with the walkthrough using_the_URL_backend. I tried dropping files that I had after doing an "git annex get ." which have the URL backend associated with the files it fails with + + +
+[jtang@lenny gc]$ git annex drop -v curl-7.21.4.tar.gz
+drop curl-7.21.4.tar.gz
+failed
+git-annex: 1 failed
+
+ +At first I thought it was just my OSX machine not having the coreutils stuff load up before the BSD utils, but I then tried the same thing on my archlinux machine and it showed the same behaviour, that is I could not drop a file with the URL backend as shown in the walkthrough. + +> Whoops, got some logic backwards. [[fixed|done]]! --[[Joey]] diff --git a/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn new file mode 100644 index 0000000000..a6b44cd2a3 --- /dev/null +++ b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn @@ -0,0 +1,87 @@ +Unused files with double spaces in their name are not removed by `dropunused`: + +Script: + + #!/bin/bash + + BASE=/tmp/unused-bug + + # setup + set -x + chmod -R +w $BASE + rm -rf $BASE + mkdir -p $BASE + cd $BASE + + # create annex + git init . + git annex init + + # make a file with two spaces + echo hello > 'foo bar' + + # add it + git annex add --backend WORM 'foo bar' + git commit -m 'add' + + # remove it + git rm 'foo bar' + git commit -m 'remove' + + # unused + git annex unused + git annex dropunused 1 + git annex unused + +Output: + + + chmod -R +w /tmp/unused-bug + + rm -rf /tmp/unused-bug + + mkdir -p /tmp/unused-bug + + cd /tmp/unused-bug + + git init . + Initialized empty Git repository in /tmp/unused-bug/.git/ + + git annex init + init ok + + echo hello + + git annex add --backend WORM 'foo bar' + add foo bar ok + (Recording state in git...) + + git commit -m add + [master (root-commit) 926f7f5] add + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 foo bar + + git rm 'foo bar' + rm 'foo bar' + + git commit -m remove + [master d025e3f] remove + 1 files changed, 0 insertions(+), 1 deletions(-) + delete mode 120000 foo bar + + git annex unused + unused . (checking for unused data...) (checking master...) + Some annexed data is no longer used by any files: + NUMBER KEY + 1 WORM-s6-m1322200438--foo bar + (To see where data was previously used, try: git log --stat -S'KEY') + + To remove unwanted data: git-annex dropunused NUMBER + + ok + + git annex dropunused 1 + dropunused 1 ok + + git annex unused + unused . (checking for unused data...) (checking master...) + Some annexed data is no longer used by any files: + NUMBER KEY + 1 WORM-s6-m1322200438--foo bar + (To see where data was previously used, try: git log --stat -S'KEY') + + To remove unwanted data: git-annex dropunused NUMBER + + ok + +Strange that `dropunused` still said "ok" when it didn't succeed at removing the file. + +> It was misparsing the unused file, so it thought you'd asked it to drop a +> key that didn't exist (which means already dropped) so no error. I've +> fixed the bug. [[done]] --[[Joey]] diff --git a/doc/bugs/dropunused_doesn__39__t_work_in_my_case__63__.mdwn b/doc/bugs/dropunused_doesn__39__t_work_in_my_case__63__.mdwn new file mode 100644 index 0000000000..7428b091ae --- /dev/null +++ b/doc/bugs/dropunused_doesn__39__t_work_in_my_case__63__.mdwn @@ -0,0 +1,70 @@ +What steps will reproduce the problem? + +I am unable to create a minimal setup to reproduce this unfortunately. But my case is the following: + +* Two synced repos, I have switched between direct and indirect in both, currently they are indirect +* I have two submodules in the repos (not related to the unuesd files) + +* I used "git rm -r" to remove a bunch of files along with "git mv" to move some +* I have three commits of actions like this + +* In one repo, the one where I did the deletions, I have 172 unused files, and all seem to come from the first of the three commits +* In the second repo, to which I synced, I have 188 unused files, which includes a couple of files from the third commit as well + +* If I try to dropunused either a single file or the whole range of the files, from either repo, I get git-annex telling me "ok, recording state" but when I run unused again the files are still there. And looking into .git/objects/annex/ the file is still present + +This is the debug from the drop command: + + dropunused 9 [2013-02-07 12:47:24 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","show-ref","git-annex"] + [2013-02-07 12:47:24 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","show-ref","--hash","refs/heads/git-annex"] + [2013-02-07 12:47:24 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","log","refs/heads/git-annex..5f3fc9db5c7badb5fb25c3159c20584f11dadaf9","--oneline","-n1"] + [2013-02-07 12:47:24 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","log","refs/heads/git-annex..8e5674078706864f2eade86d8aa43027e05afc1d","--oneline","-n1"] + [2013-02-07 12:47:24 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","log","refs/heads/git-annex..cbe492cfa79728698f5d891d7f716fbcd9fc29e2","--oneline","-n1"] + [2013-02-07 12:47:24 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","log","refs/heads/git-annex..48a1bdf98a10ad9a81c0587f8909e94c1c0dccc4","--oneline","-n1"] + [2013-02-07 12:47:24 CET] chat: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","cat-file","--batch"] + [2013-02-07 12:47:24 CET] read: git ["config","--null","--list"] + [2013-02-07 12:47:24 CET] chat: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","hash-object","-w","--stdin-paths"] + [2013-02-07 12:47:24 CET] feed: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","update-index","-z","--index-info"] + [2013-02-07 12:47:24 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","show-ref","--hash","refs/heads/git-annex"] + [2013-02-07 12:47:24 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","write-tree"] + [2013-02-07 12:47:24 CET] chat: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","commit-tree","76f5041bc6e8a109e0309a09b5f36cd0da8b204a","-p","refs/heads/git-annex"] + [2013-02-07 12:47:24 CET] call: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","update-ref","refs/heads/git-annex","96de755475bdd8f0f948dd6421c3956803a63e66"] + ok + (Recording state in git...) + +If I run it again, I get: + + dropunused 9 [2013-02-07 12:47:47 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","show-ref","git-annex"] + [2013-02-07 12:47:47 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","show-ref","--hash","refs/heads/git-annex"] + [2013-02-07 12:47:47 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","log","refs/heads/git-annex..96de755475bdd8f0f948dd6421c3956803a63e66","--oneline","-n1"] + [2013-02-07 12:47:48 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","log","refs/heads/git-annex..8e5674078706864f2eade86d8aa43027e05afc1d","--oneline","-n1"] + [2013-02-07 12:47:48 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","log","refs/heads/git-annex..cbe492cfa79728698f5d891d7f716fbcd9fc29e2","--oneline","-n1"] + [2013-02-07 12:47:48 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","log","refs/heads/git-annex..48a1bdf98a10ad9a81c0587f8909e94c1c0dccc4","--oneline","-n1"] + [2013-02-07 12:47:48 CET] chat: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","cat-file","--batch"] + [2013-02-07 12:47:48 CET] read: git ["config","--null","--list"] + [2013-02-07 12:47:48 CET] chat: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","hash-object","-w","--stdin-paths"] + [2013-02-07 12:47:48 CET] feed: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","update-index","-z","--index-info"] + [2013-02-07 12:47:48 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","show-ref","--hash","refs/heads/git-annex"] + [2013-02-07 12:47:48 CET] read: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","write-tree"] + [2013-02-07 12:47:48 CET] chat: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","commit-tree","e40d82db10c60519f6a3a72055e9577850972fdf","-p","refs/heads/git-annex"] + [2013-02-07 12:47:48 CET] call: git ["--git-dir=/home/arand/.git","--work-tree=/home/arand","update-ref","refs/heads/git-annex","6cf49f629251f9b39fa8b457cf6590c71c1d509b"] + ok + (Recording state in git...) + + +What version of git-annex are you using? On what operating system? + +git-annex: 3.20130124 +Debian: sid 2013-02-01 + +> I put a fix in for this in 57780cb3a4dfe1292b72e1412ec4d2a70b6d04ce +> but it was buggy and I had to revert it. +> +> The bug is caused by direct mode cache and mapping info. +> This makes getKeysPresent find keys that are not present. +> It would be expensive to make getKeysPresent check that the +> actual key files are present (it just lists the directories). +> But this seems to be needed, since direct mode can leave +> cache and mapping files behind. --[[Joey]] + +>> Now fixed properly. [[done]] --[[Joey]] diff --git a/doc/bugs/encrypted_S3_stalls.mdwn b/doc/bugs/encrypted_S3_stalls.mdwn new file mode 100644 index 0000000000..109e6e793a --- /dev/null +++ b/doc/bugs/encrypted_S3_stalls.mdwn @@ -0,0 +1,9 @@ +Sending large-ish (few megabytes) files to encrypted S3 remotes stalls out. +It works for the tiny files I was using to test while developing it, on +dialup. + +There was a similar issue with bup, which I fixed by forking a process +rather than using a thread to do some IO. Probably need the same here. +--[[Joey]] + +[[done]] --[[Joey]] diff --git a/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption.mdwn b/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption.mdwn new file mode 100644 index 0000000000..15bc95f27b --- /dev/null +++ b/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption.mdwn @@ -0,0 +1,46 @@ +What steps will reproduce the problem? + % > git annex initremote glacier type=glacier encryption=E9053BDA + -- SNIP -- + initremote glacier [2013-01-10 14:50:12 PST] read: gpg ["--quiet","--trust-model","always","--with-colons","--list-public-keys","E9053BDA"] + [2013-01-10 14:50:12 PST] chat: gpg ["--quiet","--trust-model","always","--decrypt"] + + You need a passphrase to unlock the secret key for + user: "Andrew Mark Kraut " + 4096-bit ELG key, ID 353E49B9, created 2008-11-11 (main key ID E9053BDA) + + [2013-01-10 14:50:13 PST] chat: gpg ["--quiet","--trust-model","always","--encrypt","--no-encrypt-to","--no-default-recipient","--recipient","B608B8F6E9053BDA"] + (encryption updated with gpg key B608B8F6E9053BDA) [2013-01-10 14:50:13 PST] call: glacier ["--region=us-west-1","vault","create","glacier-06D927EC-5761-447B-86AC-CA66040BAC25"] + [2013-01-10 14:50:13 PST] call: git ["--git-dir=/Users/akraut/Desktop/annex/.git","--work-tree=/Users/akraut/Desktop/annex","config","remote.glacier.annex-glacier","true"] + [2013-01-10 14:50:13 PST] call: git ["--git-dir=/Users/akraut/Desktop/annex/.git","--work-tree=/Users/akraut/Desktop/annex","config","remote.glacier.annex-uuid","06D927EC-5761-447B-86AC-CA66040BAC25"] + (gpg) [2013-01-10 14:50:13 PST] chat: gpg ["--quiet","--trust-model","always","--decrypt"] + + You need a passphrase to unlock the secret key for + user: "Andrew Mark Kraut " + 4096-bit ELG key, ID 353E49B9, created 2008-11-11 (main key ID E9053BDA) + + [2013-01-10 14:50:14 PST] chat: gpg ["--quiet","--trust-model","always","--passphrase-fd","8","--symmetric","--force-mdc"] + ok + +What is the expected output? What do you see instead? + +> I expect any transfers to this remote (glacier) to use the given gpg key, but instead it uses --symmetric, as you can see above. + +What version of git-annex are you using? On what operating system? + + git-annex version: 3.20130107 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + + +Please provide any additional information below. + +> Symmetric encryption is used as described in [[design/encryption]], +> with the symmetric key stored encrypted using your gpg key. +> +> The extra prompting described in the comments in the bug. +> +> I've reproduced this with gpg 2.0.19. It is a documented incompatability +> between gpg 1.x and 2.x; the latter needs --batch included in its +> parameters. I've put in a fix. [[done]] diff --git a/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption/comment_1_2f4ec4b7b92a0f0a0c4c0758da4a05a5._comment b/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption/comment_1_2f4ec4b7b92a0f0a0c4c0758da4a05a5._comment new file mode 100644 index 0000000000..d4c537ed98 --- /dev/null +++ b/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption/comment_1_2f4ec4b7b92a0f0a0c4c0758da4a05a5._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmRFKwny4rArBaz-36xTcsJYqKIgdDaw5Q" + nickname="Andrew" + subject="comment 1" + date="2013-01-11T00:01:08Z" + content=""" +Ok, I just reread [[design/encryption]] and perhaps this isn't a bug after all. +Though, the annoyance I experience that made me dig into this a bit perhaps is a bug. + +In my example output above, if I 'git annex copy dir_full_of_files --to=glacier', I will get the GPG agent's passphrase prompt for each file, even if I have passphrase caching turned on and (on my mac) even if I have the passphrase saved in the keychain. Additionally, GPG will successfully encrypt the file if I enter anything at all into the passphrase prompt as long as I enter something. This leads me to believe that it either doesn't actually need to decrypt my GPG private key or it's using what I enter as the symmetric encryption key. + +Ideas? +"""]] diff --git a/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption/comment_2_7c0aeae6b1b2b0338735b0231c5db7d4._comment b/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption/comment_2_7c0aeae6b1b2b0338735b0231c5db7d4._comment new file mode 100644 index 0000000000..86dc2aa718 --- /dev/null +++ b/doc/bugs/encryption_given_a_gpg_keyid_still_uses_symmetric_encryption/comment_2_7c0aeae6b1b2b0338735b0231c5db7d4._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 2" + date="2013-01-16T02:17:44Z" + content=""" +What operating system is this? + +How did you install git-annex? + +Can you still reproduce the bug? + +What happens if you run this command; does it prompt for a passphrase? + +touch foo; echo foo| gpg --symmetric --passphrase-fd=0 foo +"""]] diff --git a/doc/bugs/encryption_key_is_surprising.mdwn b/doc/bugs/encryption_key_is_surprising.mdwn new file mode 100644 index 0000000000..5b18610b55 --- /dev/null +++ b/doc/bugs/encryption_key_is_surprising.mdwn @@ -0,0 +1,24 @@ +Crypto.hs seems to imply that the cipher key it's acting upon is 512 bytes long. Because of a probable programming mistake the key that's actually used is a bit surprising. The random key is generated by this snippet in Utility/Gpg.hs: + + {- Creates a block of high-quality random data suitable to use as a cipher. + - It is armored, to avoid newlines, since gpg only reads ciphers up to the + - first newline. -} + genRandom :: Int -> IO String + genRandom size = readStrict + [ Params "--gen-random --armor" + , Param $ show randomquality + , Param $ show size + ] + where + -- 1 is /dev/urandom; 2 is /dev/random + randomquality = 1 :: Int + +This lets GPG generate the randomness and by passing armor, it avoids newlines. However, this base64 encoding is never undone on the way to Crypto.hs. Hence what cipherPassphrase and cipherHmac do is dropping or skipping the first 256 bytes of the base64 value. Base64, with its 6 bit per byte encoding, causes the Hmac function to operate on 192 bytes instead of 256 bytes. The key used by GPG will be larger. Some assertions that the resulting functions really operate on strings of the right length would've been helpful. Also GPG and HMAC get tested, the encryption/decryption part are not tested AFAICS. + +The encryption wiki page could have had more information. Enough code (sadly in Python, not reusing the Haskell code) to operate on the resulting files can be found in [this Gist](https://gist.github.com/pkern/5078559). + +-- Philipp Kern + +> In addition to the comment below, I have added a check that gpg outputs +> the expected quantity of data, and the storage of the cipher is now +> documented in [[internals]]. Think I can call this [[done]]. --[[Joey]] diff --git a/doc/bugs/encryption_key_is_surprising/comment_1_5b172830ac31d51a1687bc8b1db489f9._comment b/doc/bugs/encryption_key_is_surprising/comment_1_5b172830ac31d51a1687bc8b1db489f9._comment new file mode 100644 index 0000000000..04854b3a4d --- /dev/null +++ b/doc/bugs/encryption_key_is_surprising/comment_1_5b172830ac31d51a1687bc8b1db489f9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 1" + date="2013-03-04T00:04:53Z" + content=""" +My first concern is if this means it's insecure. Luckily it seems not; HMAC SHA1 needs only 64 bytes of entropy, which are more than provided in the 256 bytes of base64 provided. As long as both gpg and the HMAC code use the full provided key (and not just the first 64 bytes of it, say), we're ok. And as far as I can tell, both do fully consume and use the key. + +So, I don't feel the need to change the code, aside from some minor improvements to variable names. +"""]] diff --git a/doc/bugs/encryption_key_is_surprising/comment_2_5b7e6bb36c3333dfd71808e8b4544746._comment b/doc/bugs/encryption_key_is_surprising/comment_2_5b7e6bb36c3333dfd71808e8b4544746._comment new file mode 100644 index 0000000000..24f8452c30 --- /dev/null +++ b/doc/bugs/encryption_key_is_surprising/comment_2_5b7e6bb36c3333dfd71808e8b4544746._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://phil.0x539.de/" + nickname="Philipp Kern" + subject="comment 2" + date="2013-03-04T07:36:55Z" + content=""" +GPG also reduces the key material to the size of a SHA1 hash (because we're using the default option for s2k-digest-algo) to generate the symmetric key used with CAST5. So I wonder a bit why we bother with 512 bytes in the first place. Also they come from urandom (even on Linux), despite being generated once per remote. So maybe the strongness of the weakest link should be written down somewhere. +"""]] diff --git a/doc/bugs/encryption_key_is_surprising/comment_4_c5e49b3a0eceabe6d14f5226d7ba4c7a._comment b/doc/bugs/encryption_key_is_surprising/comment_4_c5e49b3a0eceabe6d14f5226d7ba4c7a._comment new file mode 100644 index 0000000000..a685626e5d --- /dev/null +++ b/doc/bugs/encryption_key_is_surprising/comment_4_c5e49b3a0eceabe6d14f5226d7ba4c7a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 4" + date="2013-03-04T17:29:28Z" + content=""" +What do you mean by \"reduces\"? I assume it doesn't just take the first N bytes? +"""]] diff --git a/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn b/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn new file mode 100644 index 0000000000..df83f4e4ed --- /dev/null +++ b/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn @@ -0,0 +1,159 @@ +I am trying to install git-annex 3.20120624 using cabal. My currently installed version of git-annex is 3.20120615. After a "cabal update", the build of git-annex fails: + + bram@falafel% cabal install git-annex + Resolving dependencies... + [1 of 4] Compiling Utility.SafeCommand ( /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/Utility/SafeCommand.hs, /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/Utility/SafeCommand.o ) + [2 of 4] Compiling Build.TestConfig ( /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/Build/TestConfig.hs, /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/Build/TestConfig.o ) + [3 of 4] Compiling Build.Configure ( /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/Build/Configure.hs, /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/Build/Configure.o ) + [4 of 4] Compiling Main ( /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/Setup.hs, /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/Main.o ) + Linking /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/setup ... + checking version... 3.20120624 + checking git... yes + checking git version... 1.7.9.5 + checking cp -a... yes + checking cp -p... yes + checking cp --reflink=auto... yes + checking uuid generator... uuid + checking xargs -0... yes + checking rsync... yes + checking curl... no + checking wget... yes + checking bup... no + checking gpg... yes + checking lsof... yes + checking ssh connection caching... yes + checking sha1... sha1sum + checking sha512... sha512sum + checking sha224... sha224sum + checking sha384... sha384sum + checking sha256... sha256sum + Configuring git-annex-3.20120624... + Building git-annex-3.20120624... + Preprocessing executable 'git-annex' for git-annex-3.20120624... + [ 1 of 183] Compiling Utility.Percentage ( Utility/Percentage.hs, dist/build/git-annex/git-annex-tmp/Utility/Percentage.o ) + [ 2 of 183] Compiling Utility.Dot ( Utility/Dot.hs, dist/build/git-annex/git-annex-tmp/Utility/Dot.o ) + [ 3 of 183] Compiling Utility.ThreadLock ( Utility/ThreadLock.hs, dist/build/git-annex/git-annex-tmp/Utility/ThreadLock.o ) + [ 4 of 183] Compiling Utility.Base64 ( Utility/Base64.hs, dist/build/git-annex/git-annex-tmp/Utility/Base64.o ) + [ 5 of 183] Compiling Utility.DataUnits ( Utility/DataUnits.hs, dist/build/git-annex/git-annex-tmp/Utility/DataUnits.o ) + [ 6 of 183] Compiling Utility.JSONStream ( Utility/JSONStream.hs, dist/build/git-annex/git-annex-tmp/Utility/JSONStream.o ) + [ 7 of 183] Compiling Messages.JSON ( Messages/JSON.hs, dist/build/git-annex/git-annex-tmp/Messages/JSON.o ) + [ 8 of 183] Compiling Build.SysConfig ( Build/SysConfig.hs, dist/build/git-annex/git-annex-tmp/Build/SysConfig.o ) + [ 9 of 183] Compiling Types.KeySource ( Types/KeySource.hs, dist/build/git-annex/git-annex-tmp/Types/KeySource.o ) + [ 10 of 183] Compiling Types.UUID ( Types/UUID.hs, dist/build/git-annex/git-annex-tmp/Types/UUID.o ) + [ 11 of 183] Compiling Utility.State ( Utility/State.hs, dist/build/git-annex/git-annex-tmp/Utility/State.o ) + [ 12 of 183] Compiling Types.Messages ( Types/Messages.hs, dist/build/git-annex/git-annex-tmp/Types/Messages.o ) + [ 13 of 183] Compiling Types.TrustLevel ( Types/TrustLevel.hs, dist/build/git-annex/git-annex-tmp/Types/TrustLevel.o ) + [ 14 of 183] Compiling Types.BranchState ( Types/BranchState.hs, dist/build/git-annex/git-annex-tmp/Types/BranchState.o ) + [ 15 of 183] Compiling Git.Index ( Git/Index.hs, dist/build/git-annex/git-annex-tmp/Git/Index.o ) + [ 16 of 183] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, dist/build/git-annex/git-annex-tmp/Utility/PartialPrelude.o ) + [ 17 of 183] Compiling Utility.Format ( Utility/Format.hs, dist/build/git-annex/git-annex-tmp/Utility/Format.o ) + [ 18 of 183] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, dist/build/git-annex/git-annex-tmp/Utility/FileSystemEncoding.o ) + [ 19 of 183] Compiling Utility.Touch ( dist/build/git-annex/git-annex-tmp/Utility/Touch.hs, dist/build/git-annex/git-annex-tmp/Utility/Touch.o ) + [ 20 of 183] Compiling Utility.Monad ( Utility/Monad.hs, dist/build/git-annex/git-annex-tmp/Utility/Monad.o ) + [ 21 of 183] Compiling Utility.Path ( Utility/Path.hs, dist/build/git-annex/git-annex-tmp/Utility/Path.o ) + [ 22 of 183] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, dist/build/git-annex/git-annex-tmp/Utility/SafeCommand.o ) + [ 23 of 183] Compiling Utility.RsyncFile ( Utility/RsyncFile.hs, dist/build/git-annex/git-annex-tmp/Utility/RsyncFile.o ) + [ 24 of 183] Compiling Utility.Exception ( Utility/Exception.hs, dist/build/git-annex/git-annex-tmp/Utility/Exception.o ) + [ 25 of 183] Compiling Utility.TempFile ( Utility/TempFile.hs, dist/build/git-annex/git-annex-tmp/Utility/TempFile.o ) + [ 26 of 183] Compiling Utility.Directory ( Utility/Directory.hs, dist/build/git-annex/git-annex-tmp/Utility/Directory.o ) + [ 27 of 183] Compiling Utility.Misc ( Utility/Misc.hs, dist/build/git-annex/git-annex-tmp/Utility/Misc.o ) + [ 28 of 183] Compiling Git.Types ( Git/Types.hs, dist/build/git-annex/git-annex-tmp/Git/Types.o ) + [ 29 of 183] Compiling Common ( Common.hs, dist/build/git-annex/git-annex-tmp/Common.o ) + [ 30 of 183] Compiling Utility.FileMode ( Utility/FileMode.hs, dist/build/git-annex/git-annex-tmp/Utility/FileMode.o ) + [ 31 of 183] Compiling Git ( Git.hs, dist/build/git-annex/git-annex-tmp/Git.o ) + [ 32 of 183] Compiling Git.Command ( Git/Command.hs, dist/build/git-annex/git-annex-tmp/Git/Command.o ) + [ 33 of 183] Compiling Git.Ref ( Git/Ref.hs, dist/build/git-annex/git-annex-tmp/Git/Ref.o ) + [ 34 of 183] Compiling Git.FilePath ( Git/FilePath.hs, dist/build/git-annex/git-annex-tmp/Git/FilePath.o ) + [ 35 of 183] Compiling Utility.Matcher ( Utility/Matcher.hs, dist/build/git-annex/git-annex-tmp/Utility/Matcher.o ) + [ 36 of 183] Compiling Utility.Gpg ( Utility/Gpg.hs, dist/build/git-annex/git-annex-tmp/Utility/Gpg.o ) + [ 37 of 183] Compiling Types.Crypto ( Types/Crypto.hs, dist/build/git-annex/git-annex-tmp/Types/Crypto.o ) + [ 38 of 183] Compiling Types.Key ( Types/Key.hs, dist/build/git-annex/git-annex-tmp/Types/Key.o ) + [ 39 of 183] Compiling Types.Backend ( Types/Backend.hs, dist/build/git-annex/git-annex-tmp/Types/Backend.o ) + [ 40 of 183] Compiling Types.Remote ( Types/Remote.hs, dist/build/git-annex/git-annex-tmp/Types/Remote.o ) + [ 41 of 183] Compiling Git.Sha ( Git/Sha.hs, dist/build/git-annex/git-annex-tmp/Git/Sha.o ) + [ 42 of 183] Compiling Git.Branch ( Git/Branch.hs, dist/build/git-annex/git-annex-tmp/Git/Branch.o ) + [ 43 of 183] Compiling Git.UpdateIndex ( Git/UpdateIndex.hs, dist/build/git-annex/git-annex-tmp/Git/UpdateIndex.o ) + [ 44 of 183] Compiling Git.Queue ( Git/Queue.hs, dist/build/git-annex/git-annex-tmp/Git/Queue.o ) + [ 45 of 183] Compiling Git.Url ( Git/Url.hs, dist/build/git-annex/git-annex-tmp/Git/Url.o ) + [ 46 of 183] Compiling Git.Construct ( Git/Construct.hs, dist/build/git-annex/git-annex-tmp/Git/Construct.o ) + [ 47 of 183] Compiling Git.Config ( Git/Config.hs, dist/build/git-annex/git-annex-tmp/Git/Config.o ) + [ 48 of 183] Compiling Git.SharedRepository ( Git/SharedRepository.hs, dist/build/git-annex/git-annex-tmp/Git/SharedRepository.o ) + [ 49 of 183] Compiling Git.Version ( Git/Version.hs, dist/build/git-annex/git-annex-tmp/Git/Version.o ) + [ 50 of 183] Compiling Utility.CoProcess ( Utility/CoProcess.hs, dist/build/git-annex/git-annex-tmp/Utility/CoProcess.o ) + [ 51 of 183] Compiling Git.HashObject ( Git/HashObject.hs, dist/build/git-annex/git-annex-tmp/Git/HashObject.o ) + [ 52 of 183] Compiling Git.CatFile ( Git/CatFile.hs, dist/build/git-annex/git-annex-tmp/Git/CatFile.o ) + [ 53 of 183] Compiling Git.UnionMerge ( Git/UnionMerge.hs, dist/build/git-annex/git-annex-tmp/Git/UnionMerge.o ) + [ 54 of 183] Compiling Git.CheckAttr ( Git/CheckAttr.hs, dist/build/git-annex/git-annex-tmp/Git/CheckAttr.o ) + [ 55 of 183] Compiling Annex ( Annex.hs, dist/build/git-annex/git-annex-tmp/Annex.o ) + [ 56 of 183] Compiling Types.Option ( Types/Option.hs, dist/build/git-annex/git-annex-tmp/Types/Option.o ) + [ 57 of 183] Compiling Types ( Types.hs, dist/build/git-annex/git-annex-tmp/Types.o ) + [ 58 of 183] Compiling Messages ( Messages.hs, dist/build/git-annex/git-annex-tmp/Messages.o ) + [ 59 of 183] Compiling Types.Command ( Types/Command.hs, dist/build/git-annex/git-annex-tmp/Types/Command.o ) + [ 60 of 183] Compiling Locations ( Locations.hs, dist/build/git-annex/git-annex-tmp/Locations.o ) + [ 61 of 183] Compiling Common.Annex ( Common/Annex.hs, dist/build/git-annex/git-annex-tmp/Common/Annex.o ) + [ 62 of 183] Compiling Annex.Exception ( Annex/Exception.hs, dist/build/git-annex/git-annex-tmp/Annex/Exception.o ) + [ 63 of 183] Compiling Annex.BranchState ( Annex/BranchState.hs, dist/build/git-annex/git-annex-tmp/Annex/BranchState.o ) + [ 64 of 183] Compiling Annex.CatFile ( Annex/CatFile.hs, dist/build/git-annex/git-annex-tmp/Annex/CatFile.o ) + [ 65 of 183] Compiling Annex.Perms ( Annex/Perms.hs, dist/build/git-annex/git-annex-tmp/Annex/Perms.o ) + [ 66 of 183] Compiling Annex.Journal ( Annex/Journal.hs, dist/build/git-annex/git-annex-tmp/Annex/Journal.o ) + [ 67 of 183] Compiling Annex.Branch ( Annex/Branch.hs, dist/build/git-annex/git-annex-tmp/Annex/Branch.o ) + [ 68 of 183] Compiling Crypto ( Crypto.hs, dist/build/git-annex/git-annex-tmp/Crypto.o ) + [ 69 of 183] Compiling Usage ( Usage.hs, dist/build/git-annex/git-annex-tmp/Usage.o ) + [ 70 of 183] Compiling Annex.CheckAttr ( Annex/CheckAttr.hs, dist/build/git-annex/git-annex-tmp/Annex/CheckAttr.o ) + [ 71 of 183] Compiling Remote.Helper.Special ( Remote/Helper/Special.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Special.o ) + [ 72 of 183] Compiling Logs.Presence ( Logs/Presence.hs, dist/build/git-annex/git-annex-tmp/Logs/Presence.o ) + [ 73 of 183] Compiling Logs.Location ( Logs/Location.hs, dist/build/git-annex/git-annex-tmp/Logs/Location.o ) + [ 74 of 183] Compiling Logs.Web ( Logs/Web.hs, dist/build/git-annex/git-annex-tmp/Logs/Web.o ) + [ 75 of 183] Compiling Annex.LockPool ( Annex/LockPool.hs, dist/build/git-annex/git-annex-tmp/Annex/LockPool.o ) + [ 76 of 183] Compiling Backend.SHA ( Backend/SHA.hs, dist/build/git-annex/git-annex-tmp/Backend/SHA.o ) + [ 77 of 183] Compiling Backend.WORM ( Backend/WORM.hs, dist/build/git-annex/git-annex-tmp/Backend/WORM.o ) + [ 78 of 183] Compiling Backend.URL ( Backend/URL.hs, dist/build/git-annex/git-annex-tmp/Backend/URL.o ) + [ 79 of 183] Compiling Assistant.ThreadedMonad ( Assistant/ThreadedMonad.hs, dist/build/git-annex/git-annex-tmp/Assistant/ThreadedMonad.o ) + [ 80 of 183] Compiling Logs.UUIDBased ( Logs/UUIDBased.hs, dist/build/git-annex/git-annex-tmp/Logs/UUIDBased.o ) + [ 81 of 183] Compiling Logs.Remote ( Logs/Remote.hs, dist/build/git-annex/git-annex-tmp/Logs/Remote.o ) + [ 82 of 183] Compiling Utility.DiskFree ( Utility/DiskFree.hs, dist/build/git-annex/git-annex-tmp/Utility/DiskFree.o ) + [ 83 of 183] Compiling Utility.Url ( Utility/Url.hs, dist/build/git-annex/git-annex-tmp/Utility/Url.o ) + [ 84 of 183] Compiling Utility.CopyFile ( Utility/CopyFile.hs, dist/build/git-annex/git-annex-tmp/Utility/CopyFile.o ) + [ 85 of 183] Compiling Git.LsFiles ( Git/LsFiles.hs, dist/build/git-annex/git-annex-tmp/Git/LsFiles.o ) + [ 86 of 183] Compiling Git.AutoCorrect ( Git/AutoCorrect.hs, dist/build/git-annex/git-annex-tmp/Git/AutoCorrect.o ) + [ 87 of 183] Compiling Git.CurrentRepo ( Git/CurrentRepo.hs, dist/build/git-annex/git-annex-tmp/Git/CurrentRepo.o ) + [ 88 of 183] Compiling Utility.Daemon ( Utility/Daemon.hs, dist/build/git-annex/git-annex-tmp/Utility/Daemon.o ) + [ 89 of 183] Compiling Utility.LogFile ( Utility/LogFile.hs, dist/build/git-annex/git-annex-tmp/Utility/LogFile.o ) + [ 90 of 183] Compiling Utility.ThreadScheduler ( Utility/ThreadScheduler.hs, dist/build/git-annex/git-annex-tmp/Utility/ThreadScheduler.o ) + [ 91 of 183] Compiling Assistant.DaemonStatus ( Assistant/DaemonStatus.hs, dist/build/git-annex/git-annex-tmp/Assistant/DaemonStatus.o ) + [ 92 of 183] Compiling Utility.Types.DirWatcher ( Utility/Types/DirWatcher.hs, dist/build/git-annex/git-annex-tmp/Utility/Types/DirWatcher.o ) + [ 93 of 183] Compiling Utility.INotify ( Utility/INotify.hs, dist/build/git-annex/git-annex-tmp/Utility/INotify.o ) + [ 94 of 183] Compiling Utility.DirWatcher ( Utility/DirWatcher.hs, dist/build/git-annex/git-annex-tmp/Utility/DirWatcher.o ) + [ 95 of 183] Compiling Utility.Lsof ( Utility/Lsof.hs, dist/build/git-annex/git-annex-tmp/Utility/Lsof.o ) + [ 96 of 183] Compiling Git.Merge ( Git/Merge.hs, dist/build/git-annex/git-annex-tmp/Git/Merge.o ) + [ 97 of 183] Compiling Git.Filename ( Git/Filename.hs, dist/build/git-annex/git-annex-tmp/Git/Filename.o ) + [ 98 of 183] Compiling Git.LsTree ( Git/LsTree.hs, dist/build/git-annex/git-annex-tmp/Git/LsTree.o ) + [ 99 of 183] Compiling Config ( Config.hs, dist/build/git-annex/git-annex-tmp/Config.o ) + [100 of 183] Compiling Annex.UUID ( Annex/UUID.hs, dist/build/git-annex/git-annex-tmp/Annex/UUID.o ) + [101 of 183] Compiling Logs.UUID ( Logs/UUID.hs, dist/build/git-annex/git-annex-tmp/Logs/UUID.o ) + [102 of 183] Compiling Backend ( Backend.hs, dist/build/git-annex/git-annex-tmp/Backend.o ) + [103 of 183] Compiling Remote.Helper.Hooks ( Remote/Helper/Hooks.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Hooks.o ) + [104 of 183] Compiling Remote.Helper.Encryptable ( Remote/Helper/Encryptable.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Encryptable.o ) + [105 of 183] Compiling Annex.Queue ( Annex/Queue.hs, dist/build/git-annex/git-annex-tmp/Annex/Queue.o ) + [106 of 183] Compiling Annex.Content ( Annex/Content.hs, dist/build/git-annex/git-annex-tmp/Annex/Content.o ) + [107 of 183] Compiling Remote.S3 ( Remote/S3.hs, dist/build/git-annex/git-annex-tmp/Remote/S3.o ) + [108 of 183] Compiling Remote.Directory ( Remote/Directory.hs, dist/build/git-annex/git-annex-tmp/Remote/Directory.o ) + [109 of 183] Compiling Remote.Rsync ( Remote/Rsync.hs, dist/build/git-annex/git-annex-tmp/Remote/Rsync.o ) + [110 of 183] Compiling Remote.Web ( Remote/Web.hs, dist/build/git-annex/git-annex-tmp/Remote/Web.o ) + [111 of 183] Compiling Remote.Hook ( Remote/Hook.hs, dist/build/git-annex/git-annex-tmp/Remote/Hook.o ) + [112 of 183] Compiling Upgrade.V2 ( Upgrade/V2.hs, dist/build/git-annex/git-annex-tmp/Upgrade/V2.o ) + [113 of 183] Compiling Assistant.Changes ( Assistant/Changes.hs, dist/build/git-annex/git-annex-tmp/Assistant/Changes.o ) + + Assistant/Changes.hs:73:30: + Not in scope: `tryReadTChan' + Perhaps you meant `readTChan' (imported from Control.Concurrent.STM) + cabal: Error: some packages failed to install: + git-annex-3.20120624 failed during the building phase. The exception was: + ExitFailure 1 + +This is using haskell-platform 2012.1.0.0~debian1 on Ubuntu 12.04. + +> Turns out it needs version 2.3 of the STM library. (libghc-stm-dev +> package). I've made cabal detect an older version and skip building +> the new `git annex watch` command, so you'll be able to build the next +> release. [[done]] --[[Joey]] diff --git a/doc/bugs/error_propigation.mdwn b/doc/bugs/error_propigation.mdwn new file mode 100644 index 0000000000..25998907e8 --- /dev/null +++ b/doc/bugs/error_propigation.mdwn @@ -0,0 +1,3 @@ +If a subcommand fails w/o throwing an error, no error is propigated to the +git-annex exit code. With --quiet, this makes it look like the command +succeeded. [[done]] diff --git a/doc/bugs/error_when_using_repositories_with_non-ASCII_characters.mdwn b/doc/bugs/error_when_using_repositories_with_non-ASCII_characters.mdwn new file mode 100644 index 0000000000..d88b86b444 --- /dev/null +++ b/doc/bugs/error_when_using_repositories_with_non-ASCII_characters.mdwn @@ -0,0 +1,62 @@ +*What steps will reproduce the problem?* + + hactar% mkdir demonstração + hactar% cd demonstração + hactar% cp ~/tmp/*(.) . + hactar% git init + Initialized empty Git repository in /tmp/demonstração/.git/ + hactar% git annex init + init ok + (Recording state in git...) + hactar% git annex add . + add Equipment Consumption.ods (checksum...) ok + add Personal.vcard (checksum...) ok + add Trampo.vcard (checksum...) ok + add blah.txt (checksum...) ok + [ more git output ] + hactar% git commit -m initial + [master (root-commit) d16bafb] initial + 42 files changed, 42 insertions(+) + [ more git output ] + hactar% cd /var/tmp + hactar% git clone ssh://localhost//tmp/demonstração demonstração + Cloning into 'demonstração'... + remote: Counting objects: 176, done. + remote: Compressing objects: 100% (134/134), done. + remote: Total 176 (delta 1), reused 0 (delta 0) + Receiving objects: 100% (176/176), 17.23 KiB, done. + Resolving deltas: 100% (1/1), done. + hactar% cd demonstração + hactar% git annex init + init ok + (Recording state in git...) + hactar% git annex status + supported backends: SHA256E SHA1E SHA512E SHA224E SHA384E SHA256 SHA1 SHA512 SHA224 SHA384 WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: (merging origin/git-annex into git-annex...) + + git-annex: fd:14: commitBuffer: invalid argument (invalid character) + failed + git-annex: status: 1 failed + +*What is the expected output? What do you see instead?* + +I expect that "git annex status" will complete successfuly and show information about all repositories. +Instead of that I get the "git-annex: fd:14: commitBuffer: invalid argument (invalid character)" error above. + +*What version of git-annex are you using? On what operating system?* + +This is with Debian's git-annex_3.20121001_i386.deb installed on an Ubuntu 12.04 system. +Using Ubuntu's original version (3.20120406) the error message is a bit different (here I used the name acentuação instead of demonstração): + + trusted repositories: git-annex-shell: //tmp/acentuação: changeWorkingDirectory: does not exist (No such file or directory) + Command ssh ["-S","/var/tmp/acentua\231\227o/.git/annex/ssh/localhost","-o","ControlMaster=auto","-o","ControlPersist=yes","localhost","git-annex-shell 'configlist' '//tmp/acentua\195\167\195\163o'"] failed; exit code 1 + +> I think this is the last unvalid utf-8 bug in git-annex. At least, +> the last one I hypothesized exists. It's in the union merge code. I will +> try to look at it again soon; the last 2 times I looked at that code +> I could not see an easy way to make it allow invalid utf-8 encoded data. +> --[[Joey]] + +>> [[done]], although I am no longer so sure this was the last utf-8 +>> encoding bug.. --[[Joey]] diff --git a/doc/bugs/error_when_using_repositories_with_non-ASCII_characters/comment_1_38cc2d2ed907649df085de8ad83cb9dd._comment b/doc/bugs/error_when_using_repositories_with_non-ASCII_characters/comment_1_38cc2d2ed907649df085de8ad83cb9dd._comment new file mode 100644 index 0000000000..69d7f1da1e --- /dev/null +++ b/doc/bugs/error_when_using_repositories_with_non-ASCII_characters/comment_1_38cc2d2ed907649df085de8ad83cb9dd._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmcYryijvlF8bJvM_eZNSrUPEkMlxMDGTQ" + nickname="Thiago" + subject="The bug is actually something completely different!" + date="2012-10-04T03:53:57Z" + content=""" +Ok, this is a mix of luser error and a bug. + +I noticed that I wasn't providing a description in git annex init. So if I do \"git annex init foo\" in the first repository and \"git annex init bar\" in the clone, then git annex status works! + +So the bug is actually that you need to provide a description in git annex init, even though the man page says a description will be generated otherwise. + +(In my defense I hit the non-ACII characters bug with the git annex version in Ubuntu, and that one still has the same error even if a description is provided so this was fixed later). +"""]] diff --git a/doc/bugs/error_with_file_names_starting_with_dash.mdwn b/doc/bugs/error_with_file_names_starting_with_dash.mdwn new file mode 100644 index 0000000000..84bf1cfa07 --- /dev/null +++ b/doc/bugs/error_with_file_names_starting_with_dash.mdwn @@ -0,0 +1,15 @@ +git annex add has problems if items start with dashes, example: + +-wut-a-directory-name-/file1 + +leads to + +[[!format bash """ +add -wut-a-directory-name-/file1 (checksum...) sha1sum: invalid option -- 'u' +„sha1sum --help“ gibt weitere Informationen. + + git-annex: : hGetLine: end of file +"""]] + +> This is fixed in git, at least I think I've found all cases where +> filenames are passed to programs and escaped them. --[[Joey]] [[done]] diff --git a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn new file mode 100644 index 0000000000..c4ee8d5bda --- /dev/null +++ b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn @@ -0,0 +1,15 @@ +When using `git annex get foo` where foo is available in a rsync remote with encryption I got an error saying that rsync cannot +find the required file but extra ' are here. + +I attached a patch for this. + +> But you didn't, sadly. :( +> +> I don't seem to see the problem, set up a rsync over ssh with encryption +> and sent over a file "foo", and then got it back from rsync, without +> trouble. +> +> Ah, you're not using rsync over ssh, but just to a local directory, +> right? --[[Joey]] + +>> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/fails_to_handle_lot_of_files.mdwn b/doc/bugs/fails_to_handle_lot_of_files.mdwn new file mode 100644 index 0000000000..470a5180f0 --- /dev/null +++ b/doc/bugs/fails_to_handle_lot_of_files.mdwn @@ -0,0 +1,445 @@ + git-annex version: 3.20111011 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +I just created a new remote on a USB drive and wanted to copy my files over. git-annex wasn't too happy about that ;) +I included a few OK transfers as there was an error before git-annex ran into a wall. As I could easily access that temp file after it aborted, I suspect something went wrong internally before git-annex started to throw those errors. + +Please note the "_n TIMES_" comments. It's how often I got the same error message... + + + + git annex copy . --to USB --fast + + copy redacted.JPG (to USB...) + redacted + 4035668 100% 77.91MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4036374 bytes received 31 bytes 8072810.00 bytes/sec + total size is 4035668 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18002094 100% 74.19MB/s 0:00:00 (xfer#1, to-check=0/1) + WARNING: redacted failed verification -- update retained (will try again). + redacted + 18002094 100% 19.60MB/s 0:00:00 (xfer#2, to-check=0/1) + rsync: open "copy_target/.git/annex/tmp/redacted_E13" failed: Permission denied (13) + + sent 36008841 bytes received 52 bytes 24005928.67 bytes/sec + total size is 18002094 speedup is 0.50 + rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1070) [sender=3.0.8] + + rsync failed -- run git annex again to resume file transfer + failed + copy redacted.JPG (to USB...) + redacted + 3687111 100% 39.16MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 3687773 bytes received 31 bytes 2458536.00 bytes/sec + total size is 3687111 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17877177 100% 79.15MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17879573 bytes received 31 bytes 11919736.00 bytes/sec + total size is 17877177 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 3694921 100% 40.14MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 3695583 bytes received 31 bytes 2463742.67 bytes/sec + total size is 3694921 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17875448 100% 71.20MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17877844 bytes received 31 bytes 11918583.33 bytes/sec + total size is 17875448 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 3833377 100% 62.49MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 3834055 bytes received 31 bytes 2556057.33 bytes/sec + total size is 3833377 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17938200 100% 65.43MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17940604 bytes received 31 bytes 11960423.33 bytes/sec + total size is 17938200 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4512557 100% 83.77MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4513319 bytes received 31 bytes 3008900.00 bytes/sec + total size is 4512557 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18001641 100% 76.16MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18004053 bytes received 31 bytes 12002722.67 bytes/sec + total size is 18001641 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4394272 100% 50.11MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4395022 bytes received 31 bytes 8790106.00 bytes/sec + total size is 4394272 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18095781 100% 73.30MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18098205 bytes received 31 bytes 12065490.67 bytes/sec + total size is 18095781 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4683795 100% 65.23MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4684577 bytes received 31 bytes 9369216.00 bytes/sec + total size is 4683795 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18172801 100% 74.25MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18175233 bytes received 31 bytes 36350528.00 bytes/sec + total size is 18172801 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4486231 100% 77.22MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4486989 bytes received 31 bytes 8974040.00 bytes/sec + total size is 4486231 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17860427 100% 68.56MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17862823 bytes received 31 bytes 35725708.00 bytes/sec + total size is 17860427 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4499768 100% 36.41MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4500530 bytes received 31 bytes 9001122.00 bytes/sec + total size is 4499768 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17840132 100% 74.48MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17842524 bytes received 31 bytes 11895036.67 bytes/sec + total size is 17840132 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4358032 100% 75.00MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4358774 bytes received 31 bytes 8717610.00 bytes/sec + total size is 4358032 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18084753 100% 61.48MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18087173 bytes received 31 bytes 12058136.00 bytes/sec + total size is 18084753 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4270213 100% 68.49MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4270947 bytes received 31 bytes 2847318.67 bytes/sec + total size is 4270213 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17661246 100% 68.34MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17663614 bytes received 31 bytes 11775763.33 bytes/sec + total size is 17661246 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4538305 100% 63.19MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4539071 bytes received 31 bytes 9078204.00 bytes/sec + total size is 4538305 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18672466 100% 68.90MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18674958 bytes received 31 bytes 12449992.67 bytes/sec + total size is 18672466 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4453445 100% 73.96MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4454199 bytes received 31 bytes 8908460.00 bytes/sec + total size is 4453445 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18495494 100% 59.28MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18497966 bytes received 31 bytes 12331998.00 bytes/sec + total size is 18495494 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4255858 100% 70.66MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4256588 bytes received 31 bytes 1702647.60 bytes/sec + total size is 4255858 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18376531 100% 69.15MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18378987 bytes received 31 bytes 36758036.00 bytes/sec + total size is 18376531 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4013365 100% 48.67MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4014067 bytes received 31 bytes 8028196.00 bytes/sec + total size is 4013365 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17606341 100% 51.73MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17608705 bytes received 31 bytes 11739157.33 bytes/sec + total size is 17606341 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4179869 100% 74.62MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4180591 bytes received 31 bytes 8361244.00 bytes/sec + total size is 4179869 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18382569 100% 67.05MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18385025 bytes received 31 bytes 12256704.00 bytes/sec + total size is 18382569 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4318363 100% 44.91MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4319101 bytes received 31 bytes 8638264.00 bytes/sec + total size is 4318363 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17715958 100% 72.69MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17718334 bytes received 31 bytes 11812243.33 bytes/sec + total size is 17715958 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4241893 100% 65.81MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4242623 bytes received 31 bytes 8485308.00 bytes/sec + total size is 4241893 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17717287 100% 71.77MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17719663 bytes received 31 bytes 11813129.33 bytes/sec + total size is 17717287 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4488380 100% 49.99MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4489138 bytes received 31 bytes 2992779.33 bytes/sec + total size is 4488380 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 17770208 100% 38.80MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 17772592 bytes received 31 bytes 11848415.33 bytes/sec + total size is 17770208 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4603958 100% 76.48MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4604732 bytes received 31 bytes 9209526.00 bytes/sec + total size is 4603958 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18744380 100% 74.66MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18746884 bytes received 31 bytes 12497943.33 bytes/sec + total size is 18744380 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4592098 100% 79.06MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4592872 bytes received 31 bytes 3061935.33 bytes/sec + total size is 4592098 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18746205 100% 43.00MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18748709 bytes received 31 bytes 12499160.00 bytes/sec + total size is 18746205 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 7493353 100% 80.85MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 7494479 bytes received 31 bytes 14989020.00 bytes/sec + total size is 7493353 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 19496768 100% 81.77MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 19499360 bytes received 31 bytes 12999594.00 bytes/sec + total size is 19496768 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 5462482 100% 82.19MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 5463360 bytes received 31 bytes 10926782.00 bytes/sec + total size is 5462482 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 19669815 100% 80.37MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 19672431 bytes received 31 bytes 13114974.67 bytes/sec + total size is 19669815 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 5449487 100% 57.40MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 5450365 bytes received 31 bytes 3633597.33 bytes/sec + total size is 5449487 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 19633259 100% 74.18MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 19635871 bytes received 31 bytes 13090601.33 bytes/sec + total size is 19633259 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 5392184 100% 62.33MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 5393054 bytes received 31 bytes 3595390.00 bytes/sec + total size is 5392184 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 18912104 100% 65.00MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 18914628 bytes received 31 bytes 12609772.67 bytes/sec + total size is 18912104 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4869300 100% 80.92MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4870106 bytes received 31 bytes 9740274.00 bytes/sec + total size is 4869300 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 20178932 100% 68.13MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 20181608 bytes received 31 bytes 13454426.00 bytes/sec + total size is 20178932 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 4995425 100% 86.05MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 4996247 bytes received 31 bytes 9992556.00 bytes/sec + total size is 4995425 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 19970679 100% 76.36MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 19973331 bytes received 31 bytes 13315574.67 bytes/sec + total size is 19970679 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 7905795 100% 66.45MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 7906973 bytes received 31 bytes 15814008.00 bytes/sec + total size is 7905795 speedup is 1.00 + ok + copy redacted.NEF (to USB...) + redacted + 21234069 100% 78.07MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 21236877 bytes received 31 bytes 8494763.20 bytes/sec + total size is 21234069 speedup is 1.00 + ok + copy redacted.JPG (to USB...) + redacted + 7963979 100% 62.51MB/s 0:00:00 (xfer#1, to-check=0/1) + + sent 7965165 bytes received 31 bytes 5310130.67 bytes/sec + total size is 7963979 speedup is 1.00 + git ["--git-dir=copy_target/.git","--work-tree=copy_target","update-index","-z","--index-info"]: Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable) + + git-annex: user error (git ["--git-dir=copy_target/.git","--work-tree=copy_target","update-index","-z","--index-info"]: Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable)) + failed + _506 TIMES_ (user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))) failed + _11 TIMES_ copy foo (createPipe: resource exhausted (Too many open files)) failed + _2 TIMES_ (user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))) failed + _8574 TIMES_: copy foo (createPipe: resource exhausted (Too many open files)) failed + git-annex: createPipe: resource exhausted (Too many open files) + failed + git-annex: 9101 failed + + % ls copy_target/.git/annex/tmp/redacted_E13 copy_target/.git/annex/tmp/SHA512E-redacted_E13 # works + % find source -type l | wc -l + 13554 + % find copy_target -type l | wc -l + 13554 + % find copy_target/.git/annex/objects -type f | wc -l + 4455 + % find source -type f | wc -l + 13554 + +> Fixed unreaped process leak. +> (This has nothing to do with NTFS). Ran test with 10k files +> [[done]] --[[Joey]] diff --git a/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment b/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment new file mode 100644 index 0000000000..587b1fd97c --- /dev/null +++ b/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-10-26T17:16:52Z" + content=""" +After another run, i am at 8909 files in the remote, now. +"""]] diff --git a/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment b/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment new file mode 100644 index 0000000000..8e83fc19f4 --- /dev/null +++ b/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-10-26T18:22:34Z" + content=""" +In case this matters, I just realized that this disk has been formatted with NTFS instead of a sane FS. +"""]] diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn new file mode 100644 index 0000000000..70ee3b369c --- /dev/null +++ b/doc/bugs/fat_support.mdwn @@ -0,0 +1,13 @@ +Klaus pointed out that there are two problems that keep +git-annex from being used on USB keys, that would typically +be VFAT formatted: + +- Use of symlinks, which VFAT does not support. Very hard to fix. + Instead, just use [[/bare_repositories]] on the key, + they're supported now. +- Use of ":" in filenames of object files, also not supported. + Could easily be fixed by reorganizing the object directory. + +[[Done]]; in annex.version 2 repos, colons are entirely avoided in +filenames. So a bare git clone can be put on VFAT, and git-annex +used to move stuff --to and --from it, for sneakernet. diff --git a/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment b/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment new file mode 100644 index 0000000000..510e449842 --- /dev/null +++ b/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="fmarier" + ip="121.73.248.43" + subject="Exporting to a FAT filesystem?" + date="2011-04-04T07:40:41Z" + content=""" +I'm using git-annex to keep my music in sync between all of my different machines. What I'd love to be able to do is to also keep it in sync with my iRiver player. Unfortunately, the firmware, Rockbox, doesn't support ext3, so I'm stuck with a FAT filesystem. + +I can see how the design of git-annex makes it rather difficult to get rid of the symlinks, so how about taking a different approach: something like a \"git annex export DEST\" which would take a destination (not a git remote) and rsync the content over to there as regular files. + +Maybe \"git annex sync DEST\" or \"git annex rsync DEST\" would be better names if we want to convey the idea that the destination will be made to look like the source repo, including performing the necessary deletions. +"""]] diff --git a/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment b/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment new file mode 100644 index 0000000000..7618c9a7b6 --- /dev/null +++ b/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-04T18:20:45Z" + content=""" +Hey @fmarier. Well, this bug report is closed because you can already get rid of the symlinks. Just put a bare git repo on your fat filesystem, and use git-annex copy --to/--from there. + +Now, that puts all the files that are on the device in .git/annex/objects/xx/yy/blah.mp3 -- how well rockbox would support that I don't know. And if it tries to modify or delete those files, git annex also can't help you manage those changes. + +Another recent option is the [[special_remotes/directory]] special remote type, which again uses \"xx/yy/blah.mp3\" and can't track changes made to the files. This could perhaps be extended in the direction you suggest, although trying to fit this into the special remote infrastructure might not be a good fit really. + +The most likely way this has to get dealt with is really by using [[todo/smudge]] filters, which would eliminate the symlinks and allow copying a non-bare git repo onto vfat. +"""]] diff --git a/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment b/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment new file mode 100644 index 0000000000..f3db75c2f6 --- /dev/null +++ b/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="fmarier" + subject="comment 3" + date="2011-04-05T10:00:21Z" + content=""" +Thanks for the reply @joey. + +While it would certainly be possible for a bare repo to exist on my iRiver, the problem is that the music player uses the filesystem to organize files into directories like \"Artist/Album/Track.ogg\". So replacing that with \"..../xx/yy/Track.ogg\" would make it fairly difficult to browse my music collection and select the album/track I want to listen to :) + +So unless I have the files physically organized like the symlinks, then it's probably not going to work very for that particular workflow. Smudge filters are interesting though. In the meantime, I'll look into rsyncing from another box which has the right filesystem layout onto my iRiver directly. +"""]] diff --git a/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment b/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment new file mode 100644 index 0000000000..722cbdd9e7 --- /dev/null +++ b/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://ethan.betacantrips.com/" + nickname="ethan.glasser.camp" + subject="no symlinks" + date="2011-06-08T20:59:38Z" + content=""" +If you try to clone a git repo that has a symlink over to a VFAT filesystem, you get (in its place) a regular file that contains the name of the symlink target. So why can't git-annex use that? I could still do git annex get on this file, git annex would still \"know\" that it's a symlink, and could replace it with a copy of the real file (instead of putting it in .git/annex). + +I know if it were that simple, someone would have done it already, so what am I missing? I guess trying to get the file FROM the repository would fail because it wouldn't find the file in .git/annex? Couldn't you store a reverse mapping? You wouldn't be able to move the file around, but you already lose that once you give up symlinks. It would also be a little harder to tell which symlinks were \"dangling\"; I don't see an easy way to get around that. It would still be better than a bare repo.. +"""]] diff --git a/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment b/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment new file mode 100644 index 0000000000..1063b0f910 --- /dev/null +++ b/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 5" + date="2011-06-10T16:41:43Z" + content=""" +@ethan the reason that wouldn't work is because git would then see a file that was checked in and had its one line symlinkish content replaced with a huge binary blob. And git commit would try to commit that etc. The potential for foot-shooting is too high. +"""]] diff --git a/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment b/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment new file mode 100644 index 0000000000..c7defd13ef --- /dev/null +++ b/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkZRoTRyW3tox-FD2DQWxskgI6_tkEtHL4" + nickname="Ben" + subject="comment 6" + date="2012-07-23T16:11:52Z" + content=""" +The above would work fine for me but the files in my annex (e.g. .git/annex/objects/xx/yy/blah.ogg) don't have extensions like that, so my media player doesn't recognize them as media files. How do I get the files under \"objects\" to keep the extensions of the original files like in Joey's example? +"""]] diff --git a/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment b/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment new file mode 100644 index 0000000000..11668615e6 --- /dev/null +++ b/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 7" + date="2012-07-24T14:51:50Z" + content=""" +You can get the extensions by migrating to the SHA1E (or SHA256E) backend. +"""]] diff --git a/doc/bugs/fat_support/comment_8_acc947643a635eb10a1bff92083a3506._comment b/doc/bugs/fat_support/comment_8_acc947643a635eb10a1bff92083a3506._comment new file mode 100644 index 0000000000..558e0ca105 --- /dev/null +++ b/doc/bugs/fat_support/comment_8_acc947643a635eb10a1bff92083a3506._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmcYryijvlF8bJvM_eZNSrUPEkMlxMDGTQ" + nickname="Thiago" + subject="POSIX layer on top of VFAT using FUSE" + date="2012-11-24T00:21:23Z" + content=""" +I just found out about this project and didn't try it, but it looks like it would allow using git-annex on an usb stick with a normal repository: + + +"""]] diff --git a/doc/bugs/fatal:_empty_ident_name.mdwn b/doc/bugs/fatal:_empty_ident_name.mdwn new file mode 100644 index 0000000000..2414772877 --- /dev/null +++ b/doc/bugs/fatal:_empty_ident_name.mdwn @@ -0,0 +1,51 @@ +**What steps will reproduce the problem?** + + stone@skynet ~/annex $ git init + Initialized empty Git repository in /home/stone/annex/.git/ + stone@skynet ~/annex $ git annex init "work" + init work + *** Please tell me who you are. + + Run + + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + + to set your account's default identity. + Omit --global to set the identity only in this repository. + + fatal: empty ident name (for ) not allowed + git-annex: git ["--git-dir=/home/stone/annex/.git","--work-tree=/home/stone/annex","commit-tree","4b825dc652cb6eb9a060e64bf8d69288fbee4904"] exited 128 + stone@skynet ~/annex $ git config -l + user.email=stone@nospam.hu + user.name=Stone + core.editor=nano + color.ui=auto + core.repositoryformatversion=0 + core.filemode=true + core.bare=false + core.logallrefupdates=true + annex.uuid=499fb545-0b98-4bfc-816c-fb3704f3aaa0 + stone@skynet ~/annex $ cat ~/.gitconfig + [user] + email = stone@nospam.hu + name = Stone + [core] + editor = nano + [color] + ui = auto + stone@skynet ~/annex $ + +**What is the expected output? What do you see instead?** + + +**What version of git-annex are you using? On what operating system?** + +commit 56c037c69e75def74d6ea90de8aa8a1954c52178 Arch Linux + +**Please provide any additional information below.** + +> [[done]] by adding name to the user, in /etc/passwd. --Stone + +>> Actually, [[done]] by avoiding clobbering HOME when running some git +>> commands. --[[Joey]] diff --git a/doc/bugs/fatal:_empty_ident_name/comment_1_ceae87308fb75a1f79c7c8d63ec47226._comment b/doc/bugs/fatal:_empty_ident_name/comment_1_ceae87308fb75a1f79c7c8d63ec47226._comment new file mode 100644 index 0000000000..798658a46b --- /dev/null +++ b/doc/bugs/fatal:_empty_ident_name/comment_1_ceae87308fb75a1f79c7c8d63ec47226._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.246.16" + subject="comment 1" + date="2012-08-31T15:04:09Z" + content=""" +This is an error message from git. [Git faq](https://git.wiki.kernel.org/index.php/GitFaq#Git_commit_is_dying_telling_me_.22fatal:_empty_ident_.3Cuser.40myhost.3E_not_allowed.22.2C_what.27s_wrong.3F). I don't understand why git is doing it given the configuration shown, but I suppose it would do the same thing if you run git commit by hand. +"""]] diff --git a/doc/bugs/fatal:_empty_ident_name/comment_2_68832ee3e0e7244ce62bccabe2e52630._comment b/doc/bugs/fatal:_empty_ident_name/comment_2_68832ee3e0e7244ce62bccabe2e52630._comment new file mode 100644 index 0000000000..934fcdfaf0 --- /dev/null +++ b/doc/bugs/fatal:_empty_ident_name/comment_2_68832ee3e0e7244ce62bccabe2e52630._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl1yBP_JDsO1TWSC1usoHXpfDRU01u_GXY" + nickname="Péter Károly" + subject="comment 2" + date="2012-08-31T18:30:15Z" + content=""" +I use git on the same machine nearly every day, it does not complain on commit. + +(On the same session after \"git annex init\" failed...) + + stone@skynet ~/annex $ echo stone > bu + stone@skynet ~/annex $ git add bu + stone@skynet ~/annex $ git commit -a + [master (root-commit) ae5d41f] ds + 1 file changed, 1 insertion(+) + create mode 100644 bu + stone@skynet ~/annex $ git log + commit ae5d41fdd0b7082740633cf7931bb5a07be0fc5e + Author: Stone + Date: Fri Aug 31 20:26:45 2012 +0200 + + ds + stone@skynet ~/annex $ + +"""]] diff --git a/doc/bugs/fatal:_empty_ident_name/comment_3_ed31ad316747343d7730e4c2d7dacd24._comment b/doc/bugs/fatal:_empty_ident_name/comment_3_ed31ad316747343d7730e4c2d7dacd24._comment new file mode 100644 index 0000000000..ff7f39e443 --- /dev/null +++ b/doc/bugs/fatal:_empty_ident_name/comment_3_ed31ad316747343d7730e4c2d7dacd24._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.246.16" + subject="comment 3" + date="2012-08-31T18:52:53Z" + content=""" +Why don't you try the identical command that git-annex is running that fails: + +`git --git-dir=/home/stone/annex/.git --work-tree=/home/stone/annex commit-tree 4b825dc652cb6eb9a060e64bf8d69288fbee4904` +"""]] diff --git a/doc/bugs/fatal:_empty_ident_name/comment_4_b812d6f30e8a866bce7260a9ee3218e3._comment b/doc/bugs/fatal:_empty_ident_name/comment_4_b812d6f30e8a866bce7260a9ee3218e3._comment new file mode 100644 index 0000000000..9e8a1900e0 --- /dev/null +++ b/doc/bugs/fatal:_empty_ident_name/comment_4_b812d6f30e8a866bce7260a9ee3218e3._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl1yBP_JDsO1TWSC1usoHXpfDRU01u_GXY" + nickname="Péter Károly" + subject="comment 4" + date="2012-09-01T15:05:52Z" + content=""" +Finally I managed to get it working. + +My use on the computer didn't had name, so the 5th colum of my /etc/password file was empty. After I filled in my name everything worked like charm. + +Interesting that not my user's name from /etc/passwd get into the git log but the one that was in my ~/.gitconfig. + +"""]] diff --git a/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn b/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn new file mode 100644 index 0000000000..fee00855eb --- /dev/null +++ b/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn @@ -0,0 +1,36 @@ +Since the watch branch is now merged into master, it doesn't quite build cleanly on non-linux systems anymore. So here's a small change to the Makefile to work around the problem for now. + +
+From 707cb47744775c324060febe11987db5f10ed9ff Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Mon, 18 Jun 2012 09:20:35 +0100
+Subject: [PATCH] Teach _Makefile_ to only do _-DWITH_INOTIFY_ when on a Linux
+ machine.
+
+---
+ Makefile |    8 +++++++-
+ 1 file changed, 7 insertions(+), 1 deletion(-)
+
+diff --git a/Makefile b/Makefile
+index 6d36e8b..8884b5c 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,6 +1,12 @@
++OS:=$(shell uname | sed 's/[-_].*//')
++
++ifeq ($(OS),Linux)
++BASEFLAGS_OPTS+=-DWITH_INOTIFY
++endif
++
+ PREFIX=/usr
+ IGNORE=-ignore-package monads-fd -ignore-package monads-tf
+-BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 -DWITH_INOTIFY
++BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 $(BASEFLAGS_OPTS)
+ GHCFLAGS=-O2 $(BASEFLAGS)
+ 
+ ifdef PROFILE
+-- 
+1.7.10.4
+
+ +[[done]], thanks --[[Joey]] diff --git a/doc/bugs/free_space_checking.mdwn b/doc/bugs/free_space_checking.mdwn new file mode 100644 index 0000000000..92e8be40d1 --- /dev/null +++ b/doc/bugs/free_space_checking.mdwn @@ -0,0 +1,21 @@ +Should check that there is enough free space before trying to copy a +file around. + +* Need a way to tell how much free space is available on the disk containing + a given repository. + +* And, need a way to tell the size of a file before copying it from + a remote, to check local disk space. + + As of annex.version 2, this metadata can be available for any type + of backend. Newly added files will always have file size metadata, + while files that used a SHA backend and were added before the upgrade + won't. + + So, need a migration process from eg SHA1 to SHA1+filesize. It will + find files that lack size info, and rename their keys to add the size + info. Users with old repos can run this on them, to get the missing + info recorded. + +> [[done]]; no migtation process for old SHA1 keys from v1 repo though. +> --[[Joey]] diff --git a/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment b/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment new file mode 100644 index 0000000000..954433deb4 --- /dev/null +++ b/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-15T14:11:27Z" + content=""" +Keep in mind that lots of small files may have significant overhead, so a warning that it's not possible to make sure there's enough space would make sense for certain corner cases. Actually finding out the exact overhead is beyond git-annex' scope and, given transparent compression etc, ability, but a warning, optionally with a \"do you want to continue\" prompt can't hurt. + +-- RichiH +"""]] diff --git a/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment b/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment new file mode 100644 index 0000000000..9a43fe3f27 --- /dev/null +++ b/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-03-16T03:04:50Z" + content=""" +Right. You probably don't want git-annex to fill up your entire drive anyway, so if it tries to reseve 10 mb or 1% or whatever (probably configurable) for overhead, that should be good enough. +"""]] diff --git a/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment b/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment new file mode 100644 index 0000000000..ea4fb6c23e --- /dev/null +++ b/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 3" + date="2011-03-16T15:40:56Z" + content=""" +Sometimes, I might want to fill up the disk as much as possible. Thus, a warning is preferable to erroring out too early, imo -- Richard +"""]] diff --git a/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn new file mode 100644 index 0000000000..c649ff9f7c --- /dev/null +++ b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn @@ -0,0 +1,8 @@ +git annex carefully setup restrictive permissions of .git/annex directories and files. + +The fsck command should check that they are still correct. +The fix command should fix them. + +PS: Thanks for this nice tool! + +> Good idea, [[done]] (actually, fsck just fixes them too)! --[[Joey]] diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn new file mode 100644 index 0000000000..fe6536b6a7 --- /dev/null +++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn @@ -0,0 +1,57 @@ + (checksum...) failed + fsck foo (fixing location log) + Only 1 of 2 trustworthy copies exist of foo + Back it up with git-annex copy. + +> You've given me severely partial output, and no test case, but until +> it says "fsck foo", the output is pertaining to some other file than foo. +> As far as I can see, there is no bug here. --[[Joey]] + +>> Sorry, I thought it would be obvious, but that's no excuse for not +>> providing additional explanation. The problem is that fsck tells me a +>> file's fsck has failed without printing extra details. In this case, the +>> checksum is OK while I don't have enough copies to satisfy the fsck. The +>> fact that I don't have enough copies is obviously relevant, but I would +>> still like to know if the checksums are OK. -- Richard + +>>> I think you're misreading the truncated output you posted. The actual, +>>> full output would make much more sense. --[[Joey]] + +>>>> No. I have a total of 14908 annex keys, 3333 of which are on a remote. The only message other than 'checksum OK' and the above is 'git-annex: 11577 failed'. +>>>> I checked several files manually, their checksums are OK so `git annex +>>>> fsck` is reporting those files as completely failed when they "only" miss copies. -- Richard + +>>>>> fsck considers not enough copies to be a failure condition; it prints +>>>>> error messages about it etc. That has nothing to do with checksums. +>>>>> --[[Joey]] + +>>>>>> I get that. Still, I think it would be _extremely_ useful to know what failures occurred, exactly. Not having enough copies is Not Good, yet not having enough copies and a locally correct file is _lot_ better than having not enough copies and a broken file. I.e. I would prefer: + + (checksum...) OK + Not enough copies: Only 1 of 2 trustworthy copies exist of foo + +>>>>>> or similar and at the end + + git-annex: 0 wrong checksums + git-annex: 11577 with too few copies + +>>>>>> In the end, it comes down to the distinction of different failure classes. -- Richard + +>>>>>>> For the third, and final time: +>>>>>>> # You are misreading the truncated output you posted +>>>>>>> The "checksum" line is regarding **different** file than the +>>>>>>> not enough copies message. fsck does not attempt to checksum a file +>>>>>>> that is not present. [[done]] --[[Joey]] + + +>>>>>>>> I realized early on that I pasted the wrong cross-passage, but as there is a ton of the same output, I didn't think it would matter. I wasn't aware that it does not try to checksum when there aren't enough copies. To be fair, you only just mentioned that. +>>>>>>>> Personally, I think that's a bug as it makes ensuring local correctness before copying a file to remotes impossible. +>>>>>>>> Either way, I really didn't know it actually _skipped_ checksumming; that part was missing. +>>>>>>>> For the benefit of anyone else who might read this, this is the correct order: + + fsck foo (fixing location log) + Only 1 of 2 trustworthy copies exist of foo + Back it up with git-annex copy. + (checksum...) failed + +>>>>>>>> If you would like to keep things this way, fine. I think it's less than ideal, but I don't want to argue, either. -- Richard diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn new file mode 100644 index 0000000000..1b00dd7b37 --- /dev/null +++ b/doc/bugs/fsck_output.mdwn @@ -0,0 +1,46 @@ +When you check several files and the fsck fails, you get confusing output: + +
+O fsck test1 (checksum...) 
+E  Only 1 of 2 trustworthy copies of test1 exist.
+E  Back it up with git-annex copy.
+O
+O failed
+O fsck test2 (checksum...) 
+E  Only 1 of 2 trustworthy copies of test2 exist.
+E  Back it up with git-annex copy.
+O 
+O failed
+
+ +The newline is in the wrong place and confuses the user. It should be printed _after_ "failed". + +> This is a consequence of part of the output being printed to stderr, and +> part to stdout. I've marked the lines above with E and O. +> +> Normally a "failed" is preceeded by a message output to stdout desribing +> the problem; such a message will not be "\n" terminated, so a newline +> is always displayed before "failed". In this case, since the message +> is sent to stderr, it is newline terminated. +> +> Fixing this properly would involve storing state, or rethinking +> when git-annex displays newlines (and I rather like its behavior +> otherwise). +> +> A related problem occurs if an error message is unexpetedly printed. +> Dummying up an example: +> +> O get test1 (from foo...) E git-annex: failed to run ssh +> failed +> +> --[[Joey]] + +>> Well, I fixed this in all cases except a thrown non-IO error (last +>> example aboce), which output is printed by haskell's runtime. I'd +>> have to add a second error handler to handle those, and it's not +>> clear what it would do. Often an error will occur before anything +>> else is printed, and then the current behavior is right; if something +>> has been printed it would be nice to have a newline before the error, +>> but by the time the error is caught we'd be out of the annex monad +>> and not really have any way to know if something has been printed. +>> I think my fix is good enough [[done]] --[[Joey]] diff --git a/doc/bugs/fsck_should_double-check_when_a_content-check_fails.mdwn b/doc/bugs/fsck_should_double-check_when_a_content-check_fails.mdwn new file mode 100644 index 0000000000..dba775d37e --- /dev/null +++ b/doc/bugs/fsck_should_double-check_when_a_content-check_fails.mdwn @@ -0,0 +1,3 @@ +git annex fsck marks files as bad when the checksumming fails. But this could also be due to a read error when the actual data stored is correct. So, fsck should check twice when a checksum fails. + +> [[done]]; apparently problem was caused by bad RAM. --[[Joey]] diff --git a/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_1_03af24b70adbcd9f4b94d009f6b71d0a._comment b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_1_03af24b70adbcd9f4b94d009f6b71d0a._comment new file mode 100644 index 0000000000..543777e260 --- /dev/null +++ b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_1_03af24b70adbcd9f4b94d009f6b71d0a._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 1" + date="2013-01-14T16:26:59Z" + content=""" +Why stop at checking twice? The second check could also fail with a read error, and perhaps the third one would succeed. :P + +Seriously, I doubt that this is likely to be a benefit with a modern drive. If the file has a read error once, then error correction has already failed, and it's likely to fail again. Even if it managed to succeed the second time, you have a file that is being read wrong some of the time, which is not a good thing for fsck to leave unnoticed. + +Fsck moves bad files to `.git/annex/bad`, so the data in them can be recovered if it comes to that. Hopefully +though, there's a copy of the file in another repository, so git-annex can just get it from there instead. +"""]] diff --git a/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_2_41214a7d18c66b694645248d6ebeadbf._comment b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_2_41214a7d18c66b694645248d6ebeadbf._comment new file mode 100644 index 0000000000..ea9518cb6b --- /dev/null +++ b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_2_41214a7d18c66b694645248d6ebeadbf._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkHDm_DOFRcHYebCnnYKKyIwiPD4iOiiIU" + nickname="Jörn" + subject="comment 2" + date="2013-01-14T17:37:45Z" + content=""" +Maybe I was too quick in blaming the hard drive. It might be my problem is somewhere else. Let me do what I should have done in the first place and give you a detailed problem description: + +I have got three hard drives, two internal, one external connected via USB. I have got a couple of repositories with small files (mp3, JPEGs and so on). Those are fine, fsck never complains about them. +But in one repository with video files (i.e. much bigger files than in the other repos), git-annex fsck will always find some broken files. I run git-annex get to retrieve the broken files from other sources. Then I run +fsck again - and it complains about some other files. This happens on all drives. + +This could mean: + +- all my drives are broken. However, SMART data are unsuspicious, and one of the drives is just a couple of days old. +- git-annex fsck is broken +- read errors like I mentioned in my first post +- some process actually _altering_ the files (should not happen when the files are locked, right?) +- something completely different? Some possibly dangerous source of radiation? :) + +Any ideas on this? Maybe I should hash the data in .git/annex/bad and check which value I get - can I tell git-annex to do so? + +Thanks, +Jörn +"""]] diff --git a/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_3_e7ddd77ea35994f2051f840e9b4c7e0c._comment b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_3_e7ddd77ea35994f2051f840e9b4c7e0c._comment new file mode 100644 index 0000000000..f7d6221dec --- /dev/null +++ b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_3_e7ddd77ea35994f2051f840e9b4c7e0c._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 3" + date="2013-01-14T19:06:37Z" + content=""" +I would doubt you'd have three broken drives, or read errors from 3 drives. + +You can use `sha256sum` to checksum the files in `bad` yourself, and compare the results with their names. +If it matches, that would point to some kind of bug in fsck. +"""]] diff --git a/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_4_36a70d5a378983a76fcdbb7fba044044._comment b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_4_36a70d5a378983a76fcdbb7fba044044._comment new file mode 100644 index 0000000000..64df24bdb1 --- /dev/null +++ b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_4_36a70d5a378983a76fcdbb7fba044044._comment @@ -0,0 +1,32 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkHDm_DOFRcHYebCnnYKKyIwiPD4iOiiIU" + nickname="Jörn" + subject="comment 4" + date="2013-01-14T20:04:48Z" + content=""" +On the internal disk some files match, others don't: + + 42c95aee60c41157ee6d092eeea7fab73dff7b45a6e16ba77a02e2a6fec01c15 SHA256-s1099424853--42c95aee60c41157ee6d092eeea7fab73dff7b45a6e16ba77a02e2a6fec01c15 + 44c4d3b0f29d86597ae09cf52563e0a2adec7074f5359901fde8dbf88f9bc8ad SHA256-s1173029201--bb981fddd750db1255cc1fda1f9612250f766e4da946eac4c294d8071196615c + a9d804fe633ae69d0c926dea41059c211f78cf891abfa275c579bcc8e12fa700 SHA256-s1363601770--a9d804fe633ae69d0c926dea41059c211f78cf891abfa275c579bcc8e12fa700 + bac5f30b696b643c76a456e3018214ce64be7d289afaac96a7fdba03eccf73fd SHA256-s1422005768--5754211ac3e7e6314de9aa34f7b1cfd3c9aed432f52b26189c4587dc505a30e8 + be953e962d9c6a2699f361e468023aae34b1f7c66ad93852d8f88846b60ecbcc SHA256-s1424026123--258dbe5704a46ecd3190d278efdd2adfeee9ad81843bfaa0ed7adbc10ccd5362 + b9f06383c76a0161312aaf6298eb764dbfb8e1e51555ec06d8c85c9f5df9fe20 SHA256-s1448369674--a703dfd46c147396def6e1db22c7917dbdf010658aa32df1cf598e37a04d5897 + e2d748ee5c5098316bbe20c3f771f7f552b07c394a1e9932cba2a90883b6fb5c SHA256-s1480008949--505c7d2685290069140b2f03fa72a5b5e3f59c14961c60c5fd6532be05ad1c84 + ac0d49461ca67ca2de0588491c39089634ae194870594dd869fbc84d42831baa SHA256-s1489602933--f3ff3fc71b4f5493db476f29f373a4b3bf8808e9a7487589fee632ea58a415af + b9b1d133772c88a1c5f856e2d4bcb696a453c795337b9f97cfa4c00876f49b88 SHA256-s1512599506--78dc0bdf0b4e382283a770a36c7f699c83611a0436059e6bb44c8e94d26cca5f + 497af65e2d400e5651d9b850bc5efbaa006cec08f3f0dfce5d09b8e3f994a80b SHA256-s1524648153--85548bcb693491eeae1d1315a6037d5a9a15d2a10066f04f6e7ce75b6eb1ca47 + cb793e7ac6df827e96f7dfa34a10f13a899140372c4db58b08650314596d30fa SHA256-s1541427964--16b8f7d5e5e12af88e947b26ca4d38d6b6692de92a77d9ffef0c8c1d560db727 + 27ea446f928239ade9c3c1f38119fa38da50a74036f9987d57cce3b52ae632e9 SHA256-s1561020163--cfce86c309d16af772e68527ba365ceac5db447d5bbfc335e063a2a4e9d513d6 + 940d2c7ff3bde5662d71a1a463f6949c6dd6cf244d43ab7d48014a10d0fa5d6a SHA256-s1562397113--a057d447b5b32d2f1a440247d9dc3c3ca697798d73dd740670a3033dd0fcbb68 + [...] + + +On the usb drive, hashes match: + + [joern@heracles bad]$ sha256sum * + e306e07a09af9be9a974632c777935ddd0f5333b603b5f205dba03753619d682 SHA256-s1564783770--e306e07a09af9be9a974632c777935ddd0f5333b603b5f205dba03753619d682 + 8fe78a325a08099e56df44ff46d482607723dfc9402bcd8f4850907327603f05 SHA256-s207925248--8fe78a325a08099e56df44ff46d482607723dfc9402bcd8f4850907327603f05 + 8823bd4ccf5f6ff2fe0976de6a856490ef3de0ab1c1e128bde770b732e86d94c SHA256-s923668098--8823bd4ccf5f6ff2fe0976de6a856490ef3de0ab1c1e128bde770b732e86d94c + +"""]] diff --git a/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_5_899c4afbc988d81984c5c3397285bb01._comment b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_5_899c4afbc988d81984c5c3397285bb01._comment new file mode 100644 index 0000000000..fb38385115 --- /dev/null +++ b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_5_899c4afbc988d81984c5c3397285bb01._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 5" + date="2013-01-14T21:57:30Z" + content=""" +If you have both the corrupted file and the good file you can use the `cmp` command to show exactly how they differ: + + cmp -l file1 file2 + +If files are regularly marked as corrupted you might have bad ram. +"""]] diff --git a/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_6_dbff51d00c5645eb1832aa4644889c5e._comment b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_6_dbff51d00c5645eb1832aa4644889c5e._comment new file mode 100644 index 0000000000..724be0df06 --- /dev/null +++ b/doc/bugs/fsck_should_double-check_when_a_content-check_fails/comment_6_dbff51d00c5645eb1832aa4644889c5e._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkHDm_DOFRcHYebCnnYKKyIwiPD4iOiiIU" + nickname="Jörn" + subject="comment 6" + date="2013-01-15T09:38:34Z" + content=""" +That was a good advice. I have just run memtest, and it seems my memory is broken. So this was probably the cause for all this trouble. + +Thanks! +"""]] diff --git a/doc/bugs/fsck_thinks_file_content_is_bad_when_it_isn__39__t.mdwn b/doc/bugs/fsck_thinks_file_content_is_bad_when_it_isn__39__t.mdwn new file mode 100644 index 0000000000..e9051f9f34 --- /dev/null +++ b/doc/bugs/fsck_thinks_file_content_is_bad_when_it_isn__39__t.mdwn @@ -0,0 +1,35 @@ +What steps will reproduce the problem? +What is the expected output? What do you see instead? + +I can reproduce it locally, but don't know what's causing it. The file content is the same, I checked with md5sum and sha512sum. But fsck still thinks the content is different. Are there other factors I could check which fsck looks at? I'm using SHA512E backend. + +What version of git-annex are you using? On what operating system? + +git-annex version: 3.20120807 Ubuntu 12.04 updated on Aug 20th annex was installed via cabal on Aug 20th, all other packages are from ubuntu. + +> What is the error message from fsck? --[[Joey]] + +This is the output: + +> reinject ....Moon.avi (checksum...) +> Bad file content; moved to /mnt/.../.git/annex/bad/SHA512E-s94402560--ead9db1f34739014a216239d9624bce74d92fe723de06505f9b94cb4c063142ba42b04546f11d3d33869b736e40ded2ff779cb32b26aa10482f09407df0f3c8d.Moon.avi +failed +> (Recording state in git...) +> git-annex: reinject: 1 failed + +The original file also has sha512 ead9db1f34739014a216239d9624bce74d92fe723de06505f9b94cb4c063142ba42b04546f11d3d33869b736e40ded2ff779cb32b26aa10482f09407df0f3c8d + +>> And what sha512 does the file in .git/annex/bad have **now**? (fsck +>> preserves the original filename; this says nothing about what the +>> current checksum is, if the file has been corrupted). --[[Joey]] + +The same, as it's the file I was trying to inject: + +ead9db1f34739014a216239d9624bce74d92fe723de06505f9b94cb4c063142ba42b04546f11d3d33869b736e40ded2ff779cb32b26aa10482f09407df0f3c8d .git/annex/bad/SHA512E-s94402560--ead9db1f34739014a216239d9624bce74d92fe723de06505f9b94cb4c063142ba42b04546f11d3d33869b736e40ded2ff779cb32b26aa10482f09407df0f3c8d.Moon.avi + +That's what puzzles me, it is the same file, but for some weird reason git annex thinks it's not. + +> Ok, reproduced and fixed the bug. The "E" backends recently got support +> for 2 levels of filename extensions, but were not made to drop them both +> when fscking. [[done]] (I'll release a fixed version probably tomorrow; +> fix is in git now.) --[[Joey]] diff --git a/doc/bugs/fsck_thinks_file_content_is_bad_when_it_isn__39__t/comment_1_cafb58eca97a0a66110ac39b169d8de3._comment b/doc/bugs/fsck_thinks_file_content_is_bad_when_it_isn__39__t/comment_1_cafb58eca97a0a66110ac39b169d8de3._comment new file mode 100644 index 0000000000..9f63dbfa9a --- /dev/null +++ b/doc/bugs/fsck_thinks_file_content_is_bad_when_it_isn__39__t/comment_1_cafb58eca97a0a66110ac39b169d8de3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawla3gLc6_rHuggFfy7o7eGMPvPztFZTrUQ" + nickname="Florian" + subject="comment 1" + date="2012-08-27T08:59:23Z" + content=""" +Works! +"""]] diff --git a/doc/bugs/get_failed__44___but_remote_has_the_file.mdwn b/doc/bugs/get_failed__44___but_remote_has_the_file.mdwn new file mode 100644 index 0000000000..7f0b8612ba --- /dev/null +++ b/doc/bugs/get_failed__44___but_remote_has_the_file.mdwn @@ -0,0 +1,35 @@ +Not sure what caused this, but I have a file that exists on two remotes, but I can only get it from one of them. If I try to get it from the other, it fails immediately (without even connecting): + + pilot:~/vid/tv/Show$ git annex whereis Show\ -\ S03E08.mp4 + whereis Show - S03E08.mp4 (2 copies) + 09c0b436-f8de-11e0-842f-b7644539d57f -- psychosis + 82814942-f8e0-11e0-b053-e70a61e98e19 -- bucket + ok + + pilot:~/vid/tv/Show$ git annex fsck Show\ -\ S03E08.mp4 + fsck Show - S03E08.mp4 ok + + pilot:~/vid/tv/Show$ git annex get --from bucket Show\ -\ S03E08.mp4 + get Show - S03E08.mp4 failed + git-annex: get: 1 failed + + pilot:~/vid/tv/Show$ git annex get --debug --from bucket Show\ -\ S03E08.mp4 + [2013-01-17 19:05:13 EST] read: git ["--git-dir=/home/jim/vid/.git","--work-tree=/home/jim/vid","show-ref","git-annex"] + [2013-01-17 19:05:13 EST] read: git ["--git-dir=/home/jim/vid/.git","--work-tree=/home/jim/vid","show-ref","--hash","refs/heads/git-annex"] + [2013-01-17 19:05:13 EST] read: git ["--git-dir=/home/jim/vid/.git","--work-tree=/home/jim/vid","log","refs/heads/git-annex..e41c3b1ee9127129f2c9fc3fa5d4771afcb5ffd7","--oneline","-n1"] + [2013-01-17 19:05:13 EST] read: git ["--git-dir=/home/jim/vid/.git","--work-tree=/home/jim/vid","log","refs/heads/git-annex..a7ae08bccede282f46c2073f6c3e52685a593482","--oneline","-n1"] + [2013-01-17 19:05:13 EST] read: git ["--git-dir=/home/jim/vid/.git","--work-tree=/home/jim/vid","log","refs/heads/git-annex..ae0f84e906423f4da465e3d3df9d46545684d3f5","--oneline","-n1"] + [2013-01-17 19:05:13 EST] chat: git ["--git-dir=/home/jim/vid/.git","--work-tree=/home/jim/vid","cat-file","--batch"] + [2013-01-17 19:05:13 EST] read: git ["--git-dir=/home/jim/vid/.git","--work-tree=/home/jim/vid","ls-files","--cached","-z","--","Show - S03E08.mp4"] + get Show - S03E08.mp4 failed + git-annex: get: 1 failed + +Same `git annex version` on both `pilot` and `bucket`, and I ran `git annex sync` on both. + + git-annex version: 3.20130114 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +How should I debug this? diff --git a/doc/bugs/get_failed__44___but_remote_has_the_file/comment_1_55c8b73ce05dfca11a393bb296b99b9a._comment b/doc/bugs/get_failed__44___but_remote_has_the_file/comment_1_55c8b73ce05dfca11a393bb296b99b9a._comment new file mode 100644 index 0000000000..24825b2eb4 --- /dev/null +++ b/doc/bugs/get_failed__44___but_remote_has_the_file/comment_1_55c8b73ce05dfca11a393bb296b99b9a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.3.194" + subject="comment 1" + date="2013-01-18T20:25:01Z" + content=""" +One completely legitimate way for this to happen would be if bucket is a removable drive or other filesystem, and it is not mounted (or its directory doesn't contain a git repo for whatever reason). It would look identical to what you showed happening. + +Otherwise, what kind of remote is bucket? Special remote or git remote? +"""]] diff --git a/doc/bugs/get_failed__44___but_remote_has_the_file/comment_2_474c67a421dca4c245e7bfe495d3f6d3._comment b/doc/bugs/get_failed__44___but_remote_has_the_file/comment_2_474c67a421dca4c245e7bfe495d3f6d3._comment new file mode 100644 index 0000000000..a7e3d6b174 --- /dev/null +++ b/doc/bugs/get_failed__44___but_remote_has_the_file/comment_2_474c67a421dca4c245e7bfe495d3f6d3._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck" + nickname="Jim" + subject="comment 2" + date="2013-01-19T21:40:14Z" + content=""" +They're all git remotes: + + pilot:~/vid/tv$ git remote -v show + bucket ssh://bucket/home/jim/private/vid (fetch) + bucket ssh://bucket/home/jim/private/vid (push) + origin https://git/jim/annex.git (fetch) + origin https://git/jim/annex.git (push) + psychosis ssh://psychosis/vid/annex (fetch) + psychosis ssh://psychosis/vid/annex (push) + +And it never even attempts a ssh connection to bucket. +"""]] diff --git a/doc/bugs/get_failed__44___but_remote_has_the_file/comment_3_845e8a23d63fb0b071c63ee736697d26._comment b/doc/bugs/get_failed__44___but_remote_has_the_file/comment_3_845e8a23d63fb0b071c63ee736697d26._comment new file mode 100644 index 0000000000..0ddbe14bf3 --- /dev/null +++ b/doc/bugs/get_failed__44___but_remote_has_the_file/comment_3_845e8a23d63fb0b071c63ee736697d26._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck" + nickname="Jim" + subject="comment 3" + date="2013-01-19T22:21:04Z" + content=""" +Wait, that's weird, it works today. I'm thinking that maybe there was another copy of `git-annex` backgrounded and holding a lock, since that reproduces this behavior: + + pilot:~/vid/tv/Show$ git annex get --from bucket Show\ -\ S03E08.mp4 + get Show - S03E08.mp4 (from bucket...) + Enter passphrase for key '/home/jim/.ssh/id_rsa': + SHA256E-s358393024--efda17d23d68b85d47ad342f8e41f79ac04d4a65d7ef654b4838b995b86bdefe.mp4 + 96043008 26% 10.27MB/s 0:00:24 ^Z + [1]+ Stopped git annex get --from bucket Show\ -\ S03E08.mp4 + pilot:~/vid/tv/Show$ git annex get --from bucket Show\ -\ S03E08.mp4 + get Show - S03E08.mp4 failed + git-annex: get: 1 failed + +So, I guess this is probably \"not a bug\", but having a more specific error would be really helpful. +"""]] diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn new file mode 100644 index 0000000000..f9a61a8590 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn @@ -0,0 +1,75 @@ +I ran git-annex (git version) on three machines with ghc-7.0.2 for about a month, but recently (no more than a week ago) I've started getting this error for every file on "git annex get": + + git-annex-shell: internal error: evacuate(static): strange closure type 30799 + (GHC version 7.0.2 for i386_unknown_linux) + Please report this as a GHC bug: http://www.haskell.org/ghc/reportabug + +There were no changes to ghc or it's modules, so I assume something has changed in git-annex itself. + +strace shows "git annnex get" (on "host1") performing following exec's: + + [pid 9481] execve("/usr/bin/rsync", ["rsync", "-p", "--progress", "--inplace", "-e", "'ssh' 'user@host2' 'git-annex-shell ''sendkey'' ''/remote/path'' ''SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b'' ''--'''", ":", "/local/path/.git/annex/tmp/SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b"], [/* 41 vars */]) = 0 + [pid 9482] execve("/usr/bin/ssh", ["ssh", "user@host2", "git-annex-shell 'sendkey' '/remote/path' 'SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b' '--'", "", "rsync", "--server", "--sender", "-vpe.Lsf", "--inplace", ".", ""], [/* 41 vars */] + +I've tried running the second command directly from the shell and got the same error message from a remote GHC. +Adding strace before git-annex-shell to remote command yielded something like this in the end: + + stat64("/local/path.git", 0xb727d610) = -1 ENOENT (No such file or directory) + stat64("/local/path.git", 0xb727d6b0) = -1 ENOENT (No such file or directory) + waitpid(7525, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0) = 7525 + chdir("/home/user") = 0 + rt_sigprocmask(SIG_BLOCK, [INT], [], 8) = 0 + write(2, "git-annex-shell: internal error: ", 33git-annex-shell: internal error: ) = 33 + ... + +Note that "/local/path" here is not what's specified in rsync arguments at all, and git repo with files-to-be-fetched on "host2" is in "/remote/path", but "/local/path" is present in git remotes there since I mount it via nfs from "host1" (yes, to the same path as it's there): + + [remote "nfs"] + url = /local/path + fetch = +refs/heads/*:refs/remotes/nfs/* + push = refs/heads/*:refs/remotes/host2/* + annex-uuid = 0a4e14ba-5236-11e0-9004-7f24452c0f05 + +If I comment that remote out from "/remote/path/.git/config", "git annex get" works fine. +The only git-command git-annex-shell seem to exec there (on "host2") is "git config --list", so it's shouldn't be git trying to do something with it's remotes - it's git-annex itself, right? + +Anyways, looks like a simple path-joining error, if "/local/path.git" should be "/local/path/.git" there. + +I'm actually quite confused about what it's trying to do with that path. +Connect from "host1" to "host2" just to connect back to "host1"? +What for, when it should just fetch files from "host2"? + +> git-annex (and git-annex shell) always start up by learning what git +> remotes are locally configured, and this includes checking them to +> try to look up their annex.uuid setting. +> +> Since git will, given a remote like "url = /foo", first look in +> "/foo.git" for a bare git repository, so too does git-annex. +> I do not think this is a path joining error. That seems likely to +> be a red herring. --[[Joey]] + +Not sure if it's a bug or I'm doing something wrong, but if git-annex really need to check something in git remotes' paths, error message (the one at the top of this post) can be a more descriptive, I guess. +Something like "error: failed to do something with git remote X on a remote host" would've been a lot less confusing than that GHC thing. + +Thanks! + +> I've never seen anything like this error message. I don't know if the +> problem is caused by building with GHC 7, or what. You didn't say what +> OS you're using. Searching for the error message, it seems to involve +> Mac OS X. + +> For example: +>> The error "strange closure type" indicates some kind of memory corruption, which can have many different causes, from bugs in the GC to hardware failures. +> +> You said that you'd been using git-annex built with that version of GHC +> successfully before. Perhaps you could use `git bisect` to see if you can +> identify a point in git-annex's history where this started happening? +> Since you can reproduce the problem by just running git-annex-shell at +> the command line with the right parameters, it should be easy to bisect it. +> +> Probably your best bet will be changing to a different version or build of +> GHC.. --[[Joey]] + +--- + +forwarded to GHC upstream; closing [[done]] --[[Joey]] diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment new file mode 100644 index 0000000000..98f0adc3db --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Bisect it is, then" + date="2011-04-03T04:45:49Z" + content=""" +Hm, if path's ok, guess there's no way around git-bisect indeed. Wonder if there's some kind of ccache for haskell... + +OS is linux, amd64 on \"host1\" and i386 on \"host2\" where git-annex-shell is crashing. +I'll try to come up with a commit, thanks for clarifications. +"""]] diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment new file mode 100644 index 0000000000..fb36581912 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment @@ -0,0 +1,66 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Bisect results" + date="2011-04-03T06:22:15Z" + content=""" +Completed git-bisect twice, getting roughly the same results: + + 828a84ba3341d4b7a84292d8b9002a8095dd2382 is the first bad commit + commit 828a84ba3341d4b7a84292d8b9002a8095dd2382 + Author: Joey Hess + Date: Sat Mar 19 14:33:24 2011 -0400 + + Add version command to show git-annex version as well as repository version information. + + :040000 040000 ed849b7b6e9b177d6887ecebd6a0f146357824f3 1c98699dfd3fc3a3e2ce6b55150c4ef917de96e9 M Command + :100644 100644 b9c22bdfb403b0bdb1999411ccfd34e934f45f5c adf07e5b3e6260b296c982a01a73116b8a9a023c M GitAnnex.hs + :100644 100644 76dd156f83f3d757e1c20c80d689d24d0c533e16 d201cc73edb31f833b6d00edcbe4cf3f48eaecb0 M Upgrade.hs + :100644 100644 5f414e93b84589473af5b093381694090c278e50 d4a58d77a29a6a02daf13cec0df08b5aab74f65e M Version.hs + :100644 100644 f5c2956488a7afafd20374873d79579fb09b1677 f8cd577e992d38c7ec1438ce5c141eb0eb410243 M configure.hs + :040000 040000 f9b7295e997c0a5b1dda352f151417564458bd6e a30008475c1889f4fd8d60d4d9c982563380a692 M debian + :040000 040000 9d87a5d8b9b9fe7b722df303252ffd5760d66f75 08834f61a10d36651b3cdcc38389f45991acdf5e M doc + +contents of final refs/bisect: + + bad (828a84ba3341d4b7a84292d8b9002a8095dd2382) + good-33cb114be5135ce02671d8ce80440d40e97ca824 + good-942480c47f69e13cf053b8f50c98c2ce4eaa256e + good-ca48255495e1b8ef4bda5f7f019c482d2a59b431 + +\"roughly\" because second bisect gave two commits as a result, failing to build one of them (missing .o file on link, guess it's because of -j4 and bad deps in that version's build system): + + There are only 'skip'ped commits left to test. + The first bad commit could be any of: + 828a84ba3341d4b7a84292d8b9002a8095dd2382 + 5022a69e45a073046a2b14b6a4e798910c920ee9 + We cannot bisect more! + +Also noticed that \"git-annex-shell ...\" command succeeds if ran as root user, while failing from unprivileged one. +There are no permission/access errors in \"strace -f git-annex-shell ...\", so I guess it could be some bug in the GHC indeed. + +JIC, logged a whole second bisect operation. +Resulting log: [http://fraggod.net/static/share/git-annex-bisect.log](http://fraggod.net/static/share/git-annex-bisect.log) + +Bisect script I've used (git-annex-shell dies with error code 134 - SIGABRT on GHC error): + + res= + while true; do + if [[ -n \"$res\" ]]; then + cd /var/tmp/paludis/build/dev-scm-git-annex-scm.bak/work/git-annex-scm + echo \"---=== BISECT ($res) ===---\"; git bisect \"$res\" 2>&1; echo '---=== /BISECT ===---' + cd + rm -Rf /var/tmp/paludis/build/dev-scm-git-annex-scm + cp -a --reflink=auto /var/tmp/paludis/build/dev-scm-git-annex-scm{.bak,} + chown -R paludisbuild: /var/tmp/paludis/build/dev-scm-git-annex-scm + fi + res= + cave resolve -zx1 git-annex --skip-until-phase configure || res=skip + if [[ -z \"$res\" ]]; then + cd /remote/path + sudo -u user git-annex-shell 'sendkey' '/remote/path' 'SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b' '--' rsync --server --sender -vpe.Lsf --inplace . '' + if [[ $? -eq 134 ]]; then res=bad; else res=good; fi + cd + fi + done 2>&1 | tee ~/git-annex-bisect.log + +"""]] diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment new file mode 100644 index 0000000000..491b537862 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment @@ -0,0 +1,38 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="comment 3" + date="2011-04-03T06:57:02Z" + content=""" +Repeated bisect with -j1, just to be sure it's not a random error, and it gave me 828a84ba3341d4b7a84292d8b9002a8095dd2382 again. +Guess I'll look through the changes there a bit later and try to revert these until it works. + +Not sure if it's repeatable by anyone but me (and hence worth fixing), but here's a bit more of info about the system: + + Exherbo linux + Linux sacrilege 2.6.38.2-fg.roam #4 SMP PREEMPT Mon Mar 28 21:08:47 YEKST 2011 i686 GNU/Linux + + dev-lang/ghc-7.0.2:7.0.2::installed + dev-haskell/HUnit-1.2.2.3:1.2.2.3::installed + dev-haskell/MissingH-1.1.0.3:1.1.0.3::installed + dev-haskell/QuickCheck-2.4.0.1:2.4.0.1::installed + dev-haskell/array-0.3.0.2:0.3.0.2::installed + dev-haskell/bytestring-0.9.1.7:0.9.1.7::installed + dev-haskell/containers-0.4.0.0:0.4.0.0::installed + dev-haskell/extensible-exceptions-0.1.1.2:0.1.1.2::installed + dev-haskell/filepath-1.2.0.0:1.2.0.0::installed + dev-haskell/hslogger-1.1.3:0::installed + dev-haskell/mtl-2.0.1.0:2.0.1.0::installed + dev-haskell/network-2.3.0.1:2.3.0.1::installed + dev-haskell/old-locale-1.0.0.2:1.0.0.2::installed + dev-haskell/parsec-3.1.0:3.1.0::installed + dev-haskell/pcre-light-0.4:0::installed + dev-haskell/regex-base-0.93.2:0.93.2::installed + dev-haskell/regex-compat-0.93.1:0.93.1::installed + dev-haskell/regex-posix-0.94.4:0.94.4::installed + dev-haskell/syb-0.3:0.3::installed + dev-haskell/transformers-0.2.2.0:0.2.2.0::installed + dev-haskell/utf8-string-0.3.6:0.3.6::installed + +(some stuff listed here as ::installed, but contains no files, since these packages detect whether ghc-7.0.2 already comes with the same/newer package version) + +"""]] diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment new file mode 100644 index 0000000000..45d3d8bac4 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-04-03T16:06:34Z" + content=""" +Nice work on the bisection. It's obviously a compiler bug. Having two test cases that differ in only as trivial and innocous a commit as 828a84ba3341d4b7a84292d8b9002a8095dd2382 might help a GHC developer track it down. + +We should probably forward this as a GHC bug. I hope you can find a different version or build of GHC to build git-annex with. +"""]] diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment new file mode 100644 index 0000000000..bffa9bb868 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Reported the issue to GHC" + date="2011-04-07T13:44:36Z" + content=""" +Finally got around to [report the issue to GHC tracker](http://hackage.haskell.org/trac/ghc/ticket/5085#comment:7). + +Looks quite alike (at least to the haskell-illiterate person like me) to a highest-priority issue that's hanging right at the top of the list. +There are other similar reports, but they seem to be either related to PowerPC Macs, closed as invalid or due to needinfo inactivity. + +Guess any further discussion belongs there, unless ghc developers will bounce it back. +Thanks a lot for your help, Joey, and for sharing a great thing that git-annex is. +"""]] diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn new file mode 100644 index 0000000000..862259422d --- /dev/null +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn @@ -0,0 +1,34 @@ +What steps will reproduce the problem? + + alip@hayalet /tmp/aaa (git)-[master] % git annex init aaa + init aaa ok + (Recording state in git...) + alip@hayalet /tmp/aaa (git)-[master] % git remote add çüş /tmp/çüş + alip@hayalet /tmp/aaa (git)-[master] % git annex sync --debug + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","symbolic-ref","HEAD"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","show-ref","git-annex"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","show-ref","--hash","refs/heads/git-annex"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","log","refs/heads/git-annex..bc45cd9c2cb7c9b0c7a12a4c0210fe6a262abac9","--oneline","-n1"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","log","refs/heads/git-annex..9220bfedd1e13b2d791c918e2d59901af353825f","--oneline","-n1"] + (merging origin/git-annex into git-annex...) + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","cat-file","--batch"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","update-index","-z","--index-info"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","diff-index","--raw","-z","-r","--no-renames","-l0","--cached","9220bfedd1e13b2d791c918e2d59901af353825f"] + git-annex: Cannot decode byte '\xfc': Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream + 1 alip@hayalet /tmp/aaa (git)-[master] % + +What is the expected output? What do you see instead? + +Syncing a repository under a path with utf-8 characters in its name fails. + +What version of git-annex are you using? On what operating system? + +git-annex version: 3.20120624 + +On Exherbo, linux-3.4 + +Please provide any additional information below. + +'\xfc' is valid UTF-8: 'LATIN SMALL LETTER U WITH DIAERESIS' + +> closing as non-reproducible and presumably fixed. [[done]] --[[Joey]] diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_1_f1a7352b04f395e06e0094c1f51b6fff._comment b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_1_f1a7352b04f395e06e0094c1f51b6fff._comment new file mode 100644 index 0000000000..28faa7b45d --- /dev/null +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_1_f1a7352b04f395e06e0094c1f51b6fff._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 1" + date="2012-06-27T02:48:31Z" + content=""" +I don't think this has to do with the path name of the repository containing utf-8 at all. + +Your recipe for reproducing this depends on some pre-existing repository that I don't know how to set up to reproduce this bug. All I can guess is that, based on the \"decodeUtf8\" in the error message, it's coming from the one part of the code that still uses that, the union merger. + + +"""]] diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_2_c1890067079cd99667f31cbb4d2e4545._comment b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_2_c1890067079cd99667f31cbb4d2e4545._comment new file mode 100644 index 0000000000..3486be7337 --- /dev/null +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_2_c1890067079cd99667f31cbb4d2e4545._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 2" + date="2012-06-27T03:08:13Z" + content=""" +Since I can't reproduce it I am not sure, but it may be fixed by the commits I've just made. +"""]] diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_3_213c96085c60c8e52cd803df07240158._comment b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_3_213c96085c60c8e52cd803df07240158._comment new file mode 100644 index 0000000000..48a3820296 --- /dev/null +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_3_213c96085c60c8e52cd803df07240158._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkWzAq6TusMi9zI3FLkDOETRIAUTtmGZVg" + nickname="Ali" + subject="comment 3" + date="2012-06-27T12:56:37Z" + content=""" +Yes, the problem is fixed. + +The repository was a normal git repository with path /tmp/çüş (git init) +and with annex description \"çüş\" (git annex init çüş) + +afaict, i can't reproduce the problem anymore either :-) +"""]] diff --git a/doc/bugs/git-annex:_Not_in_a_git_repository._.mdwn b/doc/bugs/git-annex:_Not_in_a_git_repository._.mdwn new file mode 100644 index 0000000000..fd90406cf4 --- /dev/null +++ b/doc/bugs/git-annex:_Not_in_a_git_repository._.mdwn @@ -0,0 +1,19 @@ +What steps will reproduce the problem? + +As a default user i want to start git-annex assistent with + +`$ git-annex webapp` + +`git-annex: Not in a git repository.` + +What is the expected output? What do you see instead? + +I would expect the assistent to popup in a opened browser window. + +What version of git-annex are you using? On what operating system? + +Debian wheezy with git-annex version: 3.20130114 + +Please provide any additional information below. + +Its working if i start `git-annex webapp` as root. I had the same error on previous version. diff --git a/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_1_e10363a912953a646b87c824d1c6e5d4._comment b/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_1_e10363a912953a646b87c824d1c6e5d4._comment new file mode 100644 index 0000000000..ea2b1fbf3d --- /dev/null +++ b/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_1_e10363a912953a646b87c824d1c6e5d4._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.7.238" + subject="comment 1" + date="2013-01-15T20:20:22Z" + content=""" +Sounds like a permissions error. Take a look at the repository in `~/Desktop/annex/` or `~/annex/` , and see if it contains files owned by root, or has bad directory permissions that would prevent your normal user from accessing it. Don't forget to check in its `.git` directory. + +I'd recommend not running the git-annex webapp as root. (There may be valid use cases for root to use git-annex in command-line mode.) +"""]] diff --git a/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_2_9e96063a664b2be8a36d7940e7632d3f._comment b/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_2_9e96063a664b2be8a36d7940e7632d3f._comment new file mode 100644 index 0000000000..4b054b1204 --- /dev/null +++ b/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_2_9e96063a664b2be8a36d7940e7632d3f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmaMxYm33s0H-nxBo5uzYUzdIECoyR8Ug8" + nickname="Stefan" + subject="removing config" + date="2013-02-11T20:42:29Z" + content=""" +What fixed this for me was to remove .config/git-annex. +"""]] diff --git a/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_3_8c9bd76b0e1200723ec13fbef943a2cc._comment b/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_3_8c9bd76b0e1200723ec13fbef943a2cc._comment new file mode 100644 index 0000000000..4039338866 --- /dev/null +++ b/doc/bugs/git-annex:_Not_in_a_git_repository._/comment_3_8c9bd76b0e1200723ec13fbef943a2cc._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.183" + subject="comment 3" + date="2013-02-12T15:42:57Z" + content=""" +What can happen is that `.config/git-annex/autostart` can list a repository that is somehow trashed. For example, I've seen this when I let the webapp make a repository, and then manually deleted the directory, while the webapp was running. The webapp then re-creates the directory, but it's not a valid git repo, just a mostly empty directory. + +Tend to see this as user error though.. +"""]] diff --git a/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__.mdwn b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__.mdwn new file mode 100644 index 0000000000..976109c797 --- /dev/null +++ b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__.mdwn @@ -0,0 +1,13 @@ +What steps will reproduce the problem? + Start "./git-annex-webapp" + +What is the expected output? What do you see instead? + The webapp should start, but I get the error "git-annex: getUserEntryForID: failed (Success)" + +What version of git-annex are you using? On what operating system? + 3.20121017 on "Ubuntu 10.04.4 LTS" 32-Bit + +Please provide any additional information below. + + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_1_11a1615962325327466895d03e3d2379._comment b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_1_11a1615962325327466895d03e3d2379._comment new file mode 100644 index 0000000000..ef8800c21e --- /dev/null +++ b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_1_11a1615962325327466895d03e3d2379._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.118" + subject="comment 1" + date="2012-10-25T18:52:52Z" + content=""" +This means it has been unable to look up your home directory in /etc/passwd. I wonder, are you using NIS or a similar thing that keeps your user entry out of /etc/passwd? +"""]] diff --git a/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_2_eac51c3299e9fc04025675360969d537._comment b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_2_eac51c3299e9fc04025675360969d537._comment new file mode 100644 index 0000000000..dde7c0814a --- /dev/null +++ b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_2_eac51c3299e9fc04025675360969d537._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo" + nickname="Tobias" + subject="comment 2" + date="2012-10-25T21:29:05Z" + content=""" +Yes, the system is using LDAP as user backend... Any idea how I can use git-annex with LDAP as user backend? +"""]] diff --git a/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_3_c23dc02c7487d63b0905f1b7f3ca59f5._comment b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_3_c23dc02c7487d63b0905f1b7f3ca59f5._comment new file mode 100644 index 0000000000..3b358b7ea1 --- /dev/null +++ b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_3_c23dc02c7487d63b0905f1b7f3ca59f5._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.0.118" + subject="comment 3" + date="2012-10-25T22:18:55Z" + content=""" +Well, git-annex needs to know the user name, and the home directory. I've made it use +USER, and HOME, when set, and only fall back to getpwent otherwise. +"""]] diff --git a/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_4_0e8b28de5c173bc60ecc0126fb2209ca._comment b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_4_0e8b28de5c173bc60ecc0126fb2209ca._comment new file mode 100644 index 0000000000..63c3b474a4 --- /dev/null +++ b/doc/bugs/git-annex:_getUserEntryForID:_failed___40__Success__41__/comment_4_0e8b28de5c173bc60ecc0126fb2209ca._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawniayrgSdVLUc3c6bf93VbO-_HT4hzxmyo" + nickname="Tobias" + subject="comment 4" + date="2012-10-26T05:49:47Z" + content=""" +I think you mean the environment variables with \"use USER, and HOME\"? So I checked them I they are correct. +Reading man(3) getpwent says \"from the password database (e.g., the local password file /etc/passwd, NIS, and LDAP)\", so it should be no problem with the LDAP backend I'm using to log-in... +One other special thing: The home directory is on a NFS share. +"""]] diff --git a/doc/bugs/git-annex_3.20130216.1_tests_are_broken.mdwn b/doc/bugs/git-annex_3.20130216.1_tests_are_broken.mdwn new file mode 100644 index 0000000000..6df6bf489b --- /dev/null +++ b/doc/bugs/git-annex_3.20130216.1_tests_are_broken.mdwn @@ -0,0 +1,43 @@ + $ pwd + [bla]/git-annex-3.20130216.1 + $ runhaskell Setup configure --prefix=/usr --libdir=/usr/lib64 --docdir=/usr/share/doc/git-annex-3.20130216.1-r2 \ + --htmldir=/usr/share/doc/git-annex-3.20130216.1-r2/html --with-compiler=ghc-7.6.2 --enable-shared \ + --disable-executable-stripping --global --verbose --enable-tests --flags=S3 --flags=-WebDAV --flags=-Inotify \ + --flags=Dbus --flags=-Assistant --flags=-Webapp --flags=-Pairing --flags=-XMPP --flags=-DNS + $ runhaskell Setup.hs build + Building git-annex-3.20130217... + Preprocessing test suite 'test' for git-annex-3.20130217... + + Annex/UUID.hs:30:8: + Could not find module `System.Random' + It is a member of the hidden package `random-1.0.1.1'. + Perhaps you need to add `random' to the build-depends in your .cabal file. + Use -v to see a list of the files searched for. + +Adding `random` to the dependencies of the test suite results in: + + $ runhaskell Setup.hs build + Building git-annex-3.20130217... + Preprocessing test suite 'test' for git-annex-3.20130217... + + Annex/UUID.hs:29:18: + Could not find module `Data.UUID' + It is a member of the hidden package `uuid-1.2.9'. + Perhaps you need to add `uuid' to the build-depends in your .cabal file. + Use -v to see a list of the files searched for. + +Adding `uuid` results in: + + $ runhaskell Setup.hs build + Building git-annex-3.20130217... + Preprocessing test suite 'test' for git-annex-3.20130217... + + Command/Add.hs:25:8: + Could not find module `Utility.Touch' + Use -v to see a list of the files searched for. + + +Also: you included ".git-annex.cabal.swp" in the tarball. + +> These problems in the cabal file were fixed the other day. [[done]] +> --[[Joey]] diff --git a/doc/bugs/git-annex___38___rsync_can__39__t_copy_files_with___39__:__39___in_their_names.mdwn b/doc/bugs/git-annex___38___rsync_can__39__t_copy_files_with___39__:__39___in_their_names.mdwn new file mode 100644 index 0000000000..b55493dc4d --- /dev/null +++ b/doc/bugs/git-annex___38___rsync_can__39__t_copy_files_with___39__:__39___in_their_names.mdwn @@ -0,0 +1,38 @@ +What steps will reproduce the problem? + +Send a file with the character ':' in it. rsync fails to send those files from the command line as well +confusing them with hostnames. As far as I know a workaround is prepending the pathname with './' +(for rsync commandline invocation that is) + +What is the expected output? What do you see instead? + + copy müzik/Mixxx/Recordings/2013-01-15_16h:13m:03s.mp3 (checking kaotik...) (to kaotik...) + git-annex: //home/alip/kaotika/.git/annex/transfer/download/effe4eef-926f-494c-a3b6-eeecdc208fb9/SHA256-s36349200--ce51eaf316b19c61831 + 41f0bda1c54be7e590e5999753a4b1c16bafab93a3fc1: commitBuffer: invalid argument (invalid character) + git-annex-shell: recvkey: 1 failed + protocol version mismatch -- is your shell clean? + (see the rsync man page for an explanation) + rsync error: protocol incompatibility (code 2) at compat.c(174) [sender=3.0.9] + + +What version of git-annex are you using? On what operating system? + + git-annex version: 3.20120807 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +Operating system is Linux (Exherbo) although this isn't a packaged installation. +I installed it by hand from git sources. + +Please provide any additional information below. + +Thanks for the wonderful tool! + +> This is a duplicate of +> [[bugs/commitBuffer:_invalid_argument___40__invalid_character__41__]], +> which was fixed in version 3.20120924. You need to upgrade the git-annex +> on kaotik. [[done]] +> +> (It has nothing to do with rsync or colon in filenames.) --[[Joey]] diff --git a/doc/bugs/git-annex_branch_corruption.mdwn b/doc/bugs/git-annex_branch_corruption.mdwn new file mode 100644 index 0000000000..9c864d85f0 --- /dev/null +++ b/doc/bugs/git-annex_branch_corruption.mdwn @@ -0,0 +1,95 @@ +Below is a test case which shows a way that the git-annex branch +can become corrupted and lose data, including location log records and +uuid.log lines. + +At the end, a commit on the git-annex branch removes one of the 2 lines +from the uuid.log; which should never happen. + +The actual problem occurs earlier, at the "push point". Here a repo is +cloned from the main one, initialized (adding the last uuid.log line), +and then pushed back to the main one. That push is a fast-forward, so is +allowed to directly update the git-annex branch in the main repo: + + b884fe5..c497739 git-annex -> git-annex + +Now the git-annex branch has a change that is not reflected in +`.git/annex/index`, so the next time a change is made, it's committed +using the out of date index, which causes a reversion of the changes +that were pushed to the branch. + +--- + +## Thoughts + +This is essentially the same reason why git blocks pushes to the checked-out +branch of a non-bare repository. + +This problem only affects workflows that involve pushing. Pulling workflows +do not directly update the local git-annex branch, so avoid the problem. + +And while bare repos are pushed to, they rarely have changes made directly +to their git-annex branches, so while I think the same problem could +happen with pushing to a bare repo, it's unlikely. + +None of which is to say this is not a bad bug that needs to be comprehensively +fixed. + +Probably git-annex needs to record which ref of the git-annex branch +corresponds to its index, and if the branch is at a different ref, +merge it into the index. + +> And now that's [[done]]. I managed to do it with very little slowdown. +> +> A side benefit is that users can now safely check out the git-annex +> branch and commit changes to it, and git-annex will notice them. +> Before, it was documented to ignore such changes. +> --[[Joey]] + +--- + +## Workaround + +Users who want to prevent this bug from occuring when pushing to their +non-bare repositories can install this script as `.git/hooks/update` + +
+#!/bin/sh
+if [ "$1" = refs/heads/git-annex ]; then
+	exit 1
+fi
+
+ +--[[Joey]] + +--- + +## Test Case +
+#!/bin/sh
+mkdir annextest
+cd annextest
+
+git init dir1
+cd dir1
+git annex init
+touch foo 
+echo hi > bar
+git annex add
+git commit -m add
+
+cd ..
+git clone dir1 dir2
+cd dir2
+git annex init otherdir
+git annex get
+# push point
+git push
+
+cd ..
+cd dir1
+echo "before"
+git show git-annex:uuid.log
+git annex drop foo --force
+echo "after"
+git show git-annex:uuid.log
+
diff --git a/doc/bugs/git-annex_branch_push_race.mdwn b/doc/bugs/git-annex_branch_push_race.mdwn new file mode 100644 index 0000000000..013ff70dd5 --- /dev/null +++ b/doc/bugs/git-annex_branch_push_race.mdwn @@ -0,0 +1,45 @@ +The fix for the [[git-annex_branch_corruption]] bug is subject to a race. +With that fix, git-annex does this when committing a change to the branch: + +1. lock the journal file (this avoids git-annex racing itself, FWIW) +2. check what the head of the branch points to, to see if a newer branch + has appeared +3. if so, updates the index file from the branch +4. stages changes in the index +5. commits to the branch using the index file + +If a push to the branch comes in during 2-5, then +[[git-annex_branch_corruption]] could still occur. + +--- + +## approach 1, using locking + +Add an update hook and a post-update hook. The update hook +will use locking to ensure that no git-annex is currently running +a commit, and block any git-annex's from starting one. It +will background itself, and remain running during the push. +The post-update hook will signal it to exit. + +I don't like this approach much, since it involves a daemon, two hooks, +and lots of things to go wrong. And it blocks using git-annex during a +push. This approach should be a last resort. + +## approach 2, lockless method + +After a commit is made to the branch, check to see if the parent of +the commit is the same ref that the index file was last updated to. If it's +not, then the race occurred. + +How to recover from the race? Well, just union merging the parent of the +commit into the index file and re-committing should work, I think. When +the race occurs, the commit reverts its parent's changes, and this will +redo them. + +(Of course, this re-commit will also be subject to the race, and +will need the same check for the race as the other commits. It won't loop +forever, I hope.) + +> [[done]] and tested. + +--[[Joey]] diff --git a/doc/bugs/git-annex_direct_fails_on_repositories_with_a_partial_set_of_files.mdwn b/doc/bugs/git-annex_direct_fails_on_repositories_with_a_partial_set_of_files.mdwn new file mode 100644 index 0000000000..de88b02467 --- /dev/null +++ b/doc/bugs/git-annex_direct_fails_on_repositories_with_a_partial_set_of_files.mdwn @@ -0,0 +1,29 @@ +## What steps will reproduce the problem? + +Running the following in an annex with an archive directory with all the files dropped and located offsite + + git annex direct + +It seems that if not all the files are in the annex, then the direct mode files. + +## What is the expected output? What do you see instead? + +The expectation is that either direct mode reverts its changes if it fails instead of + + url: createLink: does not exist (No such file or directory) + failed + git-annex: direct: 1 failed + +It leaves the annex in the indirect mode, but there are a bunch of .map files lying around in git-annex's control directory. + +## What version of git-annex are you using? On what operating system? + +Running 3.20130102 on OSX + +## Please provide any additional information below. + +> More specifically, git annex direct fails, on OSX only, when there are two +> files that both have the same content. Apparently OSX doesn't allow +> hard linking two symbolic links together. There was no harm in it doing that +> otherwise, but then again no reason for it to do so, so I've put in a fix. +> [[done]] --[[Joey]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn new file mode 100644 index 0000000000..db6a35293c --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn @@ -0,0 +1,100 @@ +Currently the hashed directories in .git-annex allow for upper and lower case directory names... on linux (or any case sensitive filesystem) the directory names such as 'Gg' and 'GG' are different and unique. However on systems like OSX (and probably windows if it is ever supported) the directory names 'Gg' is the same as 'GG' + +In one of the annex'd repos that I have this has occured... + +
+$ git add -i                                                                                          
+           staged     unstaged path
+  1:    unchanged        +1/-1 .git-annex/GM/GV/WORM-s183630166-m1301072171--somefile.log
+  2:    unchanged        +1/-1 .git-annex/Gm/GV/WORM-s183630166-m1301072171--somefile.log
+
+ + +this has somewhat confused git when it tries to stage/merge files, I didn't notice this at first, but it is definately a problem for someone using case insensitive filesystems like the default OSX HFS+ formats or vfat/fat32. + +> I feel a bit stupid to not have considered case-insensative filesystems. +> They are just so far from where I have lived for 20 years that it's hard +> to keep them in mind. +> +> I guess that +> [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]] is +> somehow a consequence (or cause?) of this, but I don't quite understand +> how this is causing git to fail to stage files, or stage the same file +> twice under different capitalizations. git-annex always will run git add +> on the path with the "correct" capitalization. So unless something else +> has added the path with the other capitalization (perhaps git add +> .git-annex manually?) I don't understand how you get to this state. +> --[[Joey]] + +>> I think I got myself into this situation when I copied some files over from a HFS+ partition to a GPFS network share (which is pretty posix compliant) over samba. It probably is related to the [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]]. I thought they were unique enough to have two bug reports logged as one is a git behavioural thing and the other is git-annex specific. + +>>> If you copied `.git/` over, perhaps you got a git repo without +>>> core.ignorecase set right for the filesystem it landed on? + +>>>> I usually git clone or do a fresh repository and pull things in, I was also unaware of this ignorecase setting as well. + +>>> +>>> Something like this might reproduce it: + +
+# mkdir test; cd test; git init
+# git config core.ignorecase false
+# mkdir Foo
+# touch Foo/bar
+# git add Foo/bar
+# git add foo/bar
+# git add fOo/bar
+# git status
+# touch foo/other
+# git add fOo/other
+# git status
+
+ +>>>> And then either git commit or git clone would probably get confused +>>>> if it thought 3 distinct files had been committed. +>>>> --[[Joey]] + +>>>>> Doing the above test on a HFS+ partition yields this + +
+## with ignorecase=false
+commit bb024c6fd7482b2d10f60ae899cb7a949aca1ad8
+Author: Jimmy Tang 
+Date:   Sun Mar 27 18:40:24 2011 +0100
+
+    commit
+
+diff --git a/Foo/bar b/Foo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/fOo/bar b/fOo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/fOo/other b/fOo/other
+new file mode 100644
+index 0000000..e69de29
+diff --git a/foo/bar b/foo/bar
+new file mode 100644
+index 0000000..e69de29
+
+ +>>>>> and without changing ignorecase + +
+commit 909a089158ffb98f8e91f98905e2bfdc7234666f
+Author: Jimmy Tang 
+Date:   Sun Mar 27 18:46:57 2011 +0100
+
+    commit
+
+diff --git a/Foo/bar b/Foo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/Foo/other b/Foo/other
+new file mode 100644
+index 0000000..e69de29
+
+ +> Closing this bug, as it seems I have dealt with it adequately now. +> [[done]] +> --[[Joey]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment new file mode 100644 index 0000000000..c3e6b5e598 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 10" + date="2011-04-01T16:11:52Z" + content=""" +No, I don't need a copy of your repo now. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment new file mode 100644 index 0000000000..db605f9650 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 11" + date="2011-04-02T17:53:58Z" + content=""" +I have pushed out a preliminary fix. The old mixed-case directories will be left where they are, and still read from by git-annex. New data will be written to new, lower-case directories. I think that once git stops seeing changes being made +to mixed-case, colliding directories, the bugs you ran into won't manifest any more. + +You will need to find a way to get your git repository out of the state where it complains about uncommitted files (and won't let you commit them). I have not found a reliable way to do that; git reset --hard worked in one case but not in another. May need to clone a fresh git repository. + +Let me know how it works out. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment new file mode 100644 index 0000000000..5f9a0ae275 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 12" + date="2011-04-02T17:58:24Z" + content=""" +Also, you can delete `.git-annex/??` if you want to, then running `git annex fsck --fast` in each of your clones would regenerate the data using only the lower-case hash directories. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment new file mode 100644 index 0000000000..b4a5a72d01 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 13" + date="2011-04-03T07:43:37Z" + content=""" +Ok, thanks for the fix. It seems the fix isn't too reliable with my repos, I get different numbers of \"** No known copies of...\" in the various cloned repos that I have. After all the \"messing\" that I have done to my repos I think git-annex has gotten very confused. I will just leave things as they are and let git-annex slowly migrate over to the new format or re-clone from a linux source and see how things go. I will report back on this issue in abit after I use it more to see. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment new file mode 100644 index 0000000000..b92c3ab4ab --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 14" + date="2011-04-03T08:24:17Z" + content=""" +I meant to say in it wasn't reliable when I was following the instructions for \"Comment 12\". I did find that just doing a \"git annex copy -t externalusb .\" then a \"git annex drop .\" from the root of my cloned and \"none trusted\" annexed repos to be more reliable, it just means I temporarily need a load of space to get myself out of my earlier mess. + +On testing this bug fix, I found a minor behavioural issue with [[git annex copy -f REMOTE . doesn't work as expected]] +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment new file mode 100644 index 0000000000..d722d546a3 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="gernot" + ip="213.168.117.192" + subject="comment 15" + date="2011-04-03T15:41:00Z" + content=""" +I also ran into problems on a case-insensitive HFS+ file system, it seems. I +tried following the instructions in comment 12: + + 1. Remove everything in .git-annex besides uuid.log and trust.log + 2. git annex fsck --fast + 3. Commit + +However, I still see upper and lower case directories in .git-annex. Did I +misunderstand that they should all be lower case now? + +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment new file mode 100644 index 0000000000..97eab78c91 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 16" + date="2011-04-03T16:02:33Z" + content=""" +I think the correct steps should be, make a backup first :) then ... + +1. git pull # update your clone, and commit everything so you don't lose anything +2. git annex fsck --fast # check the repo first, just in case +3. rm -rf .git-annex/?? # remove the old metadata +4. git annex fsck --fast # get git annex to regenerate it all +5. push your changes out to your other repos, you will need to make sure git-annex is updated everywhere if there are remotes in your setup. + +I eventually migrated all of my own annex'd repos and I no longer have the old hashed directories but the new ones in the form + + .git/annex/aaa/bbb/foo.log + +I did lose some tracking information but not data (as far as I can see for now), but that was quickly fixed by pushing and pulling to my bare repo which tracks most of my data. + +I also found that it worked a bit more reliably for me on the copies of repos that were located on case sensitive filesystems, but I guess that was expected. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment new file mode 100644 index 0000000000..f7feac67cf --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 17" + date="2011-04-03T16:53:51Z" + content=""" +@gernot step 0 is to upgrade git-annex to current git, on all systems where you use it, in case that wasn't clear. + +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment new file mode 100644 index 0000000000..550558ec16 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="gernot" + ip="213.168.117.192" + subject="comment 18" + date="2011-04-03T19:46:16Z" + content=""" +Joey, sorry, I got it wrong. I thought upgrading git didn't help and you +adjusted things in git-annex instead. + +Anyway, can I get around upgrading on all hosts by reformatting the drive to +case-sensitive HFS+? Or will I have to upgrade git (currently version 1.7.2.5) +eventually anyway? + +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment new file mode 100644 index 0000000000..2676b35897 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 19" + date="2011-04-03T19:53:44Z" + content=""" +Git does not need to be upgraded. Git-annex needs to be upgraded to git rev 616e6f8a840ef4d99632d12a2e7ea15c3cfb1805 or newer, on all machines. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment new file mode 100644 index 0000000000..aa5e46ca2b --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2011-03-28T07:23:41Z" + content=""" +One possible work around is to just create a loopback file system with a case sensitive filesystem. I think I might do that for anything that I really care about for now. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment new file mode 100644 index 0000000000..8f0f5ef180 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="ssqq" + ip="208.70.196.4" + subject="Still a problem on 0.20110523" + date="2011-06-02T20:31:55Z" + content=""" +Hi, + +(I'm new to git and git annex, so please forgive any mistakes I make...) + +My repo is messed up right now. The fact that I copied the repo with rsync -a back and forth from a case insensitive filesystem to a case sensitive one, probably didn't help. + +I believe the annexed files in .git/annex/objects/ are still using a mixed case directory hashing scheme. That's the problem I'm having. The symlinks point to the wrong case and are now broken. I don't think the latest versions of git-annex changed that (it only changed the hashing under .git-annex, right?). + +Even if I clean up my repo, I think I'm still going to have a problem because I have one repo on an OS X case insensitive filesystem and my other repos on case sensitive Linux filesystems. Potentially the directory name under .git/annex/objects will have a different case. Then the symlink might have a different case than my Linux FS. Does git-annex track changes in git by the contents of the symlink? In which case the case difference would show up as a change even though there is no change? + +Is it possible to change the directory hashing scheme under .git/annex/objects to use lowercase names? + +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment new file mode 100644 index 0000000000..453a8be11b --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 21" + date="2011-06-10T16:46:03Z" + content=""" +@seqq git-annex always uses the same case when creating and accessing the files pointed to by the symlinks. So it will not matter if it's used on a case-insensative, or case-insensative but preserving system like OSX. + +You need to fix up the cases of the files in .git/annex/objects to what it expects. I'm not sure what would be the best way to do that. The method described in [[walkthrough/recover_data_from_lost+found]] might work well. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_22_52d41afd7fd0b71a4c8e84ab1b4df5bd._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_22_52d41afd7fd0b71a4c8e84ab1b4df5bd._comment new file mode 100644 index 0000000000..7fa1e7468a --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_22_52d41afd7fd0b71a4c8e84ab1b4df5bd._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawltmUlf_zHb-hDkjLLYeUxyd81YVoIgZew" + nickname="Jaen" + subject="Still somewhat broken" + date="2012-12-25T17:58:53Z" + content=""" +I moved an external HDD formatted with NTFS from Mac (case-insensitive) to Linux (case sensitive), and half of the links are broken now... What can I do to fix this? +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_23_c2cd8a69c37539c0511bae02016180ca._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_23_c2cd8a69c37539c0511bae02016180ca._comment new file mode 100644 index 0000000000..8ec1abd89b --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_23_c2cd8a69c37539c0511bae02016180ca._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawltmUlf_zHb-hDkjLLYeUxyd81YVoIgZew" + nickname="Jaen" + subject="comment 23" + date="2012-12-25T20:21:15Z" + content=""" +(to be clear, Mac put eg. hashes \"Gg\" and \"gg\" into the same directory, while Linux expects them to be in separate dirs) +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment new file mode 100644 index 0000000000..6e6e5dc6be --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment @@ -0,0 +1,185 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2011-03-28T15:09:45Z" + content=""" +I think I know how I got myself into this mess... I was on my mac workstation and I had just pulled in a change set from another repo on a linux workstation after I had a made a bunch of moves. here's a bit of a log of what happened... + + +
+jtang@x00:~/sources $ git pull cports-devel master
+Warning: untrusted X11 forwarding setup failed: xauth key data not generated
+Warning: No xauth data; using fake authentication data for X11 forwarding.
+remote: Counting objects: 4195, done.
+remote: Compressing objects: 100% (1135/1135), done.
+remote: Total 2582 (delta 866), reused 2576 (delta 860)
+Receiving objects: 100% (2582/2582), 229.42 KiB | 111 KiB/s, done.
+Resolving deltas: 100% (866/866), completed with 9 local objects.
+From cports-devel:/home/people/jtang/sources
+ * branch            master     -> FETCH_HEAD
+Updating 319df99..ab0a98c
+error: Your local changes to the following files would be overwritten by merge:
+	.git-annex/09/5X/WORM-s361516678-m1301310614--l_fcompxe_intel64_2011.2.137.tgz.log
+	.git-annex/43/2g/WORM-s19509673-m1301310496--l_fcompxe_2011.2.137_redist.tgz.log
+	.git-annex/4J/qF/WORM-s18891115-m1301310934--w_flm_p_1.0.011_ia64.zip.log
+	.git-annex/87/w1/WORM-s12212473-m1301310909--w_flm_p_1.0.011_ia32.zip.log
+	.git-annex/99/Jq/WORM-s194345957-m1301310926--l_mkl_10.3.2.137_ia32.log
+	.git-annex/99/kf/WORM-s9784531-m1301311680--l_ccompxe_2011.2.137_redist.log
+	.git-annex/FF/f3/WORM-s93033394-m1301311706--l_gen_ipp_7.0.2.137.log
+	.git-annex/MF/xZ/WORM-s515140733-m1301310936--l_cprof_p_11.1.075.log
+	.git-annex/XW/X8/WORM-s355559731-m1301310797--l_mkl_10.3.2.137.log
+	.git-annex/fJ/mZ/WORM-s1372886477-m1301313368--l_cproc_p_11.1.075.log
+	.git-annex/j7/Q9/WORM-s44423202-m1301310622--l_cprof_p_11.1.075_redist.log
+	.git-annex/k4/K7/WORM-s239539070-m1301310760--l_mkl_10.3.2.137_intel64.log
+	.git-annex/kz/01/WORM-s279573314-m1301310783--l_cprof_p_11.1.075_ia32.log
+	.git-annex/p6/Kq/WORM-s31199343-m1301311829--l_cproc_p_11.1.075_redist.log
+	.git-annex/pz/J5/WORM-s626995277-m1301312301--l_ccompxe_ia32_2011.2.137.log
+	.git-annex/v3/kX/WORM-s339693045-m1301310851--l_cprof_p_11.1.075_intel64.log
+Please, commit your changes or stash them before you can merge.
+error: Your local changes to the following files would be overwritten by merge:
+	.git-annex/12/3W/WORM-s3058814-m1276699694--Botan-1.8.9.tgz.log
+	.git-annex/1G/qV/WORM-s9122-m1251558854--Array-Compare-2.01.tar.gz.log
+	.git-annex/3W/W5/WORM-s231523-m1270740744--DBD-Pg-2.17.1.tar.gz.log
+	.git-annex/3x/PX/WORM-s380310-m1293025187--HTSeq-0.4.7.tar.gz.log
+	.git-annex/45/gk/WORM-s67337-m1248732018--ExtUtils-Install-1.54.tar.gz.log
+	.git-annex/4J/7Q/WORM-s8608-m1224694862--Algorithm-Munkres-0.08.tar.gz.log
+	.git-annex/4g/XQ/WORM-s89208-m1278682033--HTML-Parser-3.66.tar.gz.log
+	.git-annex/54/jw/WORM-s300163-m1226422051--AcePerl-1.92.tar.gz.log
+	.git-annex/63/kj/WORM-s1213460-m1262942058--DBD-SQLite-1.29.tar.gz.log
+	.git-annex/6Z/42/WORM-s4074-m943766010--File-Sync-0.09.tar.gz.log
+	.git-annex/8F/M5/WORM-s6989-m1263161127--Digest-HMAC-1.02.tar.gz.log
+	.git-annex/G2/FK/WORM-s3309-m1163872981--Bundle-BioPerl-2.1.8.tar.gz.log
+	.git-annex/Gk/XF/WORM-s23572243-m1279546902--EMBOSS-6.3.1.tar.gz.log
+	.git-annex/Jk/X6/WORM-s566429-m1279309002--DBI-1.612.tar.gz.log
+	.git-annex/K6/fV/WORM-s1561451-m1240055295--Convert-Binary-C-0.74.tar.gz.log
+	.git-annex/KM/4q/WORM-s146959-m1268515086--Graph-0.94.tar.gz.log
+	.git-annex/MF/m2/WORM-s425766-m1212514609--Data-Stag-0.11.tar.gz.log
+	.git-annex/QJ/P6/WORM-s1045868-m1282215033--9base-6.tar.gz.log
+	.git-annex/Qm/WG/WORM-s39078-m1278163547--Digest-SHA1-2.13.tar.gz.log
+	.git-annex/Wq/Fj/WORM-s45680640-m1297862101--BclConverter-1.7.1.tar.log
+	.git-annex/Wq/Wm/WORM-s263536640-m1295025537--CASAVA_v1.7.0.tar.log
+	.git-annex/XW/qm/WORM-s36609-m1276050470--Bio-ASN1-EntrezGene-1.10-withoutworldwriteables.tar.gz.log
+	.git-annex/f7/g0/WORM-s40872-m1278273227--ExtUtils-ParseXS-2.2206.tar.gz.log
+	.git-annex/j3/JF/WORM-s11753-m1232427595--Clone-0.31.tar.gz.log
+	.git-annex/kX/9g/WORM-s84690-m1229117599--GraphViz-2.04.tar.gz.log
+	.git-annex/km/z5/WORM-s44634-m1275505134--Authen-SASL-2.15.tar.gz.log
+	.git-annex/kw/J3/WORM-s132396-m1278780649--DBD-mysql-4.016.tar.gz.log
+	.git-annex/p5/1P/WORM-s53736-m1278673485--Archive-Tar-1.64.tar.gz.log
+	.git-annex/wv/zG/WORM-s30584-m1268774021--ExtUtils-CBuilder-0.2703.tar.gz.log
+	.git-annex/x5/7v/WORM-s10462526-m1254242591--BioPerl-1.6.1.tar.gz.log
+Please, commit your changes or stash them before you can merge.
+error: The following untracked working tree files would be overwritten by merge:
+	.git-annex/1g/X3/WORM-s309910751-m1301311322--l_fcompxe_ia32_2011.2.137.tgz.log
+	.git-annex/3w/Xf/WORM-s805764902-m1301312756--l_cproc_p_11.1.075_intel64.log
+	.git-annex/9Q/Wz/WORM-s1234430253-m1301311891--l_ccompxe_2011.2.137.log
+	.git-annex/FQ/4z/WORM-s318168323-m1301310848--l_cprof_p_11.1.075_ia64.log
+	.git-annex/FV/0P/WORM-s710135470-m1301311835--l_ccompxe_intel64_2011.2.137.log
+	.git-annex/Jx/qM/WORM-s599386592-m1301310731--l_fcompxe_2011.2.137.tgz.log
+	.git-annex/KX/w1/WORM-s35976002-m1301312193--l_tbb_3.0.6.174.log
+	.git-annex/Vw/jK/WORM-s15795178-m1301310913--w_flm_p_1.0.011_intel64.zip.log
+	.git-annex/jK/zK/WORM-s374617670-m1301312705--l_ipp_7.0.2.137_intel64.log
+	.git-annex/vK/kv/WORM-s584342291-m1301312669--l_cproc_p_11.1.075_ia64.log
+	.git-annex/vw/v1/WORM-s736986678-m1301312794--l_cproc_p_11.1.075_ia32.log
+	.git-annex/zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+Please move or remove them before you can merge.
+Aborting
+1|jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 2 commits.
+#
+# Changes to be committed:
+#   (use \"git reset HEAD ...\" to unstage)
+#
+#	modified:   .git-annex/09/5X/WORM-s361516678-m1301310614--l_fcompxe_intel64_2011.2.137.tgz.log
+#	modified:   .git-annex/43/2g/WORM-s19509673-m1301310496--l_fcompxe_2011.2.137_redist.tgz.log
+#	modified:   .git-annex/4J/qF/WORM-s18891115-m1301310934--w_flm_p_1.0.011_ia64.zip.log
+#	modified:   .git-annex/87/w1/WORM-s12212473-m1301310909--w_flm_p_1.0.011_ia32.zip.log
+#	modified:   .git-annex/99/Jq/WORM-s194345957-m1301310926--l_mkl_10.3.2.137_ia32.log
+#	modified:   .git-annex/99/kf/WORM-s9784531-m1301311680--l_ccompxe_2011.2.137_redist.log
+#	modified:   .git-annex/FF/f3/WORM-s93033394-m1301311706--l_gen_ipp_7.0.2.137.log
+#	modified:   .git-annex/MF/xZ/WORM-s515140733-m1301310936--l_cprof_p_11.1.075.log
+#	modified:   .git-annex/XW/X8/WORM-s355559731-m1301310797--l_mkl_10.3.2.137.log
+#	modified:   .git-annex/fJ/mZ/WORM-s1372886477-m1301313368--l_cproc_p_11.1.075.log
+#	modified:   .git-annex/j7/Q9/WORM-s44423202-m1301310622--l_cprof_p_11.1.075_redist.log
+#	modified:   .git-annex/k4/K7/WORM-s239539070-m1301310760--l_mkl_10.3.2.137_intel64.log
+#	modified:   .git-annex/kz/01/WORM-s279573314-m1301310783--l_cprof_p_11.1.075_ia32.log
+#	modified:   .git-annex/p6/Kq/WORM-s31199343-m1301311829--l_cproc_p_11.1.075_redist.log
+#	modified:   .git-annex/pz/J5/WORM-s626995277-m1301312301--l_ccompxe_ia32_2011.2.137.log
+#	modified:   .git-annex/v3/kX/WORM-s339693045-m1301310851--l_cprof_p_11.1.075_intel64.log
+#
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   .git-annex/12/3W/WORM-s3058814-m1276699694--Botan-1.8.9.tgz.log
+#	modified:   .git-annex/1G/qV/WORM-s9122-m1251558854--Array-Compare-2.01.tar.gz.log
+#	modified:   .git-annex/3W/W5/WORM-s231523-m1270740744--DBD-Pg-2.17.1.tar.gz.log
+#	modified:   .git-annex/3x/PX/WORM-s380310-m1293025187--HTSeq-0.4.7.tar.gz.log
+#	modified:   .git-annex/45/gk/WORM-s67337-m1248732018--ExtUtils-Install-1.54.tar.gz.log
+#	modified:   .git-annex/4J/7Q/WORM-s8608-m1224694862--Algorithm-Munkres-0.08.tar.gz.log
+#	modified:   .git-annex/4g/XQ/WORM-s89208-m1278682033--HTML-Parser-3.66.tar.gz.log
+#	modified:   .git-annex/54/jw/WORM-s300163-m1226422051--AcePerl-1.92.tar.gz.log
+#	modified:   .git-annex/63/kj/WORM-s1213460-m1262942058--DBD-SQLite-1.29.tar.gz.log
+#	modified:   .git-annex/6Z/42/WORM-s4074-m943766010--File-Sync-0.09.tar.gz.log
+#	modified:   .git-annex/8F/M5/WORM-s6989-m1263161127--Digest-HMAC-1.02.tar.gz.log
+#	modified:   .git-annex/G2/FK/WORM-s3309-m1163872981--Bundle-BioPerl-2.1.8.tar.gz.log
+#	modified:   .git-annex/Gk/XF/WORM-s23572243-m1279546902--EMBOSS-6.3.1.tar.gz.log
+#	modified:   .git-annex/Jk/X6/WORM-s566429-m1279309002--DBI-1.612.tar.gz.log
+#	modified:   .git-annex/K6/fV/WORM-s1561451-m1240055295--Convert-Binary-C-0.74.tar.gz.log
+#	modified:   .git-annex/KM/4q/WORM-s146959-m1268515086--Graph-0.94.tar.gz.log
+#	modified:   .git-annex/MF/m2/WORM-s425766-m1212514609--Data-Stag-0.11.tar.gz.log
+#	modified:   .git-annex/QJ/P6/WORM-s1045868-m1282215033--9base-6.tar.gz.log
+#	modified:   .git-annex/Qm/WG/WORM-s39078-m1278163547--Digest-SHA1-2.13.tar.gz.log
+#	modified:   .git-annex/Wq/Fj/WORM-s45680640-m1297862101--BclConverter-1.7.1.tar.log
+#	modified:   .git-annex/Wq/Wm/WORM-s263536640-m1295025537--CASAVA_v1.7.0.tar.log
+#	modified:   .git-annex/XW/qm/WORM-s36609-m1276050470--Bio-ASN1-EntrezGene-1.10-withoutworldwriteables.tar.gz.log
+#	modified:   .git-annex/Zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+#	modified:   .git-annex/f7/g0/WORM-s40872-m1278273227--ExtUtils-ParseXS-2.2206.tar.gz.log
+#	modified:   .git-annex/j3/JF/WORM-s11753-m1232427595--Clone-0.31.tar.gz.log
+#	modified:   .git-annex/kX/9g/WORM-s84690-m1229117599--GraphViz-2.04.tar.gz.log
+#	modified:   .git-annex/km/z5/WORM-s44634-m1275505134--Authen-SASL-2.15.tar.gz.log
+#	modified:   .git-annex/kw/J3/WORM-s132396-m1278780649--DBD-mysql-4.016.tar.gz.log
+#	modified:   .git-annex/p5/1P/WORM-s53736-m1278673485--Archive-Tar-1.64.tar.gz.log
+#	modified:   .git-annex/wv/zG/WORM-s30584-m1268774021--ExtUtils-CBuilder-0.2703.tar.gz.log
+#	modified:   .git-annex/x5/7v/WORM-s10462526-m1254242591--BioPerl-1.6.1.tar.gz.log
+#
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	.git-annex/1G/X3/
+#	.git-annex/3W/Xf/
+#	.git-annex/9q/Wz/
+#	.git-annex/Fq/4z/
+#	.git-annex/Jk/zK/
+#	.git-annex/Kx/w1/
+#	.git-annex/VK/kv/
+#	.git-annex/fv/0P/
+#	.git-annex/jX/qM/
+#	.git-annex/vW/jK/
+#	.git-annex/vW/v1/
+jtang@x00:~/sources $ git commit -a -m \"snap\"
+[master 45f254a] snap
+ 47 files changed, 64 insertions(+), 30 deletions(-)
+jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 3 commits.
+#
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	.git-annex/1G/X3/
+#	.git-annex/3W/Xf/
+#	.git-annex/9q/Wz/
+#	.git-annex/Fq/4z/
+#	.git-annex/Jk/zK/
+#	.git-annex/Kx/w1/
+#	.git-annex/VK/kv/
+#	.git-annex/fv/0P/
+#	.git-annex/jX/qM/
+#	.git-annex/vW/jK/
+#	.git-annex/vW/v1/
+nothing added to commit but untracked files present (use \"git add\" to track)
+jtang@x00:~/sources $ git pull
+
+"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment new file mode 100644 index 0000000000..00988ab58c --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-28T15:25:18Z" + content=""" +So, there is evidence here of a circumstance caused by the [[other_bug|git-annex_has_issues_with_git_when_staging__47__commiting_logs]], as I suspected. + +I don't think that manual `git commit -a` caused the problem. I suspect it was a subsequent `git add` that caused git to follow the wrong case paths and add the files in the wrong place. Ie, when you run \"git add .git-annex\", it recurses into `.git-annex/Gm/`, and adds files using that case, that were previously added from `.git-annex/GM/`. + +For completeness, can you verify this repo's core.ignorecase setting? + +--- + +I hate that you are stuck using loop filesystems to work around this bug. If my guess is correct, you don't need to, as long as you avoid manually running \"git add .git-annex\". I take this bug seriously. While I'm currently very involved in adding Amazon S3 support to git-annex (which will take days more of solid work), I do plan to make a loop filesystem of my own, probably vfat, so I can try and reproduce this on a case-insensative filesystem. If you could confirm my above hypothesis, that would speed things up for me. + +It's possible I will have to tweak the hash directories. Hopefully if so, I will only tweak them for *new* keys; if I had to do a v3 backend just to fix this stupid thing, I'd be sad -- upgrading all my offline disks from v1 to v2 took me many days. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment new file mode 100644 index 0000000000..d045f71205 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2011-03-28T15:41:56Z" + content=""" +In my \"sources\" repo on x00, the current setting is this \"ignorecase = true\" it was the first repo that I created before I clone it elsewhere and pull my changes back, it is on a HFS+ partition which is case insensitive and it is replicated on a portable hdd with a bare repo on a exfat partition. I wonder if my portable disk has a partially borked repo :P +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment new file mode 100644 index 0000000000..7127a6eef8 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment @@ -0,0 +1,37 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 5" + date="2011-03-28T15:51:11Z" + content=""" +I also failed to mention, that in the case when i have stray log files after what has happened in comment 2, I get this left over after a commit when git is confused... + + +
+jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 1 commit.
+#
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   .git-annex/1G/X3/WORM-s309910751-m1301311322--l_fcompxe_ia32_2011.2.137.tgz.log
+#	modified:   .git-annex/3W/Xf/WORM-s805764902-m1301312756--l_cproc_p_11.1.075_intel64.log
+#	modified:   .git-annex/9Q/Wz/WORM-s1234430253-m1301311891--l_ccompxe_2011.2.137.log
+#	modified:   .git-annex/FQ/4z/WORM-s318168323-m1301310848--l_cprof_p_11.1.075_ia64.log
+#	modified:   .git-annex/FV/0P/WORM-s710135470-m1301311835--l_ccompxe_intel64_2011.2.137.log
+#	modified:   .git-annex/Jk/zK/WORM-s374617670-m1301312705--l_ipp_7.0.2.137_intel64.log
+#	modified:   .git-annex/Jx/qM/WORM-s599386592-m1301310731--l_fcompxe_2011.2.137.tgz.log
+#	modified:   .git-annex/KX/w1/WORM-s35976002-m1301312193--l_tbb_3.0.6.174.log
+#	modified:   .git-annex/VK/kv/WORM-s584342291-m1301312669--l_cproc_p_11.1.075_ia64.log
+#	modified:   .git-annex/Vw/jK/WORM-s15795178-m1301310913--w_flm_p_1.0.011_intel64.zip.log
+#	modified:   .git-annex/Zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+#	modified:   .git-annex/vW/v1/WORM-s736986678-m1301312794--l_cproc_p_11.1.075_ia32.log
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+
+ + +Up until now I have just been updating the status of the staged files by hand and commiting it on my mac x00, this probably isn't helping. I'd rather not lose the tracking information. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment new file mode 100644 index 0000000000..aae020972c --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 6" + date="2011-03-31T18:02:42Z" + content=""" +Alright, I have created a case-insensative HFS+ filesystem here on my linux laptop. + +I have not been able to trick git into staging the same file with 2 different capitalizations yet. + +It might be helpful if you can send me a copy of a git repository where 'git add -i' shows the same file staged with two capitalizations. Leaving out .git/annex of course. (joey@kitenet.net; a tarball would probably work) + +It seems that `git add` only started properly working on case insensative filesystems quite recently. The commit in question is 5e738ae820ec53c45895b029baa3a1f63e654b1b, \"Support case folding for git add when core.ignorecase=true\", which was first released in git 1.7.4, January 30, 2011. If you don't yet have that version, that could explain the problem entirely. In about half an hour (dialup!) I will have downloaded an older git and will see if I can reproduce the problem with it. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment new file mode 100644 index 0000000000..c3aee6c579 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment @@ -0,0 +1,68 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 7" + date="2011-03-31T19:08:01Z" + content=""" +git 1.7.4 does not make things better. With it, if I add first \"X/foo\" and then \"x/bar\", it commits \"X/bar\". + +That will *certianly* cause problems when interoperating with a repo clone on a case-sensative filesystem, since +git-annex there will not see the location log that git committed to the wrong case directory. + +It's possible there is some interoperability problem when pulling from linux like you did, onto HFS+, too. I am not quite sure. Ah, I did find one.. if I clone the repo with \"X/foo\" in it to a case-sensative filesystem, and add a \"x/foo\" there, +and pull that commit back to HFS+, git says: + +
+ * branch            master     -> FETCH_HEAD
+Updating 8754149..e3d4640
+Fast-forward
+ x/foo |    1 +
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 100644 x/foo
+joey@gnu:/mnt/r4>ls
+X/
+joey@gnu:/mnt/r4>git st
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory
+
+#	modified:   X/foo
+
+ +Aha -- that lets me reproduce your problem with the same file being staged twice with different capitalizations, too: + +
+joey@gnu:/mnt/r4>echo haaai >| x/foo
+joey@gnu:/mnt/r4>git st
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   X/bar
+#	modified:   X/foo
+#	modified:   x/foo
+#
+joey@gnu:/mnt/r4>git commit -a
+fatal: Will not add file alias 'X/Bar' ('x/Bar' already exists in index)
+
+ +And modified files that git refuses to commit, which entirely explains [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]]. + +
+joey@gnu:/mnt/r4>git add X/foo
+joey@gnu:/mnt/r4>git commit X/foo
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   X/bar
+#	modified:   X/foo
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+
+ +I think git is frankly, buggy. It seems I will need to work around this by stopping using mixed case hashing for location logs. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment new file mode 100644 index 0000000000..05fe4658df --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 8" + date="2011-03-31T19:28:02Z" + content=""" +I've posted about this on the git mailing list. It's possible that these bugs, which can be shown to affect things other than just git-annex, will be fixed in git. + +I will wait a while to see. But am considering making git-annex use all-lowercase hash dirs for the log files. Maybe it could first look for .git-annex/aaaa/bbbb/foo.log, but also look for, read, and merge in any info from +.git-annex/Aa/Bb/foo.log. And always write to the new style filenames. This would avoid confusing git with changes to +mixed-case files, and avoid another massive transition. +"""]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment new file mode 100644 index 0000000000..8dfe746420 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 9" + date="2011-03-31T21:32:10Z" + content=""" +I'm was running git 1.7.4.1 at the time when I came across it, I have just upgraded to 1.7.4.2. I've also just moved to using a loopback fs for the stuff i care about. Do you still want a repo that exhibits the problem (excluding the .git/annex data) ??? I'm also not sure if 1.7.4.2 has corrected the problem yet as I haven't done much with my repos since. I suspect just making all the .git-annex hashed directories seems to be lower case might be better in the long run. +"""]] diff --git a/doc/bugs/git-annex_dropunused_has_no_effect.mdwn b/doc/bugs/git-annex_dropunused_has_no_effect.mdwn new file mode 100644 index 0000000000..ffa1869acf --- /dev/null +++ b/doc/bugs/git-annex_dropunused_has_no_effect.mdwn @@ -0,0 +1,10 @@ +Hi Joey, + +I have a repository with many thousands of unused files. It's hard to know exactly how many because it takes up to 5 seconds to print the name of every single one, so I'm largely guessing based on my knowledge of what I've recently deleted. + +When I run `git annex dropunused FOO`, it doesn't matter what `FOO` is -- a number, a range, the word "foo" -- the `dropunused` command returns to the prompt instantly in all cases. + +What can I do to drop all these unused files eating up i-nodes? Is there a debug flag I can turn on? + +Thanks, + John diff --git a/doc/bugs/git-annex_dropunused_has_no_effect/comment_1_66b581eb7111a9e98c6406ec75b899cf._comment b/doc/bugs/git-annex_dropunused_has_no_effect/comment_1_66b581eb7111a9e98c6406ec75b899cf._comment new file mode 100644 index 0000000000..0624eb27ce --- /dev/null +++ b/doc/bugs/git-annex_dropunused_has_no_effect/comment_1_66b581eb7111a9e98c6406ec75b899cf._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://launchpad.net/~arand" + nickname="arand" + subject="comment 1" + date="2013-03-03T22:34:27Z" + content=""" +Vhat kind of annex is this (direct/indirect, on what filesystem, etc.)? + +Also what version of git-annex? + +I know that direct mode didn't support dropunused, at least before, and on earlier versions of annex switching back to indirect after deleting things in direct didn't work either for dropunused, if I recall correctly. +"""]] diff --git a/doc/bugs/git-annex_dropunused_has_no_effect/comment_2_11c46cd2087511c3d22b7ce7c149b3e9._comment b/doc/bugs/git-annex_dropunused_has_no_effect/comment_2_11c46cd2087511c3d22b7ce7c149b3e9._comment new file mode 100644 index 0000000000..22a8a0c4f7 --- /dev/null +++ b/doc/bugs/git-annex_dropunused_has_no_effect/comment_2_11c46cd2087511c3d22b7ce7c149b3e9._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89" + nickname="John" + subject="comment 2" + date="2013-03-03T22:43:41Z" + content=""" +It's a ZFS filesystem, using indirect mode. Here are my version numbers: + + Vulcan:~/src/fpco $ git annex version + git-annex version: 4.20130227 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 4 + upgrade supported from repository versions: 0 1 2 + +"""]] diff --git a/doc/bugs/git-annex_dropunused_has_no_effect/comment_3_b1c3d8c6ec4b20727aaa9c4b746531b0._comment b/doc/bugs/git-annex_dropunused_has_no_effect/comment_3_b1c3d8c6ec4b20727aaa9c4b746531b0._comment new file mode 100644 index 0000000000..b1c2ce085f --- /dev/null +++ b/doc/bugs/git-annex_dropunused_has_no_effect/comment_3_b1c3d8c6ec4b20727aaa9c4b746531b0._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + nickname="joey" + subject="comment 3" + date="2013-03-03T23:56:17Z" + content=""" +dropunused will do nothing if the number provided to drop is not listed in `.git/annex/unused` + +The file should be generated when you run `git annex unused` +"""]] diff --git a/doc/bugs/git-annex_fix_not_noticing_file_renames.mdwn b/doc/bugs/git-annex_fix_not_noticing_file_renames.mdwn new file mode 100644 index 0000000000..a68533980e --- /dev/null +++ b/doc/bugs/git-annex_fix_not_noticing_file_renames.mdwn @@ -0,0 +1,36 @@ +What steps will reproduce the problem? + + ~$ mkdir testannex + ~$ cd testannex/ + testannex$ git init + Initialized empty Git repository in /Users/ed/testannex/.git/ + testannex$ git annex init "test annex" + init test annex ok + (Recording state in git...) + testannex$ echo "file1" > file1 + testannex$ git annex add file1 + add file1 (checksum...) ok + (Recording state in git...) + testannex$ mkdir directory + testannex$ mv file1 directory/ + testannex$ cat directory/file1 + cat: directory/file1: No such file or directory + testannex$ git annex fix directory/file1 + git-annex: directory/file1 not found + + +What is the expected output? What do you see instead? + + git annex fix should fix the symlink. It looks like maybe it's *following* the symlink? + +What version of git-annex are you using? On what operating system? + + checkout: 20d195f compiled on OS X 10.7 using cabal. + +Please provide any additional information below. + + git annex assistant is not noticing file renames either. + +> git-annex commands (other than `git annex add`) only operate on files +> checked into git, which `directory/file1` is not, since you did not use +> `git mv`. Once you `git add` the file, it'll work. [[done]] --[[Joey]] diff --git a/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_1_4edd95200d59ec5a5426167b8da8e3f9._comment b/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_1_4edd95200d59ec5a5426167b8da8e3f9._comment new file mode 100644 index 0000000000..589dfbf061 --- /dev/null +++ b/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_1_4edd95200d59ec5a5426167b8da8e3f9._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 1" + date="2012-12-13T20:14:37Z" + content=""" +Thanks for the response! It's good to know how it works right now. + +The reason I was expecting it to work on the files even if I moved them without git rename, is this: + +Imaginary use case: + +I am using git annex assistant, and not using the command line at all. Maybe I don't know anything about git. I am on a machine in the \"client\" group, so it drops content in \"archive\" subdirectories after storing it safely in a repository in the \"backup\" group. + +To add content to git annex, I drag and drop it into the git annex directory. Assistant notices, and it gets added to the annex (it is now a symlink). Yay! + +To archive content, I drag and drop the file (its symlink actually) from the git annex directory to an annex/archive directory, assistant notices, and it gets moved off to backup directories, and this symlink becomes dead. This doesn't work because git annex assistant doesn't notice renames. + +To retrieve content from an archive, I drag and drop the dead link in the archive directory to a parent directory, and git annex notices and grabs the content from a backup somewhere. This doesn't work, because git annex assistant doesn't notice renames. + +These kinds of operations are necessary if I'm going to archive and unarchive files, or otherwise move and manage them, while the assistant is running, without using git from the command line. + + +"""]] diff --git a/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_2_a9a44debefb3bdd4b8ed2d1cf53f2338._comment b/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_2_a9a44debefb3bdd4b8ed2d1cf53f2338._comment new file mode 100644 index 0000000000..33f05eac7d --- /dev/null +++ b/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_2_a9a44debefb3bdd4b8ed2d1cf53f2338._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.8.117" + subject="comment 2" + date="2012-12-13T20:24:32Z" + content=""" +The assistant does notice renames, and also automatically fixes links. +"""]] diff --git a/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_3_0efb11f35b872b75a3fbc4ebb71ac827._comment b/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_3_0efb11f35b872b75a3fbc4ebb71ac827._comment new file mode 100644 index 0000000000..75c02dda1e --- /dev/null +++ b/doc/bugs/git-annex_fix_not_noticing_file_renames/comment_3_0efb11f35b872b75a3fbc4ebb71ac827._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 3" + date="2012-12-13T20:39:57Z" + content=""" +The assistant doesn't seem to be noticing renames or fixing files anymore, for me; that was what got me started on this bug report, though it sounds like I took it in an irrelevant direction by focusing on the command line and \"git annex fix.\" + +I'll double check, and submit a new bug report if I can confirm that assistant isn't doing what it should. +"""]] diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn new file mode 100644 index 0000000000..ed629c4240 --- /dev/null +++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn @@ -0,0 +1,34 @@ +After a series of pretty convoluted copying files around between annex'd repos and pulling changes around between repos. I noticed that occassionally when git-annex tries to stage files (the `.git-annex/*/*/*logs`) git some times gets wedged and doing a "git commit -a" doesn't seem to work or files might not get added thus leaving a bunch of untracked files or modified files that aren't staged for a commit. + +I tried running a *`git rm --cached -f -r *`* then *git add -u .git-annex/* or the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users. + +For now it's just a bit of extra work for me when it does occur but it does not seem to occur often. + +> What do you mean when you say that git "got wedged"? It hung somehow? +> +> If git-annex runs concurrently with another git command that locks +> the repository, its git add of log files can fail. +> +> Update: Also, of course, if you are running a "got annex get" or +> similar, and ctrl-c it after it has gotten some files, it can +> end up with unstaged or in some cases un-added log files that git-annex +> wrote -- since git-annex only stages log files in git on shutdown, and +> ctrl-c bypasses that. +> --[[Joey]] + +>> It "got wedged" as in git doesn't let me commit anything, even though it tells me that there is stuff to be committed in the staging area. + +>>> I've never seen git refuse to commit staged files. There would have to +>>> be some error message? --[[Joey]] + +>>>> there were no error messages at all + +>>>>> Can I see a transcript? I'm having difficulty getting my head around +>>>>> what git is doing. Sounds like the files could just not be `git +>>>>> added` yet, but I get the impression from other things that you say +>>>>> that it's not so simple. --[[Joey]] + +This turns out to be a bug in git, and I have posted a bug report on the mailing list. +The git-annex behavior that causes this situation is being handled as +another bug, [[git-annex directory hashing problems on osx]]. +So, closing this bug report. [[done]] --[[Joey]] diff --git a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn new file mode 100644 index 0000000000..c94952b490 --- /dev/null +++ b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn @@ -0,0 +1,59 @@ +I have a git remote in a git-annex-enabled repository. Here's what it looks like in .git/config: + +
+[remote "renaissance"]
+        url = ssh://[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]/home/paulproteus/Music/annex/
+        fetch = +refs/heads/*:refs/remotes/renaissance/*
+        annex-uuid = 2992752e-1a13-11e0-ba68-57d3c800da64
+
+ +I wanted to "git annex get" some data. git-annex appears to pass incorrectly-formatted IPv6 addresses to rsync: + +
+get primary/emusiq/Arab Strap/Monday At The Hug And Pint/01-The Shy Retirer.mp3 (copying from renaissance...) 
+ssh: Could not resolve hostname [2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]: Name or service not known
+rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]
+rsync error: unexplained error (code 255) at io.c(601) [Receiver=3.0.7]
+
+  rsync failed -- run git annex again to resume file transfer
+  Unable to access these remotes: renaissance
+  Try making some of these repositories available:
+  	2992752e-1a13-11e0-ba68-57d3c800da64
+failed
+
+ +In this case, the square brackets should not be there. + +I tried changing the .git/config syntax slightly, and got a different, also-incorrect behavior: + +
+[remote "renaissance"]
+        url = [2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulproteus/Music/annex/
+        fetch = +refs/heads/*:refs/remotes/renaissance/*
+        annex-uuid = 2992752e-1a13-11e0-ba68-57d3c800da64
+
+ +
+paulproteus@pathi:~/Music/annex$ git annex get
+git-annex: bad url ssh://[2001/~/0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulproteus/Music/annex/
+
+ +(Note that both these .git/config entries work fine with "git fetch".) + +-- Asheesh. + +> Technically, this seems to be a bug in the haskell URI library; it honors +> the `[]` in parsing, but does not remove them when the URI is queried for +> the host part. + +
+Prelude Network.URI> let (Just u) = parseURI "http://foo@[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]/bar"
+Prelude Network.URI> let (Just a) = uriAuthority u
+Prelude Network.URI> uriRegName a
+"[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]"
+Prelude Network.URI> isIPv6address $ uriRegName a
+False
+
+ +> I have filed a [bug upstream](http://trac.haskell.org/network/ticket/40), and put a workaround in git-annex. [[done]] +> --[[Joey]] diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn new file mode 100644 index 0000000000..8df3608dbe --- /dev/null +++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn @@ -0,0 +1,103 @@ +Somehow git-annex has again lost a complete rsync remote with encryption enabled... + +git-annex version was 3.20111111 + +> "once again" ? When did it do it before? + +>> It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]] + +> "lost" ? How is the remote lost? + +>> git-annex is not able to find any files on the encrypted rsync remote anymore. +>> Copy does not copy the content again but drop doesn't find it, thus it's somehow "lost" and in an strange state. +>> I've also had the state where the content was already on the remote side but git-annex copy would copy it again, +>> ignoring all the data on the remote side. --[[gebi]] + +Both *remoteserver* and *localserver* are rsync remotes with enabled encryption. +All commands are executed on the git repository on my laptop. +Target of origin is a gitolite repository without annex support (thus the two rsync remotes). + +Is there a way in git-annex to verify that all files fullfill the numcopies, in my case +numcopies=2, and can be read from the remotes their are on? +I thought that *copy* would verify that, but seems not. + + % g a copy --to remoteserver tools + copy tools/md5_sha1_utility.exe (gpg) (checking remoteserver...) ok + copy tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking remoteserver...) ok + + % g a copy --to localserver tools + copy tools/md5_sha1_utility.exe (gpg) (checking localserver...) ok + copy tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking localserver...) ok + + % g a drop tools + drop tools/md5_sha1_utility.exe (gpg) (checking localserver...) (checking remoteserver...) (unsafe) + Could only verify the existence of 1 out of 2 necessary copies + + Try making some of these repositories available: + 718a9b5c-1b4a-11e1-8211-6f094f20e050 -- remoteserver (remote backupserver) + + (Use --force to override this check, or adjust annex.numcopies.) + failed + drop tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking localserver...) (checking remoteserver...) (unsafe) + Could only verify the existence of 1 out of 2 necessary copies + + Try making some of these repositories available: + 718a9b5c-1b4a-11e1-8211-6f094f20e050 -- remoteserver (remote backupserver) + + (Use --force to override this check, or adjust annex.numcopies.) + failed + git-annex: drop: 2 failed + + % g a fsck tools + fsck tools/md5_sha1_utility.exe (checksum...) ok + fsck tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checksum...) ok + +> Copy does do an explicit check that the content is present on remoteserver, +> and based on the above, the content was found to be already there, +> which is why it did not copy it again. +> +> Drop does an indentical check that the content is present, and +> since it failed to find it, I am left thinking something must have +> happened to the remove in between the copy and the drop to cause the +> content to go away. +> +> What happens if you copy the data to remoteserver again? --[[Joey]] + +The commands above are executed within a few seconds and completely repeatable. --[[gebi]] + +> In that case, why don't you run the commands with `-d` to see the actual +> rsync command it's running to check if the content is present. +> Then you can try repeatedly running the command by hand and see why it +> sometimes succeeds and sometimes fail. + +The commands fail and succeed consistently, not either or. +git annex copy succeeds consistently with not copying the content to remote because it checks and it's already there. + +git annex drop fails consistently with error because content is missing on the exact same remote git annex copy checks +and thinks the content is there. --[[gebi]] + +> The command will be something like this: +> `rsync --quiet hostname:/dir/file 2>/dev/null` +> +> The exit status is what's used to see if content is present -- and +> currently any failure even a failure to connect is taken to mean it's not +> present. --[[Joey]] + +hm... thats interesting, git annex drop and git annex copy check for different hashes on the same file at the same remote... + +git annex drop -d tools/md5_sha1_utility.exe +> Running: sh ["-c","rsync --quiet 'REMOVED_HOST:annex/work/JF/z7/'\"'\"'GPGHMACSHA1--7ffb3840f0e37aee964352e98808403655e8473a/GPGHMACSHA1--7ffb3840f0e37aee964352e98808403655e8473a'\"'\"'' 2>/dev/null"] + +git annex copy --to remoteserver -d tools/md5_sha1_utility.exe +> Running: sh ["-c","rsync --quiet 'REMOVED_HOST:annex/work/1F/PQ/'\"'\"'GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35/GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35'\"'\"'' 2>/dev/null"] + +And yes, only the hash *annex copy* is checking for exists on the remote side. --[[gebi]] + +> Ok, this is due to too aggressive caching of the decrypted cipher +> for a remote. When dopping, it decrypts localserver's cipher, +> caches it, and then when checking remoteserver it says hey, +> here's an already decrypted cipher -- it must be the right one! +> +> Problem reproduced here, and fixed. [[done]] --[[Joey]] + +THX Joey! -- [[gebi]] diff --git a/doc/bugs/git-annex_on_crippled_filesystem_can_still_failed_due_to_case_.mdwn b/doc/bugs/git-annex_on_crippled_filesystem_can_still_failed_due_to_case_.mdwn new file mode 100644 index 0000000000..53b993de26 --- /dev/null +++ b/doc/bugs/git-annex_on_crippled_filesystem_can_still_failed_due_to_case_.mdwn @@ -0,0 +1,32 @@ +What steps will reproduce the problem? + + $ git clone ~/corbeau/travail/ travail + Cloning into 'travail'... + done. + Checking out files: 100% (8670/8670), done. + $ cd travail + $ git annex init "portable USB drive" + init portable USB drive + Detected a crippled filesystem. + + Enabling direct mode. + + git-annex: /media/LACIE/travail/.git/annex/objects/k1: createDirectory: already exists (File exists) + failed + git-annex: init: 1 failed + +What version of git-annex are you using? On what operating system? + $ apt-cache policy git-annex + git-annex: + Installé : 3.20130216 + +This on a amd64 debian sid recently updated + + +Please provide any additional information below. + +The problem is that git annex already created a /media/LACIE/travail/.git/annex/objects/K1 file (same name in uppercase) and FAT isn't realy case sensitive. + + +> I *think* I've found the place that used createDirectory +> rather than createDirectoryIfMissing and fixed it. [[done]] --[[Joey]] diff --git a/doc/bugs/git-annex_sync_broken_on_squeeze_backports.mdwn b/doc/bugs/git-annex_sync_broken_on_squeeze_backports.mdwn new file mode 100644 index 0000000000..4e307fd68f --- /dev/null +++ b/doc/bugs/git-annex_sync_broken_on_squeeze_backports.mdwn @@ -0,0 +1,20 @@ +What steps will reproduce the problem? + + git-annex sync + +What is the expected output? What do you see instead? + +The following error is mixed in with the output, took me a while to notice it: + + Running: git ["--git-dir=/spare/annex/.git","--work-tree=/spare/annex","merge","--no-edit","refs/remotes/pip/synced/master"] + error: unknown option `no-edit' + +What version of git-annex are you using? On what operating system? + +3.20120629~bpo60+1 on debian squeeze + +Please provide any additional information below. + +Installing git from backports as well cleared up the problem. + +> Uploading a fix for this now. [[done]] Thanks for reporting. --[[Joey]] diff --git a/doc/bugs/git-annex_webapp_command_not_found.mdwn b/doc/bugs/git-annex_webapp_command_not_found.mdwn new file mode 100644 index 0000000000..6854d541c7 --- /dev/null +++ b/doc/bugs/git-annex_webapp_command_not_found.mdwn @@ -0,0 +1,25 @@ +###What steps will reproduce the problem? + + greg@x220:~/Music$ git-annex version + git-annex version: 3.20121017 + + greg@x220:~/Documents$ git-annex watch + greg@x220:~/Documents$ git-annex: Daemon is already running. + + greg@x220:~/Documents$ git-annex assistant + greg@x220:~/Documents$ git-annex: Daemon is already running. + + greg@x220:~/Documents$ git-annex webapp + git-annex: unknown command webapp + +This is on an Ubuntu (12.10) machine, using git-annex from ubuntu+1 (13.04), which is just an autosync from sid: + +https://launchpad.net/ubuntu/+source/git-annex/3.20121017 + +###What is the expected output? What do you see instead? +I expect my browser to open the webapp. + + +(Also, it is odd how the output of "Daemon is already running" appears after my prompt is given back to me) + +> [[closing|done]]; forwarded upstream --[[Joey]] diff --git a/doc/bugs/git-annex_webapp_command_not_found/comment_1_6fa63ae1a7affb2351eda57ab3b4eda1._comment b/doc/bugs/git-annex_webapp_command_not_found/comment_1_6fa63ae1a7affb2351eda57ab3b4eda1._comment new file mode 100644 index 0000000000..d60432a8d9 --- /dev/null +++ b/doc/bugs/git-annex_webapp_command_not_found/comment_1_6fa63ae1a7affb2351eda57ab3b4eda1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 1" + date="2012-11-29T19:45:25Z" + content=""" +Ubuntu is apparently building git-annex without the webapp. + +(Message display fixed.) +"""]] diff --git a/doc/bugs/git-annex_webapp_command_not_found/comment_2_d25232bb5eaff725281869d7681e81ad._comment b/doc/bugs/git-annex_webapp_command_not_found/comment_2_d25232bb5eaff725281869d7681e81ad._comment new file mode 100644 index 0000000000..eaf95ceb1b --- /dev/null +++ b/doc/bugs/git-annex_webapp_command_not_found/comment_2_d25232bb5eaff725281869d7681e81ad._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://grossmeier.net/" + nickname="greg" + subject="comment 2" + date="2012-11-29T19:55:25Z" + content=""" +Thanks, Joey. I've reported it [downstream in LP](https://bugs.launchpad.net/ubuntu/+source/git-annex/+bug/1084693). +"""]] diff --git a/doc/bugs/git_annex_add_..._adds_too_much.mdwn b/doc/bugs/git_annex_add_..._adds_too_much.mdwn new file mode 100644 index 0000000000..4eb46455f9 --- /dev/null +++ b/doc/bugs/git_annex_add_..._adds_too_much.mdwn @@ -0,0 +1,25 @@ +When a hidden file (starting with a dot) is git-annex add'ed, other non-tracked files are also added + +What steps will reproduce the problem? + +$ touch a .b + +$ git annex add .b + +add a (checksum...) ok + +add .b (checksum...) ok + +(Recording state in git...) + + +What is the expected output? What do you see instead? + +Only file .b should be added. + +What version of git-annex are you using? On what operating system? + +3.20120406 +(same problem with version 3.20120123) on Debian. + +> Thanks for reporting this bug, I've fixed it in git. [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn new file mode 100644 index 0000000000..d17e569f17 --- /dev/null +++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn @@ -0,0 +1,14 @@ +Recently I ran into the following situation under Ubuntu with an encrypted home directory (which shortens the length that filenames can be): + + $ git annex add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif + add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed + git-annex: /home/lhuhn/annex/.git/annex/tmp/155_518_WORM-s426663-m1310064100--687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long) + git-annex: 1 failed + +The file seems to be completely gone. It no longer exists in the current directory, or under .git/annex. + +I don't mind horribly that git-annex failed due to the name length limit, but it shouldn't have deleted my file in the process (fortunately the file wasn't very important, or hard to recover). + +> [[done]], as noted it did not delete content and now it makes the symlink +> before trying to write to the location log, avoiding that gotcha. +> --[[Joey]] diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment new file mode 100644 index 0000000000..1df159181d --- /dev/null +++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-07-07T20:27:33Z" + content=""" +When I reproduce this, the file is not gone, it's been moved under .git/annex/objects. There is no way an add can delete a file, since all it does is rename it. It would be good for it to error unwind and move the file back though. + +
+joey@gnu:~/tmp/a>touch 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+joey@gnu:~/tmp/a>git annex add *.gif
+add 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+git-annex: /home/joey/tmp/a/.git/annex/tmp/8e2_6a4_WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+joey@gnu:~/tmp/a>touch 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+joey@gnu:~/tmp/a>git annex add *.gif
+add 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+git-annex: /home/joey/tmp/a/.git/annex/tmp/8e2_6a4_WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+joey@gnu:~/tmp/a>find .git/annex/objects -type f
+.git/annex/objects/Mk/92/WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif/WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+
+"""]] diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment new file mode 100644 index 0000000000..bd53627bbc --- /dev/null +++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w" + nickname="Rafaël" + subject="this happens also when the user has not the permission to set the file mode" + date="2011-07-08T00:21:31Z" + content=""" +For example if the file is owned by root, I guess git-annex fails when it tries to remove write permissions (I retested with the last version of today (whose \"version\" subcommand still outputs 3.20110702)).By the way, it would be nice to have a log file created containing the list of all failures, to avoid having to scan manually all the output of a long git-annex operation. +"""]] diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment new file mode 100644 index 0000000000..f9d1b5d682 --- /dev/null +++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w" + nickname="Rafaël" + subject="comment 3" + date="2011-07-08T00:45:30Z" + content=""" +comment on the output of 'git-annex version' (from my last comment): now I get the right version 3.20110707. But I checked in my console that the three commands \"git checkout 3.20110707\", \"make\" and \"./git-annex version\" gave me before 3.20110702, I don't know why... +"""]] diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment new file mode 100644 index 0000000000..1ba57c1992 --- /dev/null +++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-07-08T01:32:30Z" + content=""" +Indeed, I've made it even more robust now, handling the case where the file has weird permissions too, and undoing the failed add so the file is always back at the start state. Had to add a dependency on another haskell module to allow this, so it took some time to figure out how to do it.. + +"""]] diff --git a/doc/bugs/git_annex_add_memory_leak.mdwn b/doc/bugs/git_annex_add_memory_leak.mdwn new file mode 100644 index 0000000000..4bcffb17bf --- /dev/null +++ b/doc/bugs/git_annex_add_memory_leak.mdwn @@ -0,0 +1,39 @@ +For the record, `git annex add` has had a series of memory leaks. +Mostly these are minor -- until you need to check in a few +million files in a single operation. + +If this happens to you, git-annex will run out of memory and stop. +(Generally well before your system runs out of memory, since it has some +built-in ulimits.) You can recover by just re-running the `git annex add` +-- it will automatically pick up where it left off. + +A history of the leaks: + +* Originally, `git annex add` remembered all the files + it had added, and fed them to git at the end. Of course + that made its memory use grow, so it was fixed to periodically + flush its buffer. Fixed in version 0.20110417. + +* Something called a "lazy state monad" caused "thunks" to build + up and memory to leak. Also affected other git annex commands + than `add`. Adding files using a SHA* backend hit the worst. + Fixed in versions afer 3.20120123. + +* Committing journal files turned out to have another memory leak. + After adding a lot of files ran out of memory, this left the journal + behind and could affect other git-annex commands. Fixed in versions afer + 3.20120123. + +* The count of the number of failed commands was updated lazily, which + caused a slow leak when running on a lot of files. Fixed in versions afer + 3.20120123. + +* (Note that `git ls-files --others`, which is used to find files to add, + also uses surpsisingly large amounts + of memory when you have a lot of files. It buffers + the entire list, so it can compare it with the files in the index, + before outputting anything. + This is Not Our Problem, but I'm sure the git developers + would appreciate a patch that fixes it.) + +[[done]] diff --git a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn new file mode 100644 index 0000000000..9b84c21fdb --- /dev/null +++ b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn @@ -0,0 +1,22 @@ +Workflow: + + % git annex add + # list new files + % git commit -a -m "foo" + # commit summary + % git annex copy . --to remote --fast + # all files listed with "ok" + % git annex copy . --to remote + # again, lists all files, _but the new ones are actually copied, this time_. + +This happens no matter if I + + % git push + +before copy or not. + +PS: Arguably, a copy should push automagically. + +> Whups, not supposed to be that fast! [[Fixed|done]], and +> you should run `git annex fsck --fast` on the repo you ran the +> copy in. --[[Joey]] diff --git a/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn new file mode 100644 index 0000000000..3bda451499 --- /dev/null +++ b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn @@ -0,0 +1,18 @@ +I was testing out the fix/workaround for [[git-annex directory hashing problems on osx]] and I tried using the short forms of some of the commands i.e. + + git annex copy -f externalusb . + +which gives me + + git-annex: user error (option `-f' is ambiguous; could be one of: + -f --force allow actions that may lose annexed data + -f REMOTE --from=REMOTE specify from where to transfer content + + +I would have expected that since *--to* is the same as *-t* and *--from* is the same as *-f* as the in program documentation suggests. But *-f* clashes with the force command, I would suggest that the short form of *--force* be changed to *-F* and possibly rename the *Fast* commands to *Quick* and use *-Q* as the short form of the *Quick* operations. I didn't try the *-f* option with the move command, but it probably suffers from the same issue. It's probably better to avoid clashing short forms of command options. + +I guess this issue is just a documentation issue and a minor interface change if needed and not a bug of git-annex, but a quirk. + +> Yeah, -f needs to be from; -F was already --fast. I have made --force not +> have any short option abbreviation, I think it's entirely reasonable to +> avoid fat-fingering an option that can lose data. [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_does_nothing_useful.mdwn b/doc/bugs/git_annex_does_nothing_useful.mdwn new file mode 100644 index 0000000000..be14733ed9 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful.mdwn @@ -0,0 +1,62 @@ +As you can see, I'm running a pretty recent build of git-annex (ac799c3f363e0008b23e9c174e6fedc35e6fa92a), + + $ git annex version + git-annex version: 3.20120630 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +We have a file here which isn't currently available yet isn't +currently available (the link is shown in red), + + $ ls -l plot.py + lrwxrwxrwx 1 ben ben 77 Jul 6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py + $ + +Yet git-annex should be able to tell us where it is, + + $ git-annex whereis plot.py + $ + +Hmm, well that's strange. What's happening here, + + $ git-annex whereis plot.py -d + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","git-annex"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","--hash","refs/heads/git-annex"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","log","refs/heads/git-annex..d5582e05f41011b571a17003934fe9e40859e4be","--oneline","-n1"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","cat-file","--batch"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"] + $ + +Alright, well maybe `git-annex get` will work, + + $ git annex get plot.py -d + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"] + $ ls -l plot.py + lrwxrwxrwx 1 ben ben 77 Jul 6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py + +Nope, the link is still shown in red. + +Alright, what about `git-annex copy`? + + $ git annex copy plot.py --from=goldnerlab --to=here -d + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","git-annex"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","--hash","refs/heads/git-annex"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","log","refs/heads/git-annex..d5582e05f41011b571a17003934fe9e40859e4be","--oneline","-n1"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","cat-file","--batch"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"] + $ ls -l plot.py + lrwxrwxrwx 1 ben ben 77 Jul 6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py + +Still red. + +Alright, what if I just try to get a non-existent file? + + $ git annex get adsflkah -d + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","adsflkah"] + $ + +Alright, it didn't fail with an error, that's very strange. What is going on here? + +[[!meta title="v1 file is ignored"]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_10_457354dc0018333002dc5049935c0feb._comment b/doc/bugs/git_annex_does_nothing_useful/comment_10_457354dc0018333002dc5049935c0feb._comment new file mode 100644 index 0000000000..266cff3dbb --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_10_457354dc0018333002dc5049935c0feb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 10" + date="2012-07-10T14:17:42Z" + content=""" +Hmm, the commands above seem to have worked on both machines (both running 3.20120630). I guess I should probably just try rebuilding my data/ repository from scratch, eh? +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_11_8a6d244165dd238ddf9dd629795de2f6._comment b/doc/bugs/git_annex_does_nothing_useful/comment_11_8a6d244165dd238ddf9dd629795de2f6._comment new file mode 100644 index 0000000000..1de08ae605 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_11_8a6d244165dd238ddf9dd629795de2f6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 11" + date="2012-07-10T14:26:06Z" + content=""" +I suppose.. joey can probably help you investigate exactly what went wrong. You might want to save an empty clone of the git repository for later.. + +The easiest way to fix the data is probably to run a `git annex uninit` in the old repository which will put the files back how they were before and then `git-annex import` them into a new repository. +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_12_30d06bc0f1c37d988a1a31962b57533c._comment b/doc/bugs/git_annex_does_nothing_useful/comment_12_30d06bc0f1c37d988a1a31962b57533c._comment new file mode 100644 index 0000000000..3de1577529 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_12_30d06bc0f1c37d988a1a31962b57533c._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="165.98.113.100" + subject="comment 12" + date="2012-07-11T23:23:04Z" + content=""" +Looking at this a leetle more closely, you had: + +
+lrwxrwxrwx 1 ben ben 77 Jul  6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py
+
+ +Well, that is not how a git-annex symlink currently looks, so it ignores it. + +Apparenly this repository was created with an old version of git-annex, possibly version 1, and you've dropped in the current version, but the normal upgrade machinery failed. This could happen if you made a new clone of a version 1 bare repository. + +I suggest you first find out what version of git-annex was originally used to create this repository (ie, version 0, 1, or 2 ... probably 1). Then make a clone, and \"git config annex.version $N\" where N=the version used). Then \"git annex upgrade\" and you should be good to go. Remember to push or sync the upgrade back to the bare repo so you don't need to do this again. +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_1_fc4f51ddcbc69631e2835b86c3489c8e._comment b/doc/bugs/git_annex_does_nothing_useful/comment_1_fc4f51ddcbc69631e2835b86c3489c8e._comment new file mode 100644 index 0000000000..6bf6e96f6f --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_1_fc4f51ddcbc69631e2835b86c3489c8e._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 1" + date="2012-07-09T23:16:32Z" + content=""" +`git ls-files` is not listing your file. Perhaps your file is not checked into git? +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_2_9bb1647e6c59f1ed7b13b81ecc33f920._comment b/doc/bugs/git_annex_does_nothing_useful/comment_2_9bb1647e6c59f1ed7b13b81ecc33f920._comment new file mode 100644 index 0000000000..3423bfae45 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_2_9bb1647e6c59f1ed7b13b81ecc33f920._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 2" + date="2012-07-09T23:31:08Z" + content=""" +Not really sure what to say about that other than, + + $ git --git-dir=/home/ben/lori/analysis/data/.git --work-tree=/home/ben/lori/analysis/data ls-files --cached -- plot.py + plot.py + $ + +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_3_d434f5c614a27b75d73530b5b918b851._comment b/doc/bugs/git_annex_does_nothing_useful/comment_3_d434f5c614a27b75d73530b5b918b851._comment new file mode 100644 index 0000000000..f03aa27459 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_3_d434f5c614a27b75d73530b5b918b851._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="Remotes? " + date="2012-07-10T00:23:11Z" + content=""" +What does + + git-annex status + +Show? + +Do you have any remotes configured? It looks like you don't somehow. +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_4_998e33219d29ea41b0b2a5d2955a9862._comment b/doc/bugs/git_annex_does_nothing_useful/comment_4_998e33219d29ea41b0b2a5d2955a9862._comment new file mode 100644 index 0000000000..fc9f6c30cb --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_4_998e33219d29ea41b0b2a5d2955a9862._comment @@ -0,0 +1,46 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 4" + date="2012-07-10T01:46:23Z" + content=""" + $ git annex status + supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 3 + 00000000-0000-0000-0000-000000000001 -- web + 02e4ea72-a77c-11e1-bbd7-0749b04e4b59 -- goldnerlab (Data for Goldner) + 3c1fd026-c794-11e1-8ebb-dbe8684e8a73 -- here + untrusted repositories: 0 + dead repositories: 0 + transfers in progress: none + available local disk space: 16 gigabytes (+1 megabyte reserved) + local annex keys: 0 + local annex size: 0 bytes + known annex keys: 0 + known annex size: 0 bytes + bloom filter size: 16 mebibytes (0% full) + backend usage: + $ git remote + goldnerlab + $ git remote show goldnerlab + * remote goldnerlab + Fetch URL: goldnerlab:data + Push URL: goldnerlab:data + HEAD branch (remote HEAD is ambiguous, may be one of the following): + master + synced/master + Remote branches: + git-annex tracked + master tracked + synced/master tracked + Local branch configured for 'git pull': + master merges with remote master + Local refs configured for 'git push': + git-annex pushes to git-annex (up to date) + master pushes to master (up to date) + synced/master pushes to synced/master (up to date) + + +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_5_c72e2571e5b8c06bbfa2276a7ad1e8a6._comment b/doc/bugs/git_annex_does_nothing_useful/comment_5_c72e2571e5b8c06bbfa2276a7ad1e8a6._comment new file mode 100644 index 0000000000..90159b5b46 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_5_c72e2571e5b8c06bbfa2276a7ad1e8a6._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 5" + date="2012-07-10T03:03:27Z" + content=""" +Well that's odd. You have remotes but no annexed files.. + +Can you post the commands you used to arrive at this situation? I'm not sure how you would have done that.. Maybe you just need a + + git-annex sync + +to get things going? + +I think somehow you cloned the git repo but not the annex stuff. +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_6_bc8b42432ba25de8f972c192bc3cdff6._comment b/doc/bugs/git_annex_does_nothing_useful/comment_6_bc8b42432ba25de8f972c192bc3cdff6._comment new file mode 100644 index 0000000000..ad98e6874e --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_6_bc8b42432ba25de8f972c192bc3cdff6._comment @@ -0,0 +1,44 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 6" + date="2012-07-10T03:26:35Z" + content=""" +I can easily reproduce the issue as follows, + + $ git clone goldnerlab:data + Cloning into 'data'... + remote: Counting objects: 61902, done. + remote: Compressing objects: 100% (61354/61354), done. + remote: Total 61902 (delta 356), reused 61902 (delta 356) + Receiving objects: 100% (61902/61902), 5.50 MiB | 894 KiB/s, done. + Resolving deltas: 100% (356/356), done. + $ cd data + $ git annex sync + (merging origin/git-annex into git-annex...) + commit + (Recording state in git...) + # On branch master + nothing to commit (working directory clean) + ok + pull origin + ok + push origin + Counting objects: 8, done. + Delta compression using up to 2 threads. + Compressing objects: 100% (5/5), done. + Writing objects: 100% (6/6), 726 bytes, done. + Total 6 (delta 1), reused 1 (delta 0) + Auto packing the repository for optimum performance. + warning: There are too many unreachable loose objects; run 'git prune' to remove them. + To goldnerlab:data + d5582e0..aaddf3c git-annex -> git-annex + ok + +Everything looks good so far. I verify that alex/plot.py doesn't exist. Now let's try getting it, + + $ git annex get alex/plot.py -d + git [\"--git-dir=/home/ben/data/.git\",\"--work-tree=/home/ben/data\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"alex/plot.py\"] + +Uh oh. ls confirms that get was unsucessful. +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_7_e7469a4c5e45078ade775f5cbdd17cfc._comment b/doc/bugs/git_annex_does_nothing_useful/comment_7_e7469a4c5e45078ade775f5cbdd17cfc._comment new file mode 100644 index 0000000000..c40e4e2cf7 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_7_e7469a4c5e45078ade775f5cbdd17cfc._comment @@ -0,0 +1,67 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 7" + date="2012-07-10T12:37:43Z" + content=""" +But how was the goldnerlab:data repository created? That looks to be where the problem is.. + +I have a slightly older version, but in general it should work the same.. +you can see right away, when I do git annex status it shows \"known annex keys: 1\". +if you do git annex status on goldnerlab, does it say you have any annex keys? + + + $ git-annex version + git-annex version: 3.20120614~bpo60+1 + $ mkdir a + $ cd a + $ git init + Initialized empty Git repository in /tmp/a/.git/ + $ git annex init a + init a ok + (Recording state in git...) + $ echo hi > file + $ git annex add file + add file (checksum...) ok + (Recording state in git...) + $ git commit -m added + fatal: No HEAD commit to compare with (yet) + fatal: No HEAD commit to compare with (yet) + [master (root-commit) cfa9049] added + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 file + $ cd .. + $ git clone a a_clone + Cloning into a_clone... + done. + $ cd a_clone + $ git annex status + (merging origin/git-annex into git-annex...) + supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL + supported remote types: git bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 3 + 00000000-0000-0000-0000-000000000001 -- web + 445d616e-ca8b-11e1-b170-ff8b03c54243 -- origin (a) + 5d3db51c-ca8b-11e1-bbc3-039dd06ab47b -- here + untrusted repositories: 0 + dead repositories: 0 + available local disk space: 63 megabytes (+1 megabyte reserved) + local annex keys: 0 + local annex size: 0 bytes + known annex keys: 1 + known annex size: 3 bytes + backend usage: + SHA256: 1 + (Recording state in git...) + $ ls + file + $ cat file + cat: file: No such file or directory + $ git annex get file + get file (from origin...) ok + (Recording state in git...) + $ cat file + hi + +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_8_bc9e6fd284440a59ffe4e4ed1f73f7d7._comment b/doc/bugs/git_annex_does_nothing_useful/comment_8_bc9e6fd284440a59ffe4e4ed1f73f7d7._comment new file mode 100644 index 0000000000..85d03f04bf --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_8_bc9e6fd284440a59ffe4e4ed1f73f7d7._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 8" + date="2012-07-10T13:02:37Z" + content=""" +On goldnerlab, + + $ git annex status + supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 4 + 00000000-0000-0000-0000-000000000001 -- web + 02e4ea72-a77c-11e1-bbd7-0749b04e4b59 -- here (Data for Goldner) + 351f3ddc-ca3e-11e1-a3fc-6338ef4724a7 + 3c1fd026-c794-11e1-8ebb-dbe8684e8a73 + untrusted repositories: 0 + dead repositories: 0 + transfers in progress: none + available local disk space: 2 terabytes (+1 megabyte reserved) + local annex keys: 19101 + local annex size: 41 gigabytes + known annex keys: 19122 + known annex size: 41 gigabytes + bloom filter size: 16 mebibytes (3.8% full) + backend usage: + WORM: 38223 + +"""]] diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_9_38a2dbeee3750d79ca9a943a02fceb29._comment b/doc/bugs/git_annex_does_nothing_useful/comment_9_38a2dbeee3750d79ca9a943a02fceb29._comment new file mode 100644 index 0000000000..dc3206ac5c --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_9_38a2dbeee3750d79ca9a943a02fceb29._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 9" + date="2012-07-10T14:08:10Z" + content=""" +Can you run the series of commands I had above on your two machines? I figure there are two possibilities: + +1. There is something wrong with the git-annex versions you are using. +2. There is something wrong with your repository. (\"warning: There are too many unreachable loose objects\"?) + +so if you can make a temp repository on goldnerlab, then clone it on the other machine and see where it fails, that would be helpful. + +after cloning git-annex status should hopefully say that you have 1 known key, not 0. + +Obviously this won't fix the problem, but it will at least narrow it down. +"""]] diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn new file mode 100644 index 0000000000..9a044860ae --- /dev/null +++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn @@ -0,0 +1,21 @@ +What is says on the tin: + +git annex fsck is a no-op in bare repos + +See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html + +> Thinking about this some more, it would be difficult to do anything +> when bad content is found, since it also cannot update the location log. +> +> So this may be another thing blocked by [[todo/branching]], assuming +> that is fixed in a way that makes `.git-annex` available to bare repos. +> --[[Joey]] + +>> Even if there is nothing it can _do_, knowing that the data is intact, +>> or not, is valuable in and as of itself. -- RichiH + +>>> While storing the data is no longer an issue in bare repos, fsck would +>>> need a special mode that examines all the location logs, since it +>>> cannot run thru the checked out files. --[[Joey]] + +>>>> [[done]]! --[[Joey]] diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment new file mode 100644 index 0000000000..d50938a784 --- /dev/null +++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-06-13T16:58:52Z" + content=""" +And, maybe, a way to start a fsck from remote? At least when the other side is a ssh or git annex shell, this would work. +"""]] diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment new file mode 100644 index 0000000000..b01590a7a7 --- /dev/null +++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://ertai.myopenid.com/" + nickname="npouillard" + subject="git annex fsck --from remote" + date="2011-06-25T16:20:44Z" + content=""" +Currently fsck silently ignores --to/--from. +It should at least complain if it is not supported. +"""]] diff --git a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn new file mode 100644 index 0000000000..92cc9170f9 --- /dev/null +++ b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn @@ -0,0 +1,13 @@ +when i want to + + git annex get file + +on repo ssh://host-without-port/annex, it works, but if i want to get a file from ssh://host:5122/annex, it tries to run command +ssh ["host:5122", "git-annex-shell 'configlist' '/annex/file'"] and fails. ssh needs the -p option to set the default port, it doesn't support host:port notation. +this is confusing because git can handle this url correctly, and will happily clone/push/pull to/from these url. + +temporary workaround is to use ssh://host/annex as url and define remote.name.annex-ssh-options to "-p 5122", but we need to use this workaround when doing annex get and undo the workaround when pushing/cloning. + +if i had more time, i would have learned haskell and provided a patch ;) + +> Fixed in git! --[[Joey]] [[done]] diff --git a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn new file mode 100644 index 0000000000..d35282e750 --- /dev/null +++ b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn @@ -0,0 +1,34 @@ +For test.com//test, I get this: + + % git annex copy . --to test.com//test + (getting UUID for test...) git-annex: there is no git remote named "test.com//test" + +And my .git/config changes from + + [remote "test.com//test"] + url = richih@test.com:/test + fetch = +refs/heads/*:refs/remotes/test.com//test/* + +to + + [remote "test.com//test"] + url = richih@test.com:/test + fetch = +refs/heads/*:refs/remotes/test.com//test/* + annex-uuid = xyz + [remote "test"] + annex-uuid = xyz + + +Unless I am misunderstanding something, git annex gets confused about what the name of the remote it supposed to be, truncates at the dot for some operations and uses the full name for others. + +> I've fixed this bug. [[done]] +> +> However, using "/" in a remote name seems likely to me to confuse +> git's own remote branch handling. Although I've never tried it. +> --[[Joey]] + +>> From what I can see, git handles / just fine, but would get upset about : which is why it's not allowed in a remote's name. +>> My naming scheme is host//path/to/annex. It sorts nicely and gives all important information left to right with the most specific parts at the beginning and end. +>> If you have any other ideas or scheme, I am all ears :) +>> Either way, thanks for fixing this so quickly. +>> -- RichiH diff --git a/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn b/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn new file mode 100644 index 0000000000..6b57f8ce5f --- /dev/null +++ b/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn @@ -0,0 +1,65 @@ +_git annex initremote_ without a complete command set still adds an entry to the uuid.log etc... and thus clutters up the state of the annex. I would not have expected this behaviour as a user. + +_initremote_ should fail and not do anything if the commands that it has been given are incomplete or incorrect. I was initialising a few rsync repos and noticed that i ended up having mutiple rsync remotes with the same name but different uuid's. I know its hard if not impossible to remove these uuid's so I have just marked them as "dead" in my live annexes. + +Here's a transcript of the problem + +
+x00:sandbox jtang$ mkdir atest
+x00:sandbox jtang$ cd atest/
+x00:atest jtang$ git init
+Initialized empty Git repository in /Users/jtang/sandbox/atest/.git/
+x00:atest jtang$ git annex init 
+init  ok
+(Recording state in git...)
+x00:atest jtang$ git annex status
+supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL
+supported remote types: git S3 bup directory rsync web hook
+trusted repositories: 0
+semitrusted repositories: 2
+	00000000-0000-0000-0000-000000000001 -- web
+ 	cbb58e1c-d737-11e1-b682-83239d5ff2e0 -- here
+untrusted repositories: 0
+dead repositories: 0
+transfers in progress: none
+available local disk space: 185 gigabytes (+1 megabyte reserved)
+local annex keys: 0
+local annex size: 0 bytes
+known annex keys: 0
+known annex size: 0 bytes
+bloom filter size: 16 mebibytes (0% full)
+backend usage: 
+x00:atest jtang$ git annex initremote foo
+git-annex: Specify the type of remote with type=
+x00:atest jtang$ git annex initremote foo type=rsync
+(Recording state in git...)
+initremote foo git-annex: Specify encryption=key or encryption=none or encryption=shared
+x00:atest jtang$ git annex initremote foo type=rsync
+(Recording state in git...)
+initremote foo git-annex: Specify encryption=key or encryption=none or encryption=shared
+x00:atest jtang$ git annex status
+supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL
+supported remote types: git S3 bup directory rsync web hook
+trusted repositories: (Recording state in git...)
+0
+semitrusted repositories: 5
+	00000000-0000-0000-0000-000000000001 -- web
+ 	cbb58e1c-d737-11e1-b682-83239d5ff2e0 -- here
+ 	d3adfcd0-d737-11e1-b15b-b7032388f8aa -- foo
+ 	d6d8e1e0-d737-11e1-956a-0b3d3451226a -- foo
+ 	d78d795c-d737-11e1-ac98-4fe3d6fdfd54 -- foo
+untrusted repositories: 0
+dead repositories: 0
+transfers in progress: none
+available local disk space: 185 gigabytes (+1 megabyte reserved)
+local annex keys: 0
+local annex size: 0 bytes
+known annex keys: 0
+known annex size: 0 bytes
+bloom filter size: 16 mebibytes (0% full)
+backend usage: 
+x00:atest jtang$ 
+
+ +> Indeed, I broke that in June by making it record the name in a much too +> early stage. Now fixed. [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn new file mode 100644 index 0000000000..acd369bded --- /dev/null +++ b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn @@ -0,0 +1,19 @@ +a issue: `git annex initremote` (in particular, adding +a key as described in [[encryption]] -- `git annex initremote my_remote +encryption=my_key`) seems to iterate over the `.git-annex/???/???/*.log` files +with lstat (tested using strace). + +in a 50k key git-annex on a slow disk, this takes quite a while, while not +seeming necessary (it's just re-encrypting the shared secret, is it?). + +could you verify the observed behavior? + +> This is due to `git commit` being called. `git commit` exposes git's +> rather innefficient handling of the index; in order to make a commit +> it has to write a new index file, and it does this by scanning every +> file in the repository. I think that git generally needs its index +> file handleing overhauled, particularly to deal with repositories with +> large numbers of files. git-annex is seems to already be running +> `git commit` in its most efficient mode, by specifying exactly what file +> to commit. [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn new file mode 100644 index 0000000000..24f04eeb54 --- /dev/null +++ b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn @@ -0,0 +1,46 @@ +I discovered a problem with `git annex map` and relative urls containing `~`. +In this case i have a remote `noam` configured with the the following urls: + + zsh» git remote show noam | head -3 + * remote noam + Fetch URL: noam:bare-annex + Push URL: noam:bare-annex + +If i try to run `git annex map` i get the following error: + + zsh» git annex map + map /home/esc/annex ok + map noam (sshing...) + bash: line 0: cd: /~/bare-annex/: No such file or directory + Command ssh ["noam","cd '/~/bare-annex/' && git config --list"] failed; exit code 1 + (sshing...) + ok + + running: dot -Tx11 map.dot + + ok + +If i run the failing command manually, i get: + + zsh» ssh noam "cd ~/bare-annex && git config --list" + core.repositoryformatversion=0 + core.filemode=true + core.bare=true + annex.uuid=f267f55c-0732-11e1-a93b-93119f9aaf54 + annex.version=3 + +Also i can change the remote url to an absolute one, in which case `git annex +map` works too: + + zsh» git remote set-url noam noam:/home/esc/bare-annex + zsh» git annex map + map /home/esc/annex ok + map noam (sshing...) + ok + + running: dot -Tx11 map.dot + + ok + + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn new file mode 100644 index 0000000000..263338d64f --- /dev/null +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -0,0 +1,19 @@ +`git annex migrate` leaves old, unlinked backend versions lying around. It +would be great if these were purged automatically somehow. + +> Yes, this is an issue mentioned in the +> [[tips/migrating_data_to_a_new_backend]]. +> +> Since multiple files can point to the same content, it could be that +> only one file has been migrated, and the content is still used. So +> the content either has to be retained, or an operation as expensive +> as `git annex unused` used to find if something else still uses it. +> +> Rather than adding such an +> expensive operation to each call to migrate, I focused on hard-linking +> the values for the old and new keys, so that the old keys don't actually +> use any additional resources (beyond an extra inode). +> +> This way a lot of migrations can be done, and only when you're done you +> can do the more expensive cleanup pass if you want to. --[[Joey]] +> [[done]] diff --git a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn new file mode 100644 index 0000000000..a92f5871b5 --- /dev/null +++ b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn @@ -0,0 +1,11 @@ +I have this line in the .gitignore file of one of my repos: +*log + +So the command 'git annex init name' fails to add the file ".git-annex/uuid.log", and the same problem happens when git-annex-add'ing files. + +> This is avoided on the v3 branch, which does not store these files in the +> same branch as your repository. + +Also, when a file is git-ignored, it should be possible to 'git annex add' it with a -f/--force option, the same way git does it. + +> Reasonable, [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment new file mode 100644 index 0000000000..c556fbd771 --- /dev/null +++ b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w" + nickname="Rafaël" + subject="comment 1" + date="2011-07-03T11:56:45Z" + content=""" +And what about emitting a warning, as git does, that some files were not annex-added (when not using --force)? +"""]] diff --git a/doc/bugs/git_annex_unlock_is_not_atomic.mdwn b/doc/bugs/git_annex_unlock_is_not_atomic.mdwn new file mode 100644 index 0000000000..6d324ff500 --- /dev/null +++ b/doc/bugs/git_annex_unlock_is_not_atomic.mdwn @@ -0,0 +1,7 @@ +Running a command like + +git annex unlock myfile + +is not atomic, that is if the execution is aborted you may end up with an incomplete version of myfile in the directory. If you don't notice this you may lock it again and then propagate this bad version of the file to your other repositories. A simple workaround is to simply name it something else while unlocking and then rename it to the correct filename once it's completely copied. I don't know Haskel yet so I can not fix this issue otherwise I would sure try. A part from this, I love git annex. + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn new file mode 100644 index 0000000000..fb0bdb0936 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn @@ -0,0 +1,15 @@ +What steps will reproduce the problem? +I don't know exactly when it started + +What is the expected output? What do you see instead? +When I run git annex unused I get + + unused . (checking for unused data...) (checking master...) git-annex: Cannot decode byte '\xb4': Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream + +Most likely I have added some file with a strange encoding that git-annex can't decode. The problem is that the unused process aborts because of this. + +What version of git-annex are you using? On what operating system? + 3.20120522, Debian testing + +> I've just fixed this bug in git, will be in the next release. --[[Joey]] +> [[done]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_1_8ba4fdb9f2d3bd44db5e910526cb9124._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_1_8ba4fdb9f2d3bd44db5e910526cb9124._comment new file mode 100644 index 0000000000..ddea8225e2 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_1_8ba4fdb9f2d3bd44db5e910526cb9124._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.6" + subject="comment 1" + date="2012-06-20T14:30:27Z" + content=""" +Try running `git annex unused --debug`; this will tell us the git command that's outputing the data it cannot process. Then you can try running that git command and see what the problem filename is. +"""]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_2_2a4a2b3e287a0444a1c8e8d98768a206._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_2_2a4a2b3e287a0444a1c8e8d98768a206._comment new file mode 100644 index 0000000000..8afe3143ce --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_2_2a4a2b3e287a0444a1c8e8d98768a206._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.6" + subject="comment 2" + date="2012-06-20T14:34:23Z" + content=""" +Your `locale` setting may also be relevant. FWIW, I've tried to create a file with `\xb4` in its name and have not gotten git-annex unused to crash on it. +"""]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_3_dacfdb8322045fc4ceefc9128bf7c505._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_3_dacfdb8322045fc4ceefc9128bf7c505._comment new file mode 100644 index 0000000000..8e2aa285a9 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_3_dacfdb8322045fc4ceefc9128bf7c505._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" + nickname="Kristian" + subject="comment 3" + date="2012-06-20T14:37:09Z" + content=""" +This is what happens when I add the debug parameter + +git annex unused --debug + +unused . (checking for unused data...) git [\"--git-dir=/home/kristian/AnnexMedia/.git\",\"--work-tree=/home/kristian/AnnexMedia\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"/home/kristian/AnnexMedia\"] +git [\"--git-dir=/home/kristian/AnnexMedia/.git\",\"--work-tree=/home/kristian/AnnexMedia\",\"show-ref\"] +(checking master...) git [\"--git-dir=/home/kristian/AnnexMedia/.git\",\"--work-tree=/home/kristian/AnnexMedia\",\"ls-tree\",\"--full-tree\",\"-z\",\"-r\",\"--\",\"refs/heads/master\"] +git [\"--git-dir=/home/kristian/AnnexMedia/.git\",\"--work-tree=/home/kristian/AnnexMedia\",\"cat-file\",\"--batch\"] +git-annex: Cannot decode byte '\xb4': Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream + +"""]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_4_7889a3ff5ce80c6322448aa674df8525._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_4_7889a3ff5ce80c6322448aa674df8525._comment new file mode 100644 index 0000000000..da97b12f7d --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_4_7889a3ff5ce80c6322448aa674df8525._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.6" + subject="comment 4" + date="2012-06-20T14:49:09Z" + content=""" +Ah, reproduced it; need to use the WORM backend and have the file present in another branch.. + + +"""]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_5_6d28c2537ce24eeb3496ca349823defd._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_5_6d28c2537ce24eeb3496ca349823defd._comment new file mode 100644 index 0000000000..fafd1d248c --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_5_6d28c2537ce24eeb3496ca349823defd._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" + nickname="Kristian" + subject="comment 5" + date="2012-06-20T14:55:33Z" + content=""" +I checkout out the git annex branch and using + + find * | grep -P \"[\xb4]\" + +I found a file + + 43e/b16/WORM-s4118528-m1245167306--Jerry Lee Lewis - Whole Lotta Shakin\302\264 Going\302\264 On.mp3.log + +The corresponding file also existed in the master branch (as a link). + +I moved both these files to a folder outside my repository and synched my git-annex branch with by master server. I still get the same error. Is there any other place where information about this file is stored? + +"""]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_6_4bf14ecef622988e80976c0fb55c24b9._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_6_4bf14ecef622988e80976c0fb55c24b9._comment new file mode 100644 index 0000000000..b35e31da62 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_6_4bf14ecef622988e80976c0fb55c24b9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.6" + subject="comment 6" + date="2012-06-20T16:59:53Z" + content=""" +git-annex was not crashing due to content in the git-annex branch, but due to a symlink in one of your regular git branches, probably master and origin/master. + +This bug is fixed in git master, if you need the fix before the next release. +"""]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_7_d2e5382fe0f38fb9dd9ee69901c68151._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_7_d2e5382fe0f38fb9dd9ee69901c68151._comment new file mode 100644 index 0000000000..65a02fed96 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_7_d2e5382fe0f38fb9dd9ee69901c68151._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg" + nickname="Kristian" + subject="comment 7" + date="2012-06-20T20:49:06Z" + content=""" +Thank you +"""]] diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_8_b282757537cda863d3dc6d0bbfd6b656._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_8_b282757537cda863d3dc6d0bbfd6b656._comment new file mode 100644 index 0000000000..ff1de6b2e8 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_8_b282757537cda863d3dc6d0bbfd6b656._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" + nickname="Kristian" + subject="comment 8" + date="2012-06-21T07:08:22Z" + content=""" +Confirmed. I built the newest version of git-annex and it solved the issue :) +"""]] diff --git a/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent.mdwn b/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent.mdwn new file mode 100644 index 0000000000..1c3d297879 --- /dev/null +++ b/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent.mdwn @@ -0,0 +1,102 @@ +The "git annex unused" command considers remote branches as well as local branches. This means that an +object may be considered unused or not depending on what remotes are present and when they were last synced. + +I ran into this issue when experimenting with using repos on removable storage. I'll post more about +what I was trying to do in the forum. I'm posting this in bugs as I believe the inconsistent behavior +should probably be considered a bug. + +#What steps will reproduce the problem? + +Here is a sample session illustrating the problem. At the end, you can see that the object is +not shown as unused, then the remote is removed and it is shown as unused, then the remote is added +back and the file is once again not shown as unused. + + /tmp/git $ mkdir 1 2 + /tmp/git $ cd 1 + /tmp/git/1 $ git init + Initialized empty Git repository in /tmp/git/1/.git/ + /tmp/git/1 $ git annex init 1 + init 1 ok + (Recording state in git...) + /tmp/git/1 $ git remote add 2 ../2 + /tmp/git/1 $ dd if=/dev/urandom of=file.bin count=100 + 100+0 records in + 100+0 records out + 51200 bytes (51 kB) copied, 0.0113172 s, 4.5 MB/s + /tmp/git/1 $ git annex add file.bin + add file.bin (checksum...) ok + (Recording state in git...) + /tmp/git/1 $ git commit -m 'added file' + [master (root-commit) 3c1ad30] added file + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 file.bin + /tmp/git/1 $ cd ../2 + /tmp/git/2 $ git init + Initialized empty Git repository in /tmp/git/2/.git/ + /tmp/git/2 $ git annex init 2 + init 2 ok + (Recording state in git...) + /tmp/git/2 $ git remote add 1 ../1 + /tmp/git/2 $ git fetch 1 + warning: no common commits + remote: Counting objects: 13, done. + remote: Compressing objects: 100% (9/9), done. + remote: Total 13 (delta 0), reused 0 (delta 0) + Unpacking objects: 100% (13/13), done. + From ../1 + * [new branch] git-annex -> 1/git-annex + * [new branch] master -> 1/master + /tmp/git/2 $ git checkout -b master 1/master + Branch master set up to track remote branch master from 1. + Already on 'master' + /tmp/git/2 $ cd ../1 + /tmp/git/1 $ git fetch 2 + remote: Counting objects: 5, done. + remote: Compressing objects: 100% (3/3), done. + remote: Total 5 (delta 0), reused 0 (delta 0) + Unpacking objects: 100% (5/5), done. + From ../2 + * [new branch] git-annex -> 2/git-annex + * [new branch] master -> 2/master + /tmp/git/1 $ git rm file.bin + rm 'file.bin' + /tmp/git/1 $ git commit -m 'rmed file' + [master ab242b0] rmed file + 1 files changed, 0 insertions(+), 1 deletions(-) + delete mode 120000 file.bin + /tmp/git/1 $ git annex unused + unused . (checking for unused data...) (checking master...) (checking 2/master...) ok + /tmp/git/1 $ git remote rm 2 + /tmp/git/1 $ git annex unused + unused . (checking for unused data...) (checking master...) + Some annexed data is no longer used by any files: + NUMBER KEY + 1 SHA256E-s51200--e400e5abea095ad4364d8f97c5fe1a3f8a6db670b2dfee951d7c9674afc9a21d.bin + (To see where data was previously used, try: git log --stat -S'KEY') + + To remove unwanted data: git-annex dropunused NUMBER + + ok + /tmp/git/1 $ git remote add 2 ../2 + /tmp/git/1 $ git fetch 2 + From ../2 + * [new branch] git-annex -> 2/git-annex + * [new branch] master -> 2/master + /tmp/git/1 $ git annex unused + unused . (checking for unused data...) (checking master...) (checking 2/master...) ok + /tmp/git/1 $ + + +#What is the expected output? What do you see instead? + +I expected that the object's unused status would not change based on which remotes this particular +repo knows about. In other words, I expected the unused status to be based on the local branches +and possibly information in the git-annex branch. + +#What version of git-annex are you using? On what operating system? + +Gentoo Linux, git annex version 3.20121211 + +#Please provide any additional information below. + +The forum post describing what I was trying to accomplish is [[forum/Best way to manage files on removable media?]] diff --git a/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent/comment_1_a636ffe55b11c46a0afcc0b9a3a88cd4._comment b/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent/comment_1_a636ffe55b11c46a0afcc0b9a3a88cd4._comment new file mode 100644 index 0000000000..97af66bc71 --- /dev/null +++ b/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent/comment_1_a636ffe55b11c46a0afcc0b9a3a88cd4._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.246.126" + subject="comment 1" + date="2012-12-23T19:51:52Z" + content=""" +The goal is not to consider an object unused that some other remote is known to rely on. We try as hard as we can to avoid losing data, at the expense of possibly not dropping unused content as early as possible. + +Running `git annex sync` or similar to get current with the state of all remotes before dropping objects they might still rely on seems reasonable from this perspective. +"""]] diff --git a/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent/comment_2_5e1ad57420efd16ae09c9e5cad55b5f2._comment b/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent/comment_2_5e1ad57420efd16ae09c9e5cad55b5f2._comment new file mode 100644 index 0000000000..ffc0359a87 --- /dev/null +++ b/doc/bugs/git_annex_unused_considers_remote_branches_which_makes_it_inconsistent/comment_2_5e1ad57420efd16ae09c9e5cad55b5f2._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="Steve" + ip="92.104.175.136" + subject="comment 2" + date="2013-01-01T23:51:53Z" + content=""" +I filed the bug because I thought the inconsistent behavior was bad. For example; If you move the data to a repo which doesn't know about the unsynced remote, it'll happily get dropped with dropunused as opposed to being considered in use in the current repo. + +There probably aren't too many people who want to have remotes set that are usually unreachable, and I can work around it now that I know it exists. + +If it is expected behavior, feel free to close the bug. + +"""]] diff --git a/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn new file mode 100644 index 0000000000..05aa695727 --- /dev/null +++ b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn @@ -0,0 +1,15 @@ +[[!meta title="`git annex unused` fails on empty repository"]] + +The ``git annex unused`` command fails on a git-annex repository, if there are no objects yet: + + $ git annex unused + unused (checking for unused data...) + git-annex: /tmp/annextest/other_annex/.git/annex/objects: getDirectoryContents: does not exist (No such file or directory) + git-annex: 1 failed + $ + +This can give a user (especially one that wants to try out simple commands with his newly created repo) the impression that something is wrong, while it is not. I'd expect the program either to show the same message ``git annex unused`` shows when everything is ok (since it is, or should be). + +This can be a bug in the ``unused`` subcommand (that fails to accept the absence of an objects directory) or in the ``init`` subcommand (that fails to create it). + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn new file mode 100644 index 0000000000..f2d301cd7c --- /dev/null +++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn @@ -0,0 +1,39 @@ +When I run `git annex unused` from my repository's root it shows everything ok: + + ~/annex$ git annex unused + unused (checking for unused data...) ok + +But... When I run it from a subdirectory, it shows a lot: + + ~/annex/Software$ git annex unused + unused (checking for unused data...) + Some annexed data is no longer pointed to by any files in the repository: + NUMBER KEY + 1 SHA1:######################################## + ... + 921 SHA1:######################################## + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused NUMBER) + ok + +Is this a bug or by design? By removing these "unused" files with `dropunused` I've just lost the only copy of 160 files. + +I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b + +> I'm very sorry you lost data. +> +> But, git annex unused absolutely does not let the current directory +> influence what it does. It always scans the entire repo from the top. +> And I've tested it just now to make sure that in a subdirectory +> it does the same thing as at the top. +> +> There are only two ways this could happen that I can think of: +> +> 1. If "Software" were a separate git repository than "~/annex". +> 2. If gitignores or something made `git ls-files` +> not list the files when ran in the subdir. This seems *possible*, +> but I don't know how to construct such an ignore. +> +> --[[Joey]] + +>> Closing as there is no followup. [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn new file mode 100644 index 0000000000..9daf8a0cb4 --- /dev/null +++ b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn @@ -0,0 +1,33 @@ +"git annex upgrade" has lost track of some of my files. Most of them have "&" characters. The others contain "%" characters (I haven't tried the testcase below with "%" however). + +Testcase: + + # (With git annex v2) + mkdir ~/testannex1 + cd ~/testannex1 + git init + git annex init "testannex1" + touch '02 - Afternoons & Coffeespoons.mp3' + touch 'no ampersand.mp3' + git annex add '02 - Afternoons & Coffeespoons.mp3' + git annex add 'no ampersand.mp3' + git commit -m added + git annex whereis '02 - Afternoons & Coffeespoons.mp3' + git annex whereis 'no ampersand.mp3' + # (Upgrade git-annex binary to v3 and then...) + git annex upgrade + git annex whereis '02 - Afternoons & Coffeespoons.mp3' + git annex whereis 'no ampersand.mp3' + +This produces: + + 12:38:40 ~/testannex1 (master)$ git annex whereis '02 - Afternoons & Coffeespoons.mp3' + whereis 02 - Afternoons & Coffeespoons.mp3 (0 copies) + failed + git-annex: 1 failed + 12:38:40 ~/testannex1 (master)$ git annex whereis 'no ampersand.mp3' + whereis no ampersand.mp3 (1 copy) + a7b680fc-a8d0-11e0-b0fe-4f94e86d1fb7 -- testannex1 <-- here + ok + +[[!tag done]] diff --git a/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment new file mode 100644 index 0000000000..194b36ac10 --- /dev/null +++ b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-07-07T21:04:23Z" + content=""" +What an evil little bug. In retrospect, this probably bit my own test upgrades, but I ran `git annex fsck` everywhere and so avoided the location log breakage. + +I've fixed the bug, which also involved files with other punctuation in their names [&:%] when using the WORM backend. + +The only way I have to recover repos that have already been upgraded is to run `git annex fsck --fast` in each clone of such a repo, which will let it rebuild the location log information. I think that is the best way to recover; ie I can't think of a way to recover that doesn't need to do everything fsck does anyway. +"""]] diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn new file mode 100644 index 0000000000..ec8a10915e --- /dev/null +++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn @@ -0,0 +1,15 @@ +Upgrading from v1 to v3: + + upgrade . (v1 to v2...) (moving content...) (updating symlinks...) (moving location logs...) (v2 to v3...) .............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................. + git-annex branch created + Be sure to push this branch when pushing to remotes. + ok + +A whirly would be preferable, imo. + +> Erm, I'm pretty sure you were the one who asked for there to be some +> progress dots, Richard. +> +> I'm not particularly interested in implementing a whirley that would only +> be used in this one place, in code that very few users are going to run +> again. I could remove the dots.. [[done]] --[[Joey]] diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment new file mode 100644 index 0000000000..4f9565517d --- /dev/null +++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-10-29T17:03:26Z" + content=""" +I could dig it out, but I am sure I said dots are fine and a whirly better. + +Still, WONTFIX is fine. +"""]] diff --git a/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn new file mode 100644 index 0000000000..5c995852b1 --- /dev/null +++ b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn @@ -0,0 +1,7 @@ +was checking the version of git-annex on a machine before cloning a repo... + + $ git annex version + git-annex: Not in a git repository. + +> made difficult by the Annex monad, but I made it work! --[[Joey]] +> [[done]] diff --git a/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment b/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment new file mode 100644 index 0000000000..ab30d8a452 --- /dev/null +++ b/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 1" + date="2011-11-16T03:24:30Z" + content=""" +oh, and that probably goes for 'help' and other subcommands as well. +"""]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive.mdwn b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive.mdwn new file mode 100644 index 0000000000..6e1c007816 --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive.mdwn @@ -0,0 +1,60 @@ +One of my remotes, on a USB drive, is behaving exceedingly strangely. Files sometimes refuse to copy to it - whether I copy to it from my home annex, or whether I "cd" to that USB drive and try to "get" files to it. + +Note that the external HD is a FAT32 filesystem. This has never caused problems in the past, but I am wondering if some of the recent work on "crippled" filesystems might have caused breakage on existing repositories which had been working well on FAT32 filesystems? + +What steps will reproduce the problem? + +On my annex, something like this: + +
+Talislanta Books$ git annex whereis talislanta_fantasy_roleplaying.pdf
+whereis talislanta_fantasy_roleplaying.pdf (2 copies) 
+  	d16d0d1a-3cdd-11e2-9161-67c83599f720 -- homeworld
+   	fa2bd02e-3ce2-11e2-a675-47389975a32e -- here (macbook)
+ok
+Talislanta Books$ git annex copy --to=toshiba talislanta_fantasy_roleplaying.pdf
+copy talislanta_fantasy_roleplaying.pdf ok
+Talislanta Books$ git annex whereis talislanta_fantasy_roleplaying.pdf
+whereis talislanta_fantasy_roleplaying.pdf (2 copies) 
+  	d16d0d1a-3cdd-11e2-9161-67c83599f720 -- homeworld
+   	fa2bd02e-3ce2-11e2-a675-47389975a32e -- here (macbook)
+ok
+Talislanta Books$ cd /Volumes/TOSHIBAEXT/annex/Books/archive/Talislanta\ Books/
+Talislanta Books$ git annex whereis talislanta_fantasy_roleplaying.pdf
+whereis talislanta_fantasy_roleplaying.pdf (2 copies) 
+  	d16d0d1a-3cdd-11e2-9161-67c83599f720 -- homeworld
+   	fa2bd02e-3ce2-11e2-a675-47389975a32e -- macbook
+ok
+Talislanta Books$ git annex get talislanta_fantasy_roleplaying.pdf
+Talislanta Books$ git annex whereis talislanta_fantasy_roleplaying.pdf
+whereis talislanta_fantasy_roleplaying.pdf (2 copies) 
+  	d16d0d1a-3cdd-11e2-9161-67c83599f720 -- homeworld
+   	fa2bd02e-3ce2-11e2-a675-47389975a32e -- macbook
+ok
+Talislanta Books$
+
+ + +What is the expected output? What do you see instead? + +I should be able to copy files to my external hard drive, /Volumes/TOSHIBAEXT/annex + + +What version of git-annex are you using? On what operating system? + +
+Talislanta Books$ git annex version
+git-annex version: 3.20130216
+local repository version: 3
+default repository version: 3
+supported repository versions: 3
+upgrade supported from repository versions: 0 1 2
+
+ +OS X 10.6 (lion) + +Please provide any additional information below. + +Most files are affected by this, a few are not. I don't see any pattern to which is which. + +> Both bugs reported here are now [[done]]. --[[Joey]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_1_7707017fbf3d92ee21d600fe0aefce4f._comment b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_1_7707017fbf3d92ee21d600fe0aefce4f._comment new file mode 100644 index 0000000000..2f157417ff --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_1_7707017fbf3d92ee21d600fe0aefce4f._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 1" + date="2013-02-25T06:18:58Z" + content=""" +Just tried downgrading to 3.20130207 and the behavior is the same. :( + + +"""]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_2_f3392ec3ca7392823cbad2cc9b77f54e._comment b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_2_f3392ec3ca7392823cbad2cc9b77f54e._comment new file mode 100644 index 0000000000..17379f3543 --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_2_f3392ec3ca7392823cbad2cc9b77f54e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 2" + date="2013-02-25T12:44:18Z" + content=""" +Update: git annex fsck --fast --from=toshiba fixed this. Guess I was up too late to think about the obvious. Sorry bout that. + +"""]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_3_b3d016a487b12748fe2c4d14300eb158._comment b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_3_b3d016a487b12748fe2c4d14300eb158._comment new file mode 100644 index 0000000000..f891af92e5 --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_3_b3d016a487b12748fe2c4d14300eb158._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 3" + date="2013-02-25T14:24:29Z" + content=""" +This is still weird though. + +Setup: I have my home drive, an ssh remote, and a usb remote. + +Home drive is a \"client\" and the other two are \"backups.\" numcopies = 2. + +WHen I start the assistant and watch it in the webapp, it starts furiously copying content in \"archive\" subdirectories from the USB remote to the home drive, and then dropping it from the USB remote! + +I then kill the assistant because I do not want content dropped from my backup, that's why it's a backup. + +Is there any way to tell *why* the assistant is doing this crazy thing? + + +"""]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_4_61f600511a3172f0707e5809fc444d0c._comment b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_4_61f600511a3172f0707e5809fc444d0c._comment new file mode 100644 index 0000000000..c8b6788f3d --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_4_61f600511a3172f0707e5809fc444d0c._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 4" + date="2013-02-25T14:57:20Z" + content=""" +Yeah, the assistant wants to drop *everything* on my USB drive. When I vicfg I see that the usb drive repo is \"untrusted/backup/standard\". Why in the world would the assistant want to drop everything on it? + +"""]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_5_8cf029ac7bf3c19dcb0b613eed3b52ac._comment b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_5_8cf029ac7bf3c19dcb0b613eed3b52ac._comment new file mode 100644 index 0000000000..2c133e8d0b --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_5_8cf029ac7bf3c19dcb0b613eed3b52ac._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 5" + date="2013-02-25T16:02:44Z" + content=""" +Downgrading to 3.20130207 and testing.... this seems to have been a regression since 3.20130207. 3.20130207 seems right now to be behaving normally, not trying to drop things it shouldn't. Is it possible something went terribly wrong with preferred content settings since 3.20130207? + +Still testing, but things look good so far. +"""]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_6_e40d88eba7d8aec1530ce1d32d1c85f2._comment b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_6_e40d88eba7d8aec1530ce1d32d1c85f2._comment new file mode 100644 index 0000000000..830a1c6310 --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_6_e40d88eba7d8aec1530ce1d32d1c85f2._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.210" + subject="comment 6" + date="2013-02-26T17:20:13Z" + content=""" +The recent switch to using the Glob library seems to be responsible for this problem. It seems that +with Glob, \"\*\" matches \"foo\", but not \"directory/foo\", so the \"\*\" in preferred content for backup repositories matches only files in the top directory! + +Writing test cases and fixing this now. +"""]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_7_b101fab9e690d1b335a1a29abab68d6c._comment b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_7_b101fab9e690d1b335a1a29abab68d6c._comment new file mode 100644 index 0000000000..178b62fa78 --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_7_b101fab9e690d1b335a1a29abab68d6c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.210" + subject="comment 7" + date="2013-02-26T18:41:57Z" + content=""" +Getting back to the original problem, it seemed to be that the remote already had the files, but the local location log was not aware of this. Perhaps because the remote got the files from somewhere else and `git annex sync` or similar had not been run to get that into into the local repo recently? Anyway, copying files to the remote was correctly detected to be unnecessary. + +Currently, when that happens, it does not update the local location log. That is a change made fairly recently, in 40df26757a61d4f057bcbf38cd5fe949d1c9be95, as a kind of optimisation -- I'd seen it updating the location log during mass copies when it didn't need to, which just bloats `.git`. Seems that optimisation went too far: It should avoid updating the location log when it was correct, but if the location log is wrong, it should optimistically update it. +"""]] diff --git a/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_8_b30d32086314a7e357f3dd6608828ee5._comment b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_8_b30d32086314a7e357f3dd6608828ee5._comment new file mode 100644 index 0000000000..32a8012246 --- /dev/null +++ b/doc/bugs/git_annex_won__39__t_copy_files_to_my_usb_drive/comment_8_b30d32086314a7e357f3dd6608828ee5._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 8" + date="2013-02-26T21:55:23Z" + content=""" +Cool, glad this led to an important fix, even if it turned out to be unrelated to the thing I opened the bug report for. + +"""]] diff --git a/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn new file mode 100644 index 0000000000..181b02b5c1 --- /dev/null +++ b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn @@ -0,0 +1,15 @@ +This doesn't look right: + + simons 11148 0.0 0.0 15572 1268 pts/1 SN+ 04:00 0:00 | \_ git annex unannex stuff + simons 11150 0.5 0.0 130504 11212 pts/1 SN+ 04:00 3:40 | | \_ git-annex unannex stuff + simons 11152 0.0 0.1 39536 23932 pts/1 SN+ 04:00 0:00 | | \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex ls-files --cached -z -- stuff + simons 11288 0.0 0.0 0 0 pts/1 ZN+ 04:01 0:00 | | \_ [git] + simons 11339 0.0 0.0 0 0 pts/1 ZN+ 04:02 0:00 | | \_ [git-annex] + simons 11442 0.0 0.0 0 0 pts/1 ZN+ 04:06 0:00 | | \_ [git] + simons 11443 0.0 0.0 0 0 pts/1 ZN+ 04:06 0:05 | | \_ [git] + simons 16541 0.0 0.0 0 0 pts/1 ZN+ 04:14 0:00 | | \_ [git] + simons 16543 0.3 0.0 15644 1744 pts/1 SN+ 04:14 2:13 | | \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex cat-file --batch + simons 14224 0.0 0.0 100744 796 pts/1 SN+ 14:10 0:00 | | \_ xargs -0 git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex + simons 14225 0.4 0.1 32684 18652 pts/1 DN+ 14:10 0:00 | | \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a + +> [[Fixed|done]] --[[Joey]] diff --git a/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__.mdwn b/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__.mdwn new file mode 100644 index 0000000000..14eb3b3292 --- /dev/null +++ b/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__.mdwn @@ -0,0 +1,34 @@ +What steps will reproduce the problem? + +run git annex assistant, add a file, which is picked up and pushed by the assistant. + +What is the expected output? What do you see instead? + +a ps -ef shows a large number of defunct git processes.. for example: +
+nelg      9622     1  0 02:01 ?        00:00:01 git-annex assistant
+nelg      9637  9622  0 02:01 ?        00:00:00 git --git-dir=/home/nelg/Downloads/test2/.git --work-tree=/home/nelg/Downloads/test2 cat-file --batch
+nelg     12080  9622  0 02:19 ?        00:00:00 [git] <defunct>
+nelg     12082  9622  0 02:19 ?        00:00:00 [git] <defunct>
+nelg     12083  9622  0 02:19 ?        00:00:00 [git] <defunct>
+nelg     12084  9622  0 02:19 ?        00:00:00 [git] <defunct>
+-----
+
+ +What version of git-annex are you using? On what operating system? + +Compiled git annex from git (cbcd208d158f8e42dda03a5eeaf1bac21045a140), on Mandriva 2010.2, 32 bit, using ghc-7.4.1. + git version 1.7.1 + + +Please provide any additional information below. + +I also found that the version of git I have does not support the option: --allow-empty-message +So, suggest that if the version of git installed is an older version, that the params in Assistant/Threads/Committer.hs +are changed to [ Param "-m", Param "git assistant".... or something like that. + +I have done this on my copy for testing it. + +For testing, I am also using two repositories on the same computer. I set this up from the command line, as the web app does not seem to support syncing to two different git folders on the same computer. + +> [[done]]; all zombies are squelched now in the assistant. --[[Joey]] diff --git a/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__/comment_1_5e3f4b63db5cd32b63fb3e6a78f9b093._comment b/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__/comment_1_5e3f4b63db5cd32b63fb3e6a78f9b093._comment new file mode 100644 index 0000000000..f70cdd1bdc --- /dev/null +++ b/doc/bugs/git_defunct_processes___40__child_of_git-annex_assistant__41__/comment_1_5e3f4b63db5cd32b63fb3e6a78f9b093._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.84" + subject="comment 1" + date="2012-09-19T17:02:47Z" + content=""" +Thanks for the note about --allow-empty-message. That appeared in 1.7.2, I've added a fallback for the old version. + +As far as I know, the assistant always reaps old children eventually, but it does it somewhat lazily. Effectively each time it starts up a new set of git children it reaps the old set. So you should not see them grow without bounds or anything like that. +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move.mdwn b/doc/bugs/git_rename_detection_on_file_move.mdwn new file mode 100644 index 0000000000..76f1e098e5 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move.mdwn @@ -0,0 +1,13 @@ +It's unfortunate that git-annex sorta defeats git's rename detection. + +When an annexed file is moved to a different directory (specifically, a +directory that is shallower or deeper than the old directory), +the symlink often has to change. And so git log cannot --follow back +through the rename history, since all it has to go on is that symlink, +which it effectively sees as a one line file containing the symlink target. + +One way to fix this might be to do the `git annex fix` *after* the rename +is committed. This would mean that a commit would result in new staged +changes for another commit, which is perhaps startling behavior. + +The other way to fix it is to stop using symlinks, see [[todo/smudge]]. diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment b/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment new file mode 100644 index 0000000000..6ea2677289 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w" + nickname="Rafael" + subject="comment 10" + date="2012-05-15T07:36:25Z" + content=""" +Won't git itself be fixed on this issue? It was on my plans to look into that, however I don't know how difficult it will be. +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment b/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment new file mode 100644 index 0000000000..8fec6bad72 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="use mini-branches" + date="2011-03-09T23:47:48Z" + content=""" +if you go for the two-commits version, small intermediate branches (or git-commit-tree) could be used to create a tree like this: + + + * commit 106eef2 + |\ Merge: 436e46f 9395665 + | | + | | the main commit + | | + | * commit 9395665 + |/ + | intermediate move + | + * commit 436e46f + | + | ... + +while the first commit (436e46f) has a \"`/subdir/foo → ../.git-annex/where_foo_is`\", the intermediate (9395665) has \"`/subdir/deeper/foo → ../.git-annex/where_foo_is`\", and the inal commit (106eef2) has \"`/subdir/deeper/foo → ../../.git-annex/where_foo_is`\". + +`--follow` uses the intermediate commit to find the history, but the intermediate commit would neither show up in `git log --first-parent` nor affect `git diff HEAD^..` & co. (there could still be confusion over `git show`, though). +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment b/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment new file mode 100644 index 0000000000..7d50c58d1b --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="praet" + ip="81.240.159.215" + subject="Use variable symlinks, relative to the repo's root ?" + date="2011-03-10T16:50:28Z" + content=""" +It all boils down to the fact that the path to a relative symlink's target is determined relative to the symlink itself. + +Now, if we define the symlink's target relative to the git repo's root (eg. using the $GIT_DIR environment variable, which can be a relative or absolute path itself), this unfortunately results in an absolute symlink, which would -for obvious reasons- only be usable locally: + + user@host:~$ mkdir -p tmp/{.git/annex,somefolder} + user@host:~$ export GIT_DIR=~/tmp + user@host:~$ touch $GIT_DIR/.git/annex/realfile + user@host:~$ ln -s $GIT_DIR/.git/annex/realfile $GIT_DIR/somefolder/file + user@host:~$ ls -al $GIT_DIR/somefolder/ + total 12 + drwxr-x--- 2 user group 4096 2011-03-10 16:54 . + drwxr-x--- 4 user group 4096 2011-03-10 16:53 .. + lrwxrwxrwx 1 user group 33 2011-03-10 16:54 file -> /home/user/tmp/.git/annex/realfile + user@host:~$ + +So, what we need is the ability to record the actual variable name (instead of it's value) in our symlinks. + +It *is* possible, using [variable/variant symlinks](http://en.wikipedia.org/wiki/Symbolic_link#Variable_symbolic_links), yet I'm unsure as to whether or not this is available on Linux systems, and even if it is, it would introduce compatibility issues in multi-OS environments. + +Thoughts on this? +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment b/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment new file mode 100644 index 0000000000..534723254a --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T03:03:19Z" + content=""" +Interesting, I had not heard of variable symlinks before. AFAIK linux does not have them. +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment b/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment new file mode 100644 index 0000000000..c265b58995 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment @@ -0,0 +1,34 @@ +[[!comment format=mdwn + username="praet" + ip="81.240.27.89" + subject="Brainfart" + date="2011-03-20T20:11:27Z" + content=""" +Haven't given these any serious thought (which will become apparent in a moment) but hoping they will give birth to some less retarded ideas: + +--- + +### Bait'n'switch + +- pre-commit: Replace all staged symlinks (when pointing to annexed files) with plaintext files containing the key of their respective annexed content, re-stage, and add their paths (relative to repo root) to .gitignore. +- post-commit: Replace the plaintext files with (git annex fix'ed) symlinks. + +In doing so, the blobs to be committed can remain unaltered, irrespective of their related files' depth in the directory hierarchy. + +To prevent git from reporting ALL annexed files as unstaged changes after running post-commit hook, their paths would need to be added to .gitignore. + +This wouldn't cause any issues when adding files, very little when modifying files (would need some alterations to \"git annex unlock\"), BUT would make git totally oblivious to removals... + +--- + +### Manifest-based (re)population +- Keep a manifest of all annexed files (key + relative path) +- DON'T track the symlinks (.gitignore) +- Populate/update the directory structure using a post-commit hook. + +... thus circumventing the issue entirely, yet diffstats (et al.) would be rather uninformative. + +--- + +***Wide open to suggestions, criticism, mocking laughter and finger-pointing :)*** +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment b/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment new file mode 100644 index 0000000000..93db97e704 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="praet" + ip="81.242.56.203" + subject="comment 5" + date="2011-03-21T19:58:34Z" + content=""" +In the meantime, would it be acceptable to split the pre-commit hook +into two discrete parts? + +This would allow to (if preferred) defer \"git annex fix\" until +post-commit while still keeping the safety net for unlocked files. +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment b/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment new file mode 100644 index 0000000000..7398ac5614 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://adamspiers.myopenid.com/" + nickname="Adam" + subject="extra level of indirection" + date="2011-12-19T12:45:18Z" + content=""" +Surely this could be handled with an extra layer of indirection? + +git-annex would ensure that every directory containing annexed data contains a new symlink `.git-annex` which points to `$git_root/.git/annex`. Then every symlink to an annexed object uses a relative symlink via this: `.git_annex/objects/xx/yy/ZZZZZZZZZZ`. Even though this symlink is relative, moving it to a different directory would not break anything: if the move destination directory already contained other annexed data, it would also already contain `.git-annex` so git-annex wouldn't need to do anything. And if it didn't, git-annex would simply create a new `.git-annex` symlink there. + +These `.git-annex` symlinks could either be added to `.gitignore`, or manually/automatically checked in to the current branch - I'm not sure which would be best. There's also the option of using multiple levels of indirection: + + foo/bar/baz/.git-annex -> ../.git-annex + foo/bar/.git-annex -> ../.git-annex + foo/.git-annex -> ../.git-annex + .git-annex -> .git/annex + +I'm not sure whether this would bring any advantages. It might bring a performance hit due to the kernel having to traverse more symlinks, but without benchmarking it's difficult to say how much. I'd expect it only to be an issue with a large number of deep directory trees. +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment b/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment new file mode 100644 index 0000000000..0a045feb63 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 7" + date="2011-12-19T18:22:25Z" + content=""" +That seems an excellent idea, also eliminating the need for git annex fix after moving. + +However, I think CVS and svn have taught us the pain associated with a version control system putting something in every subdirectory. Would this pain be worth avoiding the minor pain of needing git annex fix and sometimes being unable to follow renames? +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment b/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment new file mode 100644 index 0000000000..d53022302d --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="http://adamspiers.myopenid.com/" + nickname="Adam" + subject="comment 8" + date="2011-12-20T12:00:11Z" + content=""" +Personally I'd rather have working rename detection but I agree it's not 100% ideal to be littering multiple directories like this, so perhaps you could make it optional, e.g. based on a git config setting? + +Here are a few more considerations, some in defence of the approach, some against it: + +* `.git-annex` is hidden; `CVS/` is not. +* Unlike `CVS/` and `.svn/`, it's only a symlink, not a directory containing other files. +* It doesn't contain any data specific to that directory and could easily be regenerated if deleted accidentally or otherwise. +* If a whole directory containing `.git-annex` was moved within the repository: + * git-annex would need to fix up these symlinks if and only if it's moved to a different depth within the tree. + * However, if the multi-level indirection approach is used, `.git-annex` in any subdirectory is *always* a symlink to `../.git-annex` so instead you would need to check that all of the new ancestors contain this symlink too, and optionally remove any no longer needed symlinks. + * In either case, git-annex already goes to the trouble of fixing symlinks, and if anything, I *think* this approach would reduce the number of symlinks which need checking (right?) +* find `$git_root/foo -follow`, `diff -r` etc. would traverse into `$git_root/.git/annex` + +This last point is the only downside to this approach I can think of which gives me any noticeable cause for concern. However, people are already use to working around this from CVS and svn days, e.g. `diff -r -x .svn` so I don't think it's anywhere near bad enough to rule it out. +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment b/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment new file mode 100644 index 0000000000..919455bdcc --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 9" + date="2011-12-20T14:56:12Z" + content=""" +Git can follow the rename fine if the file is committed before `git annex fix` (you can git commit -n to see this), so +making git-annex pre-commit generate a fixup commit before the staged commit would be one way. Or the other two ways I originally mentioned when writing down this minor issue. I like all those approaches better than .git-annex clutter. +"""]] diff --git a/doc/bugs/gix-annex_help_is_homicidal.mdwn b/doc/bugs/gix-annex_help_is_homicidal.mdwn new file mode 100644 index 0000000000..f4a8d72137 --- /dev/null +++ b/doc/bugs/gix-annex_help_is_homicidal.mdwn @@ -0,0 +1,23 @@ +> What steps will reproduce the problem? + +Run 'git-annex help' + +> What is the expected output? + +Something similar to 'git-annex --help', or a pointer to --help. + +> What do you see instead? + + git-annex: Unknown command 'help' + Did you mean one of these? + drop + dead + +> What version of git-annex are you using? On what operating system? + +git-annex version 3.20120825 on Arch Linux x86_64, installed from AUR package git-annex and using the [haskell] repository for dependencies. + +>> Lol, that's great! Also worth noting that with help.autocorrect=1, it'd +>> actually run drop. Only with --force can you lose data however. +>> +>> I've added a help command. [[done]] --[[Joey]] diff --git a/doc/bugs/glacier_from_multiple_repos.mdwn b/doc/bugs/glacier_from_multiple_repos.mdwn new file mode 100644 index 0000000000..f6ef2a4acc --- /dev/null +++ b/doc/bugs/glacier_from_multiple_repos.mdwn @@ -0,0 +1,14 @@ +glacier-cli currently relies on a local cache of +inventory information, and so other git-annexes using the same glacier +repository are not able to access stuff in it, unless and until +`glacier vault sync` is run. + +An example of this causing trouble is with the assistant. When a file is +moved into archive/, the assistant that sends it to glacier is able to +trust that it's in glacier and remove the local copy. But other assistants +that also have a copy cannot trust that, and so don't remove their copies. + +I've discussed with glacier-cli's author making git-annex store enough info +in its branch to be able to bootstrap glacier-cli to know about a file. +This seems doable and he had a design; waiting on movement +on the glacier-cli side. diff --git a/doc/bugs/gpg_bundled_with_OSX_build_fails.mdwn b/doc/bugs/gpg_bundled_with_OSX_build_fails.mdwn new file mode 100644 index 0000000000..701e67cc5f --- /dev/null +++ b/doc/bugs/gpg_bundled_with_OSX_build_fails.mdwn @@ -0,0 +1,25 @@ +What steps will reproduce the problem? + +run + + /Applications/git-annex.app/Contents/MacOS/bin/gpg + +from the terminal + +What is the expected output? What do you see instead? + +I expect to see typical gpg output. Instead, I see + + dyld: Library not loaded: /opt/local/lib/libiconv.2.dylib + Referenced from: /Applications/git-annex.app/Contents/MacOS/bin/gpg + Reason: Incompatible library version: gpg requires version 8.0.0 or later, but libiconv.2.dylib provides version 7.0.0 + Trace/BPT trap: 5 + +What version of git-annex are you using? On what operating system? + +git annex Version: 3.20121017 on Mac OS X 10.7.5 + +[[!tag /design/assistant/OSX]] + +> Libraries are now handled better in the OSX app and this should be able +> to happen anymore. [[done]] --[[Joey]] diff --git a/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_1_ec911f920db6c354ba998ffbb5886606._comment b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_1_ec911f920db6c354ba998ffbb5886606._comment new file mode 100644 index 0000000000..574a25b5f7 --- /dev/null +++ b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_1_ec911f920db6c354ba998ffbb5886606._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.194" + subject="comment 1" + date="2012-11-01T02:36:07Z" + content=""" +You need to use runshell to run the commands included in the OSX app. + +(Also, using them outside of git-annex is not really something that is intended to be supported.) +"""]] diff --git a/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_2_bf2a3ab1bbe258bd501ec4b776882adf._comment b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_2_bf2a3ab1bbe258bd501ec4b776882adf._comment new file mode 100644 index 0000000000..268a577ca1 --- /dev/null +++ b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_2_bf2a3ab1bbe258bd501ec4b776882adf._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/6xTna_B_h.ECb6_ftC2dYLytAEwrv36etg_054U-#4c1e7" + nickname="Fake" + subject="comment 2" + date="2012-11-01T11:49:55Z" + content=""" +Ok, thanks for the info. I actually tried running this directly because I was getting the same error from inside of git-annex assistant. When I add a remote server repository with encrypted rsync, I am able to add the server (\"Check this server\" works), but when I click on the \"Use an encrypted rsync repository on the server\" button, I get the following error: + + Internal Server Error + + user error (gpg [\"--quiet\",\"--trust-model\",\"always\",\"--gen-random\",\"--armor\",\"1\",\"512\"] exited 5) +"""]] diff --git a/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_3_c0142427400323c00bd8294415ae32c5._comment b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_3_c0142427400323c00bd8294415ae32c5._comment new file mode 100644 index 0000000000..163ce66c04 --- /dev/null +++ b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_3_c0142427400323c00bd8294415ae32c5._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/6xTna_B_h.ECb6_ftC2dYLytAEwrv36etg_054U-#4c1e7" + nickname="Fake" + subject="comment 3" + date="2012-11-01T11:52:30Z" + content=""" +A little more info. Here is the error from the console log when I try to add the remote server repository. + + Dyld Error Message: + Library not loaded: /opt/local/lib/libncurses.5.dylib + Referenced from: /Applications/git-annex.app/Contents/MacOS/opt/local/lib/libreadline.6.2.dylib + Reason: no suitable image found. Did find: + /usr/lib/libncurses.5.dylib: mach-o, but wrong architecture + /usr/lib/libncurses.5.dylib: mach-o, but wrong architecture +"""]] diff --git a/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_4_b56db4b5afc276f88a2b980e22fda8a0._comment b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_4_b56db4b5afc276f88a2b980e22fda8a0._comment new file mode 100644 index 0000000000..64946b41aa --- /dev/null +++ b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_4_b56db4b5afc276f88a2b980e22fda8a0._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.152.108.194" + subject="comment 4" + date="2012-11-01T13:40:39Z" + content=""" +Ah ok, thanks for the info about this problem. + +Which URL did you download the app from? We have 2 builds. +"""]] diff --git a/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_5_a4eda81e5f927c463593bc48fbe84077._comment b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_5_a4eda81e5f927c463593bc48fbe84077._comment new file mode 100644 index 0000000000..d5a583020b --- /dev/null +++ b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_5_a4eda81e5f927c463593bc48fbe84077._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/6xTna_B_h.ECb6_ftC2dYLytAEwrv36etg_054U-#4c1e7" + nickname="Fake" + subject="comment 5" + date="2012-11-01T14:29:40Z" + content=""" +This is the beta release (http://downloads.kitenet.net/git-annex/OSX/git-annex.dmg.bz2) Right now the link to the daily build is broken. + +I found the link on this page: http://git-annex.branchable.com/install/OSX/ + +Thanks! +"""]] diff --git a/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_6_2f0b9331d16a208883bac586258a7b50._comment b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_6_2f0b9331d16a208883bac586258a7b50._comment new file mode 100644 index 0000000000..abcfb6fb2a --- /dev/null +++ b/doc/bugs/gpg_bundled_with_OSX_build_fails/comment_6_2f0b9331d16a208883bac586258a7b50._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 6" + date="2012-11-26T23:53:01Z" + content=""" +The OSX app has been updated for today's release, and includes a lot of missing libraries. Your testing of it would be appreciated. +"""]] diff --git a/doc/bugs/gpg_needs_--use-agent.mdwn b/doc/bugs/gpg_needs_--use-agent.mdwn new file mode 100644 index 0000000000..d9977909ba --- /dev/null +++ b/doc/bugs/gpg_needs_--use-agent.mdwn @@ -0,0 +1,53 @@ +git-annex gpg encryption fails here when GPG_AGENT_INFO is set, it needs to be supplied --use-agent to work. + +Output from git-annex: + + copy file (to origin...) (gpg) gpg: can't query passphrase in batch mode + gpg: decryption failed: secret key not available + Command gpg ["--batch","--no-tty","--quiet","--trust-model","always","--decrypt"] failed; exit code 2 + + git-annex: user error (Command gpg ["--batch","--no-tty","--quiet","--trust-model","always","--decrypt"] failed; exit code 2) + failed + +Reproduced on command-line: + + [0 zerodogg@browncoats ~]$ echo test > testfile + [0 zerodogg@browncoats ~]$ gpg -e testfile + [0 zerodogg@browncoats ~]$ gpg --batch --no-tty --quiet --trust-model always --decrypt testfile.gpg + gpg: can't query passphrase in batch mode + gpg: decryption failed: secret key not available + [2 zerodogg@browncoats ~]$ gpg --use-agent --batch --no-tty --quiet --trust-model always --decrypt testfile.gpg + test + [0 zerodogg@browncoats ~]$ + +A patch to fix this issue: + + + From 77cb02d15245e9ad6e127388adcda960000fb3b8 Mon Sep 17 00:00:00 2001 + From: Eskild Hustvedt + Date: Fri, 17 Aug 2012 09:21:44 +0200 + Subject: [PATCH] Explicitly enable agent to ensure decryption works + + Otherwise gpg will fail when GPG_AGENT_INFO is set in certain cases. + --- + Utility/Gpg.hs | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + + diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs + index e13afe5..c28b209 100644 + --- a/Utility/Gpg.hs + +++ b/Utility/Gpg.hs + @@ -29,7 +29,7 @@ stdParams params = do + b <- getEnv "GPG_BATCH" + let batch = if isNothing e && isNothing b + then [] + - else ["--batch", "--no-tty"] + + else ["--batch", "--no-tty", "--use-agent"] + return $ batch ++ defaults ++ toCommand params + where + -- be quiet, even about checking the trustdb + -- + 1.7.10.4 + +> Thanks, [[done]].. I never noticed this since I have use-agent set in +> gpg.conf. --[[Joey] diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn new file mode 100644 index 0000000000..dfc9c4cefe --- /dev/null +++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn @@ -0,0 +1,113 @@ +Building commit 805d50c69d40be97baa28735371778df63b5fed6 + +
+x00:git-annex jtang$ cabal install
+Resolving dependencies...
+Configuring dbus-0.10...
+Building dbus-0.10...
+Preprocessing library dbus-0.10...
+[1 of 9] Compiling DBus.Types       ( lib/DBus/Types.hs, dist/build/DBus/Types.o )
+[2 of 9] Compiling DBus.Message     ( lib/DBus/Message.hs, dist/build/DBus/Message.o )
+[3 of 9] Compiling DBus.Wire        ( lib/DBus/Wire.hs, dist/build/DBus/Wire.o )
+[4 of 9] Compiling DBus.Address     ( lib/DBus/Address.hs, dist/build/DBus/Address.o )
+[5 of 9] Compiling DBus             ( lib/DBus.hs, dist/build/DBus.o )
+[6 of 9] Compiling DBus.Introspection ( lib/DBus/Introspection.hs, dist/build/DBus/Introspection.o )
+[7 of 9] Compiling DBus.Transport   ( lib/DBus/Transport.hs, dist/build/DBus/Transport.o )
+
+lib/DBus/Transport.hs:196:72: Not in scope: `getPeerCred'
+cabal: Error: some packages failed to install:
+dbus-0.10 failed during the building phase. The exception was:
+ExitFailure 1
+git-annex-3.20120721 depends on dbus-0.10 which failed to install.
+
+ +The above isn't a git-annex problem but a dbus problem, at first I thought I didn't have the network package installed, but did. I should probably report this problem to the haskell dbus author. + +On a slightly different note, based on the makefile DBUS is not enabled on OSX/BSD so I did not expect the cabal file to use dbus as well, I'm currently interested in poking at the webapp stuff ;) + +Although DBUS is available on OSX from macports I get the feeling that the haskell-dbus package might need some poking before it works properly. + +To continue, pulling, installing the dependancies (dbus is still boned) and building commit 6cecc26206c4a539999b04664136c6f785211a41 + +
+[ 92 of 205] Compiling Utility.Url      ( Utility/Url.hs, tmp/Utility/Url.o )
+
+Utility/Url.hs:39:14: Not in scope: `parseURI'
+
+Utility/Url.hs:73:14: Not in scope: `parseURI'
+
+Utility/Url.hs:88:12: Not in scope: type constructor or class `URI'
+
+Utility/Url.hs:91:30: Not in scope: type constructor or class `URI'
+
+Utility/Url.hs:107:38: Not in scope: `parseURIReference'
+
+Utility/Url.hs:111:95: Not in scope: `relativeTo'
+make: *** [git-annex] Error 1
+
+ +Which then lead me to doing a "cabal install -f-DBus" which spits out the following when trying to link the binary + +
+[206 of 206] Compiling Main             ( git-annex.hs, dist/build/git-annex/git-annex-tmp/Main.o )
+Linking dist/build/git-annex/git-annex ...
+Undefined symbols for architecture x86_64:
+  "_addfds_kqueue", referenced from:
+      _s16v6_info in Kqueue.o
+  "_init_kqueue", referenced from:
+      _s16v3_info in Kqueue.o
+  "_waitchange_kqueue", referenced from:
+      _UtilityziKqueue_zdwa1_info in Kqueue.o
+ld: symbol(s) not found for architecture x86_64
+collect2: ld returned 1 exit status
+cabal: Error: some packages failed to install:
+git-annex-3.20120721 failed during the building phase. The exception was:
+ExitFailure 1
+
+ +I then just tried to build commit with 6cecc26206c4a539999b04664136c6f785211a41 (i have the needed dependancies installed), gives me this... + +
+x00:git-annex jtang$ make
+ghc -O2 -threaded -Wall -ignore-package monads-fd -ignore-package monads-tf -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP --make git-annex Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
+
+Assistant/Threads/MountWatcher.hs:39:0:
+     warning: #warning Building without dbus support; will use mtab polling
+[ 92 of 205] Compiling Utility.Url      ( Utility/Url.hs, tmp/Utility/Url.o )
+
+Utility/Url.hs:98:65:
+    Couldn't match expected type `network-2.3.0.13:Network.URI.URI'
+                with actual type `URI'
+    In the second argument of `mkRequest', namely `u'
+    In the expression: mkRequest requesttype u :: Request_String
+    In an equation for `req':
+        req = mkRequest requesttype u :: Request_String
+make: *** [git-annex] Error 1
+
+ +The latest version of the network package in hackage is network-2.3.0.14 which I have installed, this might also be the reason why dbus is broken. removing network-2.3.0.14 at least makes it happy again. + +to remove the network-2.3.0.14 package + +
+ghc-pkg unregister network-2.3.0.14
+
+ +Hope the above isn't too random of bug/issue report. + +---- + +going through shows that getPeerCred is only available on systems where SO_PEERCRED is supported, *sigh* OSX isn't supported and thus haskell-dbus is broken. Apparently getpeerid is more portable but it isnt supported in the network package. It looks like dbus support on OSX isn't really going to work too well till haskell-dbus gets fixed on OSX (or BSD?) + +> Does OSX acually come with dbus by default, and can you +> use something like `dbus-monitor` to see events when +> plugging in removable drives? If so, this might be worth spending time +> on. + +>> No OSX does not come with dbus by default, the user must install it + +> +> Currently though, dbus is not supposed to be built on non-Linux systems. +> (Well, it might work on Freebsd or something, but I've not tried it.) +> I've fixed the cabal file to only enable it on Linux. [[done]] --[[Joey]] + diff --git a/doc/bugs/host_with_rysnc_installed__44___not_recognized.txt b/doc/bugs/host_with_rysnc_installed__44___not_recognized.txt new file mode 100644 index 0000000000..e5dd7dd4b9 --- /dev/null +++ b/doc/bugs/host_with_rysnc_installed__44___not_recognized.txt @@ -0,0 +1,12 @@ +What steps will reproduce the problem? +Set up a remote server using ssh to a FreeNAS box + +What is the expected output? What do you see instead? +Neither rsync nor git-annex are installed +rsync is in the path, user with permissions is able to run it + +What version of git-annex are you using? On what operating system? +4.20130227 OSX + +Please provide any additional information below. +ssh keys were installed to allow login, when ssh-askpass was not found on osx version diff --git a/doc/bugs/internal_server_error_creating_repo_on_ssh_server.mdwn b/doc/bugs/internal_server_error_creating_repo_on_ssh_server.mdwn new file mode 100644 index 0000000000..7235757bf7 --- /dev/null +++ b/doc/bugs/internal_server_error_creating_repo_on_ssh_server.mdwn @@ -0,0 +1,26 @@ +What steps will reproduce the problem? + +I downloaded the os x assistant today. + +I'd previously installed it but hadn't linked it to another repo. + +I tried to create a remote repo on an ssh server using the assistant. + +What is the expected output? What do you see instead? + +Creating a remote repo or giving an understandable error message. + +What version of git-annex are you using? On what operating system? + +3.20121212 +OS X 10.8.2 (Mountain Lion) + +The remote machine is running debian stable. + +Please provide any additional information below. + +Internal Server Error +user error (gpg ["--quiet","--trust-model","always","--gen-random","--armor","1","512"] exited 127) +git-annex version 3.20121212 + +> [[done]], see comments --[[Joey]] diff --git a/doc/bugs/internal_server_error_creating_repo_on_ssh_server/comment_1_4a2c9338d5c779496049d78e29cf5cbd._comment b/doc/bugs/internal_server_error_creating_repo_on_ssh_server/comment_1_4a2c9338d5c779496049d78e29cf5cbd._comment new file mode 100644 index 0000000000..04b25ecc2b --- /dev/null +++ b/doc/bugs/internal_server_error_creating_repo_on_ssh_server/comment_1_4a2c9338d5c779496049d78e29cf5cbd._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlcxKZHglATIiJXD7jcxfYhkhgeFmcVFqE" + nickname="James" + subject="ignore" + date="2013-01-24T16:33:27Z" + content=""" +I had downloaded and installed the new version of git annex but not restarted the assistant. I rebooting my computer and started the new version (3.20130114) of the assistant and I was able to create a repo. +"""]] diff --git a/doc/bugs/interrupting_migration_causes_problems.mdwn b/doc/bugs/interrupting_migration_causes_problems.mdwn new file mode 100644 index 0000000000..68426e54af --- /dev/null +++ b/doc/bugs/interrupting_migration_causes_problems.mdwn @@ -0,0 +1,52 @@ +Killing a migration from WORM to SHA256 with ^C breaks things; future attempts to do the migration fail: + + #!/bin/bash + + BASE=/tmp/migrate-bug + + set -x + + chmod -R +w $BASE + rm -rf $BASE + mkdir -p $BASE + cd $BASE + + # create annex + git init . + git annex init + + # make a big (sparse) file and add it + dd if=/dev/zero of=bigfile bs=1 count=0 seek=1G + git annex add --backend WORM bigfile + git commit -m 'added bigfile' + + # look at status + git annex status + + # now migrate it, but kill migration during checksum + # Simulate ^C by making a new process group and sending SIGINT + setsid git annex migrate --backend SHA256 bigfile & + PID=$! + sleep 1 + kill -INT -$PID + wait + + # look at status + git annex status + + # this migration fails + git annex migrate --backend SHA256 bigfile + + # but fsck says everything's OK + git annex fsck + +The error: + + migrate bigfile + git-annex: /tmp/migrate-bug/.git/annex/objects/K9/V1/WORM-s1073741824-m1321566308--bigfile/WORM-s1073741824-m1321566308--bigfile: createLink: already exists (File exists) + failed + git-annex: migrate: 1 failed + +> Fixed it to delete the stale temp file. [[done]] +> +> Thanks for making such clear test cases, Jim! --[[Joey]] diff --git a/doc/bugs/javascript_functions_qouting_issue.mdwn b/doc/bugs/javascript_functions_qouting_issue.mdwn new file mode 100644 index 0000000000..7f16462f5d --- /dev/null +++ b/doc/bugs/javascript_functions_qouting_issue.mdwn @@ -0,0 +1,44 @@ +**What is the expected output? What do you see instead?** + +SyntaxError: missing ( before formal parameters + +function longpoll_"sidebar"() { + +reposi...c5e0ead (строка 5, столбец 18) + + +**Please provide any additional information below.** + +functions have illegal characters in their names: *function longpoll_"sidebar"* + + + + + + + + +
+ +
+What excites me about GIT ANNEX is how it fundamentally tracks the +backup and availability of any data you own, and allows you to share +data with a large or small audience, ensuring that the data survives. +
+-- Jason Scott + +Seen on IRC: +
+oh my god, git-annex is amazing
+this is the revolution in fucking with gigantic piles of files that I've been waiting for
+
+ +And then my own story: I have a ton of drives. I have a lot of servers. I +live in a cabin on **dialup** and often have 1 hour on broadband in a week +to get everything I need. Without git-annex, managing all this would not be +possible. It works perfectly for me, not a surprise since I wrote it, but +still, it's a different level of "perfect" than anything I could put +together before. --[[Joey]] diff --git a/doc/tips.mdwn b/doc/tips.mdwn new file mode 100644 index 0000000000..eda84c8672 --- /dev/null +++ b/doc/tips.mdwn @@ -0,0 +1,4 @@ +This page is a place to document tips and techniques for using git-annex. + +[[!inline pages="tips/* and !tips/*/*" archive="yes" +rootpage="tips" postformtext="Add a new tip about:" show=0]] diff --git a/doc/tips/Decentralized_repository_behind_a_Firewall.mdwn b/doc/tips/Decentralized_repository_behind_a_Firewall.mdwn new file mode 100644 index 0000000000..5dbc8d71a3 --- /dev/null +++ b/doc/tips/Decentralized_repository_behind_a_Firewall.mdwn @@ -0,0 +1,59 @@ +If you're anything like me¹, you have a copy of your annex on a computer running at home², set up so you can access it from anywhere like this: + + ssh myhome.no-ip.org + +This is totally great! Except, there is no way for your home computer to pull your changes, because there is no *on-the-go.no-ip.org*. You can get clunky and use a *bare git repository and git push*, but there is a better way. + +First, install *openssh-server* on your *on-the-go* computer + + sudo apt-get install openssh-server # Adjust to your flavor of unix + +Then, log into your *home* computer, with *port forwarding*: + + ssh me@myhome.no-ip.org L 2201:localhost:22 + +Your *home* computer can now ssh into your *on-the-go* computer, as long as you keep the above shell running. + +You can now add your *on-the-go* computer as a remote on your *home* computer. Use the port forwarding shell you just connected with the command above, if you like. + + ssh-keygen -t rsa + ssh-copy-id me@localhost -p 2201 + cd ~/annex + git annex remote add on-the-go ssh://me@localhost:2201/home/myuser/annex + +Now you can run normal annex operations, as long as the port forwarding shell is running³. + + git annex sync + git annex get on-the-go some/big/file + git annex status + +You can add more computers by repeating with a different port, e.g. 2202 or 2203 (or any other). + +If you're security paranoid (like me), read on. If you're not, that's it! Thanks for reading! + +--- +Paranoid Area + +Note you're granting passwordless access to your on-the-go computer to your home computer. I believe that's all right, as long as: + +* Your home computer is really in your home, and not at a friend's house or some datacenter +* Your home computer can be accessed only by ssh, and not HTTP or Samba or NTP or (shoot me now!) FTP +* Only you (and perhaps trustworthy family) have access to your home computer +* You have reasonably strong passwords or key-only logins on both your home and on-the-go computers. +* You regularly install security updates on both computers (sudo apt-get update && sudo apt-get upgrade) + +In any case, the setup is much, much, much more secure than Dropbox. With Dropbox, you have exactly the same setup, but: + +* Your data is stored in some datacenter. It's supposed to be encrypted. It might not be. +* Lot's of people have routine access to your files, and plausible reason to. Bored employees might regularly be doing some 'maintenance work' involving your pictures. +* The dropbox software can do anything it likes on your computer, and it's closed source so you don't know if it does. A disgruntled employee could put a trojan into it. +* Dropbox might have a backdoor for employee access to any file on your computer. This might be done with the best of intentions, but a mal-intentioned or careless employee might still erase things or send sensitive files from your computer by email. +* A truly huge amount of eyes connected to incredibly smart brains have looked at openssh and found it secure. Everybody trusts openssh. With dropbox, there is, well, dropbox. Whoever that is. + +----- + +¹ Me=Carlo, not Joey. I'm pretty sure doing what I wrote here is a good idea, but in case it turns out to be catastrophically dumb, it's my fault, not his. + +² My always-on computer at home is a raspberry pi with a 32GB USB stick. Best self-hosted dropbox you could imagine. + +³ You can just forward the port, but not open a shell, by adding the -N command. This could be useful for connecting on startup, e.g. in /etc/rc.local. I prefer to open the shell to forward the ports, maybe use it, and close it to stop it. diff --git a/doc/tips/Decentralized_repository_behind_a_Firewall/comment_1_78b9035234a690ca5a7c9f3cc78fa092._comment b/doc/tips/Decentralized_repository_behind_a_Firewall/comment_1_78b9035234a690ca5a7c9f3cc78fa092._comment new file mode 100644 index 0000000000..71a1db9c88 --- /dev/null +++ b/doc/tips/Decentralized_repository_behind_a_Firewall/comment_1_78b9035234a690ca5a7c9f3cc78fa092._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.6.49" + subject="comment 1" + date="2012-11-30T16:25:58Z" + content=""" +If you don't trust your home computer with shell access, you can lock it down in `.ssh/authorized_keys` to only be able to run git-annex-shell. See [[forum/Restricting_git-annex-shell_to_a_specific_repository]] +"""]] diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn new file mode 100644 index 0000000000..97f5828d39 --- /dev/null +++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo.mdwn @@ -0,0 +1,19 @@ +I worked out how to retroactively annex a large file that had been checked into a git repo some time ago. I thought this might be useful for others, so I am posting it here. + +Suppose you have a git repo where somebody had checked in a large file you would like to have annexed, but there are a bunch of commits after it and you don't want to loose history, but you also don't want everybody to have to retrieve the large file when they clone the repo. This will re-write history as if the file had been annexed when it was originally added. + +This command works for me, it relies on the current behavior of git which is to use a directory named .git-rewrite/t/ at the top of the git tree for the extracted tree. This will not be fast and it will rewrite history, so be sure that everybody who has a copy of your repo is OK with accepting the new history. If the behavior of git changes, you can specify the directory to use with the -d option. Currently, the t/ directory is created inside the directory you specify, so "-d ./.git-rewrite/" should be roughly equivalent to the default. + +Enough with the explanation, on to the command: +
+git filter-branch --tree-filter 'for FILE in file1 file2 file3;do if [ -f "$FILE" ] && [ ! -L "$FILE" ];then git rm --cached "$FILE";git annex add "$FILE";ln -sf `readlink "$FILE"|sed -e "s:^../../::"` "$FILE";fi;done' --tag-name-filter cat -- --all
+
+ +replace file1 file2 file3... with whatever paths you want retroactively annexed. If you wanted bigfile1.bin in the top dir and subdir1/bigfile2.bin to be retroactively annexed try: +
+git filter-branch --tree-filter 'for FILE in bigfile1.bin subdir1/bigfile2.bin;do if [ -f "$FILE" ] && [ ! -L "$FILE" ];then git rm --cached "$FILE";git annex add "$FILE";ln -sf `readlink "$FILE"|sed -e "s:^../../::"` "$FILE";fi;done' --tag-name-filter cat -- --all
+
+ +**If your repo has tags** then you should take a look at the git-filter-branch man page about the --tag-name-filter option and decide what you want to do. By default this will re-write the tags "nearly properly". + +You'll probably also want to look at the git-filter-branch man page's section titled "CHECKLIST FOR SHRINKING A REPOSITORY" if you want to free up the space in the existing repo that you just changed history on. diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_1_7eaf73fb3355bd706ab18a43790b3c10._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_1_7eaf73fb3355bd706ab18a43790b3c10._comment new file mode 100644 index 0000000000..d4e34e8cda --- /dev/null +++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_1_7eaf73fb3355bd706ab18a43790b3c10._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="173.162.44.162" + subject="comment 1" + date="2012-12-16T00:11:38Z" + content=""" +Man, I wish you'd written this a couple weeks ago. :) I was never able to figure that incantation out and ended up unannexing and re-annexing the whole thing to get rid of the file I inadvertently checked into git instead of the annex. +"""]] diff --git a/doc/tips/Internet_Archive_via_S3.mdwn b/doc/tips/Internet_Archive_via_S3.mdwn new file mode 100644 index 0000000000..8c0f2dde74 --- /dev/null +++ b/doc/tips/Internet_Archive_via_S3.mdwn @@ -0,0 +1,49 @@ +[The Internet Archive](http://www.archive.org/) allows members to upload +collections using an Amazon S3 +[compatible API](http://www.archive.org/help/abouts3.txt), and this can +be used with git-annex's [[special_remotes/S3]] support. + +So, you can locally archive things with git-annex, define remotes that +correspond to "items" at the Internet Archive, and use git-annex to upload +your files to there. Of course, your use of the Internet Archive must +comply with their [terms of service](http://www.archive.org/about/terms.php). + +Sign up for an account, and get your access keys here: + + + # export AWS_ACCESS_KEY_ID=blahblah + # export AWS_SECRET_ACCESS_KEY=xxxxxxx + +Specify `host=s3.us.archive.org` when doing `initremote` to set up +a remote at the Archive. This will enable a special Internet Archive mode: +Encryption is not allowed; you are required to specify a bucket name +rather than having git-annex pick a random one; and you can optionally +specify `x-archive-meta*` headers to add metadata as explained in their +[documentation](http://www.archive.org/help/abouts3.txt). + +[[!template id=note text=""" +/!\ There seems to be a bug in either hS3 or the archive that breaks +authentication when the bucket name contains spaces or upper-case letters.. +use all lowercase and no spaces when making the bucket with `initremote`. +"""]] + + # git annex initremote archive-panama type=S3 \ + host=s3.us.archive.org bucket=panama-canal-lock-blueprints \ + x-archive-meta-mediatype=texts x-archive-meta-language=eng \ + x-archive-meta-title="original Panama Canal lock design blueprints" + initremote archive-panama (Internet Archive mode) ok + # git annex describe archive-panama "a man, a plan, a canal: panama" + describe archive-panama ok + +Then you can annex files and copy them to the remote as usual: + + # git annex add photo1.jpeg --backend=SHA1E + add photo1.jpeg (checksum...) ok + # git annex copy photo1.jpeg --fast --to archive-panama + copy (to archive-panama...) ok + +Note the use of the SHA1E [[backend|backends]]. It makes most sense +to use the WORM or SHA1E backend for files that will be stored in +the Internet Archive, since the key name will be exposed as the filename +there, and since the Archive does special processing of files based on +their extension. diff --git a/doc/tips/assume-unstaged.mdwn b/doc/tips/assume-unstaged.mdwn new file mode 100644 index 0000000000..536772c893 --- /dev/null +++ b/doc/tips/assume-unstaged.mdwn @@ -0,0 +1,31 @@ +[[!meta title="using assume-unstages to speed up git with large trees of annexed files"]] + +Git update-index's assume-unstaged feature can be used to speed +up `git status` and stuff by not statting the whole tree looking for changed +files. + +This feature works quite well with git-annex. Especially because git +annex's files are immutable, so aren't going to change out from under it, +this is a nice fit. If you have a very large tree and `git status` is +annoyingly slow, you can turn it on: + + git config core.ignoreStat true + +When git mv and git rm are used, those changes *do* get noticed, even +on assume-unchanged files. When new files are added, eg by `git annex add`, +they are also noticed. + +There are two gotchas. Both occur because `git add` does not stage +assume-unchanged files. + +1. When an annexed file is moved to a different directory, it updates + the symlink, and runs `git add` on it. So the file will move, + but the changed symlink will not be noticed by git and it will commit a + dangling symlink. +2. When using `git annex migrate`, it changes the symlink and `git adds` + it. Again this won't be committed. + +These can be worked around by running `git update-index --really-refresh` +after performing such operations. I hope that `git add` will be changed +to stage changes to assume-unchanged files, which would remove this +only complication. --[[Joey]] diff --git a/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment b/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment new file mode 100644 index 0000000000..d253feb5b8 --- /dev/null +++ b/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/2djv2EYwk43rfJIAQXjYt_vfuOU-#a11a6" + nickname="Olivier R" + subject="It doesn't work 100%" + date="2012-05-03T21:42:54Z" + content=""" +When you remove tracked files... it doesn't show the new status. it's like if the file was ignored. + + +"""]] diff --git a/doc/tips/automatically_getting_files_on_checkout.mdwn b/doc/tips/automatically_getting_files_on_checkout.mdwn new file mode 100644 index 0000000000..bbb3b302eb --- /dev/null +++ b/doc/tips/automatically_getting_files_on_checkout.mdwn @@ -0,0 +1,15 @@ +Normally git-annex does not retrieve file contents when checking out a +tree. In some use cases, it makes sense to always have the contents of +files available after a `git checkout` or `git update`. This can be +accomplished by installing the following as `.git/hooks/post-checkout` + + #!/bin/sh + # Uses git-annex to get all files in the specified directories + # (relative to the top of the repository) on checkout. + dirs=. + top="$(git rev-parse --show-toplevel)" + for dir in "$dirs"; do git annex get $top/$dir"; done + +By default, all files in the whole repository will be made available. The +`dirs` setting can be configured if you only want to get files in certian +directories. diff --git a/doc/tips/centralised_repository:_starting_from_nothing.mdwn b/doc/tips/centralised_repository:_starting_from_nothing.mdwn new file mode 100644 index 0000000000..b12246d368 --- /dev/null +++ b/doc/tips/centralised_repository:_starting_from_nothing.mdwn @@ -0,0 +1,75 @@ +If you are starting from nothing (no existing `git` or `git-annex` repository) and want to use a server as a centralised repository, try the following steps. + +On the server where you'll hold the "master" repository: + + server$ cd /one/git + server$ mkdir m + server$ cd m + server$ git init --bare + Initialized empty Git repository in /one/git/m/ + server$ git annex init origin + init origin ok + server$ + +Clone that to the laptop: + + laptop$ cd /other + laptop$ git clone ssh://server//one/git/m + Cloning into 'm'... + Warning: No xauth data; using fake authentication data for X11 forwarding. + remote: Counting objects: 5, done. + remote: Compressing objects: 100% (3/3), done. + remote: Total 5 (delta 0), reused 0 (delta 0) + Receiving objects: 100% (5/5), done. + warning: remote HEAD refers to nonexistent ref, unable to checkout. + + laptop$ cd m + laptop$ git annex init laptop + init laptop ok + laptop$ + +Merge the `git-annex` repository (this is the bit that is often +overlooked!): + + laptop$ git annex merge + merge . (merging "origin/git-annex" into git-annex...) + ok + laptop$ + +Add some content: + + laptop$ git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg + "kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg" + addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg (downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...) --2011-12-15 08:13:10-- http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg + Resolving kitenet.net (kitenet.net)... 2001:41c8:125:49::10, 80.68.85.49 + Connecting to kitenet.net (kitenet.net)|2001:41c8:125:49::10|:80... connected. + HTTP request sent, awaiting response... 200 OK + Length: 39362757 (38M) [audio/ogg] + Saving to: `/other/m/.git/annex/tmp/URL--http&c%%kitenet.net%~joey%screencasts%git-annex_coding_in_haskell.ogg' + + 100%[======================================>] 39,362,757 2.31M/s in 17s + + 2011-12-15 08:13:27 (2.21 MB/s) - `/other/m/.git/annex/tmp/URL--http&c%%kitenet.net%~joey%screencasts%git-annex_coding_in_haskell.ogg' saved [39362757/39362757] + + (checksum...) ok + (Recording state in git...) + laptop$ git commit -m 'See Joey play.' + [master (root-commit) 106e923] See Joey play. + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg + laptop$ + +All fine, now push it back to the centralised master: + + laptop$ git push + Counting objects: 20, done. + Delta compression using up to 4 threads. + Compressing objects: 100% (11/11), done. + Writing objects: 100% (18/18), 1.50 KiB, done. + Total 18 (delta 1), reused 1 (delta 0) + To ssh://server//one/git/m + 3ba1386..ad3bc9e git-annex -> git-annex + laptop$ + +You can add more "client" repositories by following the `laptop` +sequence of operations. diff --git a/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment b/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment new file mode 100644 index 0000000000..22857af3e8 --- /dev/null +++ b/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-12-23T19:19:53Z" + content=""" +See also: [[centralized_git_repository_tutorial]] +"""]] diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn new file mode 100644 index 0000000000..00283829fd --- /dev/null +++ b/doc/tips/centralized_git_repository_tutorial.mdwn @@ -0,0 +1,140 @@ +The [[walkthrough]] builds up a decentralized git repository setup, but +git-annex can also be used with a centralized bare repository, just like +git can. This tutorial shows how to set up a centralized repository hosted on +GitHub. + +## set up the repository, and make a checkout + +I've created a repository for technical talk videos, which you can +[fork on Github](https://github.com/joeyh/techtalks). +Or make your own repository on GitHub (or elsewhere) now. + +On your laptop, [[install]] git-annex, and clone the repository: + + # git clone git@github.com:joeyh/techtalks.git + # cd techtalks + +Tell git-annex to use the repository, and describe where this clone is +located: + + # git annex init 'my laptop' + init my laptop ok + +Let's tell git-annex that GitHub doesn't support running git-annex-shell there. +This means you can't store annexed file *contents* on GitHub; it would +really be better to host the bare repository on your own server, which +would not have this limitation. (If you want to do that, check out +[[using_gitolite_with_git-annex]].) + + # git config remote.origin.annex-ignore true + +## add files to the repository + +Add some files, obtained however. + + # youtube-dl -t 'http://www.youtube.com/watch?v=b9FagOVqxmI' + # git annex add *.mp4 + add Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 (checksum) ok + (Recording state in git...) + # git commit -m "added a video. I have not watched it yet but it sounds interesting" + +This file is available directly from the web; so git-annex can download it: + + # git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg + addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg + (downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...) + (checksum...) ok + (Recording state in git...) + # git commit -a -m 'added a screencast I made' + +Feel free the rename the files, etc, using normal git commands: + + # git mv Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 Haskell_Amuse_Bouche.mp4 + # git mv kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg git-annex_coding_in_haskell.ogg + # git commit -m 'better filenames' + +Now push your changes back to the central repository. This first time, +remember to push the git-annex branch, which is used to track the file +contents. + + # git push origin master git-annex + To git@github.com:joeyh/techtalks.git + * [new branch] master -> master + * [new branch] git-annex -> git-annex + +That push went fast, because it didn't upload large videos to GitHub. +To check this, you can ask git-annex where the contents of the videos are: + + # git annex whereis + whereis Haskell_Amuse_Bouche.mp4 (1 copy) + 767e8558-0955-11e1-be83-cbbeaab7fff8 -- here + ok + whereis git-annex_coding_in_haskell.ogg (2 copies) + 00000000-0000-0000-0000-000000000001 -- web + 767e8558-0955-11e1-be83-cbbeaab7fff8 -- here + ok + +## make more checkouts + +So far you have a central repository, and a checkout on a laptop. +Let's make another checkout that's used as a backup. You can put it anywhere +you like, just make it be somewhere your laptop can access. A few options: + +* Put it on a USB drive that you can plug into the laptop. +* Put it on a desktop. +* Put it on some server in the local network. +* Put it on a remote VPS. + +I'll use the VPS option, but these instructions should work for +any of the above. + + # ssh server + server# sudo apt-get install git-annex + +Clone the central repository as before. (If the clone fails, you need +to add your server's ssh public key to github -- see +[this page](http://help.github.com/ssh-issues/).) + + server# git clone git@github.com:joeyh/techtalks.git + server# cd techtalks + server# git config remote.origin.annex-ignore true + server# git annex init 'backup' + init backup (merging origin/git-annex into git-annex...) ok + +Notice that the server does not have the contents of any of the files yet. +If you run `ls`, you'll see broken symlinks. We want to populate this +backup with the file contents, by copying them from your laptop. + +Back on your laptop, you need to configure a git remote for the backup. +Adjust the ssh url as needed to point to wherever the backup is. (If it +was on a local USB drive, you'd use the path to the repository instead.) + + # git remote add backup ssh://server/~/techtalks + +Now git-annex on your laptop knows how to reach the backup repository, +and can do things like copy files to it: + + # git annex copy --to backup git-annex_coding_in_haskell.ogg + copy git-annex_coding_in_haskell.ogg (checking backup...) + 12877824 2% 255.11kB/s 00:00 + ok + +You can also `git annex move` files to it, to free up space on your laptop. +And then you can `git annex get` files back to your laptop later on, as +desired. + +After you use git-annex to move files around, remember to push, +which will broadcast its updated location information. + + # git push + +## take it farther + +Of course you can create as many checkouts as you desire. If you have a +desktop machine too, you can make a checkout there, and use `git remote +add` to also let your desktop access the backup repository. + +You can add remotes for each direct connection between machines you find you +need -- so make the laptop have the desktop as a remote, and the desktop +have the laptop as a remote, and then on either machine git-annex can +access files stored on the other. diff --git a/doc/tips/emacs_integration.mdwn b/doc/tips/emacs_integration.mdwn new file mode 100644 index 0000000000..12f16888a6 --- /dev/null +++ b/doc/tips/emacs_integration.mdwn @@ -0,0 +1,20 @@ +bergey has developed an emacs mode for browsing git-annex repositories, +dired style. + + + +Locally available files are colored differently, and pressing g runs +`git annex get` on the file at point. + +---- + +John Wiegley has developed a brand new git-annex interaction mode for +Emacs, which aims to integrate with the standard facilities +(C-x C-q, M-x dired, etc) rather than invent its own interface. + + + +He has also added support to org-attach; if +`org-attach-git-annex-cutoff' is non-nil and smaller than the size + of the file you're attaching then org-attach will `git annex add the +file`; otherwise it will "git add" it. diff --git a/doc/tips/finding_duplicate_files.mdwn b/doc/tips/finding_duplicate_files.mdwn new file mode 100644 index 0000000000..94fc85400e --- /dev/null +++ b/doc/tips/finding_duplicate_files.mdwn @@ -0,0 +1,21 @@ +Maybe you had a lot of files scattered around on different drives, and you +added them all into a single git-annex repository. Some of the files are +surely duplicates of others. + +While git-annex stores the file contents efficiently, it would still +help in cleaning up this mess if you could find, and perhaps remove +the duplicate files. + +Here's a command line that will show duplicate sets of files grouped together: + + git annex find --include '*' --format='${file} ${escaped_key}\n' | \ + sort -k2 | uniq --all-repeated=separate -f1 | \ + sed 's/ [^ ]*$//' + +Here's a command line that will remove one of each duplicate set of files: + + git annex find --include '*' --format='${file} ${escaped_key}\n' | \ + sort -k2 | uniq --repeated -f1 | sed 's/ [^ ]*$//' | \ + xargs -d '\n' git rm + +--[[Joey]] diff --git a/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment b/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment new file mode 100644 index 0000000000..d1bd4475e5 --- /dev/null +++ b/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://adamspiers.myopenid.com/" + nickname="Adam" + subject="Cool" + date="2011-12-23T19:16:50Z" + content=""" +Very nice :) Just for reference, here's [my Perl implementation](https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups). As per [this discussion](http://git-annex.branchable.com/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/#comment-fb15d5829a52cd05bcbd5dc53edaffb2) it would be interesting to benchmark these two approaches and see if one is substantially more efficient than the other w.r.t. CPU and memory usage. +"""]] diff --git a/doc/tips/finding_duplicate_files/comment_2_900eafe0a781018ff44b35ac232e3ad3._comment b/doc/tips/finding_duplicate_files/comment_2_900eafe0a781018ff44b35ac232e3ad3._comment new file mode 100644 index 0000000000..605c804dd8 --- /dev/null +++ b/doc/tips/finding_duplicate_files/comment_2_900eafe0a781018ff44b35ac232e3ad3._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="bremner" + ip="156.34.89.108" + subject="problems with spaces in filenames" + date="2012-09-05T02:12:18Z" + content=""" +note that the sort -k2 doesn't work right for filenames with spaces in them. On the other hand, git-rm doesn't seem to like the escaped names from escaped_file. +"""]] diff --git a/doc/tips/finding_duplicate_files/comment_3._comment b/doc/tips/finding_duplicate_files/comment_3._comment new file mode 100644 index 0000000000..44eeb50759 --- /dev/null +++ b/doc/tips/finding_duplicate_files/comment_3._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="mhameed" + ip="82.32.202.53" + subject="problems with spaces in filenames" + date="Wed Sep 5 09:38:56 BST 2012" + content=""" + +Spaces, and other special chars can make filename handeling ugly. +If you don't have a restriction on keeping the exact filenames, then +it might be easiest just to get rid of the problematic chars. + + #!/bin/bash + + function process() { + dir="$1" + echo "processing $dir" + pushd $dir >/dev/null 2>&1 + + for fileOrDir in *; do + nfileOrDir=`echo "$fileOrDir" | sed -e 's/\[//g' -e 's/\]//g' -e 's/ /_/g' -e "s/'//g" ` + if [ "$fileOrDir" != "$nfileOrDir" ]; then + echo renaming $fileOrDir to $nfileOrDir + git mv "$fileOrDir" "$nfileOrDir" + else + echo "skipping $fileOrDir, no need to rename." + fi + done + + find ./ -mindepth 1 -maxdepth 1 -type d | while read d; do + process "$d" + done + popd >/dev/null 2>&1 + } + + process . + +Maybe you can run something like this before checking for duplicates. + +"""]] diff --git a/doc/tips/finding_duplicate_files/comment_4_1494143a74cc1e9fbe4720c14b73d42b._comment b/doc/tips/finding_duplicate_files/comment_4_1494143a74cc1e9fbe4720c14b73d42b._comment new file mode 100644 index 0000000000..f1a86f43ce --- /dev/null +++ b/doc/tips/finding_duplicate_files/comment_4_1494143a74cc1e9fbe4720c14b73d42b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="bremner" + ip="156.34.89.108" + subject="more about spaces..." + date="2012-09-09T19:33:01Z" + content=""" +Ironically, previous renaming to remove spaces, plus some synching is how I ended up with these duplicates. For what it is worth, aspiers perl script worked out for me with a small modification. I just only printed out the duplicates with spaces in them (quoted). +"""]] diff --git a/doc/tips/migrating_data_to_a_new_backend.mdwn b/doc/tips/migrating_data_to_a_new_backend.mdwn new file mode 100644 index 0000000000..b9acb8bd15 --- /dev/null +++ b/doc/tips/migrating_data_to_a_new_backend.mdwn @@ -0,0 +1,16 @@ +Maybe you started out using the WORM backend, and have now configured +git-annex to use SHA1. But files you added to the annex before still +use the WORM backend. There is a simple command that can migrate that +data: + + # git annex migrate my_cool_big_file + migrate my_cool_big_file (checksum...) ok + +You can only migrate files whose content is currently available. Other +files will be skipped. + +After migrating a file to a new backend, the old content in the old backend +will still be present. That is necessary because multiple files +can point to the same content. The `git annex unused` subcommand can be +used to clear up that detritus later. Note that hard links are used, +to avoid wasting disk space. diff --git a/doc/tips/powerful_file_matching.mdwn b/doc/tips/powerful_file_matching.mdwn new file mode 100644 index 0000000000..d5d29377c4 --- /dev/null +++ b/doc/tips/powerful_file_matching.mdwn @@ -0,0 +1,36 @@ +git-annex has a powerful syntax for making it act on only certian files. + +The simplest thing is to exclude some files, using wild cards: + + git annex get --exclude '*.mp3' --exclude '*.ogg' + +But you can also exclude files that git-annex's [[location_tracking]] +information indicates are present in a given repository. For example, +if you want to populate newarchive with files, but not those already +on oldarchive, you could do it like this: + + git annex copy --not --in oldarchive --to newarchive + +Without the --not, --in makes it act on files that *are* in the specified +repository. So, to remove files that are on oldarchive: + + git annex drop --in oldarchive + +Or maybe you're curious which files have a lot of copies, and then +also want to know which files have only one copy: + + git annex find --copies 7 + git annex find --not --copies 2 + +The above are the simple examples of specifying what files git-annex +should act on. But you can specify anything you can dream up by combining +the things above, with --and --or -( and -). Those last two strange-looking +options are parentheses, for grouping other options. You will probably +have to escape them from your shell. + +Here are the mp3 files that are in either of two repositories, but have +less than 3 copies: + + git annex find --not --exclude '*.mp3' --and \ + -\( --in usbdrive --or --in archive -\) --and \ + --not --copies 3 diff --git a/doc/tips/recover_data_from_lost+found.mdwn b/doc/tips/recover_data_from_lost+found.mdwn new file mode 100644 index 0000000000..48ef2a1d73 --- /dev/null +++ b/doc/tips/recover_data_from_lost+found.mdwn @@ -0,0 +1,19 @@ +Suppose something goes wrong, and fsck puts all the files in lost+found. +It's actually very easy to recover from this disaster. + +First, check out the git repository again. Then, in the new checkout: + + $ mkdir recovered-content + $ sudo mv ../lost+found/* recovered-content + $ sudo chown you:you recovered-content + $ chmod -R u+w recovered-content + $ git annex add recovered-content + $ git rm recovered-content + $ git commit -m "recovered some content" + $ git annex fsck + +The way that works is that when git-annex adds the same content that was in +the repository before, all the old links to that content start working +again. This works particularly well if the SHA* backends are used, but even +with the default backend it will work pretty well, as long as fsck +preserved the modification time of the files. diff --git a/doc/tips/setup_a_public_repository_on_a_web_site.mdwn b/doc/tips/setup_a_public_repository_on_a_web_site.mdwn new file mode 100644 index 0000000000..2b2363d3ec --- /dev/null +++ b/doc/tips/setup_a_public_repository_on_a_web_site.mdwn @@ -0,0 +1,28 @@ +Let's say you want to distribute some big files to the whole world. +You can of course, just drop them onto a website. But perhaps you'd like to +use git-annex to manage those files. And as an added bonus, why not let +anyone in the world clone your site and use `git-annex get`! + +My site like this is [downloads.kitenet.net](http://downloads.kitenet.net). +Here's how I set it up. --[[Joey]] + +1. Set up a web site. I used Apache, and configured it to follow symlinks. + `Options FollowSymLinks` +2. Put some files on the website. Make sure it works. +4. `git init; git annex init` +3. We want users to be able to clone the git repository over http, because + git-annex can download files from it over http as well. For this to + work, `git update-server-info` needs to get run after commits. The + git `post-update` hook will take care of this, you just need to enable + the hook. `chmod +x .git/hooks/post-update` +5. `git annex add; git commit -m added` +6. Make sure users can still download files from the site directly. +7. Instruct advanced users to clone a http url that ends with the "/.git/" + directory. For example, for downloads.kitenet.net, the clone url + is `http://downloads.kitenet.net/.git/` + +When users clone over http, and run git-annex, it will +automatically learn all about your repository and be able to download files +right out of it, also using http. + +Enjoy! diff --git a/doc/tips/untrusted_repositories.mdwn b/doc/tips/untrusted_repositories.mdwn new file mode 100644 index 0000000000..cdb5da7c3d --- /dev/null +++ b/doc/tips/untrusted_repositories.mdwn @@ -0,0 +1,28 @@ +Suppose you have a USB thumb drive and are using it as a git annex +repository. You don't trust the drive, because you could lose it, or +accidentally run it through the laundry. Or, maybe you have a drive that +you know is dying, and you'd like to be warned if there are any files +on it not backed up somewhere else. Maybe the drive has already died +or been lost. + +You can let git-annex know that you don't trust a repository, and it will +adjust its behavior to avoid relying on that repositories's continued +availability. + + # git annex untrust usbdrive + untrust usbdrive ok + +Now when you do a fsck, you'll be warned appropriately: + + # git annex fsck . + fsck my_big_file + Only these untrusted locations may have copies of this file! + 05e296c4-2989-11e0-bf40-bad1535567fe -- portable USB drive + Back it up to trusted locations with git-annex copy. + failed + +Also, git-annex will refuse to drop a file from elsewhere just because +it can see a copy on the untrusted repository. + +It's also possible to tell git-annex that you have an unusually high +level of trust for a repository. See [[trust]] for details. diff --git a/doc/tips/using_Amazon_Glacier.mdwn b/doc/tips/using_Amazon_Glacier.mdwn new file mode 100644 index 0000000000..5e7131eeb2 --- /dev/null +++ b/doc/tips/using_Amazon_Glacier.mdwn @@ -0,0 +1,75 @@ +Amazon Glacier provides low-cost storage, well suited for archiving and +backup. But it takes around 4 hours to get content out of Glacier. + +Recent versions of git-annex support Glacier. To use it, you need to have +[glacier-cli](http://github.com/basak/glacier-cli) installed. + +First, export your Amazon AWS credentials: + + # export AWS_ACCESS_KEY_ID="08TJMT99S3511WOZEP91" + # export AWS_SECRET_ACCESS_KEY="s3kr1t" + +Now, create a gpg key, if you don't already have one. This will be used +to encrypt everything stored in Glacier, for your privacy. Once you have +a gpg key, run `gpg --list-secret-keys` to look up its key id, something +like "2512E3C7" + +Next, create the Glacier remote. + + # git annex initremote glacier type=glacier encryption=2512E3C7 + initremote glacier (encryption setup with gpg key C910D9222512E3C7) (gpg) ok + +The configuration for the Glacier remote is stored in git. So to make another +repository use the same Glacier remote is easy: + + # cd /media/usb/annex + # git pull laptop + # git annex initremote glacier + initremote glacier (gpg) ok + +Now the remote can be used like any other remote. + + # git annex move my_cool_big_file --to glacier + copy my_cool_big_file (gpg) (checking glacier...) (to glacier...) ok + +But, when you try to get a file out of Glacier, it'll queue a retrieval +job: + + # git annex get my_cool_big_file + get my_cool_big_file (from glacier...) (gpg) + glacier: queued retrieval job for archive 'GPGHMACSHA1--862afd4e67e3946587a9ef7fa5beb4e8f1aeb6b8' + Recommend you wait up to 4 hours, and then run this command again. + failed + +Like it says, you'll need to run the command again later. Let's remember to +do that: + + # at now + 4 hours + at> git annex get my_cool_big_file + +Another oddity of Glacier is that git-annex is never entirely sure +if a file is still in Glacier. Glacier inventories take hours to retrieve, +and even when retrieved do not necessarily represent the current state. + +So, git-annex plays it safe, and avoids trusting the inventory: + + # git annex copy important_file --to glacier + copy important_file (gpg) (checking glacier...) (to glacier...) ok + # git annex drop important_file + drop important_file (gpg) (checking glacier...) + Glacier's inventory says it has a copy. + However, the inventory could be out of date, if it was recently removed. + (Use --trust-glacier if you're sure it's still in Glacier.) + + (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + +Like it says, you can use `--trust-glacier` if you're sure +Glacier's inventory is correct and up-to-date. + +A final potential gotcha with Glacier is that glacier-cli keeps a local +mapping of file names to Glacier archives. If this cache is lost, or +you want to retrieve files on a different box than the one that put them in +glacier, you'll need to use `glacier vault sync` to rebuild this cache. + +See [[special_remotes/Glacier]] for details. diff --git a/doc/tips/using_Amazon_S3.mdwn b/doc/tips/using_Amazon_S3.mdwn new file mode 100644 index 0000000000..19997d0265 --- /dev/null +++ b/doc/tips/using_Amazon_S3.mdwn @@ -0,0 +1,37 @@ +git-annex extends git's usual remotes with some [[special_remotes]], that +are not git repositories. This way you can set up a remote using say, +Amazon S3, and use git-annex to transfer files into the cloud. + +First, export your Amazon AWS credentials: + + # export AWS_ACCESS_KEY_ID="08TJMT99S3511WOZEP91" + # export AWS_SECRET_ACCESS_KEY="s3kr1t" + +Now, create a gpg key, if you don't already have one. This will be used +to encrypt everything stored in S3, for your privacy. Once you have +a gpg key, run `gpg --list-secret-keys` to look up its key id, something +like "2512E3C7" + +Next, create the S3 remote, and describe it. + + # git annex initremote cloud type=S3 encryption=2512E3C7 + initremote cloud (encryption setup with gpg key C910D9222512E3C7) (checking bucket) (creating bucket in US) (gpg) ok + # git annex describe cloud "at Amazon's US datacenter" + describe cloud ok + +The configuration for the S3 remote is stored in git. So to make another +repository use the same S3 remote is easy: + + # cd /media/usb/annex + # git pull laptop + # git annex initremote cloud + initremote cloud (gpg) (checking bucket) ok + +Now the remote can be used like any other remote. + + # git annex copy my_cool_big_file --to cloud + copy my_cool_big_file (gpg) (checking cloud...) (to cloud...) ok + # git annex move video/hackity_hack_and_kaxxt.mov --to cloud + move video/hackity_hack_and_kaxxt.mov (checking cloud...) (to cloud...) ok + +See [[special_remotes/S3]] for details. diff --git a/doc/tips/using_Amazon_S3/comment_1_666a26f95024760c99c627eed37b1966._comment b/doc/tips/using_Amazon_S3/comment_1_666a26f95024760c99c627eed37b1966._comment new file mode 100644 index 0000000000..60d96cb44e --- /dev/null +++ b/doc/tips/using_Amazon_S3/comment_1_666a26f95024760c99c627eed37b1966._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY" + nickname="Matt" + subject="ANNEX_S3 vs AWS for keys" + date="2012-05-29T12:24:25Z" + content=""" +The instructions state ANNEX_S3_ACCESS_KEY_ID and ANNEX_SECRET_ACCESS_KEY but git-annex cannot connect with those constants. git-annex tells me to set both \"AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY\" instead, which works. This is with Xubuntu 12.04. +"""]] diff --git a/doc/tips/using_Amazon_S3/comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment b/doc/tips/using_Amazon_S3/comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment new file mode 100644 index 0000000000..dc809cb126 --- /dev/null +++ b/doc/tips/using_Amazon_S3/comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.81.112" + subject="comment 2" + date="2012-05-29T19:10:42Z" + content=""" +Thanks, I've fixed that. (You could have too.. this is a wiki ;) +"""]] diff --git a/doc/tips/using_Google_Cloud_Storage.mdwn b/doc/tips/using_Google_Cloud_Storage.mdwn new file mode 100644 index 0000000000..d44e4f17f8 --- /dev/null +++ b/doc/tips/using_Google_Cloud_Storage.mdwn @@ -0,0 +1,9 @@ +[Google Cloud Storage](https://cloud.google.com/products/cloud-storage) +supports the same API as Amazon S3, so the +[[S3 special remote|special_remotes/S3]] can be used with it. +Here is a configuration example: + + git annex initremote cloud type=S3 encryption=none host=storage.googleapis.com port=80 + +Thanks to jterrance for the [original tip](https://gist.github.com/4576324). +--[[Joey]] diff --git a/doc/tips/using_box.com_as_a_special_remote.mdwn b/doc/tips/using_box.com_as_a_special_remote.mdwn new file mode 100644 index 0000000000..6616d0a1e2 --- /dev/null +++ b/doc/tips/using_box.com_as_a_special_remote.mdwn @@ -0,0 +1,71 @@ +[Box.com](http://box.com/) is a file storage service, currently notable +for providing 50 gb of free storage if you sign up with its Android client. +(Or a few gb free otherwise.) + +git-annex can use Box as a [[special remote|special_remotes]]. +Recent versions of git-annex make this very easy to set up: + + WEBDAV_USERNAME=you@example.com WEBDAV_PASSWORD=xxxxxxx git annex initremote box.com type=webdav url=https://www.box.com/dav/git-annex chunksize=75mb encryption=you@example.com + +Note the use of chunksize; Box has a 100 mb maximum file size, and this +breaks up large files into chunks before that limit is reached. + +# old davfs2 method + +This method is deprecated, but still documented here just in case. +Note that the files stored using this method cannot reliably be retreived +using the webdav special remote. + +## davfs2 setup + +* First, install + the [davfs2](http://savannah.nongnu.org/projects/davfs2) program, + which can mount Box using WebDAV. On Debian, just `sudo apt-get install davfs2` +* Allow users to mount davfs filesystems, by ensuring that + `/sbin/mount.davfs` is setuid root. On Debian, just `sudo dpkg-reconfigure davfs2` +* Add yourself to the davfs2 group. + + sudo adduser $(whoami) davfs2 + +* Edit `/etc/fstab`, and add a line to mount Box using davfs. + + sudo mkdir -p /media/box.com + echo "https://www.box.com/dav/ /media/box.com davfs noauto,user 0 0" | sudo tee -a /etc/fstab + +* Create `~/.davfs2/davfs2.conf` with some important settings: + + mkdir ~/.davfs2/ + echo use_locks 0 > ~/.davfs2/davfs2.conf + echo cache_size 1 >> ~/.davfs2/davfs2.conf + echo delay_upload 0 >> ~/.davfs2/davfs2.conf + +* Create `~/.davfs2/secrets`. This file contains your Box.com login and password. + Your login is probably the email address you signed up with. + + echo "/media/box.com joey@kitenet.net mypassword" > ~/.davfs2/secrets + chmod 600 ~/.davfs2/secrets + +* Now you should be able to mount Box, as a non-root user: + + mount /media/box.com + +## git-annex setup + +You need git-annex version 3.20120303 or newer, which adds support for chunking +files larger than Box's 100 mb limit. + +Create the special remote, in your git-annex repository. +** This example is non-encrypted; fill in your gpg key ID for a securely +encrypted special remote! ** + + git annex initremote box.com type=directory directory=/media/box.com chunksize=2mb encryption=none + +Now git-annex can copy files to box.com, get files from it, etc, just like +with any other special remote. + + % git annex copy bigfile --to box.com + bigfile (to box.com...) ok + % git annex drop bigfile + bigfile (checking box.com...) ok + % git annex get bigfile + bigfile (from box.com...) ok diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn new file mode 100644 index 0000000000..594d8c480f --- /dev/null +++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn @@ -0,0 +1,59 @@ +## Intro + +This tip is based on my (Matt Ford) experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address. + +I'm not using a bare repository that allows pushing (an alternative solution) nor do I fancy allowing `git push` to run against my desktop checked out repository (perhaps I worry over nothing?) + +None of this is really `git annex` specific but I think it is useful to know... + +## Dealing with no fixed hostname + +Essentially set up two repos as per the [[walkthrough]]. + +Desktop as follows: + + cd ~/annex + git init + git annex init "desktop" + +And the laptop like this + + git clone ssh://desktop/annex + git init + git annex init "laptop" + +Now we want to add the the repos as remotes of each other. + +For the laptop it is easy: + + git remote add desktop ssh://desktop/~/annex + +However for the desktop to add an ever changing laptops hostname it's a little tricky. We make use of remote SSH tunnels to do this. Essentially we have the laptop (which always knows it's own name and address and knows the address of the desktop) create a tunnel starting on an arbitrary port at the desktop and heads back to the laptop on it's own SSH server port (22). + +To do this make part of your laptop's SSH config look like this: + + Host desktop + User matt + HostName desktop.example.org + RemoteForward 2222 localhost:22 + +Now on the desktop to connect over the tunnel to the laptop's SSH port you need this: + + Host laptop + User matt + HostName localhost + port 2222 + +So to add the desktop's remote: + +a) From the laptop ensure the tunnel is up + + ssh desktop + +b) From the desktop add the remote + + git remote add laptop ssh://laptop/~/annex + +So now you can work on the train, pop on the wifi at work upon arrival, and sync up with a `git pull && git annex get`. + +An alternative solution may be to use direct tunnels over Openvpn. diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment new file mode 100644 index 0000000000..e627ead47c --- /dev/null +++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://adamspiers.myopenid.com/" + nickname="Adam" + subject="comment 1" + date="2011-12-23T13:31:33Z" + content=""" +ControlPersist is awesome - thanks! + +Here's [an alternative, git-specific approach](http://thread.gmane.org/gmane.comp.version-control.home-dir/502). +"""]] diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn new file mode 100644 index 0000000000..301e4a063d --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex.mdwn @@ -0,0 +1,89 @@ +[Gitolite](https://github.com/sitaramc/gitolite) is a git repository +manager. Here's how to add git-annex support to gitolite, so you can +`git annex copy` files to a gitolite repository, and `git annex get` +files from it. + +Warning : The method described here works with gitolite version g2, avaible in the g2 branch on github. There is an experimental support for g3 in the git-annex branch, if you tested it please add some feedback. + +A nice feature of using gitolite with git-annex is that users can be given +read-only access to a repository, and this allows them to `git annex get` +file contents, but not change anything. + +First, you need new enough versions: + +* gitolite 2.2 is needed -- this version contains a git-annex-shell ADC + and supports "ua" ADCs. +* git-annex 3.20111016 or newer needs to be installed on the gitolite + server. Don't install an older version, it wouldn't be secure! + +And here's how to set it up. The examples are for gitolite as installed +on Debian with apt-get, but the changes described can be made to any +gitolite installation, just with different paths. + +Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so. + +
+echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;"' >>~gitolite/.gitolite.rc
+
+ +Make the ADC directory, and a "ua" subdirectory. + +
   
+mkdir -p /usr/local/lib/gitolite/adc/ua
+
+ +Install the git-annex-shell ADC into the "ua" subdirectory from the gitolie repository. + +
   
+cd /usr/local/lib/gitolite/adc/ua/
+cp gitolite/contrib/adc/git-annex-shell .
+
+ +Now all gitolite repositories can be used with git-annex just as any +ssh remote normally would be used. For example: + +
+# git clone gitolite@localhost:testing
+Cloning into testing...
+Receiving objects: 100% (18/18), done.
+# cd testing
+# git annex init
+init  ok
+# cp /etc/passwd my-cool-big-file
+# git annex add my-cool-big-file
+add my-cool-big-file ok
+(Recording state in git...)
+# git commit -m added
+[master d36c8b4] added
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 120000 my-cool-big-file
+# git push --all
+Counting objects: 17, done.
+Delta compression using up to 2 threads.
+Compressing objects: 100% (12/12), done.
+Writing objects: 100% (14/14), 1.39 KiB, done.
+Total 14 (delta 0), reused 1 (delta 0)
+To gitolite@localhost:testing
+   c552a38..db4653e  git-annex -> git-annex
+   29cd204..d36c8b4  master -> master
+# git annex copy --to origin
+copy my-cool-big-file (checking origin...) (to origin...) 
+WORM-s2502-m1318875140--my-cool-big-file
+        2502 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)
+
+sent 2606 bytes  received 31 bytes  1758.00 bytes/sec
+total size is 2502  speedup is 0.95
+ok
+
+ + +### Troubleshooting + +I got an error like this when setting up gitolite *after* setting up a local git repo and git annex: + +
+git-annex-shell: First run: git-annex init
+Command ssh ["git@git.example.com","git-annex-shell 'configlist' '/~/myrepo.git'"] failed; exit code 1
+
+ +because I forgot to "git push --all" after adding the new gitolite remote. diff --git a/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment b/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment new file mode 100644 index 0000000000..807180660b --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="http://www.openid.albertlash.com/openid/" + ip="71.178.29.218" + subject="comment 1" + date="2011-12-24T06:08:45Z" + content=""" +Looks like you are missing a closing double quote on the line: + + +echo '$GL_ADC_PATH = \"/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc + +right after /; + +I got this working by the way - great stuff. +"""]] diff --git a/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment b/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment new file mode 100644 index 0000000000..007a009ea1 --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-12-24T16:54:31Z" + content=""" +I've fixed the typo (anyone can edit pages in this wiki FWIW.) +"""]] diff --git a/doc/tips/using_gitolite_with_git-annex/comment_3_807035f38509ccb9f93f1929ecd37417._comment b/doc/tips/using_gitolite_with_git-annex/comment_3_807035f38509ccb9f93f1929ecd37417._comment new file mode 100644 index 0000000000..243764054d --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_3_807035f38509ccb9f93f1929ecd37417._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="bremner" + ip="156.34.79.193" + subject="repo name conventions?" + date="2011-12-30T21:41:13Z" + content=""" +I'm confused by the fact that the git-annex-shell adc rejects any repo names that don't start with /~/ since none of my repos start that way. It seems work ok if I just delete /\~ from the front of the regex, but I feel like I must be missing something. +"""]] diff --git a/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment b/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment new file mode 100644 index 0000000000..c53ce01d94 --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment @@ -0,0 +1,33 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-12-31T00:29:45Z" + content=""" +Well a repo url like `gitolite@localhost:testing` puts it in the gitolite user's /~/testing + +This worked when I added the gitolite stuff, anyway.. Let's see if it still does: + +
+joey@gnu:~/tmp>mkdir g
+joey@gnu:~/tmp>cd g
+joey@gnu:~/tmp/g>git init
+Initialized empty Git repository in /home/joey/tmp/g/.git/
+joey@gnu:~/tmp/g>git annex init
+init  ok
+joey@gnu:~/tmp/g>git remote add test 'gitolite@localhost:testing'
+joey@gnu:~/tmp/g>touch foo
+joey@gnu:~/tmp/g>git annex add foo
+add foo (checksum...) ok
+(Recording state in git...)
+joey@gnu:~/tmp/g>git annex copy foo --to test --debug
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"foo\"]
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"check-attr\",\"annex.numcopies\",\"-z\",\"--stdin\"]
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"show-ref\",\"git-annex\"]
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"cat-file\",\"--batch\"]
+Running: ssh [\"-4\",\"gitolite@localhost\",\"git-annex-shell 'configlist' '/~/testing'\"]
+
+ +Still seems right, the ADC's regexp will match this the git-annex shell command. +"""]] diff --git a/doc/tips/using_gitolite_with_git-annex/comment_5_f688309532d2993630e9e72e87fb9c46._comment b/doc/tips/using_gitolite_with_git-annex/comment_5_f688309532d2993630e9e72e87fb9c46._comment new file mode 100644 index 0000000000..052fc90d6b --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_5_f688309532d2993630e9e72e87fb9c46._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="bremner" + ip="156.34.79.193" + subject="gitolite gets different paths for different urls" + date="2011-12-31T01:50:49Z" + content=""" +I guess there is some path rewriting going in in gitolite proper because if try a url of the form +ssh://git@localhost/testing, then it still works with gitolite, but fails with the ADC because +the repo is passed as /testing: +
+Running: ssh [\"git@host\",\"git-annex-shell 'configlist' '/recommend'\"]
+Running: ssh [\"git@host\",\"git-annex-shell 'configlist' '/recommend'\"]
+
+ +What I have to ask Sitaram and or find in the docs is if this is a bug or a feature in gitolite. I can see how the leading slash would get swallowed up by this line +
+$repo = \"'$REPO_BASE/$repo.git'\"
+
+in gl-auth-command, but I guess that isn't the whole story. +"""]] diff --git a/doc/tips/using_gitolite_with_git-annex/comment_6_3e203e010a4df5bf03899f867718adc5._comment b/doc/tips/using_gitolite_with_git-annex/comment_6_3e203e010a4df5bf03899f867718adc5._comment new file mode 100644 index 0000000000..ce888cb135 --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_6_3e203e010a4df5bf03899f867718adc5._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="bremner" + ip="156.34.79.193" + subject="ssh://gitolite-host/repo-name is supposed to work" + date="2011-12-31T03:34:17Z" + content=""" +I confirmed with Sitaram that this is intentional, if probably under-documented. +Since the ADC strips the leading /~/ in assigning $start anyway, I guess something like the following will work +
+
+diff --git a/contrib/adc/git-annex-shell b/contrib/adc/git-annex-shell
+index 7f9f5b8..523dfed 100755
+--- a/contrib/adc/git-annex-shell
++++ b/contrib/adc/git-annex-shell
+@@ -28,7 +28,7 @@ my $cmd=$ENV{SSH_ORIGINAL_COMMAND};
+ # the second parameter.
+ # Further parameters are not validated here (see below).
+ die \"bad git-annex-shell command: $cmd\"
+-    unless $cmd =~ m#^(git-annex-shell '\w+' ')/\~/([0-9a-zA-Z][0-9a-zA-Z._\@/+-
++    unless $cmd =~ m#^(git-annex-shell '\w+' ')/(?:\~\/)?([0-9a-zA-Z][0-9a-zA-Z.
+ my $start = $1;
+ my $repo = $2;
+ my $end = $3;
+
+"""]] diff --git a/doc/tips/using_gitolite_with_git-annex/comment_7_f8fd08b6ab47378ad88c87348057220d._comment b/doc/tips/using_gitolite_with_git-annex/comment_7_f8fd08b6ab47378ad88c87348057220d._comment new file mode 100644 index 0000000000..bdbecd4d99 --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_7_f8fd08b6ab47378ad88c87348057220d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 7" + date="2011-12-31T18:32:28Z" + content=""" +That patch seems ok, it doesn't seem to allow through any repo locations that were blocked before. + +So, it has my blessing.. but the ADC is in gitolite and will need to be patched there. +"""]] diff --git a/doc/tips/using_gitolite_with_git-annex/comment_8_8249772c142117f88e37975d058aa936._comment b/doc/tips/using_gitolite_with_git-annex/comment_8_8249772c142117f88e37975d058aa936._comment new file mode 100644 index 0000000000..0717bab1c2 --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_8_8249772c142117f88e37975d058aa936._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="bremner" + ip="156.34.79.193" + subject="afaict git annex normalizes urls on the client side." + date="2011-12-31T22:29:38Z" + content=""" +After some debugging printing, here is my current understanding. + +- urls of the form git@host:~repo or ssh://git@host + + - git sends commands like \"git-receive-pack '~/repo' + - gitolite converts these to $REPO_BASE/~/repo which fails. ~/repo would also fail fwiw. + - git-annex sends seems /~/repo, which works + +- urls of the form git@host:/repo or ssh://git@host/repo + + - git sends \"git-receive-pack '/db/cs3383'\" + - gitolite converts this to $REPO_BASE/repo which works + - git annex sends \"git-annex-shell 'inannex' '/repo' ...\" which works, but only with the patch above. + +- urls of the form git@host:repo + + - git sends \"git-receive-pack 'repo' + - gitolite converts this to $REPO_BASE/repo, which works + - git-annex sends \"git-annex-shell 'inannex' '/~/db/cs3383'...\", which also works for git-annex-shell. + +So the weird case is the last one where git and git-annex are sending different things over the wire. +I don't know if you have other motivations for doing the url normalization on the client side, but it isn't needed for gitolite, and in some sense complicates things a little. On the other hand, now that I see what is going on, it isn't a big deal to just strip the leading /~ off in the adc. It does lead to the odd situation of some URLs working for git-annex but not git. +"""]] diff --git a/doc/tips/using_gitolite_with_git-annex/comment_9_28418635a6ed7231b89e02211cd3c236._comment b/doc/tips/using_gitolite_with_git-annex/comment_9_28418635a6ed7231b89e02211cd3c236._comment new file mode 100644 index 0000000000..fc297ff175 --- /dev/null +++ b/doc/tips/using_gitolite_with_git-annex/comment_9_28418635a6ed7231b89e02211cd3c236._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 9" + date="2012-01-02T16:27:55Z" + content=""" +Ah right. git-annex normalizes all git ssh style user@host:dir to valid uris, which is where the `/~/` comes from. I don't anticipate this changing on the git-annex side. +"""]] diff --git a/doc/tips/using_the_SHA1_backend.mdwn b/doc/tips/using_the_SHA1_backend.mdwn new file mode 100644 index 0000000000..70dc2ef759 --- /dev/null +++ b/doc/tips/using_the_SHA1_backend.mdwn @@ -0,0 +1,11 @@ +A handy alternative to the default [[backend|backends]] is the +SHA1 backend. This backend provides more git-style assurance that your data +has not been damaged. And the checksum means that when you add the same +content to the annex twice, only one copy need be stored in the backend. + +The only reason it's not the default is that it needs to checksum +files when they're added to the annex, and this can slow things down +significantly for really big files. To make SHA1 the default, just +add something like this to `.gitattributes`: + + * annex.backend=SHA1 diff --git a/doc/tips/using_the_web_as_a_special_remote.mdwn b/doc/tips/using_the_web_as_a_special_remote.mdwn new file mode 100644 index 0000000000..24ded90c08 --- /dev/null +++ b/doc/tips/using_the_web_as_a_special_remote.mdwn @@ -0,0 +1,57 @@ +The web can be used as a [[special_remote|special_remotes]] too. + + # git annex addurl http://example.com/video.mpeg + addurl example.com_video.mpeg (downloading http://example.com/video.mpeg) + ########################################################## 100.0% + ok + +Now the file is downloaded, and has been added to the annex like any other +file. So it can be renamed, copied to other repositories, and so on. + +Note that git-annex assumes that, if the web site does not 404, and has the +right file size, the file is still present on the web, and this counts as +one [[copy|copies]] of the file. So it will let you remove your last copy, +trusting it can be downloaded again: + + # git annex drop example.com_video.mpeg + drop example.com_video.mpeg (checking http://example.com/video.mpeg) ok + +If you don't [[trust]] the web to this degree, just let git-annex know: + + # git annex untrust web + untrust web ok + +With the result that it will hang onto files: + + # git annex drop example.com_video.mpeg + drop example.com_video.mpeg (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + Also these untrusted repositories may contain the file: + 00000000-0000-0000-0000-000000000001 -- web + (Use --force to override this check, or adjust annex.numcopies.) + failed + +You can also add urls to any file already in the annex: + + # git annex addurl --file my_cool_big_file http://example.com/cool_big_file + addurl my_cool_big_file ok + # git annex whereis my_cool_big_file + whereis my_cool_big_file (2 copies) + 00000000-0000-0000-0000-000000000001 -- web + 27a9510c-760a-11e1-b9a0-c731d2b77df9 -- here + +To add a lot of urls at once, just list them all as parameters to +`git annex addurl`. + +If you're adding a bunch of related files to a directory, or just don't +like the default filenames generated by `addurl`, you can use `--pathdepth` +to specify how many parts of the url are put in the filename. +A positive number drops that many paths from the beginning, while a negative +number takes that many paths from the end. + + # git annex addurl http://example.com/videos/2012/01/video.mpeg + addurl example.com_videos_2012_01_video.mpeg (downloading http://example.com/videos/2012/01/video.mpeg) + # git annex addurl http://example.com/videos/2012/01/video.mpeg --pathdepth=2 + addurl 2012_01_video.mpeg (downloading http://example.com/videos/2012/01/video.mpeg) + # git annex addurl http://example.com/videos/2012/video.mpeg --pathdepth=-2 + addurl 01_video.mpeg (downloading http://example.com/videos/2012/01/video.mpeg) diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_1_321a41d611c6fe45e047af9c96c5176c._comment b/doc/tips/using_the_web_as_a_special_remote/comment_1_321a41d611c6fe45e047af9c96c5176c._comment new file mode 100644 index 0000000000..ee1a271eac --- /dev/null +++ b/doc/tips/using_the_web_as_a_special_remote/comment_1_321a41d611c6fe45e047af9c96c5176c._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlc1og3PqIGudOMkFNrCCNg66vB7s-jLpc" + nickname="Paul" + subject="can addurl use hashing once the file is downloaded?" + date="2012-09-20T21:01:30Z" + content=""" +There are resources that I want to add to my annex that are currently available +via a URL, but it seems like if I add these using `git-annex addurl`, they get +symlinked to file in the annex/objects directory that starts with `URL-...`, +instead of the more typical `SHA256-...`, and this does not change even after +the files are downloaded. + +My concern is that I really want to ensure that these files don't change, which +is the appeal of content-addressable symlinking of normal files (as opposed to +URL addressable ones). + +Would there be a way to automate the injection of hash-based symlinking for +files that are added via addurl? Sometimes I add a bunch of files via ``addurl +--fast``, and after I've download them via ``get``, it would be nice to have +those files have the same level of data integrity as when I download them using +something outside of git-annex, add them to the annex, and do an ``addurl +--file`` afterward. + +Thanks for all of your hard work! + +"""]] diff --git a/doc/tips/using_the_web_as_a_special_remote/comment_2_dfe9c8c49aadff80d2020288584e0390._comment b/doc/tips/using_the_web_as_a_special_remote/comment_2_dfe9c8c49aadff80d2020288584e0390._comment new file mode 100644 index 0000000000..b015cdcece --- /dev/null +++ b/doc/tips/using_the_web_as_a_special_remote/comment_2_dfe9c8c49aadff80d2020288584e0390._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 2" + date="2012-09-20T21:55:57Z" + content=""" +`addurl` only uses the URL- keys if you run it with --fast. Otherwise it downloads the content and hashes it the same as `add` does. + +If you use `--fast`, you can go back and `git annex migrate` the file once it's been downloaded, to convert +it to the SHA backend. +"""]] diff --git a/doc/tips/visualizing_repositories_with_gource.mdwn b/doc/tips/visualizing_repositories_with_gource.mdwn new file mode 100644 index 0000000000..25a69c1b7a --- /dev/null +++ b/doc/tips/visualizing_repositories_with_gource.mdwn @@ -0,0 +1,22 @@ +[Gource](http://code.google.com/p/gource/) is an amazing animated +visualisation of a git repository. + +Normally, gource shows files being added, removed, and changed in +the repository, and the user(s) making the changes. Of course it can be +used in this way in a repository using git-annex too; just run `gource`. + +The other way to use gource with git-annex is to visualise the movement of +annexed file contents between repositories. In this view, the "users" are +repositories, and they move around the file contents that are being added +or removed from them with git-annex. + +[[!img screenshot.jpg]] + +To use gource this way, first go into the directory you want to visualize, +and use `git annex log` to make an input file for `gource`: + + git annex log --gource | tee gource.log + sort gource.log | gource --log-format custom - + +The `git annex log` can take a while, to speed it up you can use something +like `--after "4 months ago"` to limit how far back it goes. diff --git a/doc/tips/visualizing_repositories_with_gource/screenshot.jpg b/doc/tips/visualizing_repositories_with_gource/screenshot.jpg new file mode 100644 index 0000000000..9d3096b99b Binary files /dev/null and b/doc/tips/visualizing_repositories_with_gource/screenshot.jpg differ diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn b/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn new file mode 100644 index 0000000000..80cb046d90 --- /dev/null +++ b/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn @@ -0,0 +1,22 @@ +A git-annex repository on a removable USB drive is great, until the cable +falls out at the wrong time and git's repository gets trashed. The way +git checksums everything and the poor quality of USB media makes this +perhaps more likely than you would expect. If this happens to you, +here's a way to recover that makes the most of whatever data is left +on the drive. + +* First, run `git fsck`. If it does not report any problems, your data + is fine, and you don't need to proceed further. +* So `git fsck` says the git repository is corrupted. But probably the data + git-annex stored is fine. Your first step is to clone another copy + of the git repository from somewhere else. Let's call this clone + "$good", and the corrupted repository "$bad". +* Preserve your git configuration changes, and the `annex.uuid` setting: + `mv $bad/.git/config $good/.git/config` +* Move annexed data into the new repository: `mkdir $good/.git/annex; mv + $bad/.git/annex/objects $good/.git/annex/objects` +* Reinitalize git-annex: `cd $good; git annex init` +* Check for any problems with the annexed data: `cd $good; git annex fsck` +* Now you can remove the corrupted repository, the new one is ready to use. + +--[[Joey]] diff --git a/doc/tips/what_to_do_when_you_lose_a_repository.mdwn b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn new file mode 100644 index 0000000000..363eeea4e0 --- /dev/null +++ b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn @@ -0,0 +1,19 @@ +So you lost a thumb drive containing a git-annex repository. Or a hard +drive died or some other misfortune has befallen your data. + +Unless you configured backups, git-annex can't get your data back. But it +can help you deal with the loss. + +Go somewhere that knows about the lost repository, and mark it as +dead: + + git annex dead usbdrive + +This retains the [[location_tracking]] information for the repository, +but avoids trying to access it, or list it as a location where files +are present. + +If you later found the drive, you could let git-annex know it's found +like so: + + git annex semitrust usbdrive diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment new file mode 100644 index 0000000000..a7fce26ef8 --- /dev/null +++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://dlaxalde.myopenid.com/" + nickname="dl" + subject="comment 1" + date="2012-05-31T14:36:33Z" + content=""" +Is there a way to have git-annex completely ignore a repository? I see that +the `dead` command adds the uuid of the repository to `trust.log` but does +not change `uuid.log`. Is it enough to remove the corresponding line in +`uuid.log` and `trust.log`? +"""]] diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment new file mode 100644 index 0000000000..a8d044c287 --- /dev/null +++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.8.243" + subject="comment 3" + date="2012-05-31T17:01:37Z" + content=""" +`dead` is the best we can do. The automatic merging used on the git-annex branch tends to re-add lines that are deleted in one repo when merging with another that still has them. +"""]] diff --git a/doc/todo.mdwn b/doc/todo.mdwn new file mode 100644 index 0000000000..79552298b6 --- /dev/null +++ b/doc/todo.mdwn @@ -0,0 +1,4 @@ +This is git-annex's todo list. Link items to [[todo/done]] when done. + +[[!inline pages="./todo/* and !./todo/done and !link(done) +and !*/Discussion" actions=yes postform=yes show=0 archive=yes]] diff --git a/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn new file mode 100644 index 0000000000..592b5e0773 --- /dev/null +++ b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn @@ -0,0 +1,7 @@ +A failure during "make test" should be signalled to the caller by means of +a non-zero exit code. Without that signal, it's very hard to run the +regression test suite in an automated fashion. + +> git-annex used to have a Makefile that ignored make test exit status, +> but that was fixed in commit dab5bddc64ab4ad479a1104748c15d194e138847, +> in October 6th. [[done]] --[[Joey]] diff --git a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn new file mode 100644 index 0000000000..f822249916 --- /dev/null +++ b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn @@ -0,0 +1,9 @@ +Git-annex doesn't compile with the latest version of monad-control. Would it be hard to support that new version? + +> I have been waiting for it to land in Debian before trying to +> deal with its changes. +> +> There is now a branch in git called `new-monad-control` that will build +> with the new monad-control. --[[Joey]] + +>> Now merged to master. [[done]] --[[Joey]] diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn new file mode 100644 index 0000000000..7e417336f0 --- /dev/null +++ b/doc/todo/S3.mdwn @@ -0,0 +1,24 @@ +Support Amazon S3 as a file storage backend. + +There's a haskell library that looks good. Not yet in Debian. + +Multiple ways of using S3 are possible. Currently implemented as +a special type of git remote. + +Before this can be close, I need to fix: + +## encryption + +TODO + +## unused checking + +One problem is `git annex unused`. Currently it only looks at the local +repository, not remotes. But if something is dropped from the local repo, +and you forget to drop it from S3, cruft can build up there. + +This could be fixed by adding a hook to list all keys present in a remote. +Then unused could scan remotes for keys, and if they were not used locally, +offer the possibility to drop them from the remote. + +[[done]] diff --git a/doc/todo/Slow_transfer_for_a_lot_of_small_files..mdwn b/doc/todo/Slow_transfer_for_a_lot_of_small_files..mdwn new file mode 100644 index 0000000000..00cdad0fe1 --- /dev/null +++ b/doc/todo/Slow_transfer_for_a_lot_of_small_files..mdwn @@ -0,0 +1,20 @@ +What steps will reproduce the problem? +Sync a lot of small files. + +What is the expected output? What do you see instead? +The expected output is hopefully a fast transfer. + +But currently it seems like git-annex is only using one thread to transfer(per host or total?) + +An option to select number of transfer threads to use(possibly per host) would be very nice. + +> Opening a lot of connections to a single host is probably not desirable. +> +> I do want to do something to allow slow hosts to not hold up transfers to +> other hosts, which might involve running multiple queued transfers at +> once. The webapp already allows the user to force a given transfer to +> happen immediately. --[[Joey]] + +And maybe also an option to limit how long a queue the browser should show, it can become quite resource intensive with a long queue. + +> The queue is limited to 20 items for this reason. --[[Joey]] diff --git a/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs.mdwn b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs.mdwn new file mode 100644 index 0000000000..a42a81d02b --- /dev/null +++ b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs.mdwn @@ -0,0 +1,7 @@ +There are times when it is handy to be able to upload a file to a web host somewhere and share a link for that file to a select few people. + +It seems to be that the assistant could handle this scenario. It could generate a directory with a random name on the remote, and transfer the file there (using the existing filename) and the appropriate URL could be displayed in the assistant webapp to allow the user to copy the URL to send it to the appropriate people. + +Note: Joey and I had a quick chat about this use case at LCA2013. + +[[!tag design/assistant]] diff --git a/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_1_1a1f34f4f389267d67e79409c0ca8b1d._comment b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_1_1a1f34f4f389267d67e79409c0ca8b1d._comment new file mode 100644 index 0000000000..35f7351915 --- /dev/null +++ b/doc/todo/Use_a_remote_as_a_sharing_site_for_files_with_obfuscated_URLs/comment_1_1a1f34f4f389267d67e79409c0ca8b1d._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://edheil.wordpress.com/" + ip="99.54.57.201" + subject="comment 1" + date="2013-02-09T22:38:56Z" + content=""" +This would be an extremely cool feature to rip off from Dropbox. :) + +"""]] diff --git a/doc/todo/add_--exclude_option_to_git_annex_find.mdwn b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn new file mode 100644 index 0000000000..a797e97f58 --- /dev/null +++ b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn @@ -0,0 +1,4 @@ +Seems pretty self-explanatory. + +> This was already implemented, the --exclude option can be used +> for find as well as most any other subcommand. --[[Joey]] [[done]] diff --git a/doc/todo/add_-all_option.mdwn b/doc/todo/add_-all_option.mdwn new file mode 100644 index 0000000000..351d2573ee --- /dev/null +++ b/doc/todo/add_-all_option.mdwn @@ -0,0 +1,18 @@ +`--all` would make git-annex operate on either every key with content +present (or in some cases like `get` and `copy --from` on +every keys with content not present). + +This would be useful when a repository has a history with deleted files +whose content you want to keep (so you're not using `dropunused`). +Or when you have a lot of branches and just want to be able to fsck +every file referenced in any branch (or indeed, any file referenced in any +ref). It could also be useful (or even a +good default) in a bare repository. + +A problem with the idea is that `.gitattributes` values for keys not +currently in the tree would not be available (without horrific anounts of +grubbing thru history to find where/when the key used to exist). So +`numcopies` set via `.gitattributes` would not work. This would be a +particular problem for `drop` and for `--auto`. + +--[[Joey]] diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn new file mode 100644 index 0000000000..2b224710ed --- /dev/null +++ b/doc/todo/add_a_git_backend.mdwn @@ -0,0 +1,18 @@ +There should be a backend where the file content is stored.. in a git +repository! + +This way, you know your annexed content is safe & versioned, but you only +have to deal with the pain of git with large files in one place, and can +use all of git-annex's features everywhere else. + +> Speaking as a future user, do very, very much want. -- RichiH + +>> Might also be interesting to use `bup` in the git backend, to work +>> around git's big file issues there. So git-annex would pull data out +>> of the git backend using bup. --[[Joey]] + +>>> Very much so. Generally speaking, having one or more versioned storage back-ends with current data in the local annexes sounds incredibly useful. Still being able to get at old data in via the back-end and/or making offline backups of the full history are excellent use cases. -- RichiH + +[[done]], the bup special remote type is written! --[[Joey]] + +> Yay! -- RichiH diff --git a/doc/todo/add_an_icon_for_the_.desktop_file.mdwn b/doc/todo/add_an_icon_for_the_.desktop_file.mdwn new file mode 100644 index 0000000000..3be158a0aa --- /dev/null +++ b/doc/todo/add_an_icon_for_the_.desktop_file.mdwn @@ -0,0 +1 @@ +Maybe add the icon /usr/share/doc/git-annex/html/logo.svg to the .desktp file. diff --git a/doc/todo/assistant_git_sync_laddering.mdwn b/doc/todo/assistant_git_sync_laddering.mdwn new file mode 100644 index 0000000000..7dbc6480a2 --- /dev/null +++ b/doc/todo/assistant_git_sync_laddering.mdwn @@ -0,0 +1,10 @@ +When the [[design/assistant]] is running on a pair of remotes, I've seen +them get out of sync, such that every pull and merge results in a conflict, +that then has to be auto-resolved. + +This seems similar to the laddering problem described in this old bug: +[[bugs/making_annex-merge_try_a_fast-forward]] + +--[[Joey]] + +Think I've fixed this. [[done]] --[[Joey]] diff --git a/doc/todo/assistant_threaded_runtime.mdwn b/doc/todo/assistant_threaded_runtime.mdwn new file mode 100644 index 0000000000..03ba66acf1 --- /dev/null +++ b/doc/todo/assistant_threaded_runtime.mdwn @@ -0,0 +1,40 @@ +The [[design/assistant]] would be better if git-annex used ghc's threaded +runtime (`ghc -threaded`). + +Currently, whenever the assistant code runs some external command, all +threads are blocked waiting for it to finish. + +For transfers, the assistant works around this problem by forking separate +upload processes, and not waiting on them until it sees an indication that +they have finished the transfer. While this works, it's messy.. threaded +would be better. + +When pulling, pushing, and merging, the assistant runs external git +commands, and this does block all other threads. The threaded runtime would +really help here. + +[[done]]; the assistant now builds with the threaded runtime. +Some work still remains to run certian long-running external git commands +in their own threads to prevent them blocking things, but that is easy to +do, now. --[[Joey]] + +--- + +Currently, git-annex seems unstable when built with the threaded runtime. +The test suite tends to hang when testing add. `git-annex` occasionally +hangs, apparently in a futex lock. This is not the assistant hanging, and +git-annex does not otherwise use threads, so this is surprising. --[[Joey]] + +> I've spent a lot of time debugging this, and trying to fix it, in the +> "threaded" branch. There are still deadlocks. --[[Joey]] + +>> Fixed, by switching from `System.Cmd.Utils` to `System.Process` +>> --[[Joey]] + +--- + +It would be possible to not use the threaded runtime. Instead, we could +have a child process pool, with associated continuations to run after a +child process finishes. Then periodically do a nonblocking waitpid on each +process in the pool in turn (waiting for any child could break anything not +using the pool!). This is probably a last resort... diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn new file mode 100644 index 0000000000..715dea7207 --- /dev/null +++ b/doc/todo/auto_remotes.mdwn @@ -0,0 +1,29 @@ +It should be possible for clones to learn about how to contact +each other without remotes needing to always be explicitly set +up. Say that `.git-annex/remote.log` is maintained by git-annex +to contain: + + UUID hostname URI + +The URI comes from configured remotes and maybe from +`file://$(pwd)`, or even `ssh://$(hostname -f)` +for the current repo. This format will merge without +conflicts or data loss. + +Then when content is belived to be in a UUID, and no +configured remote has it, the remote.log can be consulted and +URIs that look likely tried. (file:// ones if the hostname +is the same (or maybe always -- a removable drive might tend +to be mounted at the same location on different hosts), +otherwise ssh:// ones.) + +Question: When should git-annex update the remote.log? +(If not just on init.) Whenever it reads in a repo's remotes? + +> This sounds useful and the log should be updated every time any remote is being accessed. A counter or timestamp (yes, distributed times may be wrong/different) could be used to auto-prune old entries via a global and per-remote config setting. -- RichiH + +--- + +I no longer think I'd use this myself, I find that my repositories quickly +grow the paths I actually use, somewhat organically. Unofficial paths +across university quads come to mind. [[done]] --[[Joey]] diff --git a/doc/todo/auto_remotes/discussion.mdwn b/doc/todo/auto_remotes/discussion.mdwn new file mode 100644 index 0000000000..b9e1522a8f --- /dev/null +++ b/doc/todo/auto_remotes/discussion.mdwn @@ -0,0 +1,7 @@ +Remotes log should probably be stored in ".git/annex/remote.log" +instead of ".git-annex/remote.log" to prevent leaking credentials. + +> The idea is to distribute the info between repositories, which is +> why it'd go in `.git-annex`. Of course that does mean that repository +> location information would be included, and if that'd not desirable +> this feature would need to be turned off. --[[Joey]] diff --git a/doc/todo/automatic_bookkeeping_watch_command.mdwn b/doc/todo/automatic_bookkeeping_watch_command.mdwn new file mode 100644 index 0000000000..28b02aff25 --- /dev/null +++ b/doc/todo/automatic_bookkeeping_watch_command.mdwn @@ -0,0 +1,15 @@ +A "git annex watch" command would help make git-annex usable by users who +don't know how to use git, or don't want to bother typing the git commands. +It would run, in the background, watching via inotify for changes, and +automatically annexing new files, etc. + +The blue sky goal would be something automated like dropbox, except fully +distributed. All files put into the repository would propagate out +to all the other clones of it, as network links allow. Note that while +dropbox allows modifying files, git-annex freezes them upon creation, +so this would not be 100% equivalent to dropbox. --[[Joey]] + +This is a big project with its own [[design pages|design/assistant]]. + +> [[done]].. at least, we have a watch command an an assistant, which +> is still being developed. --[[Joey]] diff --git a/doc/todo/avoid_unnecessary_union_merges.mdwn b/doc/todo/avoid_unnecessary_union_merges.mdwn new file mode 100644 index 0000000000..5cd4b64373 --- /dev/null +++ b/doc/todo/avoid_unnecessary_union_merges.mdwn @@ -0,0 +1,20 @@ +Some commands cause a union merge unnecessarily. For example, `git annex add` +modifies the location log, which first requires reading the current log (if +any), which triggers a merge. + +Would be good to avoid these unnecessary union merges. First because it's +faster and second because it avoids a possible delay when a user might +ctrl-c and leave the repo in an inconsistent state. In the case of an add, +the file will be in the annex, but no location log will exist for it (fsck +fixes that). + +It may be that all that's needed is to modify Annex.Branch.change +to read the current value, without merging. Then commands like `get`, that +query the branch, will still cause merges, and commands like `add` that +only modify it, will not. Note that for a command like `get`, the merge +occurs before it has done anything, so ctrl-c should not be a problem +there. + +This is a delicate change, I need to take care.. --[[Joey]] + +> [[done]] (assuming I didn't miss any cases where this is not safe!) --[[Joey]] diff --git a/doc/todo/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn new file mode 100644 index 0000000000..8c16b75ad0 --- /dev/null +++ b/doc/todo/backendSHA1.mdwn @@ -0,0 +1,7 @@ +This backend is not finished. + +In particular, while files can be added using it, git-annex will not notice +when their content changes, and will not create a new key for the new sha1 +of the net content. + +[[done]]; use unlock subcommand and commit changes with git diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn new file mode 100644 index 0000000000..ad7ece6f10 --- /dev/null +++ b/doc/todo/branching.mdwn @@ -0,0 +1,159 @@ +[[done]] !!! + +The use of `.git-annex` to store logs means that if a repo has branches +and the user switched between them, git-annex will see different logs in +the different branches, and so may miss info about what remotes have which +files (though it can re-learn). + +An alternative would be to store the log data directly in the git repo +as `pristine-tar` does. Problem with that approach is that git won't merge +conflicting changes to log files if they are not in the currently checked +out branch. + +It would be possible to use a branch with a tree like this, to avoid +conflicts: + +key/uuid/time/status + +As long as new files are only added, and old timestamped files deleted, +there would be no conflicts. + +A related problem though is the size of the tree objects git needs to +commit. Having the logs in a separate branch doesn't help with that. +As more keys are added, the tree object size will increase, and git will +take longer and longer to commit, and use more space. One way to deal with +this is simply by splitting the logs amoung subdirectories. Git then can +reuse trees for most directories. (Check: Does it still have to build +dup trees in memory?) + +Another approach would be to have git-annex *delete* old logs. Keep logs +for the currently available files, or something like that. If other log +info is needed, look back through history to find the first occurance of a +log. Maybe even look at other branches -- so if the logs were on master, +a new empty branch could be made and git-annex would still know where to +get keys in that branch. + +Would have to be careful about conflicts when deleting and bringing back +files with the same name. And would need to avoid expensive searching thru +all history to try to find an old log file. + +## fleshed out proposal + +Let's use one branch per uuid, named git-annex/$UUID. + +- I came to realize this would be a good idea when thinking about how + to upgrade. Each individual annex will be upgraded independantly, + so each will want to make a branch, and if the branches aren't distinct, + they will merge conflict for sure. +- TODO: What will need to be done to git to make it push/pull these new + branches? +- A given repo only ever writes to its UUID branch. So no conflicts. + - **problem**: git annex move needs to update log info for other repos! + (possibly solvable by having git-annex-shell update the log info + when content is moved using it) +- (BTW, UUIDs probably don't compress well, and this reduces the bloat of having + them repeated lots of times in the tree.) +- Per UUID branches mean that if it wants to find a file's location + amoung configured remotes, it can examine only their branches, if + desired. +- It's important that the per-repo branches propigate beyond immediate + remotes. If there is a central bare repo, that means push --all. Without + one, it means that when repo B pulls from A, and then C pulls from B, + C needs to get A's branch -- which means that B should have a tracking + branch for A's branch. + +In the branch, only one file is needed. Call it locationlog. git-annex +can cache location log changes and write them all to locationlog in +a single git operation on shutdown. + +- TODO: what if it's ctrl-c'd with changes pending? Perhaps it should + collect them to .git/annex/locationlog, and inject that file on shutdown? +- This will be less overhead than the current staging of all the log files. + +The log is not appended to, so in git we have a series of commits each of +which replaces the log's entire contens. + +To find locations of a key, all (or all relevant) branches need to be +examined, looking backward through the history of each until a log +with a indication of the presense/absense of the key is found. + +- This will be less expensive for files that have recently been added + or transfered. +- It could get pretty slow when digging deeper. +- Only 3 places in git-annex will be affected by any slowdown: move --from, + get and drop. (Update: Now also unused, whereis, fsck) + +## alternate + +As above, but use a single git-annex branch, and keep the per-UUID +info in their own log files. Hope that git can auto-merge as long as +each observing repo only writes to its own files. (Well, it can, but for +non-fast-forward merges, the git-annex branch would need to be checked out, +which is problimatic.) + +Use filenames like: + + / + +That allows one repo to record another's state when doing a +`move`. + +## outside the box approach + +If the problem is limited to only that the `.git-annex/` files make +branching difficult (and not to the related problem that commits to them +and having them in the tree are sorta annoying), then a simple approach +would be to have git-annex look in other branches for location log info +too. + +The problem would then be that any locationlog lookup would need to look in +all other branches (any branch could have more current info after all), +which could get expensive. + +## way outside the box approach + +Another approach I have been mulling over is keeping the log file +branch checked out in .git/annex/logs/ -- this would be a checkout of a git +repository inside a git repository, using "git fake bare" techniques. This +would solve the merge problem, since git auto merge could be used. It would +still mean all the log files are on-disk, which annoys some. It would +require some tighter integration with git, so that after a pull, the log +repo is updated with the data pulled. --[[Joey]] + +> Seems I can't use git fake bare exactly. Instead, the best option +> seems to be `git clone --shared` to make a clone that uses +> `.git/annex/logs/.git` to hold its index etc, but (mostly) uses +> objects from the main repo. There would be some bloat, +> as commits to the logs made in there would not be shared with the main +> repo. Using `GIT_OBJECT_DIRECTORY` might be a way to avoid that bloat. + +## notes + +Another approach could be to use git-notes. It supports merging branches +of notes, with union merge strategy (a hook would have to do this after +a pull, it's not done automatically). + +Problem: Notes are usually attached to git +objects, and there are no git objects corresponding to git-annex keys. + +Problem: Notes are not normally copied when cloning. + +------ + +## elminating the merge problem + +Most of the above options are complicated by the problem of how to merge +changes from remotes. It should be possible to deal with the merge +problem generically. Something like this: + +* We have a local branch `B`. +* For remotes, there are also `origin/B`, `otherremote/B`, etc. +* To merge two branches `B` and `foo/B`, construct a merge commit that + makes each file have all lines that were in either version of the file, + with duplicates removed (probably). Do this without checking out a tree. + -- now implemented as git-union-merge +* As a `post-merge` hook, merge `*/B` into `B`. This will ensure `B` + is always up-to-date after a pull from a remote. +* When pushing to a remote, nothing need to be done, except ensure + `B` is either successfully pushed, or the push fails (and a pull needs to + be done to get the remote's changes merged into `B`). diff --git a/doc/todo/cache_key_info.mdwn b/doc/todo/cache_key_info.mdwn new file mode 100644 index 0000000000..d4352ccf7f --- /dev/null +++ b/doc/todo/cache_key_info.mdwn @@ -0,0 +1,37 @@ +Most of git-annex is designed to be fast no matter how many other files are +in the annex. Things like add/get/drop/move/fsck have good locality; +they will only operate on as many files as you need them to. + +(git commit can get a little slow with a great deal of files, +but that's out of scope -- and recent git-annex versions use queuing +to save git add from piling up too much in the index.) + +But currently two git-annex commands are quite slow when annexes become large +in quantity of files. These are unused and status. +(Both have --fast versions that don't do as much). +> (Update: status has become acceptably fast; most of its slowdown was due to using a bad data structure; scanning the tree is not particularly slow and it no longer looks at the git-annex branch.) + +unused is slow because it needs two pieces of information that are not +quick to look up, and require examining the whole repo, very seekily: + +1. The keys present in the annex. Found by looking thru .git/annex/objects +2. The keys referenced by files in git. Found by finding every file + in git, and looking at its symlink. + +Of these, the first is less expensive (typically, an annex does not have every +key in it). It could be optimized fairly simply, by adding a database +of keys present in the annex that is optimised to list them all. The +database would be updated by the few functions that move content in and +out. + +The second is harder to optimise, because the user can delete, revert, +copy, add, etc files in git at will, and git-annex does not have a good way +to watch that and maintain a database of what keys are being referenced. + +It could use a post-commit hook and examine files changed by commits, etc. +But then staged files would be left out. It might be sufficient to +make --fast trust the database... except unused will suggest *deleting* +data if nothing references it. Or maybe it could be required to have a +clean tree with nothing staged before running git-annex unused. + +Anyway, this is a semi-longterm item for me. --[[Joey]] diff --git a/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment b/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment new file mode 100644 index 0000000000..086e7f3e84 --- /dev/null +++ b/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-05-17T07:27:02Z" + content=""" +Sounds like a good idea. + +* git annex fsck (or similar) should check/rebuild the caches +* I would simply require a clean tree with a verbose error. 80/20 rule and defaulting to save actions. +"""]] diff --git a/doc/todo/checkout.mdwn b/doc/todo/checkout.mdwn new file mode 100644 index 0000000000..50da2d62e1 --- /dev/null +++ b/doc/todo/checkout.mdwn @@ -0,0 +1,23 @@ +The checkout subcommand replaces the symlink that normally points at a +file's content, with a copy of the file. Once you've checked a file out, +you can edit it, and `git commit` it. On commit, git-annex will detect +if the file has been changed, and if it has, `add` its content to the +annex. + +> Internally, this will need to store the original symlink to the file, in +> `.git/annex/checkedout/$filename`. +> +> * git-annex uncheckout moves that back +> * git-annex pre-commit hook checks each file being committed to see if +> it has a symlink there, and if so, removes the symlink and adds the new +> content to the annex. +> +> And it seems the file content should be copied, not moved or hard linked: +> +> * Makes sure other annexes can find it if transferring it from +> this annex. +> * Ensures it's always available for uncheckout. +> * Avoids the last copy of a file's content being lost when +> the checked out file is modified. + +[[done]] diff --git a/doc/todo/direct_mode_guard.mdwn b/doc/todo/direct_mode_guard.mdwn new file mode 100644 index 0000000000..7322cec632 --- /dev/null +++ b/doc/todo/direct_mode_guard.mdwn @@ -0,0 +1,22 @@ +Currently [[/direct_mode]] allows the user to point many normally safe +git commands at his foot and pull the trigger. At LCA2013, a git-annex +user suggested modifying direct mode to make this impossible. + +One way to do it would be to move the .git directory. Instead, make there +be a .git-annex directory in direct mode repositories. git-annex would know +how to use it, and would be extended to support all known safe git +commands, passing parameters through, and in some cases verifying them. + +So, for example, `git annex commit` would run `git commit --git-dir=.git-annex` + +However, `git annex commit -a` would refuse to run, or even do something +intelligent that does not involve staging every direct mode file. + +---- + +One source of problems here is that there is some overlap between git-annex +and git commands. Ie, `git annex add` cannot be a passthrough for `git +add`. The git wrapper could instead be another program, or it could be +something like `git annex git add` + +--[[Joey]] diff --git a/doc/todo/done.mdwn b/doc/todo/done.mdwn new file mode 100644 index 0000000000..e7c98081b7 --- /dev/null +++ b/doc/todo/done.mdwn @@ -0,0 +1,4 @@ +recently fixed [[todo]] items. + +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10 +archive=yes]] diff --git a/doc/todo/exclude_files_on_a_given_remote.mdwn b/doc/todo/exclude_files_on_a_given_remote.mdwn new file mode 100644 index 0000000000..e8bb357d31 --- /dev/null +++ b/doc/todo/exclude_files_on_a_given_remote.mdwn @@ -0,0 +1,18 @@ +Say I have some files on remote A. But I'm away from it, and transferring +files from B to C. I'd like to avoid transferring any files I already have +on A. + +Something like: + + git annex copy --to C --exclude-on A + +This would not contact A, just use its cached location log info. + +I suppose I might also sometime want to only act on files that are +thought/known to be on A. + + git annex drop --only-on A + +--[[Joey]] + +[[done]] diff --git a/doc/todo/file_copy_progress_bar.mdwn b/doc/todo/file_copy_progress_bar.mdwn new file mode 100644 index 0000000000..847c1d1eb6 --- /dev/null +++ b/doc/todo/file_copy_progress_bar.mdwn @@ -0,0 +1,5 @@ +Find a way to copy a file with a progress bar, while still preserving +stat. Easiest way might be to use pv and fix up the permissions etc +after? + +[[done]] diff --git a/doc/todo/fsck.mdwn b/doc/todo/fsck.mdwn new file mode 100644 index 0000000000..1dcaad9a51 --- /dev/null +++ b/doc/todo/fsck.mdwn @@ -0,0 +1,11 @@ +add a git annex fsck that finds keys that have no referring file + +(done) + +* Need per-backend fsck support. sha1 can checksum all files in the annex. + WORM can check filesize. + +* Both can check that annex.numcopies is satisfied. Probably only + querying the locationlog, not doing an online verification. + +[[done]] diff --git a/doc/todo/fsck_special_remotes.mdwn b/doc/todo/fsck_special_remotes.mdwn new file mode 100644 index 0000000000..7196baafe6 --- /dev/null +++ b/doc/todo/fsck_special_remotes.mdwn @@ -0,0 +1,13 @@ +`git annex fsck --from remote` + +Basically, this needs to receive each file in turn from the remote, to a +temp file, and then run the existing fsck code on it. Could be quite +expensive, but sometimes you really want to check. + +An unencrypted directory special remote could be optimised, by not actually +copying the file, just dropping a symlink, etc. + +The WORM backend doesn't care about file content, so it would be nice to +avoid transferring the content at all, and only send the size. + +> [[done]] --[[Joey]] diff --git a/doc/todo/git-annex-shell.mdwn b/doc/todo/git-annex-shell.mdwn new file mode 100644 index 0000000000..a9e3b43ede --- /dev/null +++ b/doc/todo/git-annex-shell.mdwn @@ -0,0 +1,15 @@ +[[done]] + +I've been considering adding a `git-annex-shell` command. This would +be similar to `git-shell` (and in fact would pass unknown commands off to +`git-shell`). + +## Reasons + +* Allows locking down an account to only be able to use git-annex (and + git). +* Avoids needing to construct complex shell commands to run on the remote + system. (Mostly already avoided by the plumbing level commands.) +* Could possibly allow multiple things to be done with one ssh connection + in future. +* Allows expanding `~` and `~user` in repopath on the remote system. diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn new file mode 100644 index 0000000000..760a6ccf5a --- /dev/null +++ b/doc/todo/git-annex_unused_eats_memory.mdwn @@ -0,0 +1,32 @@ +`git-annex unused` has to compare large sets of data +(all keys with content present in the repository, +with all keys used by files in the repository), and so +uses more memory than git-annex typically needs. + +It used to be a lot worse (hundreds of megabytes). + +Now it only needs enough memory to store a Set of all Keys that currently +have content in the annex. On a lightly populated repository, it runs in +quite low memory use (like 8 mb) even if the git repo has 100 thousand +files. On a repository with lots of file contents, it will use more. + +Still, I would like to reduce this to a purely constant memory use, +as running in constant memory no matter the repo size is a git-annex design +goal. + +One idea is to use a bloom filter. +For example, construct a bloom filter of all keys used by files in +the repository. Then for each key with content present, check if it's +in the bloom filter. Since there can be false positives, this might +miss finding some unused keys. The probability/size of filter +could be tunable. + +> Fixed in `bloom` branch in git. --[[Joey]] +>> [[done]]! --[[Joey]] + +Another way might be to scan the git log for files that got removed +or changed what key they pointed to. Correlate with keys with content +currently present in the repository (possibly using a bloom filter again), +and that would yield a shortlist of keys that are probably not used. +Then scan thru all files in the repo to make sure that none point to keys +on the shortlist. diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn new file mode 100644 index 0000000000..be7e2dacc8 --- /dev/null +++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn @@ -0,0 +1,13 @@ +Would help alot when having to add large(ish) amounts of remotes. + +Maybe detect this kind of commit message and ask user whether to automatically add them? See [[auto_remotes]]: +> Question: When should git-annex update the remote.log? (If not just on init.) Whenever it reads in a repo's remotes? + +---- + +I'm not sure that the above suggestion is going down a path that really +makes sense. If you want a list of repository UUIDs and descriptions, +it's there in machine-usable form in `.git-annex/uuid.log`, there is no +need to try to pull this info out of git commit messages. --[[Joey]] + +[[done]] diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn new file mode 100644 index 0000000000..2fca839863 --- /dev/null +++ b/doc/todo/gitolite_and_gitosis_support.mdwn @@ -0,0 +1,39 @@ +gitosis and gitolite should support git-annex being used to send/receive +files from the repositories they manage. Users with read-only access +could only get files, while users with write access could also put and drop +files. + +Doing this right requires modifying both programs, to add [[git-annex-shell]] +to the list of things they can run, and only allow through appropriate +git-annex-shell subcommands to read-only users. + +I have posted an RFC for modifying gitolite to the +[gitolite mailing list](http://groups.google.com/group/gitolite?lnk=srg). + +> I have not developed a patch yet, but all that git-annex needs is a way +> to ssh to the server and run the git-annex-shell command there. +> git-annex-shell is very similar to git-shell. So, one way to enable +> it is simply to set GL_ADC_PATH to a directory containing git-annex-shell. +> +> But, that's not optimal, since git-annex-shell will send off receive-pack +> commands to git, which would bypass gitolite's permissions checking. +> Also, it makes sense to limit readonly users to only download, not +> upload/delete files from git-annex. Instead, I suggest adding something +> like this to gitolite's config: + + # If set, users with W access can write file contents into the git-annex, + # and users with R access can read file contents from the git-annex. + $GL_GIT_ANNEX = 0; + +> If this makes sense, I'm sure I can put a patch together for your +> review. It would involve modifying gl-auth-command so it knows how +> to run git-annex-shell, and how to parse out the "verb" from a +> git-annex-shell command line, and modifying R_COMMANDS and W_COMMANDS. + +As I don't write python, someone else is needed to work on gitosis. +--[[Joey]] + +> [[done]]; support for gitolite is in its `pu` branch, and some changes +> made to git-annefor gitolite is in its `pu` branch, and some changes +> made to git-annex. Word is gitosis is not being maintained so I won't +> worry about try to support it. --[[Joey]] diff --git a/doc/todo/gitrm.mdwn b/doc/todo/gitrm.mdwn new file mode 100644 index 0000000000..e41c334623 --- /dev/null +++ b/doc/todo/gitrm.mdwn @@ -0,0 +1,5 @@ +how to handle git rm file? (should try to drop keys that have no +referring file, if it seems safe..) + +[[done]] -- I think that git annex unused and dropunused are the best +solution to this. diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn new file mode 100644 index 0000000000..191e9c3286 --- /dev/null +++ b/doc/todo/hidden_files.mdwn @@ -0,0 +1,30 @@ +Add a `git annex hide $file` that behaves like drop, checking counter info +and updating location log to say the current repo no longer has a file -- +but does not actually remove the content. + +Then `git annex unused` can be used to clean it up later. And in the +meantime, it's still locally accessible. This can be useful if you're +planning to need to free up space later, but want to hold onto the content +for a while. Possibly you'll be disconnected later, so it's easier to push +out that intent now. + +-- + +TODO: + +* Make 100% sure this is safe. Drop, etc should never check content files + are present on other repos if the location log doesn't say the repo + has the content. + +* What will `git annex get` do if it's asked to get a file that has been + hidden? + +> Unless I am missing something: Make sure the data is correct (for SHA1 or other tracking) and restore locally. If that's not the case, delete and restore from remote. -- RichiH + +---- + +Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH + +> Nah, `git annex unused/dropunused` already exist. --[[Joey]] + +>> OK, in that case forget what I said. No idea about your internal policy, but feel free to delete this part of the page, then. -- RichiH diff --git a/doc/todo/http_headers.mdwn b/doc/todo/http_headers.mdwn new file mode 100644 index 0000000000..9f61bdc931 --- /dev/null +++ b/doc/todo/http_headers.mdwn @@ -0,0 +1,8 @@ +The IA would find it useful to be able to control the http headers +git-annex get, addurl, etc uses. This will allow setting cookies, for +example. + +* annex-web-headers=blah +* Perhaps also annex-web-headers-command=blah + +[[done]] diff --git a/doc/todo/immutable_annexed_files.mdwn b/doc/todo/immutable_annexed_files.mdwn new file mode 100644 index 0000000000..b26838e95e --- /dev/null +++ b/doc/todo/immutable_annexed_files.mdwn @@ -0,0 +1,8 @@ +> josh: Do you do anything in git-annex to try to make the files immutable? +> For instance, removing write permission, or even chattr? +> joey: I don't, but that's a very good idea +> josh: Oh, I just thought of another slightly crazy but handy idea. +> josh: I'd hate to run into a program which somehow followed the symlink and then did an unlink to replace the file. +> josh: To break that, you could create a new directory under annex's internal directory for each file, and make the directory have no write permission. + +[[done]] and done --[[Joey]] diff --git a/doc/todo/incremental_fsck.mdwn b/doc/todo/incremental_fsck.mdwn new file mode 100644 index 0000000000..7c56328b99 --- /dev/null +++ b/doc/todo/incremental_fsck.mdwn @@ -0,0 +1,24 @@ +Justin Azoff realized git-annex should have an incremental fsck. + +This requires storing the last fsck time of each object. + +I would not be strongly opposed to sqlite, but I think there are other +places the data could be stored. One possible place is the mode or mtime +of the .git/annex/objects/xx/yy/$key directories (the parent directories +of where the content is stored). Perhaps the sticky bit could be used to +indicate the content has been fsked, and the mtime indicate the time +of last fsck. Anything that dropped or put in content would need to +clear the sticky bit. --[[Joey]] + +> Basic incremental fsck is done now. +> +> Some enhancements would include: +> +> * --max-age=30d Once the incremental fsck completes and was started 30 days ago, +> start a new one. +> * --time-limit --size-limit --file-limit: Limit how long the fsck runs. + +>> Calling this [[done]]. The `--incremental-schedule` option +>> allows scheduling time between incremental fscks. `--time-limit` is +>> done. I implemented `--smallerthan` independently. Not clear what +>> `--file-limit` would be. --[[Joey]] diff --git a/doc/todo/incremental_fsck/comment_1_609b21141dd5686b2c0eaef2b8d63229._comment b/doc/todo/incremental_fsck/comment_1_609b21141dd5686b2c0eaef2b8d63229._comment new file mode 100644 index 0000000000..709ba078c0 --- /dev/null +++ b/doc/todo/incremental_fsck/comment_1_609b21141dd5686b2c0eaef2b8d63229._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 1" + date="2012-09-20T14:11:57Z" + content=""" +I have a [proof of concept written in python](https://github.com/JustinAzoff/git-annex-background-fsck/blob/master/git-annex-background-fsck). + +You can run it and point it the root of an annex or to a subdirectory. In my brief testing it seems to work :-) + +the goal would be to have options like + + git annex fsck /data/annex --check-older-than 1w --check-for 2h --max-load-avg 0.5 +"""]] diff --git a/doc/todo/link_file_to_remote_repo_feature.mdwn b/doc/todo/link_file_to_remote_repo_feature.mdwn new file mode 100644 index 0000000000..d6b41e8059 --- /dev/null +++ b/doc/todo/link_file_to_remote_repo_feature.mdwn @@ -0,0 +1,52 @@ +I have two repos, using SHA1 backend and both using git. +The first one is a laptop, the second one is a usb drive. + +When I drop a file on the laptop repo, the file is not available on that repo until I run *git annex get* +But when the usb drive is plugged in the file is actually available. + +How about adding a feature to link some/all files to the remote repo? + +e.g. +We have *railscasts/196-nested-model-form-part-1.mp4* file added to git, and only available on the usb drive: + + $ git annex whereis 196-nested-model-form-part-1.mp4 + whereis 196-nested-model-form-part-1.mp4 (1 copy) + a7b7d7a4-2a8a-11e1-aebc-d3c589296e81 -- origin (Portable usb drive) + +I can see the link with: + + $ cd railscasts + $ ls -ls 196* + 8 lrwxr-xr-x 1 framallo staff 193 Dec 20 05:49 196-nested-model-form-part-1.mp4 -> ../.git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e + +I save this in a variable just to make the example more clear: + + ID=".git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e" + +The file doesn't exist on the local repo: + + $ ls ../$ID + ls: ../$ID: No such file or directory + +however I can create a link to access that file on the remote repo. +First I create a needed dir: + + $ mkdir ../.git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/ + +Then I link to the remote file: + + $ ln -s /mnt/usb_drive/repo_folder/$ID ../$ID + +now I can open the file in the laptop repo. + + +I think it could be easy to implement. Maybe It's a naive approach, but looks apealing. +Checking if it's a real file or a link shouldn't impact on performance. +The limitation is that it would work only with remote repos on local dirs + +Also allows you to have one directory structure like AFS or other distributed FS. If the file is not local I go to the remote server. +Which is great for apps like Picasa, Itunes, and friends that depends on the file location. + +> This is a duplicate of [[union_mounting]]. So closing it: [[done]]. +> +> It's a good idea, but making sure git-annex correctly handles these links in all cases is a subtle problem that has not yet been tackled. --[[Joey]] diff --git a/doc/todo/network_remotes.mdwn b/doc/todo/network_remotes.mdwn new file mode 100644 index 0000000000..42efa832f5 --- /dev/null +++ b/doc/todo/network_remotes.mdwn @@ -0,0 +1,5 @@ +Support for remote git repositories (ssh:// specifically can be made to +work, although the other end probably needs to have git-annex +installed..) + +[[done]], at least get and put work.. diff --git a/doc/todo/object_dir_reorg_v2.mdwn b/doc/todo/object_dir_reorg_v2.mdwn new file mode 100644 index 0000000000..49666ddc79 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2.mdwn @@ -0,0 +1,25 @@ +Several things suggest now would be a good time to reorgaize the object +directory. This would be annex.version=2. It will be slightly painful for +all users, so this should be the *last* reorg in the forseeable future. + +1. Remove colons from filenames, for [[bugs/fat_support]] + +2. Add hashing, since some filesystems do suck (like er, fat at least :) + [[forum/hashing_objects_directories]] + (Also, may as well hash .git-annex/* while at it -- that's what + really gets big.) + +3. Add filesize metadata for [[bugs/free_space_checking]]. (Currently only + present in WORM, and in an ad-hoc way.) + +4. Perhaps use a generic format that will allow further metadata to be + added later. For example, + "bSHA1,s101111,kf3101c30bb23467deaec5d78c6daa71d395d1879" + + (Probably everything after ",k" should be part of the key, even if it + contains the "," separator character. Otherwise an escaping mechanism + would be needed.) + +[[done]] now! + +Although [[bugs/free_space_checking]] is not quite there --[[Joey]] diff --git a/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment b/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment new file mode 100644 index 0000000000..261c2a51f3 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-16T01:16:48Z" + content=""" +If you support generic meta-data, keep in mind that you will need to do conflict resolution. Timestamps may not be synched across all systems, so keeping a log of old metadata could be used, sorting by history and using the latest. Which leaves the situation of two incompatible changes. This would probably mean manual conflict resolution. You will probably have thought of this already, but I still wanted to make sure this is recorded. -- RichiH +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment b/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment new file mode 100644 index 0000000000..9785f1989e --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-03-16T01:19:25Z" + content=""" +Hmm, I added quite a few comments at work, but they are stuck in moderation. Maybe I forgot to log in before adding them. I am surprised this one appeared immediately. -- RichiH +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment b/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment new file mode 100644 index 0000000000..886941be72 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-15T14:08:41Z" + content=""" +What is the potential time-frame for this change? As I am not using git-annex for production yet, I can see myself waiting to avoid any potential hassle. + +Supporting generic metadata seems like a great idea. Though if you are going this path, wouldn't it make sense to avoid metastore for mtime etc and support this natively without outside dependencies? + +-- RichiH +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment b/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment new file mode 100644 index 0000000000..475359abbf --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-03-16T03:22:45Z" + content=""" +Well, I spent a few hours playing this evening in the 'reorg' branch in git. It seems to be shaping up pretty well; type-based refactoring in haskell makes these kind of big systematic changes a matter of editing until it compiles. And it compiles and test suite passes. But, so far I've only covered 1. 3. and 4. on the list, and have yet to deal with upgrades. + +I'd recommend you not wait before using git-annex. I am committed to provide upgradability between annexes created with all versions of git-annex, going forward. This is important because we can have offline archival drives that sit unused for years. Git-annex will upgrade a repository to current standard the first time it sees it, and I hope the upgrade will be pretty smooth. It was not bad for the annex.version 0 to 1 upgrade earlier. The only annoyance with upgrades is that it will result in some big commits to git, as every symlink in the repo gets changed, and log files get moved to new names. + +(The metadata being stored with keys is data that a particular backend can use, and is static to a given key, so there are no merge issues (and it won't be used to preserve mtimes, etc).) +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment b/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment new file mode 100644 index 0000000000..2032bce3c0 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 5" + date="2011-03-16T15:51:30Z" + content=""" +Hashing & segmenting seems to be around the corner, which is nice :) + +Is there a chance that you will optionally add mtime to your native metadata store? If yes, I'd rather wait for v2 to start with the native system from the start. If not, I will probably set it up tonight. + +PS: While posting from work, my comments are held for moderation once again. I am somewhat confused as to why this happens when I can just submit directly from home. And yes, I am using the same auth provider and user in both cases. +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment b/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment new file mode 100644 index 0000000000..ff86e3970b --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 6" + date="2011-03-16T16:32:52Z" + content=""" +The mtime cannot be stored for all keys. Consider a SHA1 key. The mtime is irrelevant; 2 files with different mtimes, when added to the SHA1 backend, should get the same key. + +Probably our spam filter doesn't like your work IP. +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment b/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment new file mode 100644 index 0000000000..fc866c57a6 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 7" + date="2011-03-16T21:05:38Z" + content=""" +Ah, OK. I assumed the metadata would be attached to a key, not part of the key. This seems to make upgrades/extensions down the line harder than they need to be, but you are right that this way, merges are not, and never will be, an issue. + +Though with the SHA1 backend, changing files can be tracked. This means that tracking changes in mtime or other is possible. It also means that there are potential merge issues. But I won't argue the point endlessly. I can accept design decisions :) + +The prefix at work is from a university netblock so yes, it might be on a few hundred proxy lists etc. +"""]] diff --git a/doc/todo/optimise_git-annex_merge.mdwn b/doc/todo/optimise_git-annex_merge.mdwn new file mode 100644 index 0000000000..91d18ebd77 --- /dev/null +++ b/doc/todo/optimise_git-annex_merge.mdwn @@ -0,0 +1,23 @@ +Typically `git-annex merge` is fast, but it could still be sped up. + +`git-annex merge` runs `git-hash-object` once per file that needs to be +merged. Elsewhere in git-annex, `git-hash-object` is used in a faster mode, +reading files from disk via `--stdin-paths`. But here, the data is not +in raw files on disk, and I doubt writing them is the best approach. +Instead, I'd like a way to stream multiple objects into git using stdin. +Sometime, should look at either extending git-hash-object to support that, +or possibly look at using git-fast-import instead. + +--- + +`git-annex merge` also runs `git show` once per file that needs to be +merged. This could be reduced to a single call to `git-cat-file --batch`, +There is already a Git.CatFile library that can do this easily. --[[Joey]] + +> This is now done, part above remains todo. --[[Joey]] + +--- + +Merging used to use memory proportional to the size of the diff. It now +streams data, running in constant space. This probably sped it up a lot, +as there's much less allocation and GC action. --[[Joey]] diff --git a/doc/todo/parallel_possibilities.mdwn b/doc/todo/parallel_possibilities.mdwn new file mode 100644 index 0000000000..9c0e69e294 --- /dev/null +++ b/doc/todo/parallel_possibilities.mdwn @@ -0,0 +1,13 @@ +One of my reasons for using haskell was that it provides the possibility of +some parallell processing. Although since git-annex hits the filesystem +heavily and mostly runs other git commands, maybe not a whole lot. + +Anyway, each git-annex command is broken down into a series of independant +actions, which has some potential for parallelism. + +Each action has 3 distinct phases, basically "check", "perform", and +"cleanup". The perform actions are probably parellizable; the cleanup may be +(but not if it has to run git commands to stage state; it can queue +commands though); the check should be easily parallelizable, although they +may access the disk or run minor git query commands, so would probably not +want to run too many of them at once. diff --git a/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment b/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment new file mode 100644 index 0000000000..4aceb3abd3 --- /dev/null +++ b/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 1" + date="2011-04-08T12:41:43Z" + content=""" +I also think, that fetching keys via rsync can be done by one rsync process, when the keys are fetched from one host. This would avoid establishing a new TCP connection for every file. +"""]] diff --git a/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment b/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment new file mode 100644 index 0000000000..6ecce52c42 --- /dev/null +++ b/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://ertai.myopenid.com/" + nickname="npouillard" + subject="comment 2" + date="2011-05-20T20:14:15Z" + content=""" +I agree with Christian. + +One should first make a better use of connections to remotes before exploring parallel possibilities. One should pipeline the requests and answers. + +Of course this could be implemented using parallel&concurrency features of Haskell to do this. +"""]] diff --git a/doc/todo/pushpull.mdwn b/doc/todo/pushpull.mdwn new file mode 100644 index 0000000000..6828b35b2f --- /dev/null +++ b/doc/todo/pushpull.mdwn @@ -0,0 +1,4 @@ +--push/--pull should take a reponame and files, and push those files + to that repo; dropping them from the current repo + +[[done]] (move --from/--to) diff --git a/doc/todo/redundancy_stats_in_status.mdwn b/doc/todo/redundancy_stats_in_status.mdwn new file mode 100644 index 0000000000..56095fd333 --- /dev/null +++ b/doc/todo/redundancy_stats_in_status.mdwn @@ -0,0 +1,23 @@ +Currently, `git annex status` only shows the size of 1 copy of each file. +If numcopies is being used for redundancy, much more disk can actually be +in use than status shows. + +One idea: + + known annex size: 2 terabytes (plus 4 terabytes of redundant copies) + +But, to get that number, it would have to walk every location log, +counting how many copies currently exist of each file. That would make +status a lot slower than it is. + +One option is to just put it at the end of the status: + + redundancy: 300% (4 terabytes of copies) + +And ctrl-c if it's taking too long. + +Hmm, fsck looks at that same info. Maybe it could cache the redundancy +level it discovers? Since fsck can be run incrementally, it would be tricky +to get an overall number. And the number would tend to be stale, but +then again it might also be nice if status shows how long ago the last fsck +was. diff --git a/doc/todo/redundancy_stats_in_status/comment_1_9f1c10f8cea4fa60a99cbcc8306dd5de._comment b/doc/todo/redundancy_stats_in_status/comment_1_9f1c10f8cea4fa60a99cbcc8306dd5de._comment new file mode 100644 index 0000000000..801c1da039 --- /dev/null +++ b/doc/todo/redundancy_stats_in_status/comment_1_9f1c10f8cea4fa60a99cbcc8306dd5de._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2012-10-30T08:09:13Z" + content=""" +I like the idea of using fsck as a pre-run for status. + +Basically, it's the same as `updatedb` and `locate`; locate will warn the user if it considers its cache to be too old, as well. +"""]] diff --git a/doc/todo/resuming_encrypted_uploads.mdwn b/doc/todo/resuming_encrypted_uploads.mdwn new file mode 100644 index 0000000000..b3aaa7f966 --- /dev/null +++ b/doc/todo/resuming_encrypted_uploads.mdwn @@ -0,0 +1,22 @@ +Resuming interrupted uploads to encrypted special remotes is not currently +possible, because gpg does not produce consistent output. Special remotes +that could support resuming include rsync and glacier. + +Without consistent output, git-annex would need to locally cache the encrypted +file, and reuse that cache when resuming an upload. This would make +encrypted uploads more expensive in terms of both file IO and disk space +used. + +[It would be possible to write to the cache at the same time the special +remote is being fed data, and if the special remote upload fails, continue +writing the rest of the file. That would avoid half the overhead, since +the file would not need to be read from, just written to. (Although OS +caching may accomplish the same thing.)] + +Also, `git annex unused` would need to show temp files for uploads, +the same as it currently shows temp files for downloads, and users would +sometimes need to manually dropunused old uploads, that never completed. + +The question, then, is whether resuming uploads is useful enough to add +this overhead and user-visible complexity. +--[[Joey]] diff --git a/doc/todo/resuming_encrypted_uploads/comment_1_1832a6fb78e8ad7c838582f46731ac3b._comment b/doc/todo/resuming_encrypted_uploads/comment_1_1832a6fb78e8ad7c838582f46731ac3b._comment new file mode 100644 index 0000000000..cf35de0492 --- /dev/null +++ b/doc/todo/resuming_encrypted_uploads/comment_1_1832a6fb78e8ad7c838582f46731ac3b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://phil.0x539.de/" + nickname="Philipp Kern" + subject="comment 1" + date="2012-12-28T23:23:29Z" + content=""" +Doesn't the encryption part already write an encrypted version completely to disk before starting to upload? (At least with the rsync remote?) So the disk space and I/O usage is already there. (Except that it's probably a temporary file that's deleted after it was created, so that it's not kept around by the filesystem when certain destructive events strike.) +"""]] diff --git a/doc/todo/resuming_encrypted_uploads/comment_2_2ecc8e782f49e90ed1549e9179eb1a1e._comment b/doc/todo/resuming_encrypted_uploads/comment_2_2ecc8e782f49e90ed1549e9179eb1a1e._comment new file mode 100644 index 0000000000..a2bab9244b --- /dev/null +++ b/doc/todo/resuming_encrypted_uploads/comment_2_2ecc8e782f49e90ed1549e9179eb1a1e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog" + nickname="Michael" + subject="comment 2" + date="2012-12-29T08:00:56Z" + content=""" +Being able to resume transfers of encrypted files would absolutely be useful! Disk space is cheap, but bandwidth is not. +"""]] diff --git a/doc/todo/rsync.mdwn b/doc/todo/rsync.mdwn new file mode 100644 index 0000000000..3353f19c43 --- /dev/null +++ b/doc/todo/rsync.mdwn @@ -0,0 +1,4 @@ +Transferring a file from a ssh:// remote should use rsync to allow resuming +of a prior transfer. + +[[done]] diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn new file mode 100644 index 0000000000..6103ffa61e --- /dev/null +++ b/doc/todo/smudge.mdwn @@ -0,0 +1,162 @@ +git-annex should use smudge/clean filters. + +---- + +Update: Currently, this does not look likely to work. In particular, +the clean filter needs to consume all stdin from git, which consists of the +entire content of the file. It cannot optimise by directly accessing +the file in the repository, because git may be cleaning a different +version of the file during a merge. + +So every `git status` would need to read the entire content of all +available files, and checksum them, which is too expensive. + +> Update from GitTogether: Peff thinks a new interface could be added to +> git to handle this sort of case in an efficient way.. just needs someone +> to do the work. --[[Joey]] + +---- + +The clean filter is run when files are staged for commit. So a user could copy +any file into the annex, git add it, and git-annex's clean filter causes +the file's key to be staged, while its value is added to the annex. + +The smudge filter is run when files are checked out. Since git annex +repos have partial content, this would not git annex get the file content. +Instead, if the content is not currently available, it would need to do +something like return empty file content. (Sadly, it cannot create a +symlink, as git still wants to write the file afterwards.) + +So the nice current behavior of unavailable files being clearly missing due +to dangling symlinks, would be lost when using smudge/clean filters. +(Contact git developers to get an interface to do this?) + +Instead, we get the nice behavior of not having to remeber to `git annex +add` files, and just being able to use `git add` or `git commit -a`, +and have it use git-annex when .gitattributes says to. Also, annexed +files can be directly modified without having to `git annex unlock`. + +### design + +In .gitattributes, the user would put something like "* filter=git-annex". +This way they could control which files are annexed vs added normally. + +(git-annex could have further controls to allow eg, passing small files +through to regular processing. At least .gitattributes is a special case, +it should never be annexed...) + +For files not configured this way, git-annex could continue to use +its symlink method -- this would preserve backwards compatability, +and even allow mixing the two methods in a repo as desired. + +To find files in the repository that are annexed, git-annex would do +`ls-files` as now, but would check if found files have the appropriate +filter, rather than the current symlink checks. To determine the key +of a file, rather than reading its symlink, git-annex would need to +look up the git blob associated with the file -- this can be done +efficiently using the existing code in `Branch.catFile`. + +The clean filter would inject the file's content into the annex, and hard +link from the annex to the file. Avoiding duplication of data. + +The smudge filter can't do that, so to avoid duplication of data, it +might always create an empty file. To get the content, `git annex get` +could be used (which would hard link it). A `post-checkout` hook might +be used to set up hard links for all currently available content. + +#### clean + +The trick is doing it efficiently. Since git a2b665d, v1.7.4.1, +something like this works to provide a filename to the clean script: + + git config --global filter.huge.clean huge-clean %f + +This could avoid it needing to read all the current file content from stdin +when doing eg, a git status or git commit. Instead it is passed the +filename that git is operating on, in the working directory. +(Update: No, doesn't work; git may be cleaning a different file content +than is currently on disk, and git requires all stdin be consumed too.) + +So, WORM could just look at that file and easily tell if it is one +it already knows (same mtime and size). If so, it can short-circuit and +do nothing, file content is already cached. + +SHA1 has a harder job. Would not want to re-sha1 the file every time, +probably. So it'd need a local cache of file stat info, mapped to known +objects. + +But: Even with %f, git actually passes the full file content to the clean +filter, and if it fails to consume it all, it will crash (may only happen +if the file is larger than some chunk size; tried with 500 mb file and +saw a SIGPIPE.) This means unnecessary works needs to be done, +and it slows down *everything*, from `git status` to `git commit`. +**showstopper** I have sent a patch to the git mailing list to address +this. (Update: apparently +can't be fixed.) + +#### smudge + +The smudge script can also be provided a filename with %f, but it +cannot directly write to the file or git gets unhappy. + +### dealing with partial content availability + +The smudge filter cannot be allowed to fail, that leaves the tree and +index in a weird state. So if a file's content is requested by calling +the smudge filter, the trick is to instead provide dummy content, +indicating it is not available (and perhaps saying to run "git-annex get"). + +Then, in the clean filter, it has to detect that it's cleaning a file +with that dummy content, and make sure to provide the same identifier as +it would if the file content was there. + +I've a demo implementation of this technique in the scripts below. + +---- + +### test files + +huge-smudge: + +
+#!/bin/sh
+read f
+file="$1"
+echo "smudging $f" >&2
+if [ -e ~/$f ]; then
+	cat ~/$f # possibly expensive copy here
+else
+	echo "$f not available"
+fi
+
+ +huge-clean: + +
+#!/bin/sh
+file="$1"
+cat >/tmp/file
+# in real life, this should be done more efficiently, not trying to read
+# the whole file content!
+if grep -q 'not available' /tmp/file; then
+	awk '{print $1}' /tmp/file # provide what we would if the content were avail!
+	exit 0
+fi
+echo "cleaning $file" >&2
+# XXX store file content here
+echo $file
+
+ +.gitattributes: + +
+*.huge filter=huge
+
+ +in .git/config: + +
+[filter "huge"]
+        clean = huge-clean %f
+        smudge = huge-smudge %f
+
diff --git a/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment b/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment
new file mode 100644
index 0000000000..a4eb3cf235
--- /dev/null
+++ b/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="git-add instead of git-annex-add"
+ date="2011-02-26T21:43:21Z"
+ content="""
+would, with these modifications in place, there still be a way to *really* git-add a file? (my main repository contains both normal git and git-annex files.)
+"""]]
diff --git a/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment b/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment
new file mode 100644
index 0000000000..3a223e1c7b
--- /dev/null
+++ b/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="symlinks"
+ date="2011-04-03T20:30:21Z"
+ content="""
+> (Sadly, it cannot create a symlink, as git still wants to write the file afterwards.
+> So the nice current behavior of unavailable files being clearly missing due to dangling symlinks, would be lost when using smudge/clean filters. (Contact git developers to get an interface to do this?)
+
+Have you checked what the smudge filter sees when the input is a symlink? Because git supports tracking symlinks, so it should also support pushing symlinks through a smudge filter, right?
+Either way: yes, contact the git devs, one can only ask and hope.  And if you can demonstrate the awesomeness of git-annex they might get more 1interested :)
+"""]]
diff --git a/doc/todo/special_remote_for_amazon_glacier.mdwn b/doc/todo/special_remote_for_amazon_glacier.mdwn
new file mode 100644
index 0000000000..0fa77b5275
--- /dev/null
+++ b/doc/todo/special_remote_for_amazon_glacier.mdwn
@@ -0,0 +1,30 @@
+Amazon's new glacier service would be a nice special remote to support for
+long-term archival.
+
+The main difficulty is that glacier is organized into vaults, and accessing
+a file in a vault takes ~4 hours. A naive implementation would make `git
+annex get` wait for 4 hours, which is certianly not reasonable.
+
+One approach I am pondering is to make each glacier vault a separate
+special remote. You could then request git-annex to spin up a remote, and
+come back later, and be able to access the data stored in it (need to check
+if glacier would also allow adding new data to it then). This is
+conceptually similar to using git-annex with offline removable drives,
+except with glacier, you have a controllable robot to get them plugged in. :)
+
+Ideally, git-annex would arrange for glacier to send it a message when the
+vault becomes available, and the user could queue a list of commands to
+run, or files to transfer, at that point.
+
+--[[Joey]]
+
+> [[done]]! --[[Joey]]
+
+-----
+
+> In the coming months, Amazon S3 will introduce an option that will allow customers to seamlessly move data between Amazon S3 and Amazon Glacier based on data lifecycle policies.
+
+-- 
+
+>> They did, but it's IMHO not very useful for git-annex. It's rather
+>> intended to allow aging S3 storage out to Glacier. --[[Joey]] 
diff --git a/doc/todo/special_remote_for_amazon_glacier/comment_1_68f129441eefcbfebf7a9db680f52759._comment b/doc/todo/special_remote_for_amazon_glacier/comment_1_68f129441eefcbfebf7a9db680f52759._comment
new file mode 100644
index 0000000000..68593be425
--- /dev/null
+++ b/doc/todo/special_remote_for_amazon_glacier/comment_1_68f129441eefcbfebf7a9db680f52759._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://mike.magin.org/"
+ nickname="mmagin"
+ subject="comment 1"
+ date="2012-09-14T04:19:53Z"
+ content="""
+When I first heard about Glacier, it sounded great for a cheap backup copy, and I was thinking about writing a \"hook\" remote, but once I read some better analysis of the pricing (e.g. [[http://www.daemonology.net/blog/2012-09-04-thoughts-on-glacier-pricing.html]]) I rapidly lost interest.
+"""]]
diff --git a/doc/todo/special_remote_for_amazon_glacier/comment_2_c5eeaf8ceee414fa0379831ca52e290c._comment b/doc/todo/special_remote_for_amazon_glacier/comment_2_c5eeaf8ceee414fa0379831ca52e290c._comment
new file mode 100644
index 0000000000..701047f91c
--- /dev/null
+++ b/doc/todo/special_remote_for_amazon_glacier/comment_2_c5eeaf8ceee414fa0379831ca52e290c._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="basak"
+ subject="comment 2"
+ date="2012-09-21T22:21:04Z"
+ content="""
+I've created a glacier command line interface that integrates with git-annex [here](https://github.com/basak/glacier-cli), currently using the hook special remote mechanism. To get around the time delay, operations which require a job submission will submit the job and then fail. Retrying again four hours later should then succeed. It seems to work pretty well with git-annex.
+"""]]
diff --git a/doc/todo/speed_up_fsck.mdwn b/doc/todo/speed_up_fsck.mdwn
new file mode 100644
index 0000000000..5d5e867f80
--- /dev/null
+++ b/doc/todo/speed_up_fsck.mdwn
@@ -0,0 +1,40 @@
+moving to the git-annex branch has slowed down fsck worse than most
+commands. Actually, some commands have sped up, while others like get
+are slightly slower but are swamped by the normal runtime. 
+
+For fsck though, it has to pull each file's location log info out of git.
+And, it's typically run on the entire tree.
+
+Another slow one in `git annex copy --from`.
+
+It would be possible to run a single `git cat-file --batch` and pass it
+sha1s of location logs for file that is going to be fsked (gotten via
+`read-tree`). Then just read its output until the next requested sha1 to
+chunk it, and pass this in to fsck in a closure.
+
+The difficulty, besides writing that is that everything that works with
+location logs now reads them out of git, would need to find a way to
+provide the info on a side channel of some sort.
+
+If this is implemented, the same infrastructure could be used for other
+commands like whereis and add. --[[Joey]]
+
+> Updated plan:
+> 
+> Run `git ls-file --batch`, and cache its stdin and out handles in Branch
+> state.
+> 
+> To see a git-annex branch file, send it something like
+> "git-annex:uuid.log", and read the content fron stdout handle.
+> 
+> To detect the end of content, send "TOKEN\n", and look for 
+> "TOKEN missing" in its output. A good choice for TOKEN is anything
+> that will never exist in the repo; 40 0's would be a fairly good choice,
+> but even better seems to be something completely invalid and impossible
+> to have as a sha1 or filename or ref: "".
+> 
+> Hmm, except that's actually an error message sent to stderr. Unless
+> stderr is connected to stdout, it might be better to look for a known,
+> empty object. Could just add a git-annex:empty file to that end.
+
+[[done]] --[[Joey]] 
diff --git a/doc/todo/support-non-utf8-locales.mdwn b/doc/todo/support-non-utf8-locales.mdwn
new file mode 100644
index 0000000000..da40118d52
--- /dev/null
+++ b/doc/todo/support-non-utf8-locales.mdwn
@@ -0,0 +1,26 @@
+Currenty, git-annex forces output, particularly of filenames, in a utf-8
+locale.
+
+Note that this does not mean it cannot be used with filenames in other
+encodings. git-annex is entirely encoding agnostic when it comes to 
+manipulating filenames. It just *displays* their names always converted to
+utf-8, which  may not look right when you have a non-utf8 locale.
+
+This had to be done to work around some bugs with haskell's handling
+of filename encodings. In particular,
+
+* [[bugs/unhappy_without_UTF8_locale]]: haskell crashes when told to output 
+  a string with characters > 255 in a non-utf8 locale.
+* [[bugs/problems_with_utf8_names]]: On many OSs, haskell expects
+  non-decoded raw char8 in FilePaths. In order to display a filename,
+  though, it needs to first be decoded, and git-annex currently assumes
+  it was encoded as utf8.
+
+git-annex's behavior is unlikely to improve much until haskell's
+support for utf8 filenames improves. --[[Joey]]
+
+> [[done]] -- I just turned off all encoding handling on stdout and stderr,
+> which avoids these problems nicely. Git-annex now displays just what it
+> input, at least on platforms where haskell does not decode unicode in
+> FilePaths. This will later be a problem when it gets localized, but for
+> now works great. --[[Joey]]
diff --git a/doc/todo/support_S3_multipart_uploads.mdwn b/doc/todo/support_S3_multipart_uploads.mdwn
new file mode 100644
index 0000000000..711ac41b2a
--- /dev/null
+++ b/doc/todo/support_S3_multipart_uploads.mdwn
@@ -0,0 +1,14 @@
+Did not know of this when I wrote S3 support. Ability to resume large
+uploads would be good.
+
+
+
+Also allows supporting files > 5 gb, a S3 limit I was not aware of.
+
+NB: It would work just as well to split the object and upload the N parts
+to S3, but not bother with S3's paperwork to rejoin them into one object. 
+Only reasons not to do that are a) backwards compatability with 
+the existing S3 remote and b) this would not allow accessing the content
+in S3 w/o using git-annex, which could be useful in some scenarios.
+
+--[[Joey]]
diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
new file mode 100644
index 0000000000..32ced467e0
--- /dev/null
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -0,0 +1,17 @@
+What is says on the tin:
+
+    22:56:54 < RichiH> joeyh_: by the way, i have been thinking about fsck on bare repos
+    22:57:37 < RichiH> joeyh_: the best i could come with is to have a bare and a non-bare access the same repo store
+    22:58:00 < RichiH> joeyh_: alternatively, with the SHA* backend, you have all the information to verify that the local data is correct
+    22:58:41 < RichiH> and verifying that would already be a plus. if there  really _is_ a problem, having the SHA is enough to track issues down
+    23:09:50 < joeyh_> oh, I think I have code that fsck could use on bare repos already.. just a matter of wiring it up
+    23:10:42 < joeyh_> feel free to reopen a bug or whatever so I remember.. the unused command's branch content enumeration could be used in a bare repo
+    23:14:51 < joeyh_> unused/dropunused could work in bare repos too btw
+
+> Also `status`'s total annex keys/size could be handled for bare repos. --[[Joey]] 
+
+>> Fsck is done. Rest not done yet. --[[Joey]]
+
+>>> all [[done]]! --[[Joey]] 
+
+[[!meta title="support unused, dropunused in bare repos"]]
diff --git a/doc/todo/symlink_farming_commit_hook.mdwn b/doc/todo/symlink_farming_commit_hook.mdwn
new file mode 100644
index 0000000000..3e93cb34b8
--- /dev/null
+++ b/doc/todo/symlink_farming_commit_hook.mdwn
@@ -0,0 +1,14 @@
+TODO: implement below
+
+git-annex does use a lot of symlinks. Specicially, relative symlinks,
+that are checked into git. To allow you to move those around without
+annoyance, git-annex can run as a post-commit hook. This way, you can `git mv`
+a symlink to an annexed file, and as soon as you commit, it will be fixed
+up.
+
+`git annex init` tries to set up a post-commit hook that is itself a symlink
+back to git-annex. If you want to have your own shell script in the post-commit
+hook, just make it call `git annex` with no parameters. git-annex will detect
+when it's run from a git hook and do the necessary fixups.
+
+[[done]]
diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn
new file mode 100644
index 0000000000..9019767eb9
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals.mdwn
@@ -0,0 +1,21 @@
+[[forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] is a good
+start, but Zooko points out that using Tahoe's directory translation layer
+incurs O(N^2) overhead as the number of objects grows. Also, making
+hash subdirectories in Tahoe is expensive. Instead it would be good to use
+it as a key/value store directly. The catch is that doing so involves
+sending the content to Tahoe, and getting back a key identifier.
+
+This would be fairly easy to do as a [[backend|backends]], which can assign its
+own key names (although typically done before data is stored in it),
+but a tahoe-lafs special remote would be more flexible.
+
+To support a special remote, a mapping is needed from git-annex keys to
+Tahoe keys.
+
+The best place to store this mapping is perhaps as a new field in the
+location log:
+
+	date present repo-uuid newfields
+
+This way, each remote can store its own key-specfic data in the same place
+as other key-specific data, with minimal overhead.
diff --git a/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment b/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment
new file mode 100644
index 0000000000..16ef882a42
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="zooko"
+ ip="97.118.97.117"
+ subject="performance"
+ date="2011-05-17T19:20:39Z"
+ content="""
+Hm... O(N^2)? I think it just takes O(N). To read an entry out of a directory you have to download the entire directory (and store it in RAM and parse it). The constants are basically \"too big to be good but not big enough to be prohibitive\", I think. jctang has reported that his special remote hook performs well enough to use, but it would be nice if it were faster.
+
+The Tahoe-LAFS folks are working on speeding up mutable files, by the way, after which we would be able to speed up directories.
+"""]]
diff --git a/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
new file mode 100644
index 0000000000..6dba86c47c
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-05-17T19:57:33Z"
+ content="""
+Whoops! You'd only told me O(N) twice before..
+
+So this is not too high priority. I think I would like to get the per-remote storage sorted out anyway, since probably it will be the thing needed to convert the URL backend into a special remote, which would then allow ripping out the otherwise unused pluggable backend infrastructure.
+
+Update: Per-remote storage is now sorted out, so this could be implemented
+if it actually made sense to do so.
+"""]]
diff --git a/doc/todo/union_mounting.mdwn b/doc/todo/union_mounting.mdwn
new file mode 100644
index 0000000000..c42a055021
--- /dev/null
+++ b/doc/todo/union_mounting.mdwn
@@ -0,0 +1,10 @@
+It should be possible to union mount annexes. So if multiple drives have
+content, an annex mounting them both would have available all the 
+content from all the drives.
+
+This could be done by just making .git/annex/KEY link to the actual content
+on the mounted annex.
+
+(Need to make sure the [[copy_tracking|copies]] code does not
+confused and think the symlink is a copy of the content.. Also need to make
+sure that code that writes to .git/annex does not follow symlinks.))
diff --git a/doc/todo/union_mounting/comment_1_cb08435812dd7766de26199c73f38e8b._comment b/doc/todo/union_mounting/comment_1_cb08435812dd7766de26199c73f38e8b._comment
new file mode 100644
index 0000000000..3fadf6fa3c
--- /dev/null
+++ b/doc/todo/union_mounting/comment_1_cb08435812dd7766de26199c73f38e8b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawln3ckqKx0x_xDZMYwa9Q1bn4I06oWjkog"
+ nickname="Michael"
+ subject="comment 1"
+ date="2013-03-01T01:26:36Z"
+ content="""
+This would indeed be very helpful when remotely mounting a photo/video collection over samba.
+"""]]
diff --git a/doc/todo/union_mounting/comment_2_240b1736f6bd4fbf87c372d3a46e661b._comment b/doc/todo/union_mounting/comment_2_240b1736f6bd4fbf87c372d3a46e661b._comment
new file mode 100644
index 0000000000..08901ee178
--- /dev/null
+++ b/doc/todo/union_mounting/comment_2_240b1736f6bd4fbf87c372d3a46e661b._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 2"
+ date="2013-03-01T04:50:28Z"
+ content="""
++1 this would be sweet as hell
+
+"""]]
diff --git a/doc/todo/use_cp_reflink.mdwn b/doc/todo/use_cp_reflink.mdwn
new file mode 100644
index 0000000000..39518abf18
--- /dev/null
+++ b/doc/todo/use_cp_reflink.mdwn
@@ -0,0 +1,7 @@
+The unlock command needs to copy a file, and it would be great to use this:
+	cp --reflink=auto src dst
+
+O(1) overhead on BTRFS. Needs coreutils 7.6; and remember that git-annex
+may be used on systems without coreutils..
+
+[[done]]
diff --git a/doc/todo/using_url_backend.mdwn b/doc/todo/using_url_backend.mdwn
new file mode 100644
index 0000000000..1f3cd56281
--- /dev/null
+++ b/doc/todo/using_url_backend.mdwn
@@ -0,0 +1,11 @@
+There is no way to `git annex add` a file using the URL [[backend|backends]].
+
+For now, we have to manually make the symlink. Something like this:
+
+	ln -s .git/annex/URL:http:%%www.example.com%foo.tar.gz
+
+Note the escaping of slashes.
+
+A `git annex register ` command could do this..
+
+[[done]]
diff --git a/doc/todo/windows_support.mdwn b/doc/todo/windows_support.mdwn
new file mode 100644
index 0000000000..c64e6fce5c
--- /dev/null
+++ b/doc/todo/windows_support.mdwn
@@ -0,0 +1,61 @@
+Can it be built on Windows?
+
+short answer: not yet
+
+First, you need to get some unix utilities for windows. Git of course.
+Also rsync, and a `cp` command that understands at least `cp -p`, and
+`uuid`, and `xargs` and `sha1sum`. Note that some of these could be
+replaced with haskell libraries to some degree.
+
+There are probably still some places where it assumes / as a path
+separator, although I fixed probably almost all by now.
+
+Then windows versions of these functions could be found,
+which are all the ones that need POSIX, I think. A fair amount of this,
+the stuff to do with signals and users, could be empty stubs in windows.
+The file manipulation, particularly symlinks, would probably be the main
+challenge.
+
+
+addSignal
+blockSignals
+changeWorkingDirectory
+createLink
+createSymbolicLink
+emptySignalSet
+executeFile
+fileMode
+fileSize
+forkProcess
+getAnyProcessStatus
+getEffectiveUserID
+getEnvDefault
+getFileStatus
+getProcessID
+getProcessStatus
+getSignalMask
+getSymbolicLinkStatus
+getUserEntryForID
+getUserEntryForName
+groupWriteMode
+homeDirectory
+installHandler
+intersectFileModes
+isRegularFile
+isSymbolicLink
+modificationTime
+otherWriteMode
+ownerWriteMode
+readSymbolicLink
+setEnv
+setFileMode
+setSignalMask
+sigCHLD
+sigINT
+unionFileModes
+

+
+A good starting point is
+. However, note
+that its implementations of stuff like `createSymbolicLink` are stubs.
+--[[Joey]] 
diff --git a/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment b/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment
new file mode 100644
index 0000000000..fd5b6f5cd3
--- /dev/null
+++ b/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkRITTYYsN0TFKN7G5sZ6BWGZOTQ88Pz4s"
+ nickname="Zoltán"
+ subject="cygwin"
+ date="2012-05-15T00:14:08Z"
+ content="""
+What about [Cygwin](http://cygwin.com/)? It emulates POSIX fairly well under Windows (including signals, forking, fs (also things like /dev/null, /proc), unix file permissions), has all standard gnu utilities. It also emulates symlinks, but they are unfortunately incompatible with NTFS symlinks introduced in Vista [due to some stupid restrictions on Windows](http://cygwin.com/ml/cygwin/2009-10/msg00756.html).
+
+If git-annex could be modified to not require symlinks to work, the it would be a pretty neat solution (and you get a real shell, not some command.com on drugs (aka cmd.exe))
+"""]]
diff --git a/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment b/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment
new file mode 100644
index 0000000000..e37a555756
--- /dev/null
+++ b/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk5cj-itfFHq_yhJHdzk3QOPp-PNW_MjPU"
+ nickname="Michael"
+ subject="+1 Cygwin"
+ date="2012-05-23T19:30:21Z"
+ content="""
+Windows support is a must. In my experience, binary file means proprietary editor, which means Windows.
+
+Unfortunately, there's not much overlap between people who use graphical editors in Windows all day vs. people who are willing to tolerate Cygwin's setup.exe, compile a Haskell program, learn git and git-annex's 90-odd subcommands, and use a mintty terminal to manage their repository, especially now that there's a sexy GitHub app for Windows.
+
+That aside, I think Windows-based content producers are still *the* audience for git-annex. First Windows support, then a GUI, then the world.
+"""]]
diff --git a/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment b/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment
new file mode 100644
index 0000000000..0b48db7502
--- /dev/null
+++ b/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://xolus.net/openid/max"
+ nickname="B0FH"
+ subject="What about NTFS support ?"
+ date="2012-08-02T17:45:10Z"
+ content="""
+Has git-annex been tested with an NTFS-formatted disk under Linux ? NTFS is supposed to be case-sensitive and to allow symlinks, and these are supposed to work with ntfs3g.
+"""]]
diff --git a/doc/todo/windows_support/comment_4_ad06b98b2ddac866ffee334e41fee6a8._comment b/doc/todo/windows_support/comment_4_ad06b98b2ddac866ffee334e41fee6a8._comment
new file mode 100644
index 0000000000..66f9ca71fd
--- /dev/null
+++ b/doc/todo/windows_support/comment_4_ad06b98b2ddac866ffee334e41fee6a8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlc1og3PqIGudOMkFNrCCNg66vB7s-jLpc"
+ nickname="Paul"
+ subject="Re: What about NTFS support?"
+ date="2012-08-16T19:30:38Z"
+ content="""
+I successfully use git-annex on an NTFS formatted external USB drive, so yes, it is possible and works well.
+"""]]
diff --git a/doc/todo/windows_support/comment_5_444fc7251f57db241b6e80abae41851c._comment b/doc/todo/windows_support/comment_5_444fc7251f57db241b6e80abae41851c._comment
new file mode 100644
index 0000000000..8f76ee2587
--- /dev/null
+++ b/doc/todo/windows_support/comment_5_444fc7251f57db241b6e80abae41851c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/dASECLNzvckz4VwqUGYsvthiplY.#d2c27"
+ nickname="A. D. Sicks"
+ subject="comment 5"
+ date="2012-09-09T23:48:21Z"
+ content="""
+Haskell has C++ binding, so it should be possible to port it to .net/Mono with a VB GUI for Windows users.  Windows has a primitive form of symlinks called shortcuts.  Perhaps shortcut support in Windows could replace the use of symlinks.  I've used shortcuts since XP to put my home Windows directory on another partition and never had a hitch...
+
+If anyone is interested in working on this, hit me up.  I would like to use this in my XP vbox to have access to files on my host OS...I have a student edition of Visual Studio 2005 to do an open source port...
+"""]]
diff --git a/doc/todo/windows_support/comment_6_34f1f60b570c389bb1e741b990064a7e._comment b/doc/todo/windows_support/comment_6_34f1f60b570c389bb1e741b990064a7e._comment
new file mode 100644
index 0000000000..bf9f86f419
--- /dev/null
+++ b/doc/todo/windows_support/comment_6_34f1f60b570c389bb1e741b990064a7e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnlwEMhiNYv__mEUABW4scn83yMraC3hqE"
+ nickname="Sean"
+ subject="NTFS symlinks"
+ date="2013-01-11T21:44:21Z"
+ content="""
+It seems that NTFS (from Vista forward) has full POSIX support for symlinks. At least, Wikipedia [seems to think so.](http://en.wikipedia.org/wiki/NTFS_symbolic_link). What about doing like GitHub and using MinGW for compatibility? Cygwin absolutely blows in terms of installation size and compatability with the rest of Windows.
+"""]]
diff --git a/doc/todo/wishlist:_An_--all_option_for_dropunused.mdwn b/doc/todo/wishlist:_An_--all_option_for_dropunused.mdwn
new file mode 100644
index 0000000000..7f7ac13175
--- /dev/null
+++ b/doc/todo/wishlist:_An_--all_option_for_dropunused.mdwn
@@ -0,0 +1 @@
+Cleaning out a repository is presently a fairly manual process.  Am I missing a UI trick?  "dropunsed" with no arguments prints nothing at all; I think in that case it should display the list of what could be dropped.
diff --git a/doc/todo/wishlist:_An_--all_option_for_dropunused/comment_1_d8726d108b3b40116b4ec3c9935f2dff._comment b/doc/todo/wishlist:_An_--all_option_for_dropunused/comment_1_d8726d108b3b40116b4ec3c9935f2dff._comment
new file mode 100644
index 0000000000..6c728a5d00
--- /dev/null
+++ b/doc/todo/wishlist:_An_--all_option_for_dropunused/comment_1_d8726d108b3b40116b4ec3c9935f2dff._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.0.23"
+ subject="comment 1"
+ date="2012-10-22T15:35:30Z"
+ content="""
+`git annex unused` prints the list
+"""]]
diff --git a/doc/todo/wishlist:_An_--all_option_for_dropunused/comment_2_578248f7686ba2d80d7dc8b17c0cdf52._comment b/doc/todo/wishlist:_An_--all_option_for_dropunused/comment_2_578248f7686ba2d80d7dc8b17c0cdf52._comment
new file mode 100644
index 0000000000..a87a367d61
--- /dev/null
+++ b/doc/todo/wishlist:_An_--all_option_for_dropunused/comment_2_578248f7686ba2d80d7dc8b17c0cdf52._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://hands.com/~phil/"
+ nickname="hands"
+ subject="and you can specify ranges to dropunused"
+ date="2012-11-02T09:07:48Z"
+ content="""
+so having run:
+
+    git annex unused
+
+you can then run:
+
+    git annex dropunused 1-10000
+
+or whatever, and it deletes the items in that range from the most recent unused invocation
+"""]]
diff --git a/doc/todo/wishlist:_An_option_like_--git-dir.mdwn b/doc/todo/wishlist:_An_option_like_--git-dir.mdwn
new file mode 100644
index 0000000000..cb9d374b39
--- /dev/null
+++ b/doc/todo/wishlist:_An_option_like_--git-dir.mdwn
@@ -0,0 +1,3 @@
+I'm currently integrating git-annex support into a filesystem synchronization tool that I use, and I have a use case where I'd like to run "git annex sync' on a local directory, and then automatically ssh over to remote hosts and run "git annex sync" in the related annex on that remote host.  However, while I can easily "cd" on the local, there is no really easy way to "cd" on the remote without a hack.
+
+If I could say: git annex --annex-dir=PATH sync, where PATH is the annex directory, it would solve all my problems, and would also provide a nice correlation to the --git-dir option used by most Git commands.  The basic idea is that I shouldn't have to be IN the directory to run git-annex commands, I should be able to tell git-annex which directory to apply its commands to.
diff --git a/doc/todo/wishlist:_An_option_like_--git-dir/comment_1_5d877d90b8bdf21d4b8649744d229efd._comment b/doc/todo/wishlist:_An_option_like_--git-dir/comment_1_5d877d90b8bdf21d4b8649744d229efd._comment
new file mode 100644
index 0000000000..8e7c3c03ea
--- /dev/null
+++ b/doc/todo/wishlist:_An_option_like_--git-dir/comment_1_5d877d90b8bdf21d4b8649744d229efd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="What about..."
+ date="2012-10-16T16:43:29Z"
+ content="""
+    ssh remotehost \"cd /path/to/annex && git annex sync\"
+"""]]
diff --git a/doc/todo/wishlist:_An_option_like_--git-dir/comment_2_462264821cbc48a433330cbf7ec6044d._comment b/doc/todo/wishlist:_An_option_like_--git-dir/comment_2_462264821cbc48a433330cbf7ec6044d._comment
new file mode 100644
index 0000000000..980658dc64
--- /dev/null
+++ b/doc/todo/wishlist:_An_option_like_--git-dir/comment_2_462264821cbc48a433330cbf7ec6044d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 2"
+ date="2012-10-17T18:31:58Z"
+ content="""
+You can use `GIT_DIR`. It would not be hard to add a --git-dir option, the only catch is how to communicate that state on to where it constructs its git repository data structure. (I suppose it could just set GIT_DIR..)
+"""]]
diff --git a/doc/todo/wishlist:_An_option_like_--git-dir/comment_3_0c3709b07a0a1091ceeee73b69e0f7ac._comment b/doc/todo/wishlist:_An_option_like_--git-dir/comment_3_0c3709b07a0a1091ceeee73b69e0f7ac._comment
new file mode 100644
index 0000000000..a76c42d9d1
--- /dev/null
+++ b/doc/todo/wishlist:_An_option_like_--git-dir/comment_3_0c3709b07a0a1091ceeee73b69e0f7ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2grhJvAC049fJnvALDXek.6MRZMTlg--#eec89"
+ nickname="John"
+ subject="Response"
+ date="2012-10-20T05:21:13Z"
+ content="""
+@Justin If you have full shell access on the remote your solution works fine, but not if git-annex is the only binary you are allowed to execute.
+"""]]
diff --git a/doc/todo/wishlist:_Option_to_specify_max_transfer_rate.mdwn b/doc/todo/wishlist:_Option_to_specify_max_transfer_rate.mdwn
new file mode 100644
index 0000000000..3ecb421978
--- /dev/null
+++ b/doc/todo/wishlist:_Option_to_specify_max_transfer_rate.mdwn
@@ -0,0 +1,3 @@
+A big part of my online use is done via a low-speed connection over my mobile phone, this is limited to 16KB/sec because I always use up my 500MB quota the very first day of the month. `;-/` So when I need to download big files, I first download them to my online server, then transfer the files to my laptop with git-annex. If I'm connected via GSM, this occupies all the bandwidth and everything else moves like a heavily sedated slug. So if I want to work via VNC or SSH, I have to terminate ongoing transfers with Ctrl-C and then hopefully remember to restart it when I work locally. I know git-annex is robust enough to handle this gracefully, but it would be really nice to have a continuous connection going on in the background, limited to a value I choose.
+
+rsync(1) has a `--bwlimit` (bandwidth limit) where you can specify max download/upload speed in kilobytes/sec. It would be great if a similar option was integrated into git-annex. Thanks in advance.
diff --git a/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_1_4fd870e14b5b70c8a6ade41406294387._comment b/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_1_4fd870e14b5b70c8a6ade41406294387._comment
new file mode 100644
index 0000000000..78ca76939a
--- /dev/null
+++ b/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_1_4fd870e14b5b70c8a6ade41406294387._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="trickle"
+ date="2013-01-22T22:44:52Z"
+ content="""
+not exactly integrated, but you can easily use trickle for this.
+
+    trickle -d 50 git annex ...
+"""]]
diff --git a/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_2_dd854f297ad6a94b54be9f3edfd0f766._comment b/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_2_dd854f297ad6a94b54be9f3edfd0f766._comment
new file mode 100644
index 0000000000..70f04c6168
--- /dev/null
+++ b/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_2_dd854f297ad6a94b54be9f3edfd0f766._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://sunny256.sunbase.org/"
+ nickname="sunny256"
+ subject="Yay, trickle works"
+ date="2013-01-23T01:36:21Z"
+ content="""
+Justin, thanks a lot! trickle(1) works great. I didn't know about this program, but I'm not surprised that such a program is available. It never ceases to amaze me what's possible in a *NIX environment.
+"""]]
diff --git a/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_3_a8b7e90a473d5937807cc7eb456efe33._comment b/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_3_a8b7e90a473d5937807cc7eb456efe33._comment
new file mode 100644
index 0000000000..a5f8f6a1bf
--- /dev/null
+++ b/doc/todo/wishlist:_Option_to_specify_max_transfer_rate/comment_3_a8b7e90a473d5937807cc7eb456efe33._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 3"
+ date="2013-01-24T01:55:10Z"
+ content="""
+In addition to trickle, the git-annex man page has examples of how to make rsync use --bwlimit
+
+Something like trickle is needed to limit rates for remotes not using rsync, however.
+"""]]
diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
new file mode 100644
index 0000000000..341a9afa45
--- /dev/null
+++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
@@ -0,0 +1,45 @@
+Simple, when performing various git annex command over ssh, in particular a multi-file get, and using password authentication, git annex will prompt more than once for a user password.  This makes batch updates very inconvenient.
+
+> I'd suggest using ssh-agent, or a passwordless ssh key. Possibly in
+> combination with [[git-annex-shell]] if you want to lock down a
+> particular ssh key to only being able to use git-annex and git-daemon.
+> 
+> Combining multiple operations into a single ssh is on the todo list, but
+> very far down it. --[[Joey]]
+
+>> OTOH, automatically running ssh in ControlMaster mode (and stopping it
+>> at exit) would be useful and not hard thing for git-annex to do.
+>> 
+>> It'd just need to set the appropriate config options, setting
+>> ControlPath to a per-remote socket location that includes git-annex's
+>> pid. Then at shutdown, run `ssh -O exit` on each such socket.
+>> 
+>> Complicated slightly by not doing this if the user has already set up
+>> more broad ssh connection caching.
+>> 
+>> [[done]]! --[[Joey]]
+
+---
+
+Slightly more elaborate design for using ssh connection caching:
+
+* Per-uuid ssh socket in `.git/annex/ssh/user@host.socket`
+* Can be shared amoung concurrent git-annex processes as well as ssh
+  invocations inside the current git-annex.
+* Also a lock file, `.git/annex/ssh/user@host.lock`.
+  Open and take shared lock before running ssh; store lock in lock pool.
+  (Not locking socket directly, because ssh might want to.)
+* Run ssh like: `ssh -S .git/annex/ssh/user@host.socket -o ControlMaster=auto -o ControlPersist=yes user@host`
+* At shutdown, enumerate all existing sockets, and on each:
+  1. Drop any shared lock.
+  2. Attempt to take an exclusive lock (non-blocking).
+  3. `ssh -q -S .git/annex/ssh/user@host.socket -o ControlMaster=auto -o ControlPersist=yes -O stop user@host`
+     (Will exit nonzero if ssh is not running on that socket.)
+  4. And then remove the socket and the lock file.
+* Do same *at startup*. Why? In case an old git-annex was interrupted
+  and left behind a ssh. May have moved to a different network
+  in the meantime, etc, and be stalled waiting for a response from the
+  network, or talking to the wrong interface or something.
+  (Ie, the reason why I don't use ssh connection caching by default.)
+* User should be able to override this, to use their own preferred
+  connection caching setup. `annex.sshcaching=false`
diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment
new file mode 100644
index 0000000000..2801d8e68f
--- /dev/null
+++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-05-06T18:30:02Z"
+ content="""
+Unless you are forced to use a password, you should really be using a ssh key.
+
+    ssh-keygen
+    #put local .ssh/id_?sa.pub into remote .ssh/authorized_keys (which needs to be chmod 600)
+    ssh-add
+    git annex whatever
+
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
new file mode 100644
index 0000000000..9336535788
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -0,0 +1,28 @@
+(Hi, this is paulproteus@debian, AKA Asheesh).
+
+I've been enjoying using git-annex to archive my data.
+
+It's great that, by using git-annex and the SHA1 backend, I get a space-saving kind of deduplication through the symbolic links.
+
+I'm looking for the ability to filter files, before they get added to the annex, so that I don't add new files whose content is already in the annex.look  That would help me in terms of personal file organization.
+
+It seems there is not, so this is a wishlist bug filed so that maybe such a thing might exist. What I would really like to do is:
+
+* $ git annex add --no-add-if-already-present .
+* $ git commit -m "Slurping in some photos I found on my old laptop hard drive"
+
+And then I'd do something like:
+
+* $ git clean -f
+
+to remove the files that didn't get annexed in this run. That way, only one filename would ever point to a particular SHA1.
+
+I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks. If this feature existed, I could comfortably toss them all into one git annex that grew, bit by bit, to store all of these files exactly once.
+
+(I would be even happier for "git annex add --unlink-duplicates .")
+
+(Another way to do this would be to "git annex add" them all, and then use a "git annex remove-duplicates" that could prompt me about which files are duplicates of each other, and then I could pipe that command's output into xargs git rm.)
+
+(As I write this, I realize it's possible to parse the destination of the symlink in a way that does this..)
+
+> [[done]]; see [[tips/finding_duplicate_files]] --[[Joey]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment
new file mode 100644
index 0000000000..5dbb66cf66
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment
@@ -0,0 +1,68 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 10"
+ date="2011-12-23T17:22:11Z"
+ content="""
+> Your perl script is not O(n). Inserting into perl hash tables has
+> overhead of minimum O(n log n).
+
+What's your source for this assertion?  I would expect an amortized
+average of `O(1)` per insertion, i.e. `O(n)` for full population.
+
+> Not counting the overhead of resizing hash tables,
+> the grevious slowdown if the bucket size is overcome by data (it
+> probably falls back to a linked list or something then), and the
+> overhead of traversing the hash tables to get data out.
+
+None of which necessarily change the algorithmic complexity.  However
+real benchmarks are far more useful here than complexity analysis, and
+[the dangers of premature optimization](http://c2.com/cgi/wiki?PrematureOptimization) 
+should not be forgotten.
+
+> Your memory size calculations ignore the overhead of a hash table or
+> other data structure to store the data in, which will tend to be
+> more than the actual data size it's storing. I estimate your 50
+> million number is off by at least one order of magnitude, and more
+> likely two;
+
+Sure, I was aware of that, but my point still stands.  Even 500k keys
+per 1GB of RAM does not sound expensive to me.
+
+> in any case I don't want git-annex to use 1 gb of ram.
+
+Why not?  What's the maximum it should use?  512MB?  256MB?
+32MB?  I don't see the sense in the author of a program 
+dictating thresholds which are entirely dependent on the context
+in which the program is *run*, not the context in which it's *written*.
+That's why systems have files such as `/etc/security/limits.conf`.
+
+You said you want git-annex to scale to enormous repositories.  If you
+impose an arbitrary memory restriction such as the above, that means
+avoiding implementing *any* kind of functionality which requires `O(n)`
+memory or worse.  Isn't it reasonable to assume that many users use
+git-annex on repositories which are *not* enormous?  Even when they do
+work with enormous repositories, just like with any other program,
+they would naturally expect certain operations to take longer or
+become impractical without sufficient RAM.  That's why I say that this
+restriction amounts to throwing out the baby with the bathwater.
+It just means that those who need the functionality would have to
+reimplement it themselves, assuming they are able, which is likely
+to result in more wheel reinventions.  I've already shared
+[my implementation](https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups)
+but how many people are likely to find it, let alone get it working?
+
+> Little known fact: sort(1) will use a temp file as a buffer if too
+> much memory is needed to hold the data to sort.
+
+Interesting.  Presumably you are referring to some undocumented
+behaviour, rather than `--batch-size` which only applies when merging
+multiple files, and not when only sorting STDIN.
+
+> It's also written in the most efficient language possible and has
+> been ruthlessly optimised for 30 years, so I would be very surprised
+> if it was not the best choice.
+
+It's the best choice for sorting.  But sorting purely to detect
+duplicates is a dismally bad choice.
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment
new file mode 100644
index 0000000000..286487eee5
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 11"
+ date="2011-12-23T17:52:21Z"
+ content="""
+I don't think that [[tips/finding_duplicate_files]] is hard to find, and the multiple different ways it shows to deal with the duplicate files shows the flexability of the unix pipeline approach.
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment
new file mode 100644
index 0000000000..909beed837
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 12"
+ date="2011-12-23T18:02:24Z"
+ content="""
+BTW, sort -S '90%' benchmarks consistently 2x as fast as perl's hashes all the way up to 1 million files. Of course the pipeline approach allows you to swap in perl or whatever else is best for you at scale.
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment
new file mode 100644
index 0000000000..094e4526eb
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-01-27T18:29:44Z"
+ content="""
+Hey Asheesh, I'm happy you're finding git-annex useful.
+
+So, there are two forms of duplication going on here. There's duplication of the content, and duplication of the filenames 
+pointing at that content.
+
+Duplication of the filenames is probably not a concern, although it's what I thought you were talking about at first. It's probably info worth recording that backup-2010/some_dir/foo and backup-2009/other_dir/foo are two names you've used for the same content in the past. If you really wanted to remove backup-2009/foo, you could do it by writing a script that looks at the basenames of the symlink targets and removes files that point to the same content as other files.
+
+Using SHA1 ensures that the same key is used for identical files, so generally avoids duplication of content. But if you have 2 disks with an identical file on each, and make them both into annexes, then git-annex will happily retain both
+copies of the content, one per disk. It generally considers keeping copies of content a good thing. :)
+
+So, what if you want to remove the unnecessary copies? Well, there's a really simple way:
+
+
+cd /media/usb-1
+git remote add other-disk /media/usb-0
+git annex add
+git annex drop
+

+
+This asks git-annex to add everything to the annex, but then remove any file contents that it can safely remove. What can it safely remove? Well, anything that it can verify is on another repository such as \"other-disk\"! So, this will happily drop any duplicated file contents, while leaving all the rest alone.
+
+In practice, you might not want to have all your old backup disks mounted at the same time and configured as remotes. Look into configuring [[trust]] to avoid needing do to that. If usb-0 is already a trusted disk, all you need is a simple \"git annex drop\" on usb-1.
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment
new file mode 100644
index 0000000000..04d58a4598
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
+ nickname="Asheesh"
+ subject="I actually *do* want to avoid duplication of filenames"
+ date="2011-01-28T07:30:05Z"
+ content="""
+I really do want just one filename per file, at least for some cases.
+
+For my photos, there's no benefit to having a few filenames point to the same file. As I'm putting them all into the git-annex, that is a good time to remove the pure duplicates so that I don't e.g. see them twice when browsing the directory as a gallery. Also, I am uploading my photos to the web, and I want to avoid uploading the same photo (by content) twice.
+
+I hope that makes things clearer!
+
+For now I'm just doing this:
+
+* paulproteus@renaissance:/mnt/backups-terabyte/paulproteus/sd-card-from-2011-01-06/sd-cards/DCIM/100CANON $ for file in *; do hash=$(sha1sum \"$file\"); if ls /home/paulproteus/Photos/in-flickr/.git-annex | grep -q \"$hash\"; then echo already annexed ; else flickr_upload \"$file\" && mv \"$file\" \"/home/paulproteus/Photos/in-flickr/2011-01-28/from-some-nested-sd-card-bk\" && (cd /home/paulproteus/Photos/in-flickr/2011-01-28/from-some-nested-sd-card-bk && git annex add . && git commit -m ...) ; fi; done
+
+(Yeah, Flickr for my photos for now. I feel sad about betraying the principle of autonomo.us-ness.)
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment
new file mode 100644
index 0000000000..d11119bc3d
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
+ nickname="Asheesh"
+ subject="Duplication of the filenames is what I am concerned about"
+ date="2011-04-29T11:48:22Z"
+ content="""
+For what it's worth, yes, I want to actually forget I ever had the same file in the filesystem with a duplicated name. I'm not just aiming to clean up the disk's space usage; I'm also aiming to clean things up so that navigating the filesystem is easier.
+
+I can write my own script to do that based on the symlinks' target (and I wrote something along those lines), but I still think it'd be nicer if git-annex supported this use case.
+
+Perhaps:
+
+
git annex drop --by-contents

+
+could let me remove a file from git-annex if the contents are available through a different name. (Right now, \"git annex drop\" requires the name *and* contents match.)
+
+-- Asheesh.
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment
new file mode 100644
index 0000000000..a218ee3d51
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="List the duplicate filenames, then let the user decide what to do"
+ date="2011-12-22T12:31:29Z"
+ content="""
+I have the same use case as Asheesh but I want to be able to see which filenames point to the same objects and then decide which of the duplicates to drop myself.  I think
+
+    git annex drop --by-contents
+
+would be the wrong approach because how does git-annex know which ones to drop?  There's too much potential for error.
+
+Instead it would be great to have something like
+
+    git annex finddups
+
+While it's easy enough to knock up a bit of shell or Perl to achieve this, that relies on knowledge of the annex symlink structure, so I think really it belongs inside git-annex.
+
+If this command gave output similar to the excellent `fastdup` utility:
+
+    Scanning for files... 672 files in 10.439 seconds
+    Comparing 2 sets of files...
+
+    2 files (70.71 MB/ea)
+            /home/adam/media/flat/tour/flat-tour.3gp
+            /home/adam/videos/tour.3gp
+
+    Found 1 duplicate of 1 file (70.71 MB wasted)
+    Scanned 672 files (1.96 GB) in 11.415 seconds
+
+then you could do stuff like
+
+    git annex finddups | grep /home/adam/media/flat | xargs rm
+
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment
new file mode 100644
index 0000000000..e48a4a9b38
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="Here's a Perl version"
+ date="2011-12-22T15:43:51Z"
+ content="""
+https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups
+
+but it would be better in git-annex itself ...
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
new file mode 100644
index 0000000000..5d8ac8e61b
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-12-22T16:39:24Z"
+ content="""
+My main concern with putting this in git-annex is that finding duplicates necessarily involves storing a list of every key and file in the repository, and git-annex is very carefully built to avoid things that require non-constant memory use, so that it can scale to very big repositories. (The only exception is the `unused` command, and reducing its memory usage is a continuing goal.)
+
+So I would rather come at this from a different angle.. like providing a way to output a list of files and their associated keys, which the user can then use in their own shell pipelines to find duplicate keys:
+
+      git annex find --include '*' --format='${file} ${key}\n' | sort --key 2 | uniq --all-repeated --skip-fields=1
+
+Which is implemented now!
+
+(Making that pipeline properly handle filenames with spaces is left as an exercise for the reader..)
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment
new file mode 100644
index 0000000000..a337002804
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment
@@ -0,0 +1,54 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 7"
+ date="2011-12-22T20:04:14Z"
+ content="""
+> My main concern with putting this in git-annex is that finding
+> duplicates necessarily involves storing a list of every key and file
+> in the repository
+
+Only if you want to search the *whole* repository for duplicates, and if
+you do, then you're necessarily going to have to chew up memory in
+some process anyway, so what difference whether it's git-annex or
+(say) a Perl wrapper?
+
+> and git-annex is very carefully built to avoid things that require
+> non-constant memory use, so that it can scale to very big
+> repositories.
+
+That's a worthy goal, but if everything could be implemented with an
+O(1) memory footprint then we'd be in much more pleasant world :-)
+Even O(n) isn't that bad ...
+
+That aside, I like your `--format=\"%f %k\n\"` idea a lot.  That opens
+up the \"black box\" of `.git/annex/objects` and makes nice things
+possible, as your pipeline already demonstrates.  However, I'm not
+sure why you think `git annex find | sort | uniq` would be more
+efficient.  Not only does the sort require the very thing you were
+trying to avoid (i.e. the whole list in memory), but it's also 
+O(n log n) which is significantly slower than my O(n) Perl script 
+linked above.
+
+More considerations about this pipeline:
+
+* Doesn't it only include locally available files?  Ideally it should
+  spot duplicates even when the backing blob is not available locally.
+* What's the point of `--include '*'` ?  Doesn't `git annex find` 
+  with no arguments already include all files, modulo the requirement
+  above that they're locally available?
+* Any user using this `git annex find | ...` approach is likely to
+  run up against its limitations sooner rather than later, because
+  they're already used to the plethora of options `find(1)` provides.
+  Rather than reinventing the wheel, is there some way `git annex find`
+  could harness the power of `find(1)` ?
+
+Those considerations aside, a combined approach would be to implement
+
+    git annex find --format=...
+
+and then alter my Perl wrapper to `popen(2)` from that rather than using
+`File::Find`.  But I doubt you would want to ship Perl wrappers in the
+distribution, so if you don't provide a Haskell equivalent then users
+who can't code are left high and dry.
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment
new file mode 100644
index 0000000000..5ac292afeb
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="How much memory would it actually use anyway?"
+ date="2011-12-22T20:15:22Z"
+ content="""
+Another thought - an SHA1 digest is 20 bytes.  That means you can fit over 50 million keys into 1GB of RAM.  Granted you also need memory to store the values (pathnames) which in many cases will be longer, and some users may also choose more expensive backends than SHA1 ... but even so, it seems to me that you are at risk of throwing the baby out with the bath water.  
+"""]]
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment
new file mode 100644
index 0000000000..82c6921ebb
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 9"
+ date="2011-12-23T16:07:39Z"
+ content="""
+Adam, to answer a lot of points breifly..
+
+* --include='*' makes find list files whether their contents are present or not
+* Your perl script is not O(n). Inserting into perl hash tables has overhead of minimum O(n log n). Not counting the overhead of resizing hash tables, the grevious slowdown if the bucket size is overcome by data (it probably falls back to a linked list or something then), and the overhead of traversing the hash tables to get data out.
+* I think that git-annex's set of file matching options is coming along nicely, and new ones can easily be added, so see no need to pull in unix find(1).
+* Your memory size calculations ignore the overhead of a hash table or other data structure to store the data in, which will tend to be more than the actual data size it's storing.  I estimate your 50 million number is off by at least one order of magnitude, and more likely two; in any case I don't want git-annex to use 1 gb of ram.
+* Little known fact: sort(1) will use a temp file as a buffer if too much memory is needed to hold the data to sort. It's also written in the most efficient language possible and has been ruthlessly optimised for 30 years, so I would be very surprised if it was not the best choice.
+"""]]
diff --git a/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn
new file mode 100644
index 0000000000..a04af05b42
--- /dev/null
+++ b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn
@@ -0,0 +1,10 @@
+Hello,
+
+i'm in the process of managing my music collection with git-annex. The initial "git annex add" using the sha1 banckend is quite long an i was wondering that it could be nice to launch multiple "sha1sum" processes in parallel to speed up things.
+
+Anyway, thanks for this wonderful piece of software.
+
+Jean-Baptiste
+
+> closing as dup of [[parallel possibilities]] (also see comments below)
+> [[done]] --[[Joey]] 
diff --git a/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment
new file mode 100644
index 0000000000..2364b7fb83
--- /dev/null
+++ b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-02-25T19:12:42Z"
+ content="""
+I'd expect the checksumming to be disk bound, not CPU bound, on most systems.
+
+I suggest you start off on the WORM backend, and then you can run a job later to [[migrate|walkthrough#index14h2]] to the SHA1 backend.
+"""]]
diff --git a/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment
new file mode 100644
index 0000000000..9b8240658b
--- /dev/null
+++ b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-02-25T19:54:28Z"
+ content="""
+But, see [[todo/parallel_possibilities]]
+"""]]
diff --git a/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment
new file mode 100644
index 0000000000..ee769f0ddd
--- /dev/null
+++ b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jbd"
+ ip="89.158.228.148"
+ subject="comment 3"
+ date="2011-02-26T10:26:12Z"
+ content="""
+Thank your for your answer and the link !
+"""]]
diff --git a/doc/todo/wishlist:___96__git_annex_sync_-m__96__.mdwn b/doc/todo/wishlist:___96__git_annex_sync_-m__96__.mdwn
new file mode 100644
index 0000000000..92b5dee270
--- /dev/null
+++ b/doc/todo/wishlist:___96__git_annex_sync_-m__96__.mdwn
@@ -0,0 +1,10 @@
+Similar to how
+
+    git commit -m 'foo'
+
+works, if I run 
+
+    git annex sync -m 'my hovercraft is full of eels'
+
+git annex should use that commit message instead of the default ones. That way, I could use [[sync]] directly and not be forced to commit prior to syncing just to make sure I have a useful commit message.
+
diff --git a/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__.mdwn b/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__.mdwn
new file mode 100644
index 0000000000..f2c4254ad0
--- /dev/null
+++ b/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__.mdwn
@@ -0,0 +1,25 @@
+I've seen the [[tips/using box.com as a special remote]] for using mounted WebDAV remote directory for storage of the tracked files.
+
+It's quite close to a scenario familiar to me, although with a difference.
+
+Let me describe my situation.
+
+I worked on a supplement to a set of textbooks. My work was dependent on the revision of the textbooks (for correct references, etc.). The textbooks were being edited by the author, and published in a WebDAV directory.
+
+So I set up a Git repo for my work, and also a branch to track the revisions of the textbooks which [I updated by copying them from the WebDAV directory (after mounting it)](http://unix.stackexchange.com/q/25015/4319). The branch where my work was in was either based on the textbooks branch (and rebased/merged and edited to reflect the changes in the new revisions of the textbooks when needed) or contained the repo with textbooks as a submodule.
+
+The textbooks were large files, so I didn't want them be a part of the Git repo with my supplement work when I publish the repo. But I wanted for those who looked into my public repo to understand how to get the textbooks I'm referring to.
+
+I haven't solved this problem for myself completely. Now I see I could use git-annex for this. I t would track the state of the textbooks in the repo, without actually storing them there, and whenever one would need to get the missing textbooks in a clone of the repo, git-annex could handle the download from the WebDAV directory for him, right?
+
+I simply wrote down the rules for reproducing these downloading and saving operations, together with source URL (as a [Makefile](https://gitorious.org/primary-school-informatics-problems/received_2011-11-mathinf-initial-edition/blobs/RULES/Makefile)):
+
+whenever I wanted to update the revisions of the textbooks (or to download them the first time), I would checkout the branch which included this Makefile and was for holding the textbooks, and the run:
+
+    make get
+
+-- this target had the temporary mountpoint for the remote directory as prerequisite, and there was a rule to create it, and mount the specified URL at it; then it would sync the files, and I could use Git to track the changes. After I was done inspecting the remote directory, I had to clean up the temporary mountpoint fby unmounting and deleting it. I didn't make it do this automatically after a `get` operation for performance reasons (caching of the remote directory would help if I wanted to access it once again).
+
+So, this differs from [[tips/using box.com as a special remote]] in that the tip for WebDAV suggest to handle the mounting manually, and git-annex knows nothing about the WebDAV URL where the content comes from.
+
+So here's my wish: a [[special remote|special remotes]] to track the WebDAV URLs in the repo, and mount the remote directory automatically under the hood, whenever one wants to get a file from there. (Then I assume it should also unmount it immediately in order to clean up after itself, despite possible inefficiencies).
diff --git a/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__/comment_1_f46b0c9b49607e9f4f7a27f5a331ce83._comment b/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__/comment_1_f46b0c9b49607e9f4f7a27f5a331ce83._comment
new file mode 100644
index 0000000000..32dd9c039b
--- /dev/null
+++ b/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__/comment_1_f46b0c9b49607e9f4f7a27f5a331ce83._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 1"
+ date="2012-09-25T22:56:22Z"
+ content="""
+Note that git-annex already has the git configs `remote..annex-start-command` and `remote..annex-stop-command` which can be used to handle mounting and umounting.
+"""]]
diff --git a/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__/comment_2_1b34e1dd72011c65e881dec2543a0373._comment b/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__/comment_2_1b34e1dd72011c65e881dec2543a0373._comment
new file mode 100644
index 0000000000..80c5ed2a93
--- /dev/null
+++ b/doc/todo/wishlist:_a_spec.remote_for_network_directories_that_would_mount_them_whenever_needed___40__e.g.__44___with_WebDAV__41__/comment_2_1b34e1dd72011c65e881dec2543a0373._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.56.162"
+ subject="comment 2"
+ date="2012-09-25T23:33:23Z"
+ content="""
+I see, thanks for pointing at these config options! Perhaps, that'll be enough.
+
+I'll have to see whether the information on how to access the remote copy (the source URL and how to mount it) saved in config variables will be transferred to the clones of the repo.
+
+AFAIU [[location tracking]], usually, git-annex would transfer the information on where to look for copies from one repo to another.
+"""]]
diff --git a/doc/todo/wishlist:_addurl_https:.mdwn b/doc/todo/wishlist:_addurl_https:.mdwn
new file mode 100644
index 0000000000..0a62eda6d6
--- /dev/null
+++ b/doc/todo/wishlist:_addurl_https:.mdwn
@@ -0,0 +1,11 @@
+It would be nice if "git annex addurl" allowed https: urls, rather than just http:.
+To give an example, here is a PDF file:
+
+ https://www.fbo.gov/utils/view?id=59ba4c8aa59101a09827ab7b9a787b05
+
+If you switch the https: to http: it redirects you back to https:.
+
+As more sites provide https: for non-secret traffic, this becomes more of an issue.
+
+> I've gotten rid of the use of the HTTP library, now it just uses curl.
+> [[done]] --[[Joey]]
diff --git a/doc/todo/wishlist:_addurl_https:/comment_1_4e8f5e1fc52c3000eb2a1dad0624906e._comment b/doc/todo/wishlist:_addurl_https:/comment_1_4e8f5e1fc52c3000eb2a1dad0624906e._comment
new file mode 100644
index 0000000000..fa500b1dd8
--- /dev/null
+++ b/doc/todo/wishlist:_addurl_https:/comment_1_4e8f5e1fc52c3000eb2a1dad0624906e._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="2001:4978:f:21a::2"
+ subject="comment 1"
+ date="2013-01-26T08:44:38Z"
+ content="""
+This works fine with \"git annex addurl\". 
+
+However, with --fast, it fails:
+
+  git-annex: user error (https not supported)
+
+This is because the Haskell HTTP library doesn't support https yet.  Seems to be very little momentum on fixing that, perhaps I need to switch the code to use http-enumerator, which does.
+"""]]
diff --git a/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads.mdwn b/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads.mdwn
new file mode 100644
index 0000000000..ce79aa6a63
--- /dev/null
+++ b/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads.mdwn
@@ -0,0 +1,26 @@
+A replacement for a web-browser's downloads menu that uses git-annex internally ([[`addurl`|tips/using the web as a special remote]] command for the download, and `drop` or `git rm` for the clean up) would be quite helpful:
+
+say, when working on a topic, writing a paper or similar things, I usually have a Git repo with my text, all relevant data and processing code, and possibly other background information. It's nice to store the literature you needed at the same place where you work. (So that it is easy to catch up with what I was doing and thinking over when I left this work aside for a while, perhaps after cloning the repo from another location.)
+
+When I find an interesting literature, I save the file to the directory with my work, and read it. Then I might return to it to re-read it. There might be references to this document from my work, so I'd like them to work as links (perhaps pointing at the local file, but also at the source URL for the case when my document is read by someone else not on my system).
+
+I need to keep track of the source URLs for the documents I have saved which I read and use.
+
+That's a task that fits well git-annex.
+
+Note that doing the dull work of copying and pasting the URL and the downloading it and then opening it for reading is a pain to do every time I'm interested in a document I have found on the web. (Of course, I would need to fill out the bibliogrphic information for this document if I want to refer to it, but that can be done later. Initially, I wish I just don't lose the source URL of a document at the moment when I get interested in it and start reading.)
+
+So, I could be assisted by a replacement of the "downloads" menu of, say, Firefox: whenever I want to open a file for viewing (like a PDF), it should ask me where to save it, and I'd choose the directory with my work, then it should register it with git-annex (so that the source URL is saved, and perhaps it should also write down the referring page's URL somewhere nearby automatically), download it, and open with a viewer for reading.
+
+Then I'll have the interesting literature there when I'm offline; the source URLs would be saved, so that they can be put into the references. Also, if I distribute this work with Git, at another location git-annex can be used to easily get all the literature again.
+
+(Hmmm... probably, the browser that this will be simplest for me to implement for is emacs-w3m; simply, some functions calling git-annex...)
+
+> This seems fairly doable to implement since the git-annex [[design/assistant]]
+> already has a webapp. So a javascript toolbar thing could be made that
+> submits the current url (or maybe links dragged into it?) to the webapp
+> for adding to the annex.
+> 
+> The only wrinkle is that the webapp runs under a new url each time
+> it starts, due to using a high port and embedding some auth token in the
+> url. --[[Joey]]
diff --git a/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_1_36ae3c75053d5ec278b5e6eb2084d57a._comment b/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_1_36ae3c75053d5ec278b5e6eb2084d57a._comment
new file mode 100644
index 0000000000..4c20561c75
--- /dev/null
+++ b/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_1_36ae3c75053d5ec278b5e6eb2084d57a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-09-25T23:55:44Z"
+ content="""
+You know about `git annex addurl` right?  It doesn't help with the browser integration (though I bet there are existing download manager extensions you could re-use for this) but it takes care of the other use cases.
+"""]]
diff --git a/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_be8eb800523db8cf7a2c68a28fbf5ea5._comment b/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_be8eb800523db8cf7a2c68a28fbf5ea5._comment
new file mode 100644
index 0000000000..c958fd08f7
--- /dev/null
+++ b/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_be8eb800523db8cf7a2c68a28fbf5ea5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://m-f-k.myopenid.com/"
+ ip="93.207.162.28"
+ subject="+1"
+ date="2012-10-05T20:00:31Z"
+ content="""
+Copy+Paste+Open is to much for lazy people like me;)  I like the idea of a direct browser download manager integration.  This would make it so much easier to find find the original source of a downloaded file when you're to lazy to write it down somewhere in the first place …
+"""]]
diff --git a/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_d9f725de41a8572c85e4c6d9b4bcc927._comment b/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_d9f725de41a8572c85e4c6d9b4bcc927._comment
new file mode 100644
index 0000000000..30515a49b5
--- /dev/null
+++ b/doc/todo/wishlist:_an___34__assistant__34___for_web-browsing_--_tracking_the_sources_of_the_downloads/comment_3_d9f725de41a8572c85e4c6d9b4bcc927._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.56.162"
+ subject="the scheme to follow to use the addurl command is longer"
+ date="2012-09-26T00:07:11Z"
+ content="""
+Justin, yes, I know. To use [[addurl|tips/using the web as a special remote]], I should force myself not to click on a link to view it, but rather to copy it, paste into the command line with addurl (then it will be downloaded, and I can run a command to view it...). Not terrible, probably learnable.
+"""]]
diff --git a/doc/todo/wishlist:_disable_automatic_commits.mdwn b/doc/todo/wishlist:_disable_automatic_commits.mdwn
new file mode 100644
index 0000000000..5b39c60c75
--- /dev/null
+++ b/doc/todo/wishlist:_disable_automatic_commits.mdwn
@@ -0,0 +1,36 @@
+When using the [[/assistant]] on some of my repositories, I would like to
+retain manual control over the granularity and contents of the commit
+history.  Some motivating reasons:
+
+* manually specified commit messages makes the history easier to follow
+* make a series of minor changes to a file over a period of a few hours would result in a single commit rather than capturing intermediate incomplete edits
+
+* manual choice of which files to annex (based on predicted usage) could be useful, e.g. a repo might contain a 4MB PDF which you want available in *every* remote even without `git annex get`, and also some 2MB images which are only required in some remotes
+
+> This particular case is now catered to by the "manual" repository group
+> in preferred content settings. --[[Joey]]
+
+Obviously this needs to be configurable at least per repository, and
+ideally perhaps even per remote, since usage habits can vary from machine
+to machine (e.g. I could choose to commit manually from my desktop machine
+which has a nice comfy keyboard and large screen, but this would be too
+much pain to do from my tiny netbook).
+
+In fact, this is vaguely related to [[design/assistant/partial_content]],
+since the usefulness of the commit history depends on the context of the
+data being manipulated, which in turn depends on which subdirectories are
+being touched.  So any mechanism for disabling sync per directory could
+potentially be reused for disabling auto-commit per directory.
+
+According to Joey, it should be easy to arrange for the watcher thread not
+to run, but would need some more work for the assistant to notice manual
+commits in order to sync them; however the assistant already does some
+crazy inotify watching of git refs, in order to detect incoming pushes, so
+detecting manual commits wouldn't be a stretch.
+
+[[!tag design/assistant]]
+
+> You can do this now by pausing committing via the webapp,
+> or setting `annex.autocommit=false`.
+> 
+> The assustant probably doesn't push such commits yet.
diff --git a/doc/todo/wishlist:_make_partial_files_available_during_transfer.mdwn b/doc/todo/wishlist:_make_partial_files_available_during_transfer.mdwn
new file mode 100644
index 0000000000..b021c90914
--- /dev/null
+++ b/doc/todo/wishlist:_make_partial_files_available_during_transfer.mdwn
@@ -0,0 +1,18 @@
+Imagine this situation:
+You have a laptop and a NAS.
+On your laptop you want to consume a large media file located on the NAS.
+So you type:
+
+    git annex get --from nas mediafile
+
+But now you have to wait for the download to complete, unless either 
+
+* rsync is pointed directly to the file in the object storage ("--inplace")
+or
+* the symlink temporarily points to the partial file during a transfer
+
+which would allow you instantaneous consumption of your media.
+It might make sense to make this behavior configurable, because not everyone might agree with having partial content (that mismatches its key) around.
+
+
+So what do you say?
diff --git a/doc/todo/wishlist:_make_partial_files_available_during_transfer/comment_3_1304a721da6f5133fdfa1dac507f1ecb._comment b/doc/todo/wishlist:_make_partial_files_available_during_transfer/comment_3_1304a721da6f5133fdfa1dac507f1ecb._comment
new file mode 100644
index 0000000000..8e955fd48f
--- /dev/null
+++ b/doc/todo/wishlist:_make_partial_files_available_during_transfer/comment_3_1304a721da6f5133fdfa1dac507f1ecb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.152.108.194"
+ subject="comment 3"
+ date="2012-11-04T19:58:28Z"
+ content="""
+I'm not at all comfortable with either idea. Temporarily repointing the symlink could lead to accidentially git committing a bad symlink. Or the user accidentially doing something with a partially transferred file. Running rsync in place would break lots of things that assume that, once the file is present, it can be assumed to be the full and correct file. (Obviously fsck doesn't assume that, but checks made by `git annex drop` do, for example.)
+
+However, you can access partially transferred files by key in `.git/annex/tmp`. It would be easy to write some hack that looks at the symlink to get the key, and then spits out the name of the partial file in `.git/annex/tmp.`.
+"""]]
diff --git a/doc/todo/wishlist:_more_info_in_the_standard_commit_message_of___96__sync__96__.mdwn b/doc/todo/wishlist:_more_info_in_the_standard_commit_message_of___96__sync__96__.mdwn
new file mode 100644
index 0000000000..4e5e3a171b
--- /dev/null
+++ b/doc/todo/wishlist:_more_info_in_the_standard_commit_message_of___96__sync__96__.mdwn
@@ -0,0 +1,3 @@
+Could you include the REPO and UUID information in the "automatic sync" commit message?
+
+This would make troubleshooting easier. (not there was much trouble with git-annex!)
diff --git a/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails.mdwn b/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails.mdwn
new file mode 100644
index 0000000000..e50ebbde5e
--- /dev/null
+++ b/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails.mdwn
@@ -0,0 +1,7 @@
+Right now the assistant can have a huge list of pending transfers for certain hosts if its data is a bit outdated, or a host hasn't been synced lately. When starting up it will then attempt each transfer to said host (which will in turn fail, but at times take time to time out), possibly before doing other stuff like attempting to download new files, or copy files to online hosts.
+
+I suggest that if a transfer fails for host X, and there are other pending transfers, say to host Y and from Z, then all other pending transfers to/from X gets pushed to the back of the queue, to avoid having to wait a long time for several transfers to time out before doing useful stuff.
+
+The prime example for me was this morning, when a laptop that was turned off had a huge amount of queued transfers to it, resulting in the assistant attempting a load of transfers to that host before it retrieved a new file that I had created on another machine yesterday.
+
+[[!tag design/assistant]]
diff --git a/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_1_82ee9de610a0ac55cd1c27c211079e5b._comment b/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_1_82ee9de610a0ac55cd1c27c211079e5b._comment
new file mode 100644
index 0000000000..3b6101ed58
--- /dev/null
+++ b/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_1_82ee9de610a0ac55cd1c27c211079e5b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.49"
+ subject="comment 1"
+ date="2012-12-01T19:22:03Z"
+ content="""
+There are several difficulties with reordering the queue that way. One is that the failure may be intermittent; another is that the queue is fed by a scanning process, so doesn't always have a well-defined end.
+
+Another way to deal with this problem, which I think I prefer, is to allow multiple actions from the queue to run at once. Then slow or unreachable remotes don't block it from using other remotes. 
+"""]]
diff --git a/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_2_bea55156bd32cf9e6dd9b946ba1bb8c1._comment b/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_2_bea55156bd32cf9e6dd9b946ba1bb8c1._comment
new file mode 100644
index 0000000000..62d46bccd0
--- /dev/null
+++ b/doc/todo/wishlist:_move_pending_transfers_for_a_host_to_the_end_of_the_queue_when_one_fails/comment_2_bea55156bd32cf9e6dd9b946ba1bb8c1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="EskildHustvedt"
+ ip="84.48.83.221"
+ subject="comment 2"
+ date="2012-12-01T19:31:18Z"
+ content="""
+I agree your method might be preferable, the end result is the same, and would have avoided the issues I had (and, of course, running multiple transfers at once has other benefits as well).
+
+An alternate way would be to push every transfer NOT from host X to the front of the queue (avoiding most of the \"no defined end\" issue and largely solving the problem), but if multiple actions at once is feasible then that'd still be much nicer.
+"""]]
diff --git a/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__.mdwn b/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__.mdwn
new file mode 100644
index 0000000000..545bd861d6
--- /dev/null
+++ b/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__.mdwn
@@ -0,0 +1,26 @@
+Apart from Tahoe-LAFS (covered by [[todo/tahoe lfs for reals]] and [[forum/tips: special_remotes/hook with tahoe-lafs]]), [[special remotes]] (which I understand as real storage backends) for other other [peer network data stores](http://en.wikipedia.org/wiki/Distributed_data_store#Peer_network_node_data_stores_2) would be interesting.
+
+I mean gnunet, freenet, BitTorrent (also trackerless).
+
+Before dropping a file locally, the BitTorrent client should check that all parts are still available from the peers.
+
+Of course, there is no guarantee assumed that the content won't disappear from the peer network in future: they act more like a cache rather than an archive on whose lifespan you decide. (I'm only not sure about gnunet now: whether there is a rule of dropping unused content from it, like in freenet.)
+
+So, a copy in peer networks shouldn't be counted on by git-annex as much as a copy on a storage you control: probably, by efault, it shouldn't let you delete the local copy if there is a copy in a peer network unless you saved it somewhere else. 
+
+(Think of such a scenario: I could save some of my public large data on external disks/DVDs and keep them at home, and also put them onto peer networks with the same nterface of git-annex which I would be used to; I would also use the git-annex interface to check from time to time that the content is still present, i.e. "cached", on the peer networks. Whenever I'm away from home, and unexpectedly need to show this content to someone, or have a look at it for some reason, I could get it from the peer network "cache".)
+
+Also networks like namecoin (derived from bitcoin) can be used as a key-value store. Despite being a peer network, a system like namecoin actually could offer the publisher more control over the lifespan of the content: he should be able to offer "financial" reward for others processing his key-value data. (But I'm not sure namecoin is designed reasonably for this reward system to work actually; but there might be appearing other similar systems.)
+
+## A different view: extend the key-value backends with ways to look for the content in other content-addressable storage systems
+We might want to look for the registered files in other [content-addressable storage systems](http://en.wikipedia.org/wiki/Content-addressable_storage#Open-source_implementations) (and also to be able to put the files there for storage).
+
+For example:
+
+* [**GNUnet**](http://en.wikipedia.org/wiki/Gnunet) uses its own hash format to address the content. git-annex could extend its own [[backends]] with a one to work with GNUnet, and by default have a built-in [[special remote|special remotes]] that would interact with GNUnet when looking for a content or storing some content. No special setup of the special remote in each repo should be necessary, because GNUnet is "global", so we'd just use the user's already configured GNUnet client. Just turning the builtin GNUnet special remote on or off should be an option (in the repo configuration, and when calling the commands that would query it, like `whereis`).
+* **freenet** is similar.
+* Similarly, a backend  for the hashes used in **BitTorrent** and **magnet links** could be used. If we want a trackerless mode, then probably it's a similar case for a "global"/built-in special remote that needs no local setup in each repo. Using a selected tracker would mean setting up a special remote in our repo.
+* **Git**  itself can be viwed as place to look for the content. There could be a corresponding backend and a builtin special remote (needing no extra setup) to look for the content among the objects stored in the local Git repo. (What if we have a copy of a file that we've put under the control of git-annex in a previous Git commit? We could get it from the object store of Git.)
+* **Venti**, [[**Tahoe-LAFS**|todo/tahoe lfs for reals]] would need a backend for their hashes, and a specially setup special remote in each repo where we'd like to use them--because these are not "global" system, we must setup the path to the instance of the filesystem we'd like to use.
+* probably, there must be other interesting cases of this kind...
+* (I'm also thinking about using somethng like a **bibliographic information** as a key, but then it wouldn't guarantee identical files: the same paper can be stored in different formats, etc. Cf. [**URNs**](http://en.wikipedia.org/wiki/Uniform_resource_name#Examples), via . Also, an URN like bibliographic information can't be computed from the file, it will have to be entered manually or obtained from another directory of URNs.)
diff --git a/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_1_e2c2047e7401cb95a82ffb686a732859._comment b/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_1_e2c2047e7401cb95a82ffb686a732859._comment
new file mode 100644
index 0000000000..80a245d144
--- /dev/null
+++ b/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_1_e2c2047e7401cb95a82ffb686a732859._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.14.141"
+ subject="comment 1"
+ date="2012-09-25T22:57:19Z"
+ content="""
+The best first step to adding such kinds of data stores to git-annex is probably to use the [[special_remotes/hook]] special remote to access them. 
+"""]]
diff --git a/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_2_472b576afdb169b233edd01adcb2123d._comment b/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_2_472b576afdb169b233edd01adcb2123d._comment
new file mode 100644
index 0000000000..c58f97c1cb
--- /dev/null
+++ b/doc/todo/wishlist:_spec.remotes_for_other_peer_network_data_stores___40__gnunet__44___freenet__41__/comment_2_472b576afdb169b233edd01adcb2123d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.56.162"
+ subject="comment 2"
+ date="2012-09-25T23:29:49Z"
+ content="""
+I see. But then, as with Tahoe-LAFS, they also have their own formats for checksums, keys, which could be re-used in git-annex, and that needs special treatment.
+"""]]
diff --git a/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn
new file mode 100644
index 0000000000..4227678ba8
--- /dev/null
+++ b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn
@@ -0,0 +1,12 @@
+The [[Web special remote|special remotes/web]] could possibly be improved by detecting when URLs reference a Youtube video page and using [youtube-dl](http://rg3.github.com/youtube-dl/) instead of wget to download the page. Youtube-dl can also handle several other video sites such as vimeo.com and blip.tv, so if this idea were to be implemented, it might make sense to borrow the regular expressions that youtube-dl uses to identify video URLs. A quick grep through the youtube-dl source for the identifier _VALID_URL should find those regexes (in Python's regex format).
+
+> This is something I've thought about doing for a while.. 
+> Two things I have not figured out:
+> 
+> * Seems that this should really be user-configurable or a plugin system,
+>   to handle more than just this one case.
+> * Youtube-dl breaks from time to time, I really trust these urls a lot
+>   less than regular urls. Perhaps per-url trust levels are called for by
+>   this.
+> 
+> --[[Joey]] 
diff --git a/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote/comment_1_1a383c30df4fb1767f13d8c670b0c0b5._comment b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote/comment_1_1a383c30df4fb1767f13d8c670b0c0b5._comment
new file mode 100644
index 0000000000..5569ff94a4
--- /dev/null
+++ b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote/comment_1_1a383c30df4fb1767f13d8c670b0c0b5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://rmunn.myopenid.com/"
+ nickname="rmunn"
+ subject="comment 1"
+ date="2012-06-12T15:52:35Z"
+ content="""
+* One way to handle the configuration might be with regular expressions. If the URL matches regex A, handle it with downloader A' (with option set A''). If the URL matches regex B, handle it with downloader B' and option set B''. And so on. Then if nothing is matched, the default downloader is wget/curl.
+
+* In my experience, youtube-dl breakages are fixed relatively quickly; a much more serious problem from a trust standpoint is that Youtube videos often disappear. Sometimes due to a legitimate copyright claim, sometimes due to illegitimate copyright claims. (I've seen both happen). Or because the video uploader decided to upload *other* videos that violated copyright, and Youtube closed his/her account, thereby removing *all* his/her videos from the Web. Youtube is definitely an untrustworthy repository as far as \"the file will still be there later on\" is concerned. Perhaps a default trust relationship could go along with the regexes? URLs matching regex A are semitrusted, while URLs matching regex B are untrusted.
+"""]]
diff --git a/doc/todo/wishlist:_special_remote_Ubuntu_One.mdwn b/doc/todo/wishlist:_special_remote_Ubuntu_One.mdwn
new file mode 100644
index 0000000000..b88a038eac
--- /dev/null
+++ b/doc/todo/wishlist:_special_remote_Ubuntu_One.mdwn
@@ -0,0 +1 @@
+Special remote support for [Ubuntu One](http://one.ubuntu.com) would be nice. They're [using propietary but open protocol](https://wiki.ubuntu.com/UbuntuOne/TechnicalDetails#ubuntuone-storageprotocol) based on [Google Protocol Buffers](http://code.google.com/p/protobuf/). There's [protobuf for Haskell](http://code.google.com/p/protobuf-haskell/) so it should be possible to compile [the protocol file](http://bazaar.launchpad.net/~ubuntuone-control-tower/ubuntuone-storage-protocol/trunk/view/head:/ubuntuone/storageprotocol/protocol.proto) to Haskell code and then use that to implement the native Ubuntu special remote.
diff --git a/doc/todo/wishlist:_special_remote_mega.co.nz.mdwn b/doc/todo/wishlist:_special_remote_mega.co.nz.mdwn
new file mode 100644
index 0000000000..788a3a43fa
--- /dev/null
+++ b/doc/todo/wishlist:_special_remote_mega.co.nz.mdwn
@@ -0,0 +1 @@
+mega.co.nz has 50gb for free accounts. They also have an API, so I guess it wouldn't be too hard to use it as a special remote.
diff --git a/doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn
new file mode 100644
index 0000000000..55b8120a75
--- /dev/null
+++ b/doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn
@@ -0,0 +1,22 @@
+git-annex does not seem to support all kinds of urls that git does.
+
+Specifically, if I have ~/bar set up on host foo:
+
+    [remote "foo"]
+    ## this one is not recognized as ssh url at all
+    #	url = foo:bar
+    ## this one makes git-annex try to access '/~/bar' literally
+    #	url = ssh://foo/~/bar
+    ## this one works
+    	url = ssh://foo/home/tv/bar
+
+> scp-style is now supported.
+
+> `~` expansions (for the user's home, or other users)
+> are somewhat tricky to support as they require running
+> code on the remote to lookup homedirs. If git-annex grows a
+> `git annex shell` that is run on the remote side
+> (something I am [[considering|todo/git-annex-shell]] for other reasons), it
+> could handle the expansions there. --[[Joey]]
+
+> Update: Now `~` expansions are supported. [[done]]
diff --git a/doc/todo/wishlist:_swift_backend.mdwn b/doc/todo/wishlist:_swift_backend.mdwn
new file mode 100644
index 0000000000..28bd265faf
--- /dev/null
+++ b/doc/todo/wishlist:_swift_backend.mdwn
@@ -0,0 +1,5 @@
+[swift](http://swift.openstack.org/) is the object storage of Openstack. Think S3, but fully open source. As it's backed by rackspace.com, NASA, Dell and several other major players, adoption rates will explode.
+
+I can provide a test account soonish if need be, else rackspace.com if offering swift storage. Their API gateway lives at https://auth.api.rackspacecloud.com/v1.0
+
+Richard
diff --git a/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment b/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment
new file mode 100644
index 0000000000..98a998c1cf
--- /dev/null
+++ b/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-05-14T10:04:36Z"
+ content="""
+I don't suppose this SWIFT api is compatible with the eucalytpus walrus api ?
+"""]]
diff --git a/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment b/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment
new file mode 100644
index 0000000000..97863b095f
--- /dev/null
+++ b/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-05-14T15:00:51Z"
+ content="""
+It does offer a S3 compability layer, but that is de facto non-functioning as of right now.
+"""]]
diff --git a/doc/todo/wishlist:_swift_backend/comment_3_bf8625b909c3a7321cae40e6f145e874._comment b/doc/todo/wishlist:_swift_backend/comment_3_bf8625b909c3a7321cae40e6f145e874._comment
new file mode 100644
index 0000000000..90af10c41c
--- /dev/null
+++ b/doc/todo/wishlist:_swift_backend/comment_3_bf8625b909c3a7321cae40e6f145e874._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="+1"
+ date="2012-09-18T08:52:21Z"
+ content="""
+OVH (french IT company) is migrating its could storage infrastructure to swift/openstack and the next few weeks.
+"""]]
diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
new file mode 100644
index 0000000000..d1ec5963f5
--- /dev/null
+++ b/doc/transferring_data.mdwn
@@ -0,0 +1,19 @@
+git-annex can transfer data to or from any of a repository's git remotes.
+Depending on where the remote is, the data transfer is done using rsync
+(over ssh or locally), or plain cp (with copy-on-write
+optimisations on supported filesystems), or using curl (for repositories
+on the web). Some [[special_remotes]] are also supported that are not
+traditional git remotes.
+
+If a data transfer is interrupted, git-annex retains the partial transfer
+to allow it to be automatically resumed later.
+
+It's equally easy to transfer a single file to or from a repository,
+or to launch a retrievel of a massive pile of files from whatever
+repositories they are scattered amongst.
+
+git-annex automatically uses whatever remotes are currently accessible,
+preferring ones that are less expensive to talk to.
+
+[[!img repomap.png caption="A real-world repository interconnection map
+(generated by git-annex map)"]]
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
new file mode 100644
index 0000000000..1fd47fd1d3
--- /dev/null
+++ b/doc/trust.mdwn
@@ -0,0 +1,59 @@
+Git-annex supports several levels of trust of a repository:
+
+* semitrusted (default)
+* untrusted
+* trusted
+* dead
+
+## semitrusted
+
+Normally, git-annex does not fully trust its stored [[location_tracking]]
+information. When removing content, it will directly check
+that other repositories have enough [[copies]].
+
+Generally that explicit checking is a good idea. Consider that the current
+[[location_tracking]] information for a remote may not yet have propagated
+out. Or, a remote may have suffered a catastrophic loss of data, or itself
+been lost.
+
+There is still some trust involved here. A semitrusted repository is
+depended on to retain a copy of the file content; possibly the only
+[[copy|copies]].
+
+(Being semitrusted is the default. The `git annex semitrust` command
+restores a repository to this default, when it has been overridden.
+The `--semitrust` option can temporarily restore a repository to this
+default.)
+
+## untrusted
+
+An untrusted repository is not trusted to retain data at all. Git-annex
+will retain sufficient [[copies]] of data elsewhere.
+
+This is a good choice for eg, portable drives that could get lost. Or,
+if a disk is known to be dying, you can set it to untrusted and let
+`git annex fsck` warn about data that needs to be copied off it.
+
+To configure a repository as untrusted, use the `git annex untrust`
+command.
+
+## trusted
+
+Sometimes, you may have reasons to fully trust the location tracking
+information for a repository. For example, it may be an offline
+archival drive, from which you rarely or never remove content. Deciding
+when it makes sense to trust the tracking info is up to you.
+
+One way to handle this is just to use `--force` when a command cannot
+access a remote you trust. Or to use `--trust` to specify a repisitory to
+trust temporarily.
+
+To configure a repository as fully and permanently trusted,
+use the `git annex trust` command.
+
+## dead
+
+This is used to indicate that you have no trust that the repository
+exists at all. It's appropriate to use when a drive has been lost,
+or a directory irretrevably deleted. It will make git-annex avoid
+even showing the repository as a place where data might still reside.
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
new file mode 100644
index 0000000000..6cf54477cc
--- /dev/null
+++ b/doc/upgrades.mdwn
@@ -0,0 +1,98 @@
+Occasionally improvments are made to how git-annex stores its data,
+that require an upgrade process to convert repositories made with an older
+version to be used by a newer version. It's annoying, it should happen
+rarely, but sometimes, it's worth it.
+
+There's a committment that git-annex will always support upgrades from all
+past versions. After all, you may have offline drives from an earlier
+git-annex, and might want to use them with a newer git-annex.
+
+git-annex will notice if it is run in a repository that
+needs an upgrade, and refuse to do anything. To upgrade,
+use the "git annex upgrade" command.
+
+The upgrade process is guaranteed to be conflict-free. Unless you
+already have git conflicts in your repository or between repositories.
+Upgrading a repository with conflicts is not recommended; resolve the
+conflicts first before upgrading git-annex.
+
+## Upgrade events, so far
+
+### v3 -> v4 (git-annex version 4.x)
+
+v4 is only used for [[direct_mode]], and no upgrade needs to be done from
+existing v3 repositories, they will continue to work.
+
+### v2 -> v3 (git-annex version 3.x)
+
+Involved moving the .git-annex/ directory into a separate git-annex branch.
+
+After this upgrade, you should make sure you include the git-annex branch
+when git pushing and pulling.
+
+### tips for this upgrade
+
+This upgrade is easier (and faster!) than the previous upgrades.
+You don't need to upgrade every repository at once; it's sufficient
+to upgrade each repository only when you next use it.
+	
+Example upgrade process:
+
+	cd localrepo
+	git pull
+	git annex upgrade
+	git commit -m "upgrade v2 to v3"
+	git gc
+
+### v1 -> v2 (git-annex version 0.20110316)
+
+Involved adding hashing to .git/annex/ and changing the names of all keys.
+Symlinks changed.
+
+Also, hashing was added to location log files in .git-annex/.
+And .gitattributes needed to have another line added to it.
+
+Previously, files added to the SHA [[backends]] did not have their file
+size tracked, while files added to the WORM backend did. Files added to
+the SHA backends after the conversion will have their file size tracked,
+and that information will be used by git-annex for disk free space checking.
+To ensure that information is available for all your annexed files, see
+[[upgrades/SHA_size]].
+
+### tips for this upgrade
+
+This upgrade can tend to take a while, if you have a lot of files.
+
+Each clone of a repository should be individually upgraded.
+Until a repository's remotes have been upgraded, git-annex
+will refuse to communicate with them.
+
+Start by upgrading one repository, and then you can commit
+the changes git-annex staged during upgrade, and push them out to other
+repositories. And then upgrade those other repositories. Doing it this
+way avoids git-annex doing some duplicate work during the upgrade.
+
+Example upgrade process:
+
+	cd localrepo
+	git pull
+	git annex upgrade
+	git commit -m "upgrade v1 to v2"
+	git push
+
+	ssh remote
+	cd remoterepo
+	git pull
+	git annex upgrade
+	...
+
+### v0 -> v1 (git-annex version 0.04)
+
+Involved a reorganisation of the layout of .git/annex/. Symlinks changed.
+
+Handled more or less transparently, although git-annex was just 2 weeks
+old at the time, and had few users other than Joey.
+
+Before doing this upgrade, set annex.version:
+
+	git config annex.version 0
diff --git a/doc/upgrades/SHA_size.mdwn b/doc/upgrades/SHA_size.mdwn
new file mode 100644
index 0000000000..97603ba913
--- /dev/null
+++ b/doc/upgrades/SHA_size.mdwn
@@ -0,0 +1,20 @@
+Before version 2 of the git-annex repository, files added to the SHA
+[[backends]] did not have their file size tracked, while files added to the
+WORM backend did. The file size information is used for disk free space
+checking.
+
+Files added to the SHA backends after the conversion will have their file
+size tracked automatically. This disk free space checking is an optional
+feature and since you're more likely to be using more recently added files,
+you're unlikely to see any bad effect if you do nothing.
+
+That said, if you have old files added to SHA backends that lack file size
+tracking info, here's how you can add that info. After [[upgrading|upgrades]]
+to repository version 2, in each repository run:
+
+	git annex migrate
+	git commit -m 'migrated keys for v2'
+
+The usual caveats about [[tips/migrating_data_to_a_new_backend]]
+apply; you will end up with unused keys that you can later clean up with
+`git annex unused`.
diff --git a/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment b/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment
new file mode 100644
index 0000000000..7b6be15321
--- /dev/null
+++ b/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
+ nickname="Asheesh"
+ subject="The fact that the keys changed causes merge conflicts"
+ date="2012-06-25T00:28:59Z"
+ content="""
+FYI, I have run into a problem where if you 'git annex sync' between various 'git annex v3' repositories, if the different repositories are using different encodings of the SHA1 information (one including size, one not), then the 'git merge' will declare that they conflict.
+
+There's no indication that 'git annex migrate' is the right tool to run, except from perusing the 'git annex' man page. In my opinion this is a major user interface problem.
+
+-- Asheesh.
+"""]]
diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn
new file mode 100644
index 0000000000..cdd3ea546d
--- /dev/null
+++ b/doc/use_case/Alice.mdwn
@@ -0,0 +1,24 @@
+### use case: The Nomad
+
+Alice is always on the move, often with her trusty netbook and a small
+handheld terabyte USB drive, or a smaller USB keydrive. She has a server
+out there on the net. She stores data, encrypted in the Cloud.
+
+All these things can have different files on them, but Alice no longer
+has to deal with the tedious process of keeping them manually in sync,
+or remembering where she put a file. git-annex manages all these data
+sources as if they were git remotes.  
+[[more about special remotes|special_remotes]]
+
+When she has 1 bar on her cell, Alice queues up interesting files on her
+server for later. At a coffee shop, she has git-annex download them to her
+USB drive. High in the sky or in a remote cabin, she catches up on
+podcasts, videos, and games, first letting git-annex copy them from
+her USB drive to the netbook (this saves battery power).  
+[[more about transferring data|transferring_data]]
+
+When she's done, she tells git-annex which to keep and which to remove.
+They're all removed from her netbook to save space, and Alice knows
+that next time she syncs up to the net, her changes will be synced back 
+to her server.  
+[[more about distributed version control|distributed_version_control]]
diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn
new file mode 100644
index 0000000000..42d10ea975
--- /dev/null
+++ b/doc/use_case/Bob.mdwn
@@ -0,0 +1,25 @@
+### use case: The Archivist
+
+Bob has many drives to archive his data, most of them kept offline, in a
+safe place.
+
+With git-annex, Bob has a single directory tree that includes all
+his files, even if their content is being stored offline. He can
+reorganize his files using that tree, committing new versions to git,
+without worry about accidentally deleting anything. 
+
+When Bob needs access to some files, git-annex can tell him which drive(s)
+they're on, and easily make them available. Indeed, every drive knows what
+is on every other drive.  
+[[more about location tracking|location_tracking]]
+
+Bob thinks long-term, and so he appreciates that git-annex uses a simple
+repository format. He knows his files will be accessible in the future
+even if the world has forgotten about git-annex and git.  
+[[more about future-proofing|future_proofing]]
+
+Run in a cron job, git-annex adds new files to archival drives at night. It
+also helps Bob keep track of intentional, and unintentional copies of
+files, and logs information he can use to decide when it's time to duplicate
+the content of old drives.  
+[[more about backup copies|copies]]
diff --git a/doc/users.mdwn b/doc/users.mdwn
new file mode 100644
index 0000000000..b9bab48ecf
--- /dev/null
+++ b/doc/users.mdwn
@@ -0,0 +1,9 @@
+Users of this wiki, feel free to create a subpage of this one and talk
+about yourself on it, within reason. You can link to it to sign your
+comments.
+
+List of users
+=============
+[[!inline pages="users/* and !users/*/* and !*/Discussion" 
+feeds=no archive=yes sort=title template=titlepage
+rootpage="users" postformtext="Add yourself as an git-annex user:"]]
diff --git a/doc/users/chrysn.mdwn b/doc/users/chrysn.mdwn
new file mode 100644
index 0000000000..f5c07b88b3
--- /dev/null
+++ b/doc/users/chrysn.mdwn
@@ -0,0 +1,5 @@
+* **name**: chrysn
+* **website**: 
+* **uses git-annex for**: managing the family's photos (and possibly videos and music in the future)
+* **likes git-annex because**: it adds a layer of commit semantics over a regular file system without keeping everything in duplicate locally
+* **would like git-annex to**: not be required any more as git itself learns to use cow filesystems to avoid abundant disk usage and gets better with sparser checkouts (git-annex might then still be a simpler tool that watches over what can be safely dropped for a sparser checkout)
diff --git a/doc/users/fmarier.mdwn b/doc/users/fmarier.mdwn
new file mode 100644
index 0000000000..ecf3426978
--- /dev/null
+++ b/doc/users/fmarier.mdwn
@@ -0,0 +1,6 @@
+# François Marier
+
+Free Software and Debian Developer. Lead developer of [Libravatar](http://www.libravatar.org)
+
+* [Blog](http://feeding.cloud.geek.nz)
+* [Identica](http://identi.ca/fmarier) / [Twitter](http://twitter.com/fmarier)
diff --git a/doc/users/gebi.mdwn b/doc/users/gebi.mdwn
new file mode 100644
index 0000000000..121bedbdd7
--- /dev/null
+++ b/doc/users/gebi.mdwn
@@ -0,0 +1 @@
+Michael Gebetsroither 
diff --git a/doc/users/joey.mdwn b/doc/users/joey.mdwn
new file mode 100644
index 0000000000..306e1cc768
--- /dev/null
+++ b/doc/users/joey.mdwn
@@ -0,0 +1,2 @@
+Joey Hess   
+
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
new file mode 100644
index 0000000000..c288b71ded
--- /dev/null
+++ b/doc/walkthrough.mdwn
@@ -0,0 +1,24 @@
+A walkthrough of the basic features of git-annex.
+
+[[!toc]]
+
+[[!inline feeds=no trail=yes show=0 template=walkthrough pagenames="""
+	walkthrough/creating_a_repository
+	walkthrough/adding_a_remote
+	walkthrough/adding_files
+	walkthrough/renaming_files
+	walkthrough/getting_file_content
+	walkthrough/syncing
+	walkthrough/transferring_files:_When_things_go_wrong
+	walkthrough/removing_files
+	walkthrough/removing_files:_When_things_go_wrong
+	walkthrough/modifying_annexed_files
+	walkthrough/using_ssh_remotes
+	walkthrough/moving_file_content_between_repositories
+	walkthrough/unused_data
+	walkthrough/fsck:_verifying_your_data
+	walkthrough/fsck:_when_things_go_wrong
+	walkthrough/backups
+	walkthrough/automatically_managing_content
+	walkthrough/more
+"""]]
diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn
new file mode 100644
index 0000000000..97690dfcdf
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote.mdwn
@@ -0,0 +1,19 @@
+Like any other git repository, git-annex repositories have remotes.
+Let's start by adding a USB drive as a remote.
+
+	# sudo mount /media/usb
+	# cd /media/usb
+	# git clone ~/annex
+	# cd annex
+	# git annex init "portable USB drive"
+	# git remote add laptop ~/annex
+	# cd ~/annex
+	# git remote add usbdrive /media/usb/annex
+
+This is all standard ad-hoc distributed git repository setup.
+The only git-annex specific part is telling it the name
+of the new repository created on the USB drive.
+
+Notice that both repos are set up as remotes of one another. This lets
+either get annexed files from the other. You'll want to do that even
+if you are using git in a more centralized fashion.
diff --git a/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment b/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment
new file mode 100644
index 0000000000..4b0b9c0fd2
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-19T01:18:49Z"
+ content="""
+After doing the above with two required copy per file, `git annex fsck` complained that I had only one copy per file even though I had created my clone, already. Once I `git pull`ed from the second repo, not getting any changes for obvious reasons, `git annex fsck` was happy. So I am not sure how my addition was incorrect. -- RichiH
+"""]]
diff --git a/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment b/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment
new file mode 100644
index 0000000000..015417a4f7
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-19T15:35:38Z"
+ content="""
+Yes, you have to pull down location tracking information in order for fsck to be satisfied in that situation. But since this is a walkthrough, and neither fsck or numcopies settings are mentioned until later, it's ok for this pull to be described a few steps along in [[getting file content]].
+
+"""]]
diff --git a/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment b/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment
new file mode 100644
index 0000000000..9280f2dccf
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="comment 3"
+ date="2011-04-02T20:24:33Z"
+ content="""
+ * why the `git remote add laptop ~/annex` ? this remote already exists under the name origin.
+ * doesn't the last command need to be `git remote add usbdrive /media/usb/annex`? because the actual repo would be in /media/usb/annex, not /media/usb?
+"""]]
diff --git a/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment b/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment
new file mode 100644
index 0000000000..b4dcb6422a
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-04-03T02:32:17Z"
+ content="""
+Good spotting on the last line, fixed.
+
+The laptop remote is indeed redundant, but it leads to clearer views of what is going on later in the walkthrough (\"git pull laptop master\", \"(copying from laptop...)\"). And if the original clone is made from a central bare repo, this reinforces that you'll want to set up remotes for other repos on the computer.
+"""]]
diff --git a/doc/walkthrough/adding_files.mdwn b/doc/walkthrough/adding_files.mdwn
new file mode 100644
index 0000000000..d1b5a04f77
--- /dev/null
+++ b/doc/walkthrough/adding_files.mdwn
@@ -0,0 +1,11 @@
+	# cd ~/annex
+	# cp /tmp/big_file .
+	# cp /tmp/debian.iso .
+	# git annex add .
+	add big_file (checksum...) ok
+	add debian.iso (checksum...) ok
+	# git commit -a -m added
+
+When you add a file to the annex and commit it, only a symlink to
+the annexed content is committed. The content itself is stored in
+git-annex's backend.
diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
new file mode 100644
index 0000000000..0080ebcb5e
--- /dev/null
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -0,0 +1,45 @@
+Once you have multiple repositories, and have perhaps configured numcopies,
+any given file can have many more copies than is needed, or perhaps fewer
+than you would like. How to manage this?
+
+The whereis subcommand can be used to see how many copies of a file are known,
+but then you have to decide what to get or drop. In this example, there
+are perhaps not enough copies of the first file, and too many of the second
+file.
+
+	# cd /media/usbdrive
+	# git annex whereis
+	whereis my_cool_big_file (1 copy)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+	whereis other_file (3 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- here (usb drive)
+		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
+
+What would be handy is some automated versions of get and drop, that only
+gets a file if there are not yet enough copies of it, or only drops a file
+if there are too many copies. Well, these exist, just use the --auto option.
+
+	# git annex get --auto --numcopies=2
+	get my_cool_big_file (from laptop...) ok
+	# git annex drop --auto --numcopies=2
+	drop other_file ok
+
+With two quick commands, git-annex was able to decide for you how to
+work toward having two copies of your files.
+
+	# git annex whereis
+	whereis my_cool_big_file (2 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- here (usb drive)
+	whereis other_file (2 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
+
+The --auto option can also be used with the copy command,
+again this lets git-annex decide whether to actually copy content.
+
+The above shows how to use --auto to manage content based on the number
+of copies. It's also possible to configure, on a per-repository basis,
+which content is desired. Then --auto also takes that into account
+see [[preferred_content]] for details.
diff --git a/doc/walkthrough/backups.mdwn b/doc/walkthrough/backups.mdwn
new file mode 100644
index 0000000000..9723022b4c
--- /dev/null
+++ b/doc/walkthrough/backups.mdwn
@@ -0,0 +1,25 @@
+git-annex can be configured to require more than one copy of a file exists,
+as a simple backup for your data. This is controlled by the "annex.numcopies"
+setting, which defaults to 1 copy. Let's change that to require 2 copies,
+and send a copy of every file to a USB drive.
+
+	# echo "* annex.numcopies=2" >> .gitattributes
+	# git annex copy . --to usbdrive
+
+Now when we try to `git annex drop` a file, it will verify that it
+knows of 2 other repositories that have a copy before removing its
+content from the current repository.
+
+You can also vary the number of copies needed, depending on the file name.
+So, if you want 3 copies of all your flac files, but only 1 copy of oggs:
+
+	# echo "*.ogg annex.numcopies=1" >> .gitattributes
+	# echo "*.flac annex.numcopies=3" >> .gitattributes
+
+Or, you might want to make a directory for important stuff, and configure
+it so anything put in there is backed up more thoroughly:
+
+	# mkdir important_stuff
+	# echo "* annex.numcopies=3" > important_stuff/.gitattributes
+
+For more details about the numcopies setting, see [[copies]].
diff --git a/doc/walkthrough/creating_a_repository.mdwn b/doc/walkthrough/creating_a_repository.mdwn
new file mode 100644
index 0000000000..51ff1c72b3
--- /dev/null
+++ b/doc/walkthrough/creating_a_repository.mdwn
@@ -0,0 +1,6 @@
+This is very straightforward. Just tell it a description of the repository.
+
+	# mkdir ~/annex
+	# cd ~/annex
+	# git init
+	# git annex init "my laptop"
diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn
new file mode 100644
index 0000000000..d036332fb3
--- /dev/null
+++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn
@@ -0,0 +1,16 @@
+You can use the fsck subcommand to check for problems in your data. What
+can be checked depends on the key-value [[backend|backends]] you've used
+for the data. For example, when you use the SHA1 backend, fsck will verify
+that the checksums of your files are good. Fsck also checks that the
+annex.numcopies setting is satisfied for all files.
+
+	# git annex fsck
+	fsck some_file (checksum...) ok
+	fsck my_cool_big_file (checksum...) ok
+	...
+
+You can also specify the files to check.  This is particularly useful if 
+you're using sha1 and don't want to spend a long time checksumming everything.
+
+	# git annex fsck my_cool_big_file
+	fsck my_cool_big_file (checksum...) ok
diff --git a/doc/walkthrough/fsck:_when_things_go_wrong.mdwn b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn
new file mode 100644
index 0000000000..85d9f20fe0
--- /dev/null
+++ b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn
@@ -0,0 +1,13 @@
+Fsck never deletes possibly bad data; instead it will be moved to
+`.git/annex/bad/` for you to recover. Here is a sample of what fsck
+might say about a badly messed up annex:
+
+	# git annex fsck
+	fsck my_cool_big_file (checksum...)
+	git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2
+	git-annex: ** No known copies exist of my_cool_big_file
+	failed
+	fsck important_file
+	git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up.
+	failed
+	git-annex: 2 failed
diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
new file mode 100644
index 0000000000..f41e17770a
--- /dev/null
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -0,0 +1,12 @@
+A repository does not always have all annexed file contents available.
+When you need the content of a file, you can use "git annex get" to
+make it available.
+
+We can use this to copy everything in the laptop's annex to the
+USB drive.
+
+	# cd /media/usb/annex
+	# git fetch laptop; git merge laptop/master
+	# git annex get .
+	get my_cool_big_file (from laptop...) ok
+	get iso/debian.iso (from laptop...) ok
diff --git a/doc/walkthrough/modifying_annexed_files.mdwn b/doc/walkthrough/modifying_annexed_files.mdwn
new file mode 100644
index 0000000000..1f7a7efb77
--- /dev/null
+++ b/doc/walkthrough/modifying_annexed_files.mdwn
@@ -0,0 +1,42 @@
+Normally, the content of files in the annex is prevented from being modified.
+That's a good thing, because it might be the only copy, you wouldn't
+want to lose it in a fumblefingered mistake.
+
+	# echo oops > my_cool_big_file
+	bash: my_cool_big_file: Permission denied
+
+In order to modify a file, it should first be unlocked.
+
+	# git annex unlock my_cool_big_file
+	unlock my_cool_big_file (copying...) ok
+
+That replaces the symlink that normally points at its content with a copy
+of the content. You can then modify the file like any regular file. Because
+it is a regular file.
+
+(If you decide you don't need to modify the file after all, or want to discard
+modifications, just use `git annex lock`.)
+
+When you `git commit`, git-annex's pre-commit hook will automatically
+notice that you are committing an unlocked file, and add its new content
+to the annex. The file will be replaced with a symlink to the new content,
+and this symlink is what gets committed to git in the end.
+
+	# echo "now smaller, but even cooler" > my_cool_big_file
+	# git commit my_cool_big_file -m "changed an annexed file"
+	add my_cool_big_file ok
+	[master 64cda67] changed an annexed file
+	 1 files changed, 1 insertions(+), 1 deletions(-)
+
+There is one problem with using `git commit` like this: Git wants to first
+stage the entire contents of the file in its index. That can be slow for
+big files (sorta why git-annex exists in the first place). So, the
+automatic handling on commit is a nice safety feature, since it prevents
+the file content being accidentally committed into git. But when working with
+big files, it's faster to explicitly add them to the annex yourself
+before committing.
+
+	# echo "now smaller, but even cooler yet" > my_cool_big_file
+	# git annex add my_cool_big_file
+	add my_cool_big_file ok
+	# git commit my_cool_big_file -m "changed an annexed file"
diff --git a/doc/walkthrough/modifying_annexed_files/comment_1_624b4a0b521b553d68ab6049f7dbaf8c._comment b/doc/walkthrough/modifying_annexed_files/comment_1_624b4a0b521b553d68ab6049f7dbaf8c._comment
new file mode 100644
index 0000000000..5fc26496e0
--- /dev/null
+++ b/doc/walkthrough/modifying_annexed_files/comment_1_624b4a0b521b553d68ab6049f7dbaf8c._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://lj.rossia.org/users/imz/"
+ ip="79.165.56.162"
+ subject="sorta why git-annex exists in the first place -- not only the slow index "
+ date="2012-09-25T22:04:01Z"
+ content="""
+> Git wants to first stage the entire contents of the file in its index. That can be slow for big files (sorta why git-annex exists in the first place)
+
+I think that git-annex's usefulness is not only because of the Git's index overhead: I like its idea because it will help track the copies in the \"[[special remotes]]\", which are not Git because 
+
+* they are either not under my control (e.g., web URLs), 
+* or because it's not convenient to hold a Git repo there (an external disk/DVD with files can be viewed easily by a human, but imposing a Git repo structure on it would either at least double the consume space: for the history of the commits and for the working dir, or make it \"unreadable\" for a human, if it is a bare repo);
+* or because it's nearly impossible to put a Git repo on a storage like peer networks without special tools.
+"""]]
diff --git a/doc/walkthrough/more.mdwn b/doc/walkthrough/more.mdwn
new file mode 100644
index 0000000000..0a4a5b94e8
--- /dev/null
+++ b/doc/walkthrough/more.mdwn
@@ -0,0 +1,3 @@
+So ends the walkthrough. By now you should be able to use git-annex.
+
+Want more? See [[tips]] for lots more features and advice.
diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn
new file mode 100644
index 0000000000..3ffcc11750
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn
@@ -0,0 +1,13 @@
+Often you will want to move some file contents from a repository to some
+other one. For example, your laptop's disk is getting full; time to move
+some files to an external disk before moving another file from a file
+server to your laptop. Doing that by hand (by using `git annex get` and
+`git annex drop`) is possible, but a bit of a pain. `git annex move`
+makes it very easy.
+
+	# git annex move my_cool_big_file --to usbdrive
+	move my_cool_big_file (to usbdrive...) ok
+	# git annex move video/hackity_hack_and_kaxxt.mov --from fileserver
+	move video/hackity_hack_and_kaxxt.mov (from fileserver...)
+	SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e   100%   82MB 199.1KB/s   07:02
+	ok
diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment
new file mode 100644
index 0000000000..b3dc8fe7a2
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-22T23:41:51Z"
+ content="""
+I may be missing something obvious, but when I copy to a remote repository, the object files are created, but no softlinks are created. When I pull everything from the remote, it pulls only files the local repo knows about already.
+
+        A
+       / \
+      B   C
+
+Moving from B to A creates no symlinks in A but the object files are moved to A. Copying back from A to B restores the object files in B and keeps them in A.
+
+Copying from A to an empty C does not create any object files nor symlinks. Copying from C to A creates no symlinks in A but the object files are copied to A.
+
+-- RichiH
+
+"""]]
diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment
new file mode 100644
index 0000000000..a6f8e9cf97
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-23T00:38:10Z"
+ content="""
+`git annex move` only moves content. All symlink management is handled by git, so you have to keep repositories in sync using git as you would any other repo. When you `git pull B` in A, it will get whatever symlinks were added to B.
+
+(It can be useful to use a central bare repo and avoid needing to git pull from one repo to another, then you can just always push commits to the central repo, and pull down all changes from other repos.)
+"""]]
diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment
new file mode 100644
index 0000000000..9a128f1ed6
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-23T02:07:49Z"
+ content="""
+Ah yes, I feel kinda stupid in hindsight.
+
+As the central server is most likely a common use case, would you object if I added that to the walkthrough? If you have any best practices on how to automate a push with every copy to a bare remote? AFAIK, git does not store information about bare/non-bare remotes, but this could easily be put into .git/config by git annex.
+
+-- RichiH
+"""]]
diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment
new file mode 100644
index 0000000000..8b4d9a0538
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-03-23T15:28:00Z"
+ content="""
+I would not mind if the walkthrough documented the central git repo case. But I don't want to complicate it unduely (it's long enough), and it's important that the fully distributed case be shown to work, and I assume that people already have basic git knowledge, so documenting the details of set up of a bare git repo is sorta out of scope. (There are also a lot of way to do it, using github, or gitosis, or raw git, etc.)
+"""]]
diff --git a/doc/walkthrough/removing_files.mdwn b/doc/walkthrough/removing_files.mdwn
new file mode 100644
index 0000000000..0df6e82a1f
--- /dev/null
+++ b/doc/walkthrough/removing_files.mdwn
@@ -0,0 +1,5 @@
+You can always drop files safely. Git-annex checks that some other annex
+has the file before removing it.
+
+	# git annex drop iso/debian.iso
+	drop iso/Debian_5.0.iso ok
diff --git a/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
new file mode 100644
index 0000000000..1c8719cecd
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="DavidEdmondson"
+ subject="Is it necessary to commit after the 'drop'?"
+ date="2011-09-05T15:43:25Z"
+ content="""
+In fact is it possible? Nothing changed as far as git is concerned.
+
+"""]]
diff --git a/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
new file mode 100644
index 0000000000..f5fb8dc7f5
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-05T15:59:27Z"
+ content="""
+Good catch. It used to be necessary before there was a git-annex branch, but not now.
+"""]]
diff --git a/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn b/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn
new file mode 100644
index 0000000000..2d3c0cde08
--- /dev/null
+++ b/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn
@@ -0,0 +1,24 @@
+Before dropping a file, git-annex wants to be able to look at other
+remotes, and verify that they still have a file. After all, it could
+have been dropped from them too. If the remotes are not mounted/available,
+you'll see something like this.
+
+	# git annex drop important_file other.iso
+	drop important_file (unsafe)
+	  Could only verify the existence of 0 out of 1 necessary copies
+	  Unable to access these remotes: usbdrive
+	  Try making some of these repositories available:
+	   	58d84e8a-d9ae-11df-a1aa-ab9aa8c00826  -- portable USB drive
+	   	ca20064c-dbb5-11df-b2fe-002170d25c55  -- backup SATA drive
+	  (Use --force to override this check, or adjust annex.numcopies.)
+	failed
+	drop other.iso (unsafe)
+	  Could only verify the existence of 0 out of 1 necessary copies
+          No other repository is known to contain the file.
+	  (Use --force to override this check, or adjust annex.numcopies.)
+	failed
+
+Here you might --force it to drop `important_file` if you [[trust]] your backup.
+But `other.iso` looks to have never been copied to anywhere else, so if
+it's something you want to hold onto, you'd need to transfer it to
+some other repository before dropping it.
diff --git a/doc/walkthrough/renaming_files.mdwn b/doc/walkthrough/renaming_files.mdwn
new file mode 100644
index 0000000000..85964d1ea5
--- /dev/null
+++ b/doc/walkthrough/renaming_files.mdwn
@@ -0,0 +1,13 @@
+	# cd ~/annex
+	# git mv big_file my_cool_big_file
+	# mkdir iso
+	# git mv debian.iso iso/
+	# git commit -m moved
+
+You can use any normal git operations to move files around, or even
+make copies or delete them.
+
+Notice that, since annexed files are represented by symlinks,
+the symlink will break when the file is moved into a subdirectory.
+But, git-annex will fix this up for you when you commit --
+it has a pre-commit hook that watches for and corrects broken symlinks.
diff --git a/doc/walkthrough/syncing.mdwn b/doc/walkthrough/syncing.mdwn
new file mode 100644
index 0000000000..38f0f8acc8
--- /dev/null
+++ b/doc/walkthrough/syncing.mdwn
@@ -0,0 +1,25 @@
+Notice that in the [[previous example|getting_file_content]], you had to
+git fetch and merge from laptop first. This lets git-annex know what has
+changed in laptop, and so it knows about the files present there and can
+get them.
+
+If you have a lot of repositories to keep in sync, manually fetching and 
+merging from them can become tedious. To automate it there is a handy
+sync command, which also even commits your changes for you.
+
+	# cd /media/usb/annex
+	# git annex sync
+	commit
+	nothing to commit (working directory clean)
+	ok
+	pull laptop
+	ok
+	push laptop
+	ok
+
+After you run sync, the repository will be updated with all changes made to
+its remotes, and any changes in the repository will be pushed out to its
+remotes, where a sync will get them. This is especially useful when using
+git in a distributed fashion, without a 
+[[central bare repository|tips/centralized_git_repository_tutorial]]. See
+[[sync]] for details.
diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
new file mode 100644
index 0000000000..cfb70aaf9a
--- /dev/null
+++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
@@ -0,0 +1,17 @@
+After a while, you'll have several annexes, with different file contents.
+You don't have to try to keep all that straight; git-annex does 
+[[location_tracking]] for you. If you ask it to get a file and the drive
+or file server is not accessible, it will let you know what it needs to get
+it:
+
+	# git annex get video/hackity_hack_and_kaxxt.mov
+	get video/_why_hackity_hack_and_kaxxt.mov (not available)
+	  Unable to access these remotes: usbdrive, server
+	  Try making some of these repositories available:
+	  	5863d8c0-d9a9-11df-adb2-af51e6559a49  -- my home file server
+	   	58d84e8a-d9ae-11df-a1aa-ab9aa8c00826  -- portable USB drive
+	   	ca20064c-dbb5-11df-b2fe-002170d25c55  -- backup SATA drive
+	failed
+	# sudo mount /media/usb
+	# git annex get video/hackity_hack_and_kaxxt.mov
+	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
new file mode 100644
index 0000000000..63fb9f66d9
--- /dev/null
+++ b/doc/walkthrough/unused_data.mdwn
@@ -0,0 +1,30 @@
+It's possible for data to accumulate in the annex that no files in any
+branch point to anymore. One way it can happen is if you `git rm` a file
+without first calling `git annex drop`. And, when you modify an annexed
+file, the old content of the file remains in the annex. Another way is when
+migrating between key-value [[backends]].
+
+This might be historical data you want to preserve, so git-annex defaults to
+preserving it. So from time to time, you may want to check for such data and
+eliminate it to save space.
+
+	# git annex unused
+	unused . (checking for unused data...) 
+	  Some annexed data is no longer used by any files in the repository.
+	    NUMBER  KEY
+	    1       SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e
+	    2       SHA1-s14--f1358ec1873d57350e3dc62054dc232bc93c2bd1
+	  (To see where data was previously used, try: git log --stat -S'KEY')
+	  (To remove unwanted data: git-annex dropunused NUMBER)
+	ok
+
+After running `git annex unused`, you can follow the instructions to examine
+the history of files that used the data, and if you decide you don't need that
+data anymore, you can easily remove it:
+
+	# git annex dropunused 1
+	dropunused 1 ok
+
+Hint: To drop a lot of unused data, use a command like this:
+
+	# git annex dropunused 1-1000
diff --git a/doc/walkthrough/unused_data/comment_1_684b7b652d3a8ec04f32129c5528f1ab._comment b/doc/walkthrough/unused_data/comment_1_684b7b652d3a8ec04f32129c5528f1ab._comment
new file mode 100644
index 0000000000..2be2a6463f
--- /dev/null
+++ b/doc/walkthrough/unused_data/comment_1_684b7b652d3a8ec04f32129c5528f1ab._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="bremner"
+ ip="156.34.89.108"
+ subject="finding data that isn't unused, but should be."
+ date="2012-10-17T20:32:11Z"
+ content="""
+Sometimes links to annexed data still exists on some branch, when it was supposed to be dropped. Here is how I found these; perhaps there is a simpler way.
+
+    % git annex find --format '${key}\n' | sort > /tmp/known-keys
+    % find .git/annex/objects -type f -exec basename {} \; | sort  > /tmp/local-keys
+    % comm -23 /tmp/local-keys /tmp/known-keys
+
+to look for what branch these are on, try 
+
+    % git log --stat --all -S$key
+
+for one of the keys output above. In my case it was the same remote branch keeping them all alive.
+
+
+*EDIT* sort key lists to make comm work properly
+
+"""]]
diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn
new file mode 100644
index 0000000000..3a6a8776aa
--- /dev/null
+++ b/doc/walkthrough/using_bup.mdwn
@@ -0,0 +1,37 @@
+Another [[special_remote|special_remotes]] that git-annex can use is
+a [[special_remotes/bup]] repository. Bup stores large file contents
+in a git repository of its own, with deduplication. Combined with
+git-annex, you can have git on both the frontend and the backend.
+
+Here's how to create a bup remote, and describe it.
+
+[[!template id=note text="""
+Instead of specifying a remote system, you could choose to make a bup
+remote that is only accessible on the current system, by passing
+"buprepo=/big/mybup".
+"""]]
+
+	# git annex initremote mybup type=bup encryption=none buprepo=example.com:/big/mybup
+	initremote bup (bup init)
+	Initialized empty Git repository in /big/mybup/
+	ok
+	# git annex describe mybup "my bup repository at example.com"
+	describe mybup ok
+
+Now the remote can be used like any other remote.
+
+	# git annex move my_cool_big_file --to mybup
+	move my_cool_big_file (to mybup...)
+	Receiving index from server: 1100/1100, done.
+	ok
+
+Note that, unlike other remotes, bup does not really support removing
+content from its git repositories. This is a feature. :)
+
+	# git annex move my_cool_big_file --from mybup
+	move my_cool_big_file (from mybup...)
+	  content cannot be removed from bup remote
+	failed
+	git-annex: 1 failed
+
+See [[special_remotes/bup]] for details.
diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn
new file mode 100644
index 0000000000..60011a200b
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes.mdwn
@@ -0,0 +1,33 @@
+So far in this walkthrough, git-annex has been used with a remote
+repository on a USB drive. But it can also be used with a git remote
+that is truely remote, a host accessed by ssh.
+
+Say you have a desktop on the same network as your laptop and want
+to clone the laptop's annex to it:
+
+	# git clone ssh://mylaptop/home/me/annex ~/annex
+	# cd ~/annex
+	# git annex init "my desktop"
+
+Now you can get files and they will be transferred (using `rsync` via `ssh`):
+
+	# git annex get my_cool_big_file
+	get my_cool_big_file (getting UUID for origin...) (from origin...)
+	SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e  100% 2159     2.1KB/s   00:00
+	ok
+
+When you drop files, git-annex will ssh over to the remote and make
+sure the file's content is still there before removing it locally:
+
+	# git annex drop my_cool_big_file
+	drop my_cool_big_file (checking origin..) ok
+
+Note that normally git-annex prefers to use non-ssh remotes, like
+a USB drive, before ssh remotes. They are assumed to be faster/cheaper to
+access, if available. There is a annex-cost setting you can configure in
+`.git/config` to adjust which repositories it prefers. See
+[[the_man_page|git-annex]] for details.
+
+Also, note that you need full shell access for this to work -- 
+git-annex needs to be able to ssh in and run commands. Or at least,
+your shell needs to be able to run the [[git-annex-shell]] command.
diff --git a/doc/walkthrough/using_ssh_remotes/comment_10_98e97c4d7fbbcd449eddf683967a71d6._comment b/doc/walkthrough/using_ssh_remotes/comment_10_98e97c4d7fbbcd449eddf683967a71d6._comment
new file mode 100644
index 0000000000..6cd9bb8485
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_10_98e97c4d7fbbcd449eddf683967a71d6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
+ nickname="Matthias"
+ subject="Hint for Debian/Ubuntu"
+ date="2012-11-07T13:10:15Z"
+ content="""
+In Debian/Ubuntu the default .bashrc returns immediately if the shell is non-interactive. Make sure to setup the PATH such that it is updated with the location of git-annex-shell before this check! This just cost me an hour of debugging as I didn't notice the return statement early on...
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment b/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment
new file mode 100644
index 0000000000..8973978ad8
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 2"
+ date="2012-05-27T20:53:05Z"
+ content="""
+When `git annex get` does nothing, it's because it doesn't know a place to get the file from.
+
+This can happen if the `git-annex` branch has not propigated from the place where the file was added.
+For example, if on the laptop you had run `git pull ssh master`, that would only pull the master branch, not the git-annex branch. 
+
+An easy way to ensure the git-annex branch is kept in sync is to run `git annex sync`
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment b/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment
new file mode 100644
index 0000000000..2121401968
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="cannot get files"
+ date="2012-05-27T20:33:09Z"
+ content="""
+Hi,
+
+I could successfully clone my ssh repo's annex to my laptop, following these instructions.
+I'm also able to sync the repositories (laptop and ssh) when I commit new files in the ssh repo.
+
+However, every time I try to get files from the ssh repo (using 'git annex get some_file'), nothing happens.
+Do you know what can be happening?
+
+Thanks!
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment b/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment
new file mode 100644
index 0000000000..75a133d840
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 3"
+ date="2012-05-27T21:13:50Z"
+ content="""
+Thanks for the quick replay!
+
+I already did 'git annex sync', but it didn't work. The steps were: 'git clone ssh...', then 'cd annex', then 'git annex init \"laptop\"'
+
+After that, I did a 'git annex sync', and tried to get the file, but nothing happens. That's why I found it weird.
+Any other thing that might have happened?
+
+Thanks again!
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment b/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment
new file mode 100644
index 0000000000..3df03abc2c
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 4"
+ date="2012-05-27T21:33:11Z"
+ content="""
+Try running `git annex whereis` on the file and see where it says it is.
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment b/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment
new file mode 100644
index 0000000000..703b89ebfa
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 5"
+ date="2012-05-27T21:42:56Z"
+ content="""
+Hi,
+
+I guess the problem is with git-annex-shell. I tried to do 'git annex get file --from name_ssh_repo', and I got the following:
+
+bash: git-annex-shell: command not found; failed; exit code 127
+
+The same thing happens if I try to do 'git annex whereis'
+
+git-annex-shell is indeed installed. How can I make my shell recognize this command?
+
+Thanks a lot!
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment b/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment
new file mode 100644
index 0000000000..4d5961ca90
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 6"
+ date="2012-05-27T22:08:50Z"
+ content="""
+git-annex-shell needs to be installed in the `PATH` on any host that will hold annexed files. 
+
+If you installed with cabal, it might be `.cabal/bin/`. Whereever it was installed to is apparently not on the PATH that is set when you ssh into that host. 
+
+
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment b/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment
new file mode 100644
index 0000000000..700b170ad6
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 7"
+ date="2012-05-27T23:35:17Z"
+ content="""
+Hi,
+
+It was already installed in PATH. In fact, I can call it from the command line, and it is recognized (e.g. calling 'git-annex-shell' gives me 'git-annex-shell: bad parameters'). However, every time I do a 'git annex whereis' or 'git annex get file --from repo', it gives me the following error:
+
+bash: git-annex-shell: command not found
+Command ssh [\"-S\",\"/Users/username/annex/.git/annex/ssh/username@example.edu\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"username@example.edu\",\"git-annex-shell 'configlist' '/~/annex'\"] failed; exit code 127
+
+I tried to run this ssh command, but it gives me the same 'command not found' error. It seems that the problem is with the ssh repo?
+The ssh repo has a git-annex-shell working and installed in PATH.
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment b/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment
new file mode 100644
index 0000000000..148f016db7
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmB-gCGEs--zfmvYU-__Hj2FbliUXgxMDs"
+ nickname="Jakub"
+ subject="Path problems"
+ date="2012-07-13T19:15:15Z"
+ content="""
+Hi,
+
+I have a same 'git-annex-shell command not found' problem as above. I've installed git annex via cabal into my ~/.haskell_bin directory. Then I've added this dir both to ~/.bashrc and ~/.zshrc. I can run git annex or 'git annex-shell' and everything is fine. My guess is that haskell is trying to spawn git-annex-shell with some current $PATH unaware shell like dash maybe?
+
+I've fixed this behavior by using a really ugly hack - I've symlinked ~/.haskell_bin/git-annex-shell to /usr/bin/git-annex-shell on all my machines and the problem is gone. Somehow haskell (or whatever is trying to call git-annex-shell) is unaware of path modifications from .bashrc/.zshrc
+
+Here is the path modification I've used:
+
+export PATH=~/.haskell_bin:$PATH
+"""]]
diff --git a/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment b/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment
new file mode 100644
index 0000000000..ddb96da369
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmB-gCGEs--zfmvYU-__Hj2FbliUXgxMDs"
+ nickname="Jakub"
+ subject="Fixed"
+ date="2012-07-13T19:27:46Z"
+ content="""
+Found the problem:
+
+One should never use ~ in such path:
+
+WRONG export PATH=~/somedir:$PATH
+
+Instead one should use $HOME:
+
+GOOD export PATH=$HOME/somedir:$PATH
+
+Can I surpress the message that shell failed with status 255 when a repo is unavailible? I've got two repos pointing to one machine - either via vpn or local lan and I keep getting erros if one is unavailible:
+
+ssh: connect to host 10.9.0.1 port 39882: No route to host
+Command ssh [\"-S\",\"/home/pielgrzym/annex/.git/annex/ssh/nas\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"nas\",\"git-annex-shell 'configlist' '/~/annex'\"] failed; exit code 255
+
+
+"""]]
diff --git a/ghci b/ghci
new file mode 100755
index 0000000000..206bbbc7ac
--- /dev/null
+++ b/ghci
@@ -0,0 +1,4 @@
+#!/bin/sh
+# ghci using objects built by cabal
+make dist/caballog
+$(grep 'ghc --make' dist/caballog | head -n 1 | perl -pe 's/--make/--interactive/; s/.\/[^\.\s]+.hs//; s/-package-id [^\s]+//g; s/-hide-all-packages//; s/-threaded//; s/-O//') $@
diff --git a/git-annex.cabal b/git-annex.cabal
new file mode 100644
index 0000000000..938b456389
--- /dev/null
+++ b/git-annex.cabal
@@ -0,0 +1,141 @@
+Name: git-annex
+Version: 4.20130228
+Cabal-Version: >= 1.8
+License: GPL
+Maintainer: Joey Hess 
+Author: Joey Hess
+Stability: Stable
+Copyright: 2010-2013 Joey Hess
+License-File: COPYRIGHT
+Homepage: http://git-annex.branchable.com/
+Build-type: Custom
+Category: Utility
+Synopsis: manage files with git, without checking their contents into git
+Description:
+ git-annex allows managing files with git, without checking the file
+ contents into git. While that may seem paradoxical, it is useful when
+ dealing with files larger than git can currently easily handle, whether due
+ to limitations in memory, time, or disk space.
+ .
+ Even without file content tracking, being able to manage files with git,
+ move files around and delete files with versioned directory trees, and use
+ branches and distributed clones, are all very handy reasons to use git. And
+ annexed files can co-exist in the same git repository with regularly
+ versioned files, which is convenient for maintaining documents, Makefiles,
+ etc that are associated with annexed files but that benefit from full
+ revision control.
+
+Flag S3
+  Description: Enable S3 support
+
+Flag WebDAV
+  Description: Enable WebDAV support
+
+Flag Inotify
+  Description: Enable inotify support
+
+Flag Dbus
+  Description: Enable dbus support
+
+Flag Assistant
+  Description: Enable git-annex assistant and watch command
+
+Flag Webapp
+  Description: Enable git-annex webapp
+
+Flag Pairing
+  Description: Enable pairing
+
+Flag XMPP
+  Description: Enable notifications using XMPP
+
+Flag DNS
+  Description: Enable the haskell DNS library for DNS lookup
+
+Flag Production
+  Description: Enable production build (slower build; faster binary)
+
+Flag Android
+  Description: Building for Android
+  Default: False
+
+Flag TestSuite
+  Description: Embed the test suite into git-annex
+
+Executable git-annex
+  Main-Is: git-annex.hs
+  Build-Depends: MissingH, hslogger, directory, filepath,
+   unix, containers, utf8-string, network (>= 2.0), mtl (>= 2),
+   bytestring, old-locale, time,
+   extensible-exceptions, dataenc, SHA, process, json,
+   base (>= 4.5 && < 4.8), monad-control, transformers-base, lifted-base,
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process,
+   SafeSemaphore, uuid, random, regex-compat
+  -- Need to list these because they're generated from .hsc files.
+  Other-Modules: Utility.Touch Utility.Mounts
+  Include-Dirs: Utility
+  C-Sources: Utility/libdiskfree.c Utility/libmounts.c
+  CC-Options: -Wall
+  GHC-Options: -threaded -Wall
+
+  if flag(Production)
+    GHC-Options: -O2
+
+  if flag(TestSuite)
+    Build-Depends: HUnit
+    CPP-Options: -DWITH_TESTSUITE
+
+  if flag(S3)
+    Build-Depends: hS3
+    CPP-Options: -DWITH_S3
+
+  if flag(WebDAV)
+    Build-Depends: DAV (>= 0.3), http-conduit, xml-conduit, http-types
+    CPP-Options: -DWITH_WEBDAV
+
+  if flag(Assistant) && ! os(windows) && ! os(solaris)
+    Build-Depends: async, stm (>= 2.3)
+    CPP-Options: -DWITH_ASSISTANT
+
+  if flag(Android)
+    CPP-Options: -D__ANDROID__
+
+  if os(linux) && flag(Inotify)
+    Build-Depends: hinotify
+    CPP-Options: -DWITH_INOTIFY
+  else
+    if os(darwin)
+      Build-Depends: hfsevents
+      CPP-Options: -DWITH_FSEVENTS
+    else
+      if (! os(windows) && ! os(solaris) && ! os(linux))
+        CPP-Options: -DWITH_KQUEUE
+        C-Sources: Utility/libkqueue.c
+        Includes: sys/event.h
+
+  if os(linux) && flag(Dbus)
+    Build-Depends: dbus (>= 0.10.3)
+    CPP-Options: -DWITH_DBUS
+
+  if flag(Webapp)
+    Build-Depends: yesod, yesod-static, case-insensitive,
+     http-types, transformers, wai, wai-logger, warp, blaze-builder,
+     crypto-api, hamlet, clientsession, aeson, yesod-form,
+     template-haskell, yesod-default, data-default
+    CPP-Options: -DWITH_WEBAPP
+
+  if flag(Pairing)
+    Build-Depends: network-multicast, network-info
+    CPP-Options: -DWITH_PAIRING
+
+  if flag(XMPP)
+    Build-Depends: network-protocol-xmpp, gnutls (>= 0.1.4), xml-types
+    CPP-Options: -DWITH_XMPP
+
+  if flag(DNS)
+    Build-Depends: dns
+    CPP-Options: -DWITH_DNS
+
+source-repository head
+  type: git
+  location: git://git-annex.branchable.com/
diff --git a/git-annex.hs b/git-annex.hs
new file mode 100644
index 0000000000..0f45f53ebc
--- /dev/null
+++ b/git-annex.hs
@@ -0,0 +1,34 @@
+{- git-annex main program stub
+ -
+ - Copyright 2010-2013 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+import System.Environment
+import System.FilePath
+
+import qualified GitAnnex
+import qualified GitAnnexShell
+#ifdef WITH_TESTSUITE
+import qualified Test
+#endif
+
+main :: IO ()
+main = run =<< getProgName
+  where
+	run n
+		| isshell n = go GitAnnexShell.run
+		| otherwise = go GitAnnex.run
+	isshell n = takeFileName n == "git-annex-shell"
+	go a = do
+		ps <- getArgs
+#ifdef WITH_TESTSUITE
+		if ps == ["test"]
+			then Test.main
+			else a ps
+#else
+		a ps
+#endif
diff --git a/git-union-merge.hs b/git-union-merge.hs
new file mode 100644
index 0000000000..0e4cd644ce
--- /dev/null
+++ b/git-union-merge.hs
@@ -0,0 +1,48 @@
+{- git-union-merge program
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+import System.Environment
+
+import Common
+import qualified Git.UnionMerge
+import qualified Git.Config
+import qualified Git.CurrentRepo
+import qualified Git.Branch
+import qualified Git.Index
+import qualified Git
+
+header :: String
+header = "Usage: git-union-merge ref ref newref"
+
+usage :: IO a
+usage = error $ "bad parameters\n\n" ++ header
+
+tmpIndex :: Git.Repo -> FilePath
+tmpIndex g = Git.localGitDir g  "index.git-union-merge"
+
+setup :: Git.Repo -> IO ()
+setup = cleanup -- idempotency
+
+cleanup :: Git.Repo -> IO ()
+cleanup g = nukeFile $ tmpIndex g
+
+parseArgs :: IO [String]
+parseArgs = do
+	args <- getArgs
+	if length args /= 3
+		then usage
+		else return args
+
+main :: IO ()
+main = do
+	[aref, bref, newref] <- map Git.Ref <$> parseArgs
+	g <- Git.Config.read =<< Git.CurrentRepo.get
+	_ <- Git.Index.override $ tmpIndex g
+	setup g
+	Git.UnionMerge.merge aref bref g
+	_ <- Git.Branch.commit "union merge" newref [aref, bref] g
+	cleanup g
diff --git a/standalone/android/.gitignore b/standalone/android/.gitignore
new file mode 100644
index 0000000000..e8792ba9fd
--- /dev/null
+++ b/standalone/android/.gitignore
@@ -0,0 +1,2 @@
+build-utils
+start
diff --git a/standalone/android/Makefile b/standalone/android/Makefile
new file mode 100644
index 0000000000..7b70054cf2
--- /dev/null
+++ b/standalone/android/Makefile
@@ -0,0 +1,153 @@
+# Cross-compiles utilities needed for git-annex on Android,
+# and builds the Android app.
+
+# Add Android cross-compiler to PATH (as installed by ghc-android)
+# (This directory also needs to have a cc that is a symlink to the prefixed
+# gcc cross-compiler executable.)
+ANDROID_CROSS_COMPILER?=$(HOME)/.ghc/android-14/arm-linux-androideabi-4.7/bin
+PATH:=$(ANDROID_CROSS_COMPILER):$(PATH)
+
+# Paths to the Android SDK and NDK.
+export ANDROID_SDK_ROOT?=$(HOME)/tmp/adt-bundle-linux-x86/sdk
+export ANDROID_NDK_ROOT?=$(HOME)/tmp/android-ndk-r8d
+
+# Where to store the $(GIT_ANNEX_ANDROID_SOURCETREE)s to utilities. This
+# directory will be created by `make source`.
+GIT_ANNEX_ANDROID_SOURCETREE?=../tmp/android-sourcetree
+
+GITTREE=$(GIT_ANNEX_ANDROID_SOURCETREE)/git/installed-tree
+
+build: start
+	$(MAKE) $(GIT_ANNEX_ANDROID_SOURCETREE)/openssl
+	$(MAKE) $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh
+	$(MAKE) $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox
+	$(MAKE) $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync
+	$(MAKE) $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg
+	$(MAKE) $(GIT_ANNEX_ANDROID_SOURCETREE)/git
+	$(MAKE) $(GIT_ANNEX_ANDROID_SOURCETREE)/term
+
+	# Debug build because it does not need signing keys.
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && tools/build-debug
+
+	# Install executables as pseudo-libraries so they will be
+	# unpacked from the .apk.
+	mkdir -p $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi
+	cp ../../git-annex $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.git-annex.so
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox/busybox $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.busybox.so
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh/ssh $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.ssh.so
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh/ssh-keygen $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.ssh-keygen.so
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync/rsync $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.rsync.so
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg/g10/gpg $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.gpg.so
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/git/git $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.git.so
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/git/git-shell $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.git-shell.so
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/git/git-upload-pack $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.git-upload-pack.so
+	arm-linux-androideabi-strip --strip-unneeded --remove-section=.comment --remove-section=.note $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/*
+	cp runshell $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.runshell.so
+	cp start $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.start.so
+	
+	# remove git stuff we don't need to save space
+	rm -rf $(GITTREE)/bin/git-cvsserver \
+		$(GITTREE)/libexec/git-core/git-daemon \
+		$(GITTREE)/libexec/git-core/git-show-index \
+		$(GITTREE)/libexec/git-core/mergetools \
+		$(GITTREE)/libexec/git-core/git-credential-* \
+		$(GITTREE)/libexec/git-core/git-cvsserver \
+		$(GITTREE)/libexec/git-core/git-cvsimport \
+		$(GITTREE)/libexec/git-core/git-fast-import \
+		$(GITTREE)/libexec/git-core/git-http-backend \
+		$(GITTREE)/libexec/git-core/git-imap-send \
+		$(GITTREE)/libexec/git-core/git-instaweb \
+		$(GITTREE)/libexec/git-core/git-p4 \
+		$(GITTREE)/libexec/git-core/git-remote-test* \
+		$(GITTREE)/libexec/git-core/git-submodule \
+		$(GITTREE)/libexec/git-core/git-svn \
+		$(GITTREE)/libexec/git-core/git-web--browse
+	# Most of git is in one multicall binary, but a few important
+	# commands are still shell scripts. Those are put into
+	# a tarball, along with a list of all the hard links that should be
+	# set up.
+	cd $(GITTREE) && mkdir -p links
+	cd $(GITTREE) && find -samefile bin/git -not -wholename ./bin/git > links/git
+	cd $(GITTREE) && find -samefile bin/git-shell -not -wholename ./bin/git-shell > links/git-shell
+	cd $(GITTREE) && find -samefile bin/git-upload-pack -not -wholename ./bin/git-upload-pack > links/git-upload-pack
+	cd $(GITTREE) && find -type f -not -samefile bin/git -not -samefile bin/git-shell -not -samefile bin/git-upload-pack|tar czf ../git.tar.gz -T -
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/git/git.tar.gz $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.git.tar.gz.so
+
+	git rev-parse HEAD > $(GIT_ANNEX_ANDROID_SOURCETREE)/term/libs/armeabi/lib.version.so
+
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && ant debug
+	mkdir -p ../../tmp
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/term/bin/Term-debug.apk ../../tmp/git-annex.apk
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/openssl:
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssl && CC=$$(which cc) ./Configure android
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssl && $(MAKE)
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/openssh: openssh.patch openssh.config.h
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && git reset --hard
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && ./configure --host=arm-linux-androideabi --with-ssl-dir=../openssl --without-openssl-header-check
+	cat openssh.patch | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && patch -p1)
+	cp openssh.config.h $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh/config.h
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && sed -i -e 's/getrrsetbyname.o //' openbsd-compat/Makefile
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && sed -i -e 's/auth-passwd.o //' Makefile
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh && $(MAKE) ssh ssh-keygen
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/busybox: busybox_config
+	cp busybox_config $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox/.config
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox && yes '' | $(MAKE) oldconfig
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox && $(MAKE)
+	touch $@
+	
+$(GIT_ANNEX_ANDROID_SOURCETREE)/git:
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/git && $(MAKE) install NO_OPENSSL=1 NO_GETTEXT=1 NO_GECOS_IN_PWENT=1 NO_GETPASS=1 NO_NSEC=1 NO_MKDTEMP=1 NO_PTHREADS=1 NO_PERL=1 NO_CURL=1 NO_EXPAT=1 NO_TCLTK=1 NO_ICONV=1 prefix= DESTDIR=installed-tree
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/rsync: rsync.patch
+	cat rsync.patch | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync && git reset --hard origin/master && git am)
+	cp $(GIT_ANNEX_ANDROID_SOURCETREE)/automake/lib/config.sub $(GIT_ANNEX_ANDROID_SOURCETREE)/automake/lib/config.guess $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync/
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync && ./configure --host=arm-linux-androideabi --disable-locale --disable-iconv-open --disable-iconv --disable-acl-support --disable-xattr-support
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync && $(MAKE)
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg:
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg && git checkout gnupg-1.4.13
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg && ./autogen.sh
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg && ./configure --host=arm-linux-androideabi --disable-gnupg-iconv --enable-minimal --disable-card-support --disable-agent-support --disable-photo-viewers --disable-keyserver-helpers --disable-nls
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg; $(MAKE) || true # expected failure in doc build
+	touch $@
+
+$(GIT_ANNEX_ANDROID_SOURCETREE)/term: term.patch icons
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && git reset --hard
+	cat term.patch | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && patch -p1)
+	(cd icons && tar c .) | (cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term/res && tar x)
+	# This renaming has a purpose. It makes the path to the app's
+	# /data directory shorter, which makes ssh connection caching
+	# sockets placed there have more space for their filenames.
+	# Also, it avoids overlap with the Android Terminal Emulator
+	# app, if it's also installed.
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && find -name .git -prune -o -type f -print0 | xargs -0 perl -pi -e 's/jackpal/ga/g'
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && perl -pi -e 's/Terminal Emulator/Git Annex/g' res/*/strings.xml
+	cd $(GIT_ANNEX_ANDROID_SOURCETREE)/term && tools/update.sh >/dev/null 2>&1
+	touch $@
+
+source: $(GIT_ANNEX_ANDROID_SOURCETREE)
+
+$(GIT_ANNEX_ANDROID_SOURCETREE):
+	mkdir -p $(GIT_ANNEX_ANDROID_SOURCETREE)
+	git clone --bare git://git.savannah.gnu.org/automake.git $(GIT_ANNEX_ANDROID_SOURCETREE)/automake
+	git clone --bare git://git.debian.org/git/d-i/busybox $(GIT_ANNEX_ANDROID_SOURCETREE)/busybox
+	git clone --bare git://git.kernel.org/pub/scm/git/git.git $(GIT_ANNEX_ANDROID_SOURCETREE)/git
+	git clone --bare git://git.samba.org/rsync.git $(GIT_ANNEX_ANDROID_SOURCETREE)/rsync
+	git clone --bare git://git.gnupg.org/gnupg.git $(GIT_ANNEX_ANDROID_SOURCETREE)/gnupg
+	git clone --bare git://git.openssl.org/openssl $(GIT_ANNEX_ANDROID_SOURCETREE)/openssl
+	git clone --bare git://github.com/CyanogenMod/android_external_openssh.git $(GIT_ANNEX_ANDROID_SOURCETREE)/openssh
+	git clone --bare git://github.com/jackpal/Android-Terminal-Emulator.git $(GIT_ANNEX_ANDROID_SOURCETREE)/term
+
+clean:
+	rm -rf $(GITTREE)
+	rm -f start
+
+reallyclean: clean
+	rm -rf $(GIT_ANNEX_ANDROID_SOURCETREE)
diff --git a/standalone/android/busybox_config b/standalone/android/busybox_config
new file mode 100644
index 0000000000..28ea880d98
--- /dev/null
+++ b/standalone/android/busybox_config
@@ -0,0 +1,997 @@
+# Run "make android2_defconfig", then "make".
+#
+# Tested with the standalone toolchain from ndk r6:
+# android-ndk-r6/build/tools/make-standalone-toolchain.sh --platform=android-8
+#
+CONFIG_HAVE_DOT_CONFIG=y
+
+#
+# Busybox Settings
+#
+
+#
+# General Configuration
+#
+# CONFIG_DESKTOP is not set
+# CONFIG_EXTRA_COMPAT is not set
+# CONFIG_INCLUDE_SUSv2 is not set
+# CONFIG_USE_PORTABLE_CODE is not set
+CONFIG_PLATFORM_LINUX=y
+CONFIG_FEATURE_BUFFERS_USE_MALLOC=y
+# CONFIG_FEATURE_BUFFERS_GO_ON_STACK is not set
+# CONFIG_FEATURE_BUFFERS_GO_IN_BSS is not set
+# CONFIG_SHOW_USAGE is not set
+# CONFIG_FEATURE_VERBOSE_USAGE is not set
+# CONFIG_FEATURE_COMPRESS_USAGE is not set
+CONFIG_FEATURE_INSTALLER=y
+# CONFIG_INSTALL_NO_USR is not set
+# CONFIG_LOCALE_SUPPORT is not set
+# CONFIG_UNICODE_SUPPORT is not set
+# CONFIG_UNICODE_USING_LOCALE is not set
+# CONFIG_FEATURE_CHECK_UNICODE_IN_ENV is not set
+CONFIG_SUBST_WCHAR=0
+CONFIG_LAST_SUPPORTED_WCHAR=0
+# CONFIG_UNICODE_COMBINING_WCHARS is not set
+# CONFIG_UNICODE_WIDE_WCHARS is not set
+# CONFIG_UNICODE_BIDI_SUPPORT is not set
+# CONFIG_UNICODE_NEUTRAL_TABLE is not set
+# CONFIG_UNICODE_PRESERVE_BROKEN is not set
+# CONFIG_LONG_OPTS is not set
+# CONFIG_FEATURE_DEVPTS is not set
+# CONFIG_FEATURE_CLEAN_UP is not set
+# CONFIG_FEATURE_UTMP is not set
+# CONFIG_FEATURE_WTMP is not set
+# CONFIG_FEATURE_PIDFILE is not set
+# CONFIG_FEATURE_SUID is not set
+# CONFIG_FEATURE_SUID_CONFIG is not set
+# CONFIG_FEATURE_SUID_CONFIG_QUIET is not set
+# CONFIG_SELINUX is not set
+# CONFIG_FEATURE_PREFER_APPLETS is not set
+CONFIG_BUSYBOX_EXEC_PATH="/proc/self/exe"
+CONFIG_FEATURE_SYSLOG=y
+# CONFIG_FEATURE_HAVE_RPC is not set
+
+#
+# Build Options
+#
+# CONFIG_STATIC is not set
+# CONFIG_PIE is not set
+# CONFIG_NOMMU is not set
+# CONFIG_BUILD_LIBBUSYBOX is not set
+# CONFIG_FEATURE_INDIVIDUAL is not set
+# CONFIG_FEATURE_SHARED_BUSYBOX is not set
+# CONFIG_LFS is not set
+CONFIG_CROSS_COMPILER_PREFIX="arm-linux-androideabi-"
+CONFIG_EXTRA_CFLAGS=""
+
+#
+# Debugging Options
+#
+# CONFIG_DEBUG is not set
+# CONFIG_DEBUG_PESSIMIZE is not set
+# CONFIG_WERROR is not set
+CONFIG_NO_DEBUG_LIB=y
+# CONFIG_DMALLOC is not set
+# CONFIG_EFENCE is not set
+
+#
+# Installation Options ("make install" behavior)
+#
+CONFIG_INSTALL_APPLET_SYMLINKS=y
+# CONFIG_INSTALL_APPLET_HARDLINKS is not set
+# CONFIG_INSTALL_APPLET_SCRIPT_WRAPPERS is not set
+# CONFIG_INSTALL_APPLET_DONT is not set
+# CONFIG_INSTALL_SH_APPLET_SYMLINK is not set
+# CONFIG_INSTALL_SH_APPLET_HARDLINK is not set
+# CONFIG_INSTALL_SH_APPLET_SCRIPT_WRAPPER is not set
+CONFIG_PREFIX="./_install"
+
+#
+# Busybox Library Tuning
+#
+# CONFIG_FEATURE_SYSTEMD is not set
+# CONFIG_FEATURE_RTMINMAX is not set
+CONFIG_PASSWORD_MINLEN=6
+CONFIG_MD5_SMALL=1
+# CONFIG_FEATURE_FAST_TOP is not set
+# CONFIG_FEATURE_ETC_NETWORKS is not set
+CONFIG_FEATURE_USE_TERMIOS=y
+# CONFIG_FEATURE_EDITING is not set
+CONFIG_FEATURE_EDITING_MAX_LEN=0
+# CONFIG_FEATURE_EDITING_VI is not set
+CONFIG_FEATURE_EDITING_HISTORY=0
+# CONFIG_FEATURE_EDITING_SAVEHISTORY is not set
+# CONFIG_FEATURE_TAB_COMPLETION is not set
+# CONFIG_FEATURE_USERNAME_COMPLETION is not set
+# CONFIG_FEATURE_EDITING_FANCY_PROMPT is not set
+# CONFIG_FEATURE_EDITING_ASK_TERMINAL is not set
+# CONFIG_FEATURE_NON_POSIX_CP is not set
+# CONFIG_FEATURE_VERBOSE_CP_MESSAGE is not set
+CONFIG_FEATURE_COPYBUF_KB=4
+# CONFIG_FEATURE_SKIP_ROOTFS is not set
+# CONFIG_MONOTONIC_SYSCALL is not set
+# CONFIG_IOCTL_HEX2STR_ERROR is not set
+# CONFIG_FEATURE_HWIB is not set
+
+#
+# Applets
+#
+
+#
+# Archival Utilities
+#
+CONFIG_FEATURE_SEAMLESS_XZ=y
+CONFIG_FEATURE_SEAMLESS_LZMA=y
+CONFIG_FEATURE_SEAMLESS_BZ2=y
+CONFIG_FEATURE_SEAMLESS_GZ=y
+CONFIG_FEATURE_SEAMLESS_Z=y
+CONFIG_AR=y
+CONFIG_FEATURE_AR_LONG_FILENAMES=y
+CONFIG_FEATURE_AR_CREATE=y
+CONFIG_BUNZIP2=y
+CONFIG_BZIP2=y
+CONFIG_CPIO=y
+CONFIG_FEATURE_CPIO_O=y
+CONFIG_FEATURE_CPIO_P=y
+CONFIG_DPKG=y
+CONFIG_DPKG_DEB=y
+# CONFIG_FEATURE_DPKG_DEB_EXTRACT_ONLY is not set
+CONFIG_GUNZIP=y
+CONFIG_GZIP=y
+# CONFIG_FEATURE_GZIP_LONG_OPTIONS is not set
+CONFIG_LZOP=y
+CONFIG_LZOP_COMPR_HIGH=y
+CONFIG_RPM2CPIO=y
+CONFIG_RPM=y
+CONFIG_TAR=y
+CONFIG_FEATURE_TAR_CREATE=y
+CONFIG_FEATURE_TAR_AUTODETECT=y
+CONFIG_FEATURE_TAR_FROM=y
+CONFIG_FEATURE_TAR_OLDGNU_COMPATIBILITY=y
+CONFIG_FEATURE_TAR_OLDSUN_COMPATIBILITY=y
+CONFIG_FEATURE_TAR_GNU_EXTENSIONS=y
+# CONFIG_FEATURE_TAR_LONG_OPTIONS is not set
+# CONFIG_FEATURE_TAR_TO_COMMAND is not set
+CONFIG_FEATURE_TAR_UNAME_GNAME=y
+CONFIG_FEATURE_TAR_NOPRESERVE_TIME=y
+# CONFIG_FEATURE_TAR_SELINUX is not set
+CONFIG_UNCOMPRESS=y
+CONFIG_UNLZMA=y
+CONFIG_FEATURE_LZMA_FAST=y
+CONFIG_LZMA=y
+CONFIG_UNXZ=y
+CONFIG_XZ=y
+CONFIG_UNZIP=y
+
+#
+# Coreutils
+#
+CONFIG_BASENAME=y
+CONFIG_CAT=y
+# CONFIG_DATE is not set
+# CONFIG_FEATURE_DATE_ISOFMT is not set
+# CONFIG_FEATURE_DATE_NANO is not set
+# CONFIG_FEATURE_DATE_COMPAT is not set
+# CONFIG_ID is not set
+# CONFIG_GROUPS is not set
+CONFIG_TEST=y
+CONFIG_FEATURE_TEST_64=y
+CONFIG_TOUCH=y
+CONFIG_TR=y
+CONFIG_FEATURE_TR_CLASSES=y
+CONFIG_FEATURE_TR_EQUIV=y
+CONFIG_BASE64=y
+CONFIG_CAL=y
+CONFIG_CATV=y
+CONFIG_CHGRP=y
+CONFIG_CHMOD=y
+CONFIG_CHOWN=y
+# CONFIG_FEATURE_CHOWN_LONG_OPTIONS is not set
+CONFIG_CHROOT=y
+CONFIG_CKSUM=y
+CONFIG_COMM=y
+CONFIG_CP=y
+# CONFIG_FEATURE_CP_LONG_OPTIONS is not set
+CONFIG_CUT=y
+CONFIG_DD=y
+CONFIG_FEATURE_DD_SIGNAL_HANDLING=y
+CONFIG_FEATURE_DD_THIRD_STATUS_LINE=y
+CONFIG_FEATURE_DD_IBS_OBS=y
+# CONFIG_DF is not set
+# CONFIG_FEATURE_DF_FANCY is not set
+CONFIG_DIRNAME=y
+CONFIG_DOS2UNIX=y
+CONFIG_UNIX2DOS=y
+CONFIG_DU=y
+CONFIG_FEATURE_DU_DEFAULT_BLOCKSIZE_1K=y
+CONFIG_ECHO=y
+CONFIG_FEATURE_FANCY_ECHO=y
+# CONFIG_ENV is not set
+# CONFIG_FEATURE_ENV_LONG_OPTIONS is not set
+CONFIG_EXPAND=y
+# CONFIG_FEATURE_EXPAND_LONG_OPTIONS is not set
+# CONFIG_EXPR is not set
+# CONFIG_EXPR_MATH_SUPPORT_64 is not set
+CONFIG_FALSE=y
+CONFIG_FOLD=y
+# CONFIG_FSYNC is not set
+CONFIG_HEAD=y
+CONFIG_FEATURE_FANCY_HEAD=y
+# CONFIG_HOSTID is not set
+CONFIG_INSTALL=y
+# CONFIG_FEATURE_INSTALL_LONG_OPTIONS is not set
+CONFIG_LN=y
+# CONFIG_LOGNAME is not set
+CONFIG_LS=y
+CONFIG_FEATURE_LS_FILETYPES=y
+CONFIG_FEATURE_LS_FOLLOWLINKS=y
+CONFIG_FEATURE_LS_RECURSIVE=y
+CONFIG_FEATURE_LS_SORTFILES=y
+CONFIG_FEATURE_LS_TIMESTAMPS=y
+CONFIG_FEATURE_LS_USERNAME=y
+# CONFIG_FEATURE_LS_COLOR is not set
+# CONFIG_FEATURE_LS_COLOR_IS_DEFAULT is not set
+CONFIG_MD5SUM=y
+CONFIG_MKDIR=y
+# CONFIG_FEATURE_MKDIR_LONG_OPTIONS is not set
+CONFIG_MKFIFO=y
+CONFIG_MKNOD=y
+CONFIG_MV=y
+# CONFIG_FEATURE_MV_LONG_OPTIONS is not set
+CONFIG_NICE=y
+CONFIG_NOHUP=y
+CONFIG_OD=y
+CONFIG_PRINTENV=y
+CONFIG_PRINTF=y
+CONFIG_PWD=y
+CONFIG_READLINK=y
+CONFIG_FEATURE_READLINK_FOLLOW=y
+CONFIG_REALPATH=y
+CONFIG_RM=y
+CONFIG_RMDIR=y
+# CONFIG_FEATURE_RMDIR_LONG_OPTIONS is not set
+CONFIG_SEQ=y
+CONFIG_SHA1SUM=y
+CONFIG_SHA256SUM=y
+CONFIG_SHA512SUM=y
+CONFIG_SLEEP=y
+CONFIG_FEATURE_FANCY_SLEEP=y
+CONFIG_FEATURE_FLOAT_SLEEP=y
+CONFIG_SORT=y
+CONFIG_FEATURE_SORT_BIG=y
+CONFIG_SPLIT=y
+CONFIG_FEATURE_SPLIT_FANCY=y
+# CONFIG_STAT is not set
+# CONFIG_FEATURE_STAT_FORMAT is not set
+CONFIG_STTY=y
+CONFIG_SUM=y
+CONFIG_SYNC=y
+CONFIG_TAC=y
+CONFIG_TAIL=y
+CONFIG_FEATURE_FANCY_TAIL=y
+CONFIG_TEE=y
+CONFIG_FEATURE_TEE_USE_BLOCK_IO=y
+CONFIG_TRUE=y
+# CONFIG_TTY is not set
+CONFIG_UNAME=y
+CONFIG_UNEXPAND=y
+# CONFIG_FEATURE_UNEXPAND_LONG_OPTIONS is not set
+CONFIG_UNIQ=y
+# CONFIG_USLEEP is not set
+CONFIG_UUDECODE=y
+CONFIG_UUENCODE=y
+CONFIG_WC=y
+CONFIG_FEATURE_WC_LARGE=y
+# CONFIG_WHO is not set
+CONFIG_WHOAMI=y
+CONFIG_YES=y
+
+#
+# Common options for cp and mv
+#
+CONFIG_FEATURE_PRESERVE_HARDLINKS=y
+
+#
+# Common options for ls, more and telnet
+#
+CONFIG_FEATURE_AUTOWIDTH=y
+
+#
+# Common options for df, du, ls
+#
+CONFIG_FEATURE_HUMAN_READABLE=y
+
+#
+# Common options for md5sum, sha1sum, sha256sum, sha512sum
+#
+CONFIG_FEATURE_MD5_SHA1_SUM_CHECK=y
+
+#
+# Console Utilities
+#
+CONFIG_CHVT=y
+CONFIG_FGCONSOLE=y
+CONFIG_CLEAR=y
+CONFIG_DEALLOCVT=y
+CONFIG_DUMPKMAP=y
+# CONFIG_KBD_MODE is not set
+# CONFIG_LOADFONT is not set
+CONFIG_LOADKMAP=y
+CONFIG_OPENVT=y
+CONFIG_RESET=y
+CONFIG_RESIZE=y
+CONFIG_FEATURE_RESIZE_PRINT=y
+CONFIG_SETCONSOLE=y
+# CONFIG_FEATURE_SETCONSOLE_LONG_OPTIONS is not set
+# CONFIG_SETFONT is not set
+# CONFIG_FEATURE_SETFONT_TEXTUAL_MAP is not set
+CONFIG_DEFAULT_SETFONT_DIR=""
+CONFIG_SETKEYCODES=y
+CONFIG_SETLOGCONS=y
+CONFIG_SHOWKEY=y
+# CONFIG_FEATURE_LOADFONT_PSF2 is not set
+# CONFIG_FEATURE_LOADFONT_RAW is not set
+
+#
+# Debian Utilities
+#
+CONFIG_MKTEMP=y
+CONFIG_PIPE_PROGRESS=y
+CONFIG_RUN_PARTS=y
+# CONFIG_FEATURE_RUN_PARTS_LONG_OPTIONS is not set
+CONFIG_FEATURE_RUN_PARTS_FANCY=y
+CONFIG_START_STOP_DAEMON=y
+CONFIG_FEATURE_START_STOP_DAEMON_FANCY=y
+# CONFIG_FEATURE_START_STOP_DAEMON_LONG_OPTIONS is not set
+CONFIG_WHICH=y
+
+#
+# Editors
+#
+CONFIG_PATCH=y
+CONFIG_VI=y
+CONFIG_FEATURE_VI_MAX_LEN=0
+# CONFIG_FEATURE_VI_8BIT is not set
+# CONFIG_FEATURE_VI_COLON is not set
+# CONFIG_FEATURE_VI_YANKMARK is not set
+CONFIG_FEATURE_VI_SEARCH=y
+# CONFIG_FEATURE_VI_REGEX_SEARCH is not set
+CONFIG_FEATURE_VI_USE_SIGNALS=y
+# CONFIG_FEATURE_VI_DOT_CMD is not set
+# CONFIG_FEATURE_VI_READONLY is not set
+# CONFIG_FEATURE_VI_SETOPTS is not set
+# CONFIG_FEATURE_VI_SET is not set
+CONFIG_FEATURE_VI_WIN_RESIZE=y
+# CONFIG_FEATURE_VI_ASK_TERMINAL is not set
+# CONFIG_FEATURE_VI_OPTIMIZE_CURSOR is not set
+# CONFIG_AWK is not set
+# CONFIG_FEATURE_AWK_LIBM is not set
+CONFIG_CMP=y
+CONFIG_DIFF=y
+# CONFIG_FEATURE_DIFF_LONG_OPTIONS is not set
+CONFIG_FEATURE_DIFF_DIR=y
+# CONFIG_ED is not set
+CONFIG_SED=y
+# CONFIG_FEATURE_ALLOW_EXEC is not set
+
+#
+# Finding Utilities
+#
+CONFIG_FIND=y
+CONFIG_FEATURE_FIND_PRINT0=y
+CONFIG_FEATURE_FIND_MTIME=y
+# CONFIG_FEATURE_FIND_MMIN is not set
+CONFIG_FEATURE_FIND_PERM=y
+CONFIG_FEATURE_FIND_TYPE=y
+# CONFIG_FEATURE_FIND_XDEV is not set
+# CONFIG_FEATURE_FIND_MAXDEPTH is not set
+# CONFIG_FEATURE_FIND_NEWER is not set
+# CONFIG_FEATURE_FIND_INUM is not set
+# CONFIG_FEATURE_FIND_EXEC is not set
+# CONFIG_FEATURE_FIND_USER is not set
+# CONFIG_FEATURE_FIND_GROUP is not set
+# CONFIG_FEATURE_FIND_NOT is not set
+# CONFIG_FEATURE_FIND_DEPTH is not set
+# CONFIG_FEATURE_FIND_PAREN is not set
+# CONFIG_FEATURE_FIND_SIZE is not set
+# CONFIG_FEATURE_FIND_PRUNE is not set
+# CONFIG_FEATURE_FIND_DELETE is not set
+# CONFIG_FEATURE_FIND_PATH is not set
+# CONFIG_FEATURE_FIND_REGEX is not set
+# CONFIG_FEATURE_FIND_CONTEXT is not set
+# CONFIG_FEATURE_FIND_LINKS is not set
+CONFIG_GREP=y
+# CONFIG_FEATURE_GREP_EGREP_ALIAS is not set
+# CONFIG_FEATURE_GREP_FGREP_ALIAS is not set
+# CONFIG_FEATURE_GREP_CONTEXT is not set
+CONFIG_XARGS=y
+CONFIG_FEATURE_XARGS_SUPPORT_CONFIRMATION=y
+CONFIG_FEATURE_XARGS_SUPPORT_QUOTES=y
+CONFIG_FEATURE_XARGS_SUPPORT_TERMOPT=y
+CONFIG_FEATURE_XARGS_SUPPORT_ZERO_TERM=y
+
+#
+# Init Utilities
+#
+# CONFIG_BOOTCHARTD is not set
+# CONFIG_FEATURE_BOOTCHARTD_BLOATED_HEADER is not set
+# CONFIG_FEATURE_BOOTCHARTD_CONFIG_FILE is not set
+# CONFIG_HALT is not set
+# CONFIG_FEATURE_CALL_TELINIT is not set
+# CONFIG_TELINIT_PATH=""
+# CONFIG_INIT is not set
+# CONFIG_FEATURE_USE_INITTAB is not set
+# CONFIG_FEATURE_KILL_REMOVED is not set
+# CONFIG_FEATURE_KILL_DELAY=0
+# CONFIG_FEATURE_INIT_SCTTY is not set
+# CONFIG_FEATURE_INIT_SYSLOG is not set
+# CONFIG_FEATURE_EXTRA_QUIET is not set
+# CONFIG_FEATURE_INIT_COREDUMPS is not set
+# CONFIG_FEATURE_INITRD is not set
+# CONFIG_INIT_TERMINAL_TYPE="linux"
+# CONFIG_MESG is not set
+# CONFIG_FEATURE_MESG_ENABLE_ONLY_GROUP=y
+
+#
+# Login/Password Management Utilities
+#
+# CONFIG_ADD_SHELL is not set
+# CONFIG_REMOVE_SHELL is not set
+# CONFIG_FEATURE_SHADOWPASSWDS is not set
+# CONFIG_USE_BB_PWD_GRP is not set
+# CONFIG_USE_BB_SHADOW is not set
+# CONFIG_USE_BB_CRYPT is not set
+# CONFIG_USE_BB_CRYPT_SHA is not set
+# CONFIG_ADDUSER is not set
+# CONFIG_FEATURE_ADDUSER_LONG_OPTIONS is not set
+# CONFIG_FEATURE_CHECK_NAMES is not set
+CONFIG_FIRST_SYSTEM_ID=0
+CONFIG_LAST_SYSTEM_ID=0
+# CONFIG_ADDGROUP is not set
+# CONFIG_FEATURE_ADDGROUP_LONG_OPTIONS is not set
+# CONFIG_FEATURE_ADDUSER_TO_GROUP is not set
+# CONFIG_DELUSER is not set
+# CONFIG_DELGROUP is not set
+# CONFIG_FEATURE_DEL_USER_FROM_GROUP is not set
+# CONFIG_GETTY is not set
+# CONFIG_LOGIN is not set
+# CONFIG_PAM is not set
+# CONFIG_LOGIN_SCRIPTS is not set
+# CONFIG_FEATURE_NOLOGIN is not set
+# CONFIG_FEATURE_SECURETTY is not set
+# CONFIG_PASSWD is not set
+# CONFIG_FEATURE_PASSWD_WEAK_CHECK is not set
+# CONFIG_CRYPTPW is not set
+# CONFIG_CHPASSWD is not set
+# CONFIG_SU is not set
+# CONFIG_FEATURE_SU_SYSLOG is not set
+# CONFIG_FEATURE_SU_CHECKS_SHELLS is not set
+# CONFIG_SULOGIN is not set
+# CONFIG_VLOCK is not set
+
+#
+# Linux Ext2 FS Progs
+#
+CONFIG_CHATTR=y
+# CONFIG_FSCK is not set
+CONFIG_LSATTR=y
+CONFIG_TUNE2FS=y
+
+#
+# Linux Module Utilities
+#
+# CONFIG_MODINFO is not set
+# CONFIG_MODPROBE_SMALL is not set
+# CONFIG_FEATURE_MODPROBE_SMALL_OPTIONS_ON_CMDLINE=y
+# CONFIG_FEATURE_MODPROBE_SMALL_CHECK_ALREADY_LOADED=y
+# CONFIG_INSMOD is not set
+# CONFIG_RMMOD is not set
+# CONFIG_LSMOD is not set
+# CONFIG_FEATURE_LSMOD_PRETTY_2_6_OUTPUT is not set
+# CONFIG_MODPROBE is not set
+# CONFIG_FEATURE_MODPROBE_BLACKLIST is not set
+# CONFIG_DEPMOD is not set
+
+#
+# Options common to multiple modutils
+#
+# CONFIG_FEATURE_2_4_MODULES is not set
+# CONFIG_FEATURE_INSMOD_TRY_MMAP is not set
+# CONFIG_FEATURE_INSMOD_VERSION_CHECKING is not set
+# CONFIG_FEATURE_INSMOD_KSYMOOPS_SYMBOLS is not set
+# CONFIG_FEATURE_INSMOD_LOADINKMEM is not set
+# CONFIG_FEATURE_INSMOD_LOAD_MAP is not set
+# CONFIG_FEATURE_INSMOD_LOAD_MAP_FULL is not set
+# CONFIG_FEATURE_CHECK_TAINTED_MODULE is not set
+# CONFIG_FEATURE_MODUTILS_ALIAS is not set
+# CONFIG_FEATURE_MODUTILS_SYMBOLS is not set
+# CONFIG_DEFAULT_MODULES_DIR="/lib/modules"
+# CONFIG_DEFAULT_DEPMOD_FILE="modules.dep"
+
+#
+# Linux System Utilities
+#
+CONFIG_BLOCKDEV=y
+CONFIG_REV=y
+# CONFIG_ACPID is not set
+# CONFIG_FEATURE_ACPID_COMPAT is not set
+CONFIG_BLKID=y
+# CONFIG_FEATURE_BLKID_TYPE is not set
+CONFIG_DMESG=y
+CONFIG_FEATURE_DMESG_PRETTY=y
+CONFIG_FBSET=y
+CONFIG_FEATURE_FBSET_FANCY=y
+CONFIG_FEATURE_FBSET_READMODE=y
+CONFIG_FDFLUSH=y
+CONFIG_FDFORMAT=y
+CONFIG_FDISK=y
+CONFIG_FDISK_SUPPORT_LARGE_DISKS=y
+CONFIG_FEATURE_FDISK_WRITABLE=y
+# CONFIG_FEATURE_AIX_LABEL is not set
+# CONFIG_FEATURE_SGI_LABEL is not set
+# CONFIG_FEATURE_SUN_LABEL is not set
+# CONFIG_FEATURE_OSF_LABEL is not set
+# CONFIG_FEATURE_GPT_LABEL is not set
+CONFIG_FEATURE_FDISK_ADVANCED=y
+CONFIG_FINDFS=y
+CONFIG_FLOCK=y
+CONFIG_FREERAMDISK=y
+# CONFIG_FSCK_MINIX is not set
+# CONFIG_MKFS_EXT2 is not set
+# CONFIG_MKFS_MINIX is not set
+# CONFIG_FEATURE_MINIX2 is not set
+# CONFIG_MKFS_REISER is not set
+# CONFIG_MKFS_VFAT is not set
+CONFIG_GETOPT=y
+CONFIG_FEATURE_GETOPT_LONG=y
+CONFIG_HEXDUMP=y
+CONFIG_FEATURE_HEXDUMP_REVERSE=y
+CONFIG_HD=y
+# CONFIG_HWCLOCK is not set
+# CONFIG_FEATURE_HWCLOCK_LONG_OPTIONS is not set
+# CONFIG_FEATURE_HWCLOCK_ADJTIME_FHS is not set
+# CONFIG_IPCRM is not set
+# CONFIG_IPCS is not set
+CONFIG_LOSETUP=y
+CONFIG_LSPCI=y
+CONFIG_LSUSB=y
+# CONFIG_MDEV is not set
+# CONFIG_FEATURE_MDEV_CONF is not set
+# CONFIG_FEATURE_MDEV_RENAME is not set
+# CONFIG_FEATURE_MDEV_RENAME_REGEXP is not set
+# CONFIG_FEATURE_MDEV_EXEC is not set
+# CONFIG_FEATURE_MDEV_LOAD_FIRMWARE is not set
+CONFIG_MKSWAP=y
+CONFIG_FEATURE_MKSWAP_UUID=y
+CONFIG_MORE=y
+# CONFIG_MOUNT is not set
+# CONFIG_FEATURE_MOUNT_FAKE is not set
+# CONFIG_FEATURE_MOUNT_VERBOSE is not set
+# CONFIG_FEATURE_MOUNT_HELPERS is not set
+# CONFIG_FEATURE_MOUNT_LABEL is not set
+# CONFIG_FEATURE_MOUNT_NFS is not set
+# CONFIG_FEATURE_MOUNT_CIFS is not set
+# CONFIG_FEATURE_MOUNT_FLAGS is not set
+# CONFIG_FEATURE_MOUNT_FSTAB is not set
+# CONFIG_PIVOT_ROOT is not set
+# CONFIG_RDATE is not set
+CONFIG_RDEV=y
+CONFIG_READPROFILE=y
+CONFIG_RTCWAKE=y
+CONFIG_SCRIPT=y
+CONFIG_SCRIPTREPLAY=y
+# CONFIG_SETARCH is not set
+# CONFIG_SWAPONOFF is not set
+# CONFIG_FEATURE_SWAPON_PRI is not set
+# CONFIG_SWITCH_ROOT is not set
+# CONFIG_UMOUNT is not set
+# CONFIG_FEATURE_UMOUNT_ALL is not set
+# CONFIG_FEATURE_MOUNT_LOOP is not set
+# CONFIG_FEATURE_MOUNT_LOOP_CREATE is not set
+# CONFIG_FEATURE_MTAB_SUPPORT is not set
+CONFIG_VOLUMEID=y
+
+#
+# Filesystem/Volume identification
+#
+CONFIG_FEATURE_VOLUMEID_EXT=y
+CONFIG_FEATURE_VOLUMEID_BTRFS=y
+CONFIG_FEATURE_VOLUMEID_REISERFS=y
+CONFIG_FEATURE_VOLUMEID_FAT=y
+CONFIG_FEATURE_VOLUMEID_HFS=y
+CONFIG_FEATURE_VOLUMEID_JFS=y
+CONFIG_FEATURE_VOLUMEID_XFS=y
+CONFIG_FEATURE_VOLUMEID_NTFS=y
+CONFIG_FEATURE_VOLUMEID_ISO9660=y
+CONFIG_FEATURE_VOLUMEID_UDF=y
+CONFIG_FEATURE_VOLUMEID_LUKS=y
+CONFIG_FEATURE_VOLUMEID_LINUXSWAP=y
+CONFIG_FEATURE_VOLUMEID_CRAMFS=y
+CONFIG_FEATURE_VOLUMEID_ROMFS=y
+CONFIG_FEATURE_VOLUMEID_SYSV=y
+CONFIG_FEATURE_VOLUMEID_OCFS2=y
+CONFIG_FEATURE_VOLUMEID_LINUXRAID=y
+
+#
+# Miscellaneous Utilities
+#
+# CONFIG_CONSPY is not set
+# CONFIG_NANDWRITE is not set
+CONFIG_NANDDUMP=y
+CONFIG_SETSERIAL=y
+# CONFIG_UBIATTACH is not set
+# CONFIG_UBIDETACH is not set
+# CONFIG_UBIMKVOL is not set
+# CONFIG_UBIRMVOL is not set
+# CONFIG_UBIRSVOL is not set
+# CONFIG_UBIUPDATEVOL is not set
+# CONFIG_ADJTIMEX is not set
+# CONFIG_BBCONFIG is not set
+# CONFIG_FEATURE_COMPRESS_BBCONFIG is not set
+CONFIG_BEEP=y
+CONFIG_FEATURE_BEEP_FREQ=4000
+CONFIG_FEATURE_BEEP_LENGTH_MS=30
+CONFIG_CHAT=y
+CONFIG_FEATURE_CHAT_NOFAIL=y
+# CONFIG_FEATURE_CHAT_TTY_HIFI is not set
+CONFIG_FEATURE_CHAT_IMPLICIT_CR=y
+CONFIG_FEATURE_CHAT_SWALLOW_OPTS=y
+CONFIG_FEATURE_CHAT_SEND_ESCAPES=y
+CONFIG_FEATURE_CHAT_VAR_ABORT_LEN=y
+CONFIG_FEATURE_CHAT_CLR_ABORT=y
+CONFIG_CHRT=y
+# CONFIG_CROND is not set
+# CONFIG_FEATURE_CROND_D is not set
+# CONFIG_FEATURE_CROND_CALL_SENDMAIL is not set
+CONFIG_FEATURE_CROND_DIR=""
+# CONFIG_CRONTAB is not set
+CONFIG_DC=y
+CONFIG_FEATURE_DC_LIBM=y
+# CONFIG_DEVFSD is not set
+# CONFIG_DEVFSD_MODLOAD is not set
+# CONFIG_DEVFSD_FG_NP is not set
+# CONFIG_DEVFSD_VERBOSE is not set
+# CONFIG_FEATURE_DEVFS is not set
+CONFIG_DEVMEM=y
+# CONFIG_EJECT is not set
+# CONFIG_FEATURE_EJECT_SCSI is not set
+CONFIG_FBSPLASH=y
+CONFIG_FLASHCP=y
+CONFIG_FLASH_LOCK=y
+CONFIG_FLASH_UNLOCK=y
+# CONFIG_FLASH_ERASEALL is not set
+# CONFIG_IONICE is not set
+CONFIG_INOTIFYD=y
+# CONFIG_LAST is not set
+# CONFIG_FEATURE_LAST_SMALL is not set
+# CONFIG_FEATURE_LAST_FANCY is not set
+# CONFIG_LESS is not set
+CONFIG_FEATURE_LESS_MAXLINES=0
+# CONFIG_FEATURE_LESS_BRACKETS is not set
+# CONFIG_FEATURE_LESS_FLAGS is not set
+# CONFIG_FEATURE_LESS_MARKS is not set
+# CONFIG_FEATURE_LESS_REGEXP is not set
+# CONFIG_FEATURE_LESS_WINCH is not set
+# CONFIG_FEATURE_LESS_DASHCMD is not set
+# CONFIG_FEATURE_LESS_LINENUMS is not set
+CONFIG_HDPARM=y
+CONFIG_FEATURE_HDPARM_GET_IDENTITY=y
+CONFIG_FEATURE_HDPARM_HDIO_SCAN_HWIF=y
+CONFIG_FEATURE_HDPARM_HDIO_UNREGISTER_HWIF=y
+CONFIG_FEATURE_HDPARM_HDIO_DRIVE_RESET=y
+CONFIG_FEATURE_HDPARM_HDIO_TRISTATE_HWIF=y
+CONFIG_FEATURE_HDPARM_HDIO_GETSET_DMA=y
+CONFIG_MAKEDEVS=y
+# CONFIG_FEATURE_MAKEDEVS_LEAF is not set
+CONFIG_FEATURE_MAKEDEVS_TABLE=y
+CONFIG_MAN=y
+# CONFIG_MICROCOM is not set
+# CONFIG_MOUNTPOINT is not set
+# CONFIG_MT is not set
+CONFIG_RAIDAUTORUN=y
+# CONFIG_READAHEAD is not set
+# CONFIG_RFKILL is not set
+# CONFIG_RUNLEVEL is not set
+CONFIG_RX=y
+CONFIG_SETSID=y
+CONFIG_STRINGS=y
+# CONFIG_TASKSET is not set
+# CONFIG_FEATURE_TASKSET_FANCY is not set
+CONFIG_TIME=y
+CONFIG_TIMEOUT=y
+CONFIG_TTYSIZE=y
+CONFIG_VOLNAME=y
+# CONFIG_WALL is not set
+# CONFIG_WATCHDOG is not set
+
+#
+# Networking Utilities
+#
+# CONFIG_NAMEIF is not set
+# CONFIG_FEATURE_NAMEIF_EXTENDED is not set
+CONFIG_NBDCLIENT=y
+CONFIG_NC=y
+CONFIG_NC_SERVER=y
+CONFIG_NC_EXTRA=y
+# CONFIG_NC_110_COMPAT is not set
+# CONFIG_PING is not set
+# CONFIG_PING6 is not set
+# CONFIG_FEATURE_FANCY_PING is not set
+CONFIG_WHOIS=y
+# CONFIG_FEATURE_IPV6 is not set
+# CONFIG_FEATURE_UNIX_LOCAL is not set
+# CONFIG_FEATURE_PREFER_IPV4_ADDRESS is not set
+# CONFIG_VERBOSE_RESOLUTION_ERRORS is not set
+CONFIG_ARP=y
+# CONFIG_ARPING is not set
+# CONFIG_BRCTL is not set
+# CONFIG_FEATURE_BRCTL_FANCY is not set
+# CONFIG_FEATURE_BRCTL_SHOW is not set
+CONFIG_DNSD=y
+# CONFIG_ETHER_WAKE is not set
+CONFIG_FAKEIDENTD=y
+CONFIG_FTPD=y
+CONFIG_FEATURE_FTP_WRITE=y
+CONFIG_FEATURE_FTPD_ACCEPT_BROKEN_LIST=y
+CONFIG_FTPGET=y
+CONFIG_FTPPUT=y
+# CONFIG_FEATURE_FTPGETPUT_LONG_OPTIONS is not set
+# CONFIG_HOSTNAME is not set
+CONFIG_HTTPD=y
+CONFIG_FEATURE_HTTPD_RANGES=y
+CONFIG_FEATURE_HTTPD_USE_SENDFILE=y
+CONFIG_FEATURE_HTTPD_SETUID=y
+CONFIG_FEATURE_HTTPD_BASIC_AUTH=y
+# CONFIG_FEATURE_HTTPD_AUTH_MD5 is not set
+CONFIG_FEATURE_HTTPD_CGI=y
+CONFIG_FEATURE_HTTPD_CONFIG_WITH_SCRIPT_INTERPR=y
+CONFIG_FEATURE_HTTPD_SET_REMOTE_PORT_TO_ENV=y
+CONFIG_FEATURE_HTTPD_ENCODE_URL_STR=y
+CONFIG_FEATURE_HTTPD_ERROR_PAGES=y
+CONFIG_FEATURE_HTTPD_PROXY=y
+CONFIG_FEATURE_HTTPD_GZIP=y
+CONFIG_IFCONFIG=y
+CONFIG_FEATURE_IFCONFIG_STATUS=y
+# CONFIG_FEATURE_IFCONFIG_SLIP is not set
+CONFIG_FEATURE_IFCONFIG_MEMSTART_IOADDR_IRQ=y
+CONFIG_FEATURE_IFCONFIG_HW=y
+CONFIG_FEATURE_IFCONFIG_BROADCAST_PLUS=y
+# CONFIG_IFENSLAVE is not set
+# CONFIG_IFPLUGD is not set
+CONFIG_IFUPDOWN=y
+CONFIG_IFUPDOWN_IFSTATE_PATH="/var/run/ifstate"
+CONFIG_FEATURE_IFUPDOWN_IP=y
+CONFIG_FEATURE_IFUPDOWN_IP_BUILTIN=y
+# CONFIG_FEATURE_IFUPDOWN_IFCONFIG_BUILTIN is not set
+CONFIG_FEATURE_IFUPDOWN_IPV4=y
+# CONFIG_FEATURE_IFUPDOWN_IPV6 is not set
+CONFIG_FEATURE_IFUPDOWN_MAPPING=y
+# CONFIG_FEATURE_IFUPDOWN_EXTERNAL_DHCP is not set
+# CONFIG_INETD is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_ECHO is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_DISCARD is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_TIME is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_DAYTIME is not set
+# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_CHARGEN is not set
+# CONFIG_FEATURE_INETD_RPC is not set
+CONFIG_IP=y
+CONFIG_FEATURE_IP_ADDRESS=y
+CONFIG_FEATURE_IP_LINK=y
+CONFIG_FEATURE_IP_ROUTE=y
+CONFIG_FEATURE_IP_TUNNEL=y
+CONFIG_FEATURE_IP_RULE=y
+CONFIG_FEATURE_IP_SHORT_FORMS=y
+# CONFIG_FEATURE_IP_RARE_PROTOCOLS is not set
+CONFIG_IPADDR=y
+CONFIG_IPLINK=y
+CONFIG_IPROUTE=y
+CONFIG_IPTUNNEL=y
+CONFIG_IPRULE=y
+CONFIG_IPCALC=y
+CONFIG_FEATURE_IPCALC_FANCY=y
+# CONFIG_FEATURE_IPCALC_LONG_OPTIONS is not set
+CONFIG_NETSTAT=y
+CONFIG_FEATURE_NETSTAT_WIDE=y
+CONFIG_FEATURE_NETSTAT_PRG=y
+# CONFIG_NSLOOKUP is not set
+# CONFIG_NTPD is not set
+# CONFIG_FEATURE_NTPD_SERVER is not set
+CONFIG_PSCAN=y
+CONFIG_ROUTE=y
+# CONFIG_SLATTACH is not set
+CONFIG_TCPSVD=y
+# CONFIG_TELNET is not set
+# CONFIG_FEATURE_TELNET_TTYPE is not set
+# CONFIG_FEATURE_TELNET_AUTOLOGIN is not set
+# CONFIG_TELNETD is not set
+# CONFIG_FEATURE_TELNETD_STANDALONE is not set
+# CONFIG_FEATURE_TELNETD_INETD_WAIT is not set
+# CONFIG_TFTP is not set
+# CONFIG_TFTPD is not set
+# CONFIG_FEATURE_TFTP_GET is not set
+# CONFIG_FEATURE_TFTP_PUT is not set
+# CONFIG_FEATURE_TFTP_BLOCKSIZE is not set
+# CONFIG_FEATURE_TFTP_PROGRESS_BAR is not set
+# CONFIG_TFTP_DEBUG is not set
+# CONFIG_TRACEROUTE is not set
+# CONFIG_TRACEROUTE6 is not set
+# CONFIG_FEATURE_TRACEROUTE_VERBOSE is not set
+# CONFIG_FEATURE_TRACEROUTE_SOURCE_ROUTE is not set
+# CONFIG_FEATURE_TRACEROUTE_USE_ICMP is not set
+CONFIG_TUNCTL=y
+CONFIG_FEATURE_TUNCTL_UG=y
+# CONFIG_UDHCPD is not set
+# CONFIG_DHCPRELAY is not set
+# CONFIG_DUMPLEASES is not set
+# CONFIG_FEATURE_UDHCPD_WRITE_LEASES_EARLY is not set
+# CONFIG_FEATURE_UDHCPD_BASE_IP_ON_MAC is not set
+CONFIG_DHCPD_LEASES_FILE=""
+CONFIG_UDHCPC=y
+CONFIG_FEATURE_UDHCPC_ARPING=y
+# CONFIG_FEATURE_UDHCP_PORT is not set
+CONFIG_UDHCP_DEBUG=9
+CONFIG_FEATURE_UDHCP_RFC3397=y
+CONFIG_FEATURE_UDHCP_8021Q=y
+CONFIG_UDHCPC_DEFAULT_SCRIPT="/usr/share/udhcpc/default.script"
+CONFIG_UDHCPC_SLACK_FOR_BUGGY_SERVERS=80
+CONFIG_IFUPDOWN_UDHCPC_CMD_OPTIONS="-R -n"
+# CONFIG_UDPSVD is not set
+# CONFIG_VCONFIG is not set
+CONFIG_WGET=y
+CONFIG_FEATURE_WGET_STATUSBAR=y
+CONFIG_FEATURE_WGET_AUTHENTICATION=y
+# CONFIG_FEATURE_WGET_LONG_OPTIONS is not set
+CONFIG_FEATURE_WGET_TIMEOUT=y
+# CONFIG_ZCIP is not set
+
+#
+# Print Utilities
+#
+CONFIG_LPD=y
+CONFIG_LPR=y
+CONFIG_LPQ=y
+
+#
+# Mail Utilities
+#
+CONFIG_MAKEMIME=y
+CONFIG_FEATURE_MIME_CHARSET="us-ascii"
+CONFIG_POPMAILDIR=y
+CONFIG_FEATURE_POPMAILDIR_DELIVERY=y
+CONFIG_REFORMIME=y
+CONFIG_FEATURE_REFORMIME_COMPAT=y
+CONFIG_SENDMAIL=y
+
+#
+# Process Utilities
+#
+CONFIG_IOSTAT=y
+CONFIG_MPSTAT=y
+CONFIG_NMETER=y
+CONFIG_PMAP=y
+# CONFIG_POWERTOP is not set
+CONFIG_PSTREE=y
+CONFIG_PWDX=y
+CONFIG_SMEMCAP=y
+# CONFIG_FREE is not set
+CONFIG_FUSER=y
+# CONFIG_KILL is not set
+# CONFIG_KILLALL is not set
+# CONFIG_KILLALL5 is not set
+# CONFIG_PGREP is not set
+CONFIG_PIDOF=y
+CONFIG_FEATURE_PIDOF_SINGLE=y
+CONFIG_FEATURE_PIDOF_OMIT=y
+# CONFIG_PKILL is not set
+# CONFIG_PS is not set
+# CONFIG_FEATURE_PS_WIDE is not set
+# CONFIG_FEATURE_PS_TIME is not set
+# CONFIG_FEATURE_PS_ADDITIONAL_COLUMNS is not set
+# CONFIG_FEATURE_PS_UNUSUAL_SYSTEMS is not set
+CONFIG_RENICE=y
+CONFIG_BB_SYSCTL=y
+CONFIG_TOP=y
+CONFIG_FEATURE_TOP_CPU_USAGE_PERCENTAGE=y
+CONFIG_FEATURE_TOP_CPU_GLOBAL_PERCENTS=y
+CONFIG_FEATURE_TOP_SMP_CPU=y
+CONFIG_FEATURE_TOP_DECIMALS=y
+CONFIG_FEATURE_TOP_SMP_PROCESS=y
+CONFIG_FEATURE_TOPMEM=y
+CONFIG_FEATURE_SHOW_THREADS=y
+# CONFIG_UPTIME is not set
+CONFIG_WATCH=y
+
+#
+# Runit Utilities
+#
+CONFIG_RUNSV=y
+CONFIG_RUNSVDIR=y
+# CONFIG_FEATURE_RUNSVDIR_LOG is not set
+CONFIG_SV=y
+CONFIG_SV_DEFAULT_SERVICE_DIR="/var/service"
+CONFIG_SVLOGD=y
+CONFIG_CHPST=y
+CONFIG_SETUIDGID=y
+CONFIG_ENVUIDGID=y
+CONFIG_ENVDIR=y
+CONFIG_SOFTLIMIT=y
+# CONFIG_CHCON is not set
+# CONFIG_FEATURE_CHCON_LONG_OPTIONS is not set
+# CONFIG_GETENFORCE is not set
+# CONFIG_GETSEBOOL is not set
+# CONFIG_LOAD_POLICY is not set
+# CONFIG_MATCHPATHCON is not set
+# CONFIG_RESTORECON is not set
+# CONFIG_RUNCON is not set
+# CONFIG_FEATURE_RUNCON_LONG_OPTIONS is not set
+# CONFIG_SELINUXENABLED is not set
+# CONFIG_SETENFORCE is not set
+# CONFIG_SETFILES is not set
+# CONFIG_FEATURE_SETFILES_CHECK_OPTION is not set
+# CONFIG_SETSEBOOL is not set
+# CONFIG_SESTATUS is not set
+
+#
+# Shells
+#
+CONFIG_ASH=y
+# CONFIG_ASH_BASH_COMPAT is not set
+# CONFIG_ASH_IDLE_TIMEOUT is not set
+CONFIG_ASH_JOB_CONTROL=y
+# CONFIG_ASH_ALIAS is not set
+CONFIG_ASH_GETOPTS=y
+# CONFIG_ASH_BUILTIN_ECHO is not set
+# CONFIG_ASH_BUILTIN_PRINTF is not set
+# CONFIG_ASH_BUILTIN_TEST is not set
+# CONFIG_ASH_CMDCMD is not set
+# CONFIG_ASH_MAIL is not set
+# CONFIG_ASH_OPTIMIZE_FOR_SIZE is not set
+# CONFIG_ASH_RANDOM_SUPPORT is not set
+# CONFIG_ASH_EXPAND_PRMT is not set
+CONFIG_CTTYHACK=y
+# CONFIG_HUSH is not set
+# CONFIG_HUSH_BASH_COMPAT is not set
+# CONFIG_HUSH_BRACE_EXPANSION is not set
+# CONFIG_HUSH_HELP is not set
+# CONFIG_HUSH_INTERACTIVE is not set
+# CONFIG_HUSH_SAVEHISTORY is not set
+# CONFIG_HUSH_JOB is not set
+# CONFIG_HUSH_TICK is not set
+# CONFIG_HUSH_IF is not set
+# CONFIG_HUSH_LOOPS is not set
+# CONFIG_HUSH_CASE is not set
+# CONFIG_HUSH_FUNCTIONS is not set
+# CONFIG_HUSH_LOCAL is not set
+# CONFIG_HUSH_RANDOM_SUPPORT is not set
+# CONFIG_HUSH_EXPORT_N is not set
+# CONFIG_HUSH_MODE_X is not set
+# CONFIG_MSH is not set
+CONFIG_FEATURE_SH_IS_ASH=y
+# CONFIG_FEATURE_SH_IS_HUSH is not set
+# CONFIG_FEATURE_SH_IS_NONE is not set
+# CONFIG_FEATURE_BASH_IS_ASH is not set
+# CONFIG_FEATURE_BASH_IS_HUSH is not set
+CONFIG_FEATURE_BASH_IS_NONE=y
+# CONFIG_SH_MATH_SUPPORT is not set
+# CONFIG_SH_MATH_SUPPORT_64 is not set
+# CONFIG_FEATURE_SH_EXTRA_QUIET is not set
+CONFIG_FEATURE_SH_STANDALONE=y
+# CONFIG_FEATURE_SH_NOFORK is not set
+# CONFIG_FEATURE_SH_HISTFILESIZE is not set
+
+#
+# System Logging Utilities
+#
+# CONFIG_SYSLOGD is not set
+# CONFIG_FEATURE_ROTATE_LOGFILE is not set
+# CONFIG_FEATURE_REMOTE_LOG is not set
+# CONFIG_FEATURE_SYSLOGD_DUP is not set
+# CONFIG_FEATURE_SYSLOGD_CFG is not set
+CONFIG_FEATURE_SYSLOGD_READ_BUFFER_SIZE=0
+# CONFIG_FEATURE_IPC_SYSLOG is not set
+CONFIG_FEATURE_IPC_SYSLOG_BUFFER_SIZE=0
+# CONFIG_LOGREAD is not set
+# CONFIG_FEATURE_LOGREAD_REDUCED_LOCKING is not set
+CONFIG_KLOGD=y
+CONFIG_FEATURE_KLOGD_KLOGCTL=y
+# CONFIG_LOGGER is not set
diff --git a/standalone/android/dropbear.patch b/standalone/android/dropbear.patch
new file mode 100644
index 0000000000..84c7dfb6d6
--- /dev/null
+++ b/standalone/android/dropbear.patch
@@ -0,0 +1,55 @@
+From 014dadb02fd984828a6232534c47dba8e2f7818a Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Wed, 13 Feb 2013 15:29:52 -0400
+Subject: [PATCH] android patch for dropbear
+
+* Disable HOME override
+* Use urandom to avoid blocking on every ssh connection.
+* Enable use of netbsd_getpass.c
+---
+ cli-auth.c |    1 +
+ cli-main.c |    2 --
+ options.h  |    2 +-
+ 3 files changed, 2 insertions(+), 3 deletions(-)
+
+diff --git a/cli-auth.c b/cli-auth.c
+index 4c17a21..91dfdf8 100644
+--- a/cli-auth.c
++++ b/cli-auth.c
+@@ -31,6 +31,7 @@
+ #include "ssh.h"
+ #include "packet.h"
+ #include "runopts.h"
++#include "netbsd_getpass.c"
+ 
+ void cli_authinitialise() {
+ 
+diff --git a/cli-main.c b/cli-main.c
+index 106006b..68cf023 100644
+--- a/cli-main.c
++++ b/cli-main.c
+@@ -47,8 +47,6 @@ int main(int argc, char ** argv) {
+ 	_dropbear_exit = cli_dropbear_exit;
+ 	_dropbear_log = cli_dropbear_log;
+ 
+-	putenv("HOME=/data/local");
+-
+ 	disallow_core();
+ 
+ 	cli_getopts(argc, argv);
+diff --git a/options.h b/options.h
+index 7625151..48e404d 100644
+--- a/options.h
++++ b/options.h
+@@ -159,7 +159,7 @@ etc) slower (perhaps by 50%). Recommended for most small systems. */
+  * however significantly reduce the security of your ssh connections
+  * if the PRNG state becomes guessable - make sure you know what you are
+  * doing if you change this. */
+-#define DROPBEAR_RANDOM_DEV "/dev/random"
++#define DROPBEAR_RANDOM_DEV "/dev/urandom"
+ 
+ /* prngd must be manually set up to produce output */
+ /*#define DROPBEAR_PRNGD_SOCKET "/var/run/dropbear-rng"*/
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/MissingH-1.2.0.0_0001-fix-build-not-Android-specific.patch b/standalone/android/haskell-patches/MissingH-1.2.0.0_0001-fix-build-not-Android-specific.patch
new file mode 100644
index 0000000000..50f641da7d
--- /dev/null
+++ b/standalone/android/haskell-patches/MissingH-1.2.0.0_0001-fix-build-not-Android-specific.patch
@@ -0,0 +1,34 @@
+From 8c4220e4dd48ad197aa0ad49214e6e7bd768044e Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:28:57 -0400
+Subject: [PATCH] fix build (not Android specific)
+
+---
+ src/System/Cmd/Utils.hs |    4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/System/Cmd/Utils.hs b/src/System/Cmd/Utils.hs
+index a9fa46f..6c6aba2 100644
+--- a/src/System/Cmd/Utils.hs
++++ b/src/System/Cmd/Utils.hs
+@@ -325,7 +325,7 @@ forceSuccess (PipeHandle pid fp args funcname) =
+                 Just (Exited (ExitSuccess)) -> return ()
+                 Just (Exited (ExitFailure fc)) ->
+                     cmdfailed funcname fp args fc
+-                Just (Terminated sig) ->
++                Just (Terminated sig _) ->
+                     warnfail fp args $ "Terminated by signal " ++ show sig
+                 Just (Stopped sig) ->
+                     warnfail fp args $ "Stopped by signal " ++ show sig
+@@ -351,7 +351,7 @@ safeSystem command args =
+        case ec of
+             Exited ExitSuccess -> return ()
+             Exited (ExitFailure fc) -> cmdfailed "safeSystem" command args fc
+-            Terminated s -> cmdsignalled "safeSystem" command args s
++            Terminated s _ -> cmdsignalled "safeSystem" command args s
+             Stopped s -> cmdsignalled "safeSystem" command args s
+ #endif
+ 
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/aeson-0.6.1.0_0001-disable-TH.patch b/standalone/android/haskell-patches/aeson-0.6.1.0_0001-disable-TH.patch
new file mode 100644
index 0000000000..787caf45cc
--- /dev/null
+++ b/standalone/android/haskell-patches/aeson-0.6.1.0_0001-disable-TH.patch
@@ -0,0 +1,24 @@
+From b220c377941d0b1271cf525a8d06bb8e48196d2b Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:29:04 -0400
+Subject: [PATCH] disable TH
+
+---
+ aeson.cabal |    1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/aeson.cabal b/aeson.cabal
+index 242aa67..275aa49 100644
+--- a/aeson.cabal
++++ b/aeson.cabal
+@@ -99,7 +99,6 @@ library
+     Data.Aeson.Generic
+     Data.Aeson.Parser
+     Data.Aeson.Types
+-    Data.Aeson.TH
+ 
+   other-modules:
+     Data.Aeson.Functions
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/async-2.0.1.4_0001-allow-building-with-unreleased-ghc.patch b/standalone/android/haskell-patches/async-2.0.1.4_0001-allow-building-with-unreleased-ghc.patch
new file mode 100644
index 0000000000..e959941b8c
--- /dev/null
+++ b/standalone/android/haskell-patches/async-2.0.1.4_0001-allow-building-with-unreleased-ghc.patch
@@ -0,0 +1,25 @@
+From 55f424de9946c4d1d89837bb18698437aecfcfa4 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:29:16 -0400
+Subject: [PATCH] allow building with unreleased ghc
+
+---
+ async.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/async.cabal b/async.cabal
+index 8e47d9d..ff317c7 100644
+--- a/async.cabal
++++ b/async.cabal
+@@ -70,7 +70,7 @@ source-repository head
+ 
+ library
+     exposed-modules:     Control.Concurrent.Async
+-    build-depends:       base >= 4.3 && < 4.7, stm >= 2.2 && < 2.5
++    build-depends:       base >= 4.3 && < 4.8, stm >= 2.2 && < 2.5
+ 
+ test-suite test-async
+     type:       exitcode-stdio-1.0
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/case-insensitive-0.4.0.1_0001-allow-building-with-unreleased-ghc.patch b/standalone/android/haskell-patches/case-insensitive-0.4.0.1_0001-allow-building-with-unreleased-ghc.patch
new file mode 100644
index 0000000000..2d7c45089a
--- /dev/null
+++ b/standalone/android/haskell-patches/case-insensitive-0.4.0.1_0001-allow-building-with-unreleased-ghc.patch
@@ -0,0 +1,27 @@
+From efd0e93de82c0b5554a4f3a4517e6127f405f6da Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:29:36 -0400
+Subject: [PATCH] allow building with unreleased ghc
+
+---
+ case-insensitive.cabal |    4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/case-insensitive.cabal b/case-insensitive.cabal
+index a73479d..18a1a51 100644
+--- a/case-insensitive.cabal
++++ b/case-insensitive.cabal
+@@ -25,8 +25,8 @@ source-repository head
+ 
+ Library
+   GHC-Options: -Wall
+-  build-depends: base       >= 3   && < 4.6
+-               , bytestring >= 0.9 && < 0.10
++  build-depends: base       >= 3   && < 4.8
++               , bytestring >= 0.9 && < 0.15
+                , text       >= 0.3 && < 0.12
+                , hashable   >= 1.0 && < 1.2
+   exposed-modules: Data.CaseInsensitive
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/hamlet-1.1.6.1_0001-axe-murdered.patch b/standalone/android/haskell-patches/hamlet-1.1.6.1_0001-axe-murdered.patch
new file mode 100644
index 0000000000..c1188ee142
--- /dev/null
+++ b/standalone/android/haskell-patches/hamlet-1.1.6.1_0001-axe-murdered.patch
@@ -0,0 +1,276 @@
+From 7be8bf3ba75acc5209066e6ba31ae589c541f344 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:30:01 -0400
+Subject: [PATCH] axe murdered
+
+---
+ Text/Hamlet.hs |  215 +-------------------------------------------------------
+ 1 file changed, 2 insertions(+), 213 deletions(-)
+
+diff --git a/Text/Hamlet.hs b/Text/Hamlet.hs
+index 4ac870a..bc8edd5 100644
+--- a/Text/Hamlet.hs
++++ b/Text/Hamlet.hs
+@@ -11,35 +11,22 @@
+ module Text.Hamlet
+     ( -- * Plain HTML
+       Html
+-    , shamlet
+-    , shamletFile
+-    , xshamlet
+-    , xshamletFile
+       -- * Hamlet
+     , HtmlUrl
+-    , hamlet
+-    , hamletFile
+-    , xhamlet
+-    , xhamletFile
+       -- * I18N Hamlet
+     , HtmlUrlI18n
+-    , ihamlet
+-    , ihamletFile
+       -- * Type classes
+     , ToAttributes (..)
+       -- * Internal, for making more
+     , HamletSettings (..)
+     , NewlineStyle (..)
+-    , hamletWithSettings
+-    , hamletFileWithSettings
+     , defaultHamletSettings
+     , xhtmlHamletSettings
+     , Env (..)
+     , HamletRules (..)
+-    , hamletRules
+-    , ihamletRules
+-    , htmlRules
+     , CloseStyle (..)
++    , condH
++    , maybeH
+     ) where
+ 
+ import Text.Shakespeare.Base
+@@ -90,14 +77,6 @@ type HtmlUrl url = Render url -> Html
+ -- | A function generating an 'Html' given a message translator and a URL rendering function.
+ type HtmlUrlI18n msg url = Translate msg -> Render url -> Html
+ 
+-docsToExp :: Env -> HamletRules -> Scope -> [Doc] -> Q Exp
+-docsToExp env hr scope docs = do
+-    exps <- mapM (docToExp env hr scope) docs
+-    case exps of
+-        [] -> [|return ()|]
+-        [x] -> return x
+-        _ -> return $ DoE $ map NoBindS exps
+-
+ unIdent :: Ident -> String
+ unIdent (Ident s) = s
+ 
+@@ -159,169 +138,9 @@ recordToFieldNames conStr = do
+   [fields] <- return [fields | RecC name fields <- cons, name == conName]
+   return [fieldName | (fieldName, _, _) <- fields]
+ 
+-docToExp :: Env -> HamletRules -> Scope -> Doc -> Q Exp
+-docToExp env hr scope (DocForall list idents inside) = do
+-    let list' = derefToExp scope list
+-    (pat, extraScope) <- bindingPattern idents
+-    let scope' = extraScope ++ scope
+-    mh <- [|F.mapM_|]
+-    inside' <- docsToExp env hr scope' inside
+-    let lam = LamE [pat] inside'
+-    return $ mh `AppE` lam `AppE` list'
+-docToExp env hr scope (DocWith [] inside) = do
+-    inside' <- docsToExp env hr scope inside
+-    return $ inside'
+-docToExp env hr scope (DocWith ((deref, idents):dis) inside) = do
+-    let deref' = derefToExp scope deref
+-    (pat, extraScope) <- bindingPattern idents
+-    let scope' = extraScope ++ scope
+-    inside' <- docToExp env hr scope' (DocWith dis inside)
+-    let lam = LamE [pat] inside'
+-    return $ lam `AppE` deref'
+-docToExp env hr scope (DocMaybe val idents inside mno) = do
+-    let val' = derefToExp scope val
+-    (pat, extraScope) <- bindingPattern idents
+-    let scope' = extraScope ++ scope
+-    inside' <- docsToExp env hr scope' inside
+-    let inside'' = LamE [pat] inside'
+-    ninside' <- case mno of
+-                    Nothing -> [|Nothing|]
+-                    Just no -> do
+-                        no' <- docsToExp env hr scope no
+-                        j <- [|Just|]
+-                        return $ j `AppE` no'
+-    mh <- [|maybeH|]
+-    return $ mh `AppE` val' `AppE` inside'' `AppE` ninside'
+-docToExp env hr scope (DocCond conds final) = do
+-    conds' <- mapM go conds
+-    final' <- case final of
+-                Nothing -> [|Nothing|]
+-                Just f -> do
+-                    f' <- docsToExp env hr scope f
+-                    j <- [|Just|]
+-                    return $ j `AppE` f'
+-    ch <- [|condH|]
+-    return $ ch `AppE` ListE conds' `AppE` final'
+-  where
+-    go :: (Deref, [Doc]) -> Q Exp
+-    go (d, docs) = do
+-        let d' = derefToExp scope d
+-        docs' <- docsToExp env hr scope docs
+-        return $ TupE [d', docs']
+-docToExp env hr scope (DocCase deref cases) = do
+-    let exp_ = derefToExp scope deref
+-    matches <- mapM toMatch cases
+-    return $ CaseE exp_ matches
+-  where
+-    readMay s =
+-        case reads s of
+-            (x, ""):_ -> Just x
+-            _ -> Nothing
+-    toMatch (idents, inside) = do
+-        let pat = case map unIdent idents of
+-                    ["_"] -> WildP
+-                    [str]
+-                        | Just i <- readMay str -> LitP $ IntegerL i
+-                    strs -> let (constr:fields) = map mkName strs
+-                            in ConP constr (map VarP fields)
+-        insideExp <- docsToExp env hr scope inside
+-        return $ Match pat (NormalB insideExp) []
+-docToExp env hr v (DocContent c) = contentToExp env hr v c
+-
+-contentToExp :: Env -> HamletRules -> Scope -> Content -> Q Exp
+-contentToExp _ hr _ (ContentRaw s) = do
+-    os <- [|preEscapedText . pack|]
+-    let s' = LitE $ StringL s
+-    return $ hrFromHtml hr `AppE` (os `AppE` s')
+-contentToExp _ hr scope (ContentVar d) = do
+-    str <- [|toHtml|]
+-    return $ hrFromHtml hr `AppE` (str `AppE` derefToExp scope d)
+-contentToExp env hr scope (ContentUrl hasParams d) =
+-    case urlRender env of
+-        Nothing -> error "URL interpolation used, but no URL renderer provided"
+-        Just wrender -> wrender $ \render -> do
+-            let render' = return render
+-            ou <- if hasParams
+-                    then [|\(u, p) -> $(render') u p|]
+-                    else [|\u -> $(render') u []|]
+-            let d' = derefToExp scope d
+-            pet <- [|toHtml|]
+-            return $ hrFromHtml hr `AppE` (pet `AppE` (ou `AppE` d'))
+-contentToExp env hr scope (ContentEmbed d) = hrEmbed hr env $ derefToExp scope d
+-contentToExp env hr scope (ContentMsg d) =
+-    case msgRender env of
+-        Nothing -> error "Message interpolation used, but no message renderer provided"
+-        Just wrender -> wrender $ \render ->
+-            return $ hrFromHtml hr `AppE` (render `AppE` derefToExp scope d)
+-contentToExp _ hr scope (ContentAttrs d) = do
+-    html <- [|attrsToHtml . toAttributes|]
+-    return $ hrFromHtml hr `AppE` (html `AppE` derefToExp scope d)
+-
+-shamlet :: QuasiQuoter
+-shamlet = hamletWithSettings htmlRules defaultHamletSettings
+-
+-xshamlet :: QuasiQuoter
+-xshamlet = hamletWithSettings htmlRules xhtmlHamletSettings
+-
+-htmlRules :: Q HamletRules
+-htmlRules = do
+-    i <- [|id|]
+-    return $ HamletRules i ($ (Env Nothing Nothing)) (\_ b -> return b)
+-
+-hamlet :: QuasiQuoter
+-hamlet = hamletWithSettings hamletRules defaultHamletSettings
+-
+-xhamlet :: QuasiQuoter
+-xhamlet = hamletWithSettings hamletRules xhtmlHamletSettings
+-
+ asHtmlUrl :: HtmlUrl url -> HtmlUrl url
+ asHtmlUrl = id
+ 
+-hamletRules :: Q HamletRules
+-hamletRules = do
+-    i <- [|id|]
+-    let ur f = do
+-            r <- newName "_render"
+-            let env = Env
+-                    { urlRender = Just ($ (VarE r))
+-                    , msgRender = Nothing
+-                    }
+-            h <- f env
+-            return $ LamE [VarP r] h
+-    return $ HamletRules i ur em
+-  where
+-    em (Env (Just urender) Nothing) e = do
+-        asHtmlUrl' <- [|asHtmlUrl|]
+-        urender $ \ur' -> return ((asHtmlUrl' `AppE` e) `AppE` ur')
+-    em _ _ = error "bad Env"
+-
+-ihamlet :: QuasiQuoter
+-ihamlet = hamletWithSettings ihamletRules defaultHamletSettings
+-
+-ihamletRules :: Q HamletRules
+-ihamletRules = do
+-    i <- [|id|]
+-    let ur f = do
+-            u <- newName "_urender"
+-            m <- newName "_mrender"
+-            let env = Env
+-                    { urlRender = Just ($ (VarE u))
+-                    , msgRender = Just ($ (VarE m))
+-                    }
+-            h <- f env
+-            return $ LamE [VarP m, VarP u] h
+-    return $ HamletRules i ur em
+-  where
+-    em (Env (Just urender) (Just mrender)) e =
+-          urender $ \ur' -> mrender $ \mr -> return (e `AppE` mr `AppE` ur')
+-    em _ _ = error "bad Env"
+-
+-hamletWithSettings :: Q HamletRules -> HamletSettings -> QuasiQuoter
+-hamletWithSettings hr set =
+-    QuasiQuoter
+-        { quoteExp = hamletFromString hr set
+-        }
+-
+ data HamletRules = HamletRules
+     { hrFromHtml :: Exp
+     , hrWithEnv :: (Env -> Q Exp) -> Q Exp
+@@ -333,36 +152,6 @@ data Env = Env
+     , msgRender :: Maybe ((Exp -> Q Exp) -> Q Exp)
+     }
+ 
+-hamletFromString :: Q HamletRules -> HamletSettings -> String -> Q Exp
+-hamletFromString qhr set s = do
+-    hr <- qhr
+-    case parseDoc set s of
+-        Error s' -> error s'
+-        Ok (_mnl, d) -> hrWithEnv hr $ \env -> docsToExp env hr [] d
+-
+-hamletFileWithSettings :: Q HamletRules -> HamletSettings -> FilePath -> Q Exp
+-hamletFileWithSettings qhr set fp = do
+-#ifdef GHC_7_4
+-    qAddDependentFile fp
+-#endif
+-    contents <- fmap TL.unpack $ qRunIO $ readUtf8File fp
+-    hamletFromString qhr set contents
+-
+-hamletFile :: FilePath -> Q Exp
+-hamletFile = hamletFileWithSettings hamletRules defaultHamletSettings
+-
+-xhamletFile :: FilePath -> Q Exp
+-xhamletFile = hamletFileWithSettings hamletRules xhtmlHamletSettings
+-
+-shamletFile :: FilePath -> Q Exp
+-shamletFile = hamletFileWithSettings htmlRules defaultHamletSettings
+-
+-xshamletFile :: FilePath -> Q Exp
+-xshamletFile = hamletFileWithSettings htmlRules xhtmlHamletSettings
+-
+-ihamletFile :: FilePath -> Q Exp
+-ihamletFile = hamletFileWithSettings ihamletRules defaultHamletSettings
+-
+ varName :: Scope -> String -> Exp
+ varName _ "" = error "Illegal empty varName"
+ varName scope v@(_:_) = fromMaybe (strToExp v) $ lookup (Ident v) scope
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/lifted-base-0.2.0.2_0001-hacked-for-newer-ghc.patch b/standalone/android/haskell-patches/lifted-base-0.2.0.2_0001-hacked-for-newer-ghc.patch
new file mode 100644
index 0000000000..b61dc17ba9
--- /dev/null
+++ b/standalone/android/haskell-patches/lifted-base-0.2.0.2_0001-hacked-for-newer-ghc.patch
@@ -0,0 +1,163 @@
+From 4bb0de1e6213ec925820c8b9cc3ff5f3c3c72d7a Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:31:27 -0400
+Subject: [PATCH] hacked for newer ghc
+
+---
+ Control/Concurrent/Lifted.hs |    2 +-
+ Control/Exception/Lifted.hs  |   11 ++--------
+ Setup.hs                     |   46 ++----------------------------------------
+ lifted-base.cabal            |    9 ++++-----
+ 4 files changed, 9 insertions(+), 59 deletions(-)
+
+diff --git a/Control/Concurrent/Lifted.hs b/Control/Concurrent/Lifted.hs
+index 4bc58a8..e4445e6 100644
+--- a/Control/Concurrent/Lifted.hs
++++ b/Control/Concurrent/Lifted.hs
+@@ -124,7 +124,7 @@ import Control.Concurrent.SampleVar.Lifted
+ #endif
+ import Control.Exception.Lifted ( throwTo
+ #if MIN_VERSION_base(4,6,0)
+-                                , SomeException, try, mask
++                                , SomeException, try
+ #endif
+                                 )
+ #include "inlinable.h"
+diff --git a/Control/Exception/Lifted.hs b/Control/Exception/Lifted.hs
+index 871cda7..0b9d8b7 100644
+--- a/Control/Exception/Lifted.hs
++++ b/Control/Exception/Lifted.hs
+@@ -50,8 +50,8 @@ module Control.Exception.Lifted
+       -- |The following functions allow a thread to control delivery of
+       -- asynchronous exceptions during a critical region.
+ #if MIN_VERSION_base(4,3,0)
+-    , mask, mask_
+-    , uninterruptibleMask, uninterruptibleMask_
++    , mask_
++    , uninterruptibleMask_
+     , getMaskingState
+ #if MIN_VERSION_base(4,4,0)
+     , allowInterrupt
+@@ -266,10 +266,6 @@ evaluate = liftBase ∘ E.evaluate
+ --------------------------------------------------------------------------------
+ 
+ #if MIN_VERSION_base(4,3,0)
+--- |Generalized version of 'E.mask'.
+-mask ∷ MonadBaseControl IO m ⇒ ((∀ a. m a → m a) → m b) → m b
+-mask = liftBaseOp E.mask ∘ liftRestore
+-{-# INLINABLE mask #-}
+ 
+ liftRestore ∷ MonadBaseControl IO m
+             ⇒ ((∀ a.  m a →  m a) → b)
+@@ -283,9 +279,6 @@ mask_ = liftBaseOp_ E.mask_
+ {-# INLINABLE mask_ #-}
+ 
+ -- |Generalized version of 'E.uninterruptibleMask'.
+-uninterruptibleMask ∷ MonadBaseControl IO m ⇒ ((∀ a. m a → m a) → m b) → m b
+-uninterruptibleMask = liftBaseOp E.uninterruptibleMask ∘ liftRestore
+-{-# INLINABLE uninterruptibleMask #-}
+ 
+ -- |Generalized version of 'E.uninterruptibleMask_'.
+ uninterruptibleMask_ ∷ MonadBaseControl IO m ⇒ m a → m a
+diff --git a/Setup.hs b/Setup.hs
+index 33956e1..9a994af 100644
+--- a/Setup.hs
++++ b/Setup.hs
+@@ -1,44 +1,2 @@
+-#! /usr/bin/env runhaskell
+-
+-{-# LANGUAGE NoImplicitPrelude, UnicodeSyntax #-}
+-
+-module Main (main) where
+-
+-
+--------------------------------------------------------------------------------
+--- Imports
+--------------------------------------------------------------------------------
+-
+--- from base
+-import System.IO ( IO )
+-
+--- from cabal
+-import Distribution.Simple ( defaultMainWithHooks
+-                           , simpleUserHooks
+-                           , UserHooks(haddockHook)
+-                           )
+-
+-import Distribution.Simple.LocalBuildInfo ( LocalBuildInfo(..) )
+-import Distribution.Simple.Program        ( userSpecifyArgs )
+-import Distribution.Simple.Setup          ( HaddockFlags )
+-import Distribution.PackageDescription    ( PackageDescription(..) )
+-
+-
+--------------------------------------------------------------------------------
+--- Cabal setup program which sets the CPP define '__HADDOCK __' when haddock is run.
+--------------------------------------------------------------------------------
+-
+-main ∷ IO ()
+-main = defaultMainWithHooks hooks
+-  where
+-    hooks = simpleUserHooks { haddockHook = haddockHook' }
+-
+--- Define __HADDOCK__ for CPP when running haddock.
+-haddockHook' ∷ PackageDescription → LocalBuildInfo → UserHooks → HaddockFlags → IO ()
+-haddockHook' pkg lbi =
+-  haddockHook simpleUserHooks pkg (lbi { withPrograms = p })
+-  where
+-    p = userSpecifyArgs "haddock" ["--optghc=-D__HADDOCK__"] (withPrograms lbi)
+-
+-
+--- The End ---------------------------------------------------------------------
++import Distribution.Simple
++main = defaultMain
+diff --git a/lifted-base.cabal b/lifted-base.cabal
+index 54ef418..8da5086 100644
+--- a/lifted-base.cabal
++++ b/lifted-base.cabal
+@@ -9,7 +9,7 @@ Copyright:           (c) 2011-2012 Bas van Dijk, Anders Kaseorg
+ Homepage:            https://github.com/basvandijk/lifted-base
+ Bug-reports:         https://github.com/basvandijk/lifted-base/issues
+ Category:            Control
+-Build-type:          Custom
++Build-type:          Simple
+ Cabal-version:       >= 1.9.2
+ Description:         @lifted-base@ exports IO operations from the base library lifted to
+                      any instance of 'MonadBase' or 'MonadBaseControl'.
+@@ -37,7 +37,6 @@ Library
+   Exposed-modules: Control.Exception.Lifted
+                    Control.Concurrent.MVar.Lifted
+                    Control.Concurrent.Chan.Lifted
+-                   Control.Concurrent.Lifted
+                    Data.IORef.Lifted
+                    System.Timeout.Lifted
+   if impl(ghc < 7.6)
+@@ -46,7 +45,7 @@ Library
+                    Control.Concurrent.QSemN.Lifted
+                    Control.Concurrent.SampleVar.Lifted
+ 
+-  Build-depends: base                 >= 3     && < 4.7
++  Build-depends: base                 >= 3     && < 4.8
+                , base-unicode-symbols >= 0.1.1 && < 0.3
+                , transformers-base    >= 0.4   && < 0.5
+                , monad-control        >= 0.3   && < 0.4
+@@ -64,7 +63,7 @@ test-suite test-lifted-base
+   hs-source-dirs: test
+ 
+   build-depends: lifted-base
+-               , base                 >= 3     && < 4.7
++               , base                 >= 3     && < 4.8
+                , transformers         >= 0.2   && < 0.4
+                , transformers-base    >= 0.4   && < 0.5
+                , monad-control        >= 0.3   && < 0.4
+@@ -87,7 +86,7 @@ benchmark bench-lifted-base
+   ghc-options:    -O2
+ 
+   build-depends: lifted-base
+-               , base          >= 3   && < 4.7
++               , base          >= 3   && < 4.8
+                , transformers  >= 0.2 && < 0.4
+                , criterion     >= 0.5 && < 0.7
+                , monad-control >= 0.3 && < 0.4
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/monad-control-0.3.1.4_0001-build-with-newer-ghc.patch b/standalone/android/haskell-patches/monad-control-0.3.1.4_0001-build-with-newer-ghc.patch
new file mode 100644
index 0000000000..ee1c996d80
--- /dev/null
+++ b/standalone/android/haskell-patches/monad-control-0.3.1.4_0001-build-with-newer-ghc.patch
@@ -0,0 +1,25 @@
+From 3dde0175096903207c9774d8f6bba9b81ab6c2f9 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:31:45 -0400
+Subject: [PATCH] build with newer ghc
+
+---
+ monad-control.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/monad-control.cabal b/monad-control.cabal
+index 2e3eb46..b12ffaf 100644
+--- a/monad-control.cabal
++++ b/monad-control.cabal
+@@ -56,7 +56,7 @@ Library
+ 
+   Exposed-modules: Control.Monad.Trans.Control
+ 
+-  Build-depends: base                 >= 3     && < 4.7
++  Build-depends: base                 >= 3     && < 4.8
+                , base-unicode-symbols >= 0.1.1 && < 0.3
+                , transformers         >= 0.2   && < 0.4
+                , transformers-base    >= 0.4.1 && < 0.5
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/monad-logger-0.2.3.2_0001-remove-TH-logging-stuff.patch b/standalone/android/haskell-patches/monad-logger-0.2.3.2_0001-remove-TH-logging-stuff.patch
new file mode 100644
index 0000000000..e684c67a79
--- /dev/null
+++ b/standalone/android/haskell-patches/monad-logger-0.2.3.2_0001-remove-TH-logging-stuff.patch
@@ -0,0 +1,124 @@
+From ca88563e63cc31f0b96b00d3a4fe1f0c56b1e1eb Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:32:01 -0400
+Subject: [PATCH] remove TH logging stuff
+
+---
+ Control/Monad/Logger.hs |   76 -----------------------------------------------
+ monad-logger.cabal      |    2 +-
+ 2 files changed, 1 insertion(+), 77 deletions(-)
+
+diff --git a/Control/Monad/Logger.hs b/Control/Monad/Logger.hs
+index fd1282b..80b8ed9 100644
+--- a/Control/Monad/Logger.hs
++++ b/Control/Monad/Logger.hs
+@@ -27,18 +27,6 @@ module Control.Monad.Logger
+     , LoggingT (..)
+     , runStderrLoggingT
+     , runStdoutLoggingT
+-    -- * TH logging
+-    , logDebug
+-    , logInfo
+-    , logWarn
+-    , logError
+-    , logOther
+-    -- * TH logging with source
+-    , logDebugS
+-    , logInfoS
+-    , logWarnS
+-    , logErrorS
+-    , logOtherS
+     ) where
+ 
+ import Language.Haskell.TH.Syntax (Lift (lift), Q, Exp, Loc (..), qLocation)
+@@ -91,13 +79,6 @@ import Control.Monad.Writer.Class ( MonadWriter (..) )
+ data LogLevel = LevelDebug | LevelInfo | LevelWarn | LevelError | LevelOther Text
+     deriving (Eq, Prelude.Show, Prelude.Read, Ord)
+ 
+-instance Lift LogLevel where
+-    lift LevelDebug = [|LevelDebug|]
+-    lift LevelInfo = [|LevelInfo|]
+-    lift LevelWarn = [|LevelWarn|]
+-    lift LevelError = [|LevelError|]
+-    lift (LevelOther x) = [|LevelOther $ pack $(lift $ unpack x)|]
+-
+ type LogSource = Text
+ 
+ class Monad m => MonadLogger m where
+@@ -128,63 +109,6 @@ instance (MonadLogger m, Monoid w) => MonadLogger (Strict.WriterT w m) where DEF
+ instance (MonadLogger m, Monoid w) => MonadLogger (Strict.RWST r w s m) where DEF
+ #undef DEF
+ 
+-logTH :: LogLevel -> Q Exp
+-logTH level =
+-    [|monadLoggerLog $(qLocation >>= liftLoc) $(lift level) . (id :: Text -> Text)|]
+-
+--- | Generates a function that takes a 'Text' and logs a 'LevelDebug' message. Usage:
+---
+--- > $(logDebug) "This is a debug log message"
+-logDebug :: Q Exp
+-logDebug = logTH LevelDebug
+-
+--- | See 'logDebug'
+-logInfo :: Q Exp
+-logInfo = logTH LevelInfo
+--- | See 'logDebug'
+-logWarn :: Q Exp
+-logWarn = logTH LevelWarn
+--- | See 'logDebug'
+-logError :: Q Exp
+-logError = logTH LevelError
+-
+--- | Generates a function that takes a 'Text' and logs a 'LevelOther' message. Usage:
+---
+--- > $(logOther "My new level") "This is a log message"
+-logOther :: Text -> Q Exp
+-logOther = logTH . LevelOther
+-
+-liftLoc :: Loc -> Q Exp
+-liftLoc (Loc a b c (d1, d2) (e1, e2)) = [|Loc
+-    $(lift a)
+-    $(lift b)
+-    $(lift c)
+-    ($(lift d1), $(lift d2))
+-    ($(lift e1), $(lift e2))
+-    |]
+-
+--- | Generates a function that takes a 'LogSource' and 'Text' and logs a 'LevelDebug' message. Usage:
+---
+--- > $logDebug "SomeSource" "This is a debug log message"
+-logDebugS :: Q Exp
+-logDebugS = [|\a b -> monadLoggerLogSource $(qLocation >>= liftLoc) a LevelDebug (b :: Text)|]
+-
+--- | See 'logDebugS'
+-logInfoS :: Q Exp
+-logInfoS = [|\a b -> monadLoggerLogSource $(qLocation >>= liftLoc) a LevelInfo (b :: Text)|]
+--- | See 'logDebugS'
+-logWarnS :: Q Exp
+-logWarnS = [|\a b -> monadLoggerLogSource $(qLocation >>= liftLoc) a LevelWarn (b :: Text)|]
+--- | See 'logDebugS'
+-logErrorS :: Q Exp
+-logErrorS = [|\a b -> monadLoggerLogSource $(qLocation >>= liftLoc) a LevelError (b :: Text)|]
+-
+--- | Generates a function that takes a 'LogSource', a level name and a 'Text' and logs a 'LevelOther' message. Usage:
+---
+--- > $logOther "SomeSource" "My new level" "This is a log message"
+-logOtherS :: Q Exp
+-logOtherS = [|\src level msg -> monadLoggerLogSource $(qLocation >>= liftLoc) src (LevelOther level) (msg :: Text)|]
+-
+ -- | Monad transformer that adds a new logging function.
+ --
+ -- Since 0.2.2
+diff --git a/monad-logger.cabal b/monad-logger.cabal
+index ab71424..fa3d292 100644
+--- a/monad-logger.cabal
++++ b/monad-logger.cabal
+@@ -24,4 +24,4 @@ library
+                      , transformers-base
+                      , monad-control
+                      , mtl
+-                     , bytestring
++                     , bytestring >= 0.10.3.0
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/network-2.4.1.0_0001-android-port-fixes.patch b/standalone/android/haskell-patches/network-2.4.1.0_0001-android-port-fixes.patch
new file mode 100644
index 0000000000..d7d0608d21
--- /dev/null
+++ b/standalone/android/haskell-patches/network-2.4.1.0_0001-android-port-fixes.patch
@@ -0,0 +1,1960 @@
+From 9750532bd6200353fe09dda65ee6fb59702c4ac1 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:32:15 -0400
+Subject: [PATCH] android port fixes
+
+Build note: Ensure a hsc2hs in PATH is modified to pass -x to the real
+one, to enable cross-compiling.
+---
+ Network/Socket.hsc            |   22 +-
+ Network/Socket/ByteString.hsc |    2 +-
+ Network/Socket/Internal.hsc   |    2 +-
+ Network/Socket/Types.hsc      |    4 +-
+ cbits/HsNet.c                 |   14 +
+ config.guess                  |  562 ++++++++++++++++++++++-------------------
+ config.sub                    |  384 ++++++++++++++++++++--------
+ configure                     |    1 +
+ include/HsNetworkConfig.h     |    8 +-
+ 9 files changed, 612 insertions(+), 387 deletions(-)
+
+diff --git a/Network/Socket.hsc b/Network/Socket.hsc
+index 259e843..e6c0feb 100644
+--- a/Network/Socket.hsc
++++ b/Network/Socket.hsc
+@@ -38,7 +38,7 @@ module Network.Socket
+     , SockAddr(..)
+     , SocketStatus(..)
+     , HostAddress
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORTNO)
+     , HostAddress6
+     , FlowInfo
+     , ScopeID
+@@ -55,7 +55,7 @@ module Network.Socket
+     , HostName
+     , ServiceName
+ 
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORT) || 1
+     , AddrInfo(..)
+ 
+     , AddrInfoFlag(..)
+@@ -134,7 +134,7 @@ module Network.Socket
+     -- * Special constants
+     , aNY_PORT
+     , iNADDR_ANY
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORTNO)
+     , iN6ADDR_ANY
+ #endif
+     , sOMAXCONN
+@@ -330,16 +330,6 @@ socket family stype protocol = do
+     setNonBlockIfNeeded fd
+     socket_status <- newMVar NotConnected
+     let sock = MkSocket fd family stype protocol socket_status
+-#if HAVE_DECL_IPV6_V6ONLY
+-# if defined(mingw32_HOST_OS)
+-    -- the IPv6Only option is only supported on Windows Vista and later,
+-    -- so trying to change it might throw an error
+-    when (family == AF_INET6) $
+-            E.catch (setSocketOption sock IPv6Only 0) $ (\(_ :: E.IOException) -> return ())
+-# else
+-    when (family == AF_INET6) $ setSocketOption sock IPv6Only 0
+-# endif
+-#endif
+     return sock
+ 
+ -- | Build a pair of connected socket objects using the given address
+@@ -1043,9 +1033,9 @@ aNY_PORT = 0
+ iNADDR_ANY :: HostAddress
+ iNADDR_ANY = htonl (#const INADDR_ANY)
+ 
+-foreign import CALLCONV unsafe "htonl" htonl :: Word32 -> Word32
++foreign import CALLCONV unsafe "my_htonl" htonl :: Word32 -> Word32
+ 
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORTNO)
+ -- | The IPv6 wild card address.
+ 
+ iN6ADDR_ANY :: HostAddress6
+@@ -1219,7 +1209,7 @@ unpackBits ((k,v):xs) r
+ -----------------------------------------------------------------------------
+ -- Address and service lookups
+ 
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORT) || 1
+ 
+ -- | Flags that control the querying behaviour of 'getAddrInfo'.
+ data AddrInfoFlag
+diff --git a/Network/Socket/ByteString.hsc b/Network/Socket/ByteString.hsc
+index bec2eb9..cb8ed8c 100644
+--- a/Network/Socket/ByteString.hsc
++++ b/Network/Socket/ByteString.hsc
+@@ -201,7 +201,7 @@ sendMany sock@(MkSocket fd _ _ _ _) cs = do
+       liftM fromIntegral . withIOVec cs $ \(iovsPtr, iovsLen) ->
+           throwSocketErrorWaitWrite sock "writev" $
+               c_writev (fromIntegral fd) iovsPtr
+-              (fromIntegral (min iovsLen (#const IOV_MAX)))
++              (fromIntegral (min iovsLen (0x0026)))
+ #else
+ sendMany sock = sendAll sock . B.concat
+ #endif
+diff --git a/Network/Socket/Internal.hsc b/Network/Socket/Internal.hsc
+index 96fe9c6..df5ce64 100644
+--- a/Network/Socket/Internal.hsc
++++ b/Network/Socket/Internal.hsc
+@@ -24,7 +24,7 @@ module Network.Socket.Internal
+     (
+     -- * Socket addresses
+       HostAddress
+-#if defined(IPV6_SOCKET_SUPPORT)
++#if defined(IPV6_SOCKET_SUPPORTNO)
+     , HostAddress6
+     , FlowInfo
+     , ScopeID
+diff --git a/Network/Socket/Types.hsc b/Network/Socket/Types.hsc
+index 7ad24f1..dad1d1d 100644
+--- a/Network/Socket/Types.hsc
++++ b/Network/Socket/Types.hsc
+@@ -705,8 +705,8 @@ intToPortNumber v = PortNum (htons (fromIntegral v))
+ portNumberToInt :: PortNumber -> Int
+ portNumberToInt (PortNum po) = fromIntegral (ntohs po)
+ 
+-foreign import CALLCONV unsafe "ntohs" ntohs :: Word16 -> Word16
+-foreign import CALLCONV unsafe "htons" htons :: Word16 -> Word16
++foreign import CALLCONV unsafe "my_ntohs" ntohs :: Word16 -> Word16
++foreign import CALLCONV unsafe "my_htons" htons :: Word16 -> Word16
+ --foreign import CALLCONV unsafe "ntohl" ntohl :: Word32 -> Word32
+ 
+ instance Enum PortNumber where
+diff --git a/cbits/HsNet.c b/cbits/HsNet.c
+index 86b55dc..5ea1199 100644
+--- a/cbits/HsNet.c
++++ b/cbits/HsNet.c
+@@ -6,3 +6,17 @@
+ 
+ #define INLINE
+ #include "HsNet.h"
++
++#include 
++uint16_t my_htons(uint16_t v)
++{
++	htons(v);
++}
++uint32_t my_htonl(uint32_t v)
++{
++	htonl(v);
++}
++uint16_t my_ntohs(uint16_t v)
++{
++	ntohs(v);
++}
+diff --git a/config.guess b/config.guess
+index c38553d..1804e9f 100644
+--- a/config.guess
++++ b/config.guess
+@@ -1,13 +1,14 @@
+ #! /bin/sh
+ # Attempt to guess a canonical system name.
+ #   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
+-#   2000, 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
++#   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
++#   2011, 2012, 2013 Free Software Foundation, Inc.
+ 
+-timestamp='2006-02-23'
++timestamp='2012-12-29'
+ 
+ # This file is free software; you can redistribute it and/or modify it
+ # under the terms of the GNU General Public License as published by
+-# the Free Software Foundation; either version 2 of the License, or
++# the Free Software Foundation; either version 3 of the License, or
+ # (at your option) any later version.
+ #
+ # This program is distributed in the hope that it will be useful, but
+@@ -16,26 +17,22 @@ timestamp='2006-02-23'
+ # General Public License for more details.
+ #
+ # You should have received a copy of the GNU General Public License
+-# along with this program; if not, write to the Free Software
+-# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
+-# 02110-1301, USA.
++# along with this program; if not, see .
+ #
+ # As a special exception to the GNU General Public License, if you
+ # distribute this file as part of a program that contains a
+ # configuration script generated by Autoconf, you may include it under
+-# the same distribution terms that you use for the rest of that program.
+-
+-
+-# Originally written by Per Bothner .
+-# Please send patches to .  Submit a context
+-# diff and a properly formatted ChangeLog entry.
++# the same distribution terms that you use for the rest of that
++# program.  This Exception is an additional permission under section 7
++# of the GNU General Public License, version 3 ("GPLv3").
++#
++# Originally written by Per Bothner. 
+ #
+-# This script attempts to guess a canonical system name similar to
+-# config.sub.  If it succeeds, it prints the system name on stdout, and
+-# exits with 0.  Otherwise, it exits with 1.
++# You can get the latest version of this script from:
++# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+ #
+-# The plan is that this can be called by configure scripts if you
+-# don't specify an explicit build system type.
++# Please send patches with a ChangeLog entry to config-patches@gnu.org.
++
+ 
+ me=`echo "$0" | sed -e 's,.*/,,'`
+ 
+@@ -55,8 +52,9 @@ version="\
+ GNU config.guess ($timestamp)
+ 
+ Originally written by Per Bothner.
+-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005
+-Free Software Foundation, Inc.
++Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
++2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
++2012, 2013 Free Software Foundation, Inc.
+ 
+ This is free software; see the source for copying conditions.  There is NO
+ warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
+@@ -143,7 +141,7 @@ UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+     *:NetBSD:*:*)
+ 	# NetBSD (nbsd) targets should (where applicable) match one or
+-	# more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*,
++	# more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*,
+ 	# *-*-netbsdecoff* and *-*-netbsd*.  For targets that recently
+ 	# switched to ELF, *-*-netbsd* would select the old
+ 	# object file format.  This provides both forward
+@@ -160,6 +158,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 	    arm*) machine=arm-unknown ;;
+ 	    sh3el) machine=shl-unknown ;;
+ 	    sh3eb) machine=sh-unknown ;;
++	    sh5el) machine=sh5le-unknown ;;
+ 	    *) machine=${UNAME_MACHINE_ARCH}-unknown ;;
+ 	esac
+ 	# The Operating System including object format, if it has switched
+@@ -168,7 +167,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 	    arm*|i386|m68k|ns32k|sh3*|sparc|vax)
+ 		eval $set_cc_for_build
+ 		if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
+-			| grep __ELF__ >/dev/null
++			| grep -q __ELF__
+ 		then
+ 		    # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout).
+ 		    # Return netbsd for either.  FIX?
+@@ -178,7 +177,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 		fi
+ 		;;
+ 	    *)
+-	        os=netbsd
++		os=netbsd
+ 		;;
+ 	esac
+ 	# The OS release
+@@ -199,6 +198,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 	# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
+ 	echo "${machine}-${os}${release}"
+ 	exit ;;
++    *:Bitrig:*:*)
++	UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
++	echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE}
++	exit ;;
+     *:OpenBSD:*:*)
+ 	UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
+ 	echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE}
+@@ -210,7 +213,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 	echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE}
+ 	exit ;;
+     macppc:MirBSD:*:*)
+-	echo powerppc-unknown-mirbsd${UNAME_RELEASE}
++	echo powerpc-unknown-mirbsd${UNAME_RELEASE}
+ 	exit ;;
+     *:MirBSD:*:*)
+ 	echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE}
+@@ -221,7 +224,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 		UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'`
+ 		;;
+ 	*5.*)
+-	        UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
++		UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
+ 		;;
+ 	esac
+ 	# According to Compaq, /usr/sbin/psrinfo has been available on
+@@ -267,7 +270,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 	# A Xn.n version is an unreleased experimental baselevel.
+ 	# 1.2 uses "1.2" for uname -r.
+ 	echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+-	exit ;;
++	# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
++	exitcode=$?
++	trap '' 0
++	exit $exitcode ;;
+     Alpha\ *:Windows_NT*:*)
+ 	# How do we know it's Interix rather than the generic POSIX subsystem?
+ 	# Should we change UNAME_MACHINE based on the output of uname instead
+@@ -293,12 +299,12 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 	echo s390-ibm-zvmoe
+ 	exit ;;
+     *:OS400:*:*)
+-        echo powerpc-ibm-os400
++	echo powerpc-ibm-os400
+ 	exit ;;
+     arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+ 	echo arm-acorn-riscix${UNAME_RELEASE}
+ 	exit ;;
+-    arm:riscos:*:*|arm:RISCOS:*:*)
++    arm*:riscos:*:*|arm*:RISCOS:*:*)
+ 	echo arm-unknown-riscos
+ 	exit ;;
+     SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*)
+@@ -322,14 +328,33 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ 	case `/usr/bin/uname -p` in
+ 	    sparc) echo sparc-icl-nx7; exit ;;
+ 	esac ;;
++    s390x:SunOS:*:*)
++	echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
++	exit ;;
+     sun4H:SunOS:5.*:*)
+ 	echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ 	exit ;;
+     sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*)
+ 	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ 	exit ;;
+-    i86pc:SunOS:5.*:*)
+-	echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
++    i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*)
++	echo i386-pc-auroraux${UNAME_RELEASE}
++	exit ;;
++    i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
++	eval $set_cc_for_build
++	SUN_ARCH="i386"
++	# If there is a compiler, see if it is configured for 64-bit objects.
++	# Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
++	# This test works for both compilers.
++	if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
++	    if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
++		(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
++		grep IS_64BIT_ARCH >/dev/null
++	    then
++		SUN_ARCH="x86_64"
++	    fi
++	fi
++	echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ 	exit ;;
+     sun4*:SunOS:6*:*)
+ 	# According to config.sub, this is the proper way to canonicalize
+@@ -373,23 +398,23 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+     # MiNT.  But MiNT is downward compatible to TOS, so this should
+     # be no problem.
+     atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*)
+-        echo m68k-atari-mint${UNAME_RELEASE}
++	echo m68k-atari-mint${UNAME_RELEASE}
+ 	exit ;;
+     atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*)
+ 	echo m68k-atari-mint${UNAME_RELEASE}
+-        exit ;;
++	exit ;;
+     *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*)
+-        echo m68k-atari-mint${UNAME_RELEASE}
++	echo m68k-atari-mint${UNAME_RELEASE}
+ 	exit ;;
+     milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*)
+-        echo m68k-milan-mint${UNAME_RELEASE}
+-        exit ;;
++	echo m68k-milan-mint${UNAME_RELEASE}
++	exit ;;
+     hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*)
+-        echo m68k-hades-mint${UNAME_RELEASE}
+-        exit ;;
++	echo m68k-hades-mint${UNAME_RELEASE}
++	exit ;;
+     *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*)
+-        echo m68k-unknown-mint${UNAME_RELEASE}
+-        exit ;;
++	echo m68k-unknown-mint${UNAME_RELEASE}
++	exit ;;
+     m68k:machten:*:*)
+ 	echo m68k-apple-machten${UNAME_RELEASE}
+ 	exit ;;
+@@ -459,8 +484,8 @@ EOF
+ 	echo m88k-motorola-sysv3
+ 	exit ;;
+     AViiON:dgux:*:*)
+-        # DG/UX returns AViiON for all architectures
+-        UNAME_PROCESSOR=`/usr/bin/uname -p`
++	# DG/UX returns AViiON for all architectures
++	UNAME_PROCESSOR=`/usr/bin/uname -p`
+ 	if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ]
+ 	then
+ 	    if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \
+@@ -473,7 +498,7 @@ EOF
+ 	else
+ 	    echo i586-dg-dgux${UNAME_RELEASE}
+ 	fi
+- 	exit ;;
++	exit ;;
+     M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+ 	echo m88k-dolphin-sysv3
+ 	exit ;;
+@@ -530,7 +555,7 @@ EOF
+ 		echo rs6000-ibm-aix3.2
+ 	fi
+ 	exit ;;
+-    *:AIX:*:[45])
++    *:AIX:*:[4567])
+ 	IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'`
+ 	if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then
+ 		IBM_ARCH=rs6000
+@@ -573,52 +598,52 @@ EOF
+ 	    9000/[678][0-9][0-9])
+ 		if [ -x /usr/bin/getconf ]; then
+ 		    sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
+-                    sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
+-                    case "${sc_cpu_version}" in
+-                      523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
+-                      528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
+-                      532)                      # CPU_PA_RISC2_0
+-                        case "${sc_kernel_bits}" in
+-                          32) HP_ARCH="hppa2.0n" ;;
+-                          64) HP_ARCH="hppa2.0w" ;;
++		    sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
++		    case "${sc_cpu_version}" in
++		      523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
++		      528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
++		      532)                      # CPU_PA_RISC2_0
++			case "${sc_kernel_bits}" in
++			  32) HP_ARCH="hppa2.0n" ;;
++			  64) HP_ARCH="hppa2.0w" ;;
+ 			  '') HP_ARCH="hppa2.0" ;;   # HP-UX 10.20
+-                        esac ;;
+-                    esac
++			esac ;;
++		    esac
+ 		fi
+ 		if [ "${HP_ARCH}" = "" ]; then
+ 		    eval $set_cc_for_build
+-		    sed 's/^              //' << EOF >$dummy.c
++		    sed 's/^		//' << EOF >$dummy.c
+ 
+-              #define _HPUX_SOURCE
+-              #include 
+-              #include 
++		#define _HPUX_SOURCE
++		#include 
++		#include 
+ 
+-              int main ()
+-              {
+-              #if defined(_SC_KERNEL_BITS)
+-                  long bits = sysconf(_SC_KERNEL_BITS);
+-              #endif
+-                  long cpu  = sysconf (_SC_CPU_VERSION);
++		int main ()
++		{
++		#if defined(_SC_KERNEL_BITS)
++		    long bits = sysconf(_SC_KERNEL_BITS);
++		#endif
++		    long cpu  = sysconf (_SC_CPU_VERSION);
+ 
+-                  switch (cpu)
+-              	{
+-              	case CPU_PA_RISC1_0: puts ("hppa1.0"); break;
+-              	case CPU_PA_RISC1_1: puts ("hppa1.1"); break;
+-              	case CPU_PA_RISC2_0:
+-              #if defined(_SC_KERNEL_BITS)
+-              	    switch (bits)
+-              		{
+-              		case 64: puts ("hppa2.0w"); break;
+-              		case 32: puts ("hppa2.0n"); break;
+-              		default: puts ("hppa2.0"); break;
+-              		} break;
+-              #else  /* !defined(_SC_KERNEL_BITS) */
+-              	    puts ("hppa2.0"); break;
+-              #endif
+-              	default: puts ("hppa1.0"); break;
+-              	}
+-                  exit (0);
+-              }
++		    switch (cpu)
++			{
++			case CPU_PA_RISC1_0: puts ("hppa1.0"); break;
++			case CPU_PA_RISC1_1: puts ("hppa1.1"); break;
++			case CPU_PA_RISC2_0:
++		#if defined(_SC_KERNEL_BITS)
++			    switch (bits)
++				{
++				case 64: puts ("hppa2.0w"); break;
++				case 32: puts ("hppa2.0n"); break;
++				default: puts ("hppa2.0"); break;
++				} break;
++		#else  /* !defined(_SC_KERNEL_BITS) */
++			    puts ("hppa2.0"); break;
++		#endif
++			default: puts ("hppa1.0"); break;
++			}
++		    exit (0);
++		}
+ EOF
+ 		    (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
+ 		    test -z "$HP_ARCH" && HP_ARCH=hppa
+@@ -638,7 +663,7 @@ EOF
+ 	    # => hppa64-hp-hpux11.23
+ 
+ 	    if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) |
+-		grep __LP64__ >/dev/null
++		grep -q __LP64__
+ 	    then
+ 		HP_ARCH="hppa2.0w"
+ 	    else
+@@ -709,22 +734,22 @@ EOF
+ 	exit ;;
+     C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+ 	echo c1-convex-bsd
+-        exit ;;
++	exit ;;
+     C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+ 	if getsysinfo -f scalar_acc
+ 	then echo c32-convex-bsd
+ 	else echo c2-convex-bsd
+ 	fi
+-        exit ;;
++	exit ;;
+     C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+ 	echo c34-convex-bsd
+-        exit ;;
++	exit ;;
+     C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+ 	echo c38-convex-bsd
+-        exit ;;
++	exit ;;
+     C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+ 	echo c4-convex-bsd
+-        exit ;;
++	exit ;;
+     CRAY*Y-MP:*:*:*)
+ 	echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+ 	exit ;;
+@@ -748,14 +773,14 @@ EOF
+ 	exit ;;
+     F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
+ 	FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+-        FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
+-        FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
+-        echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+-        exit ;;
++	FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
++	FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
++	echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
++	exit ;;
+     5000:UNIX_System_V:4.*:*)
+-        FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
+-        FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
+-        echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
++	FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
++	FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
++	echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+ 	exit ;;
+     i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
+ 	echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE}
+@@ -767,38 +792,51 @@ EOF
+ 	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+ 	exit ;;
+     *:FreeBSD:*:*)
+-	case ${UNAME_MACHINE} in
+-	    pc98)
+-		echo i386-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
++	UNAME_PROCESSOR=`/usr/bin/uname -p`
++	case ${UNAME_PROCESSOR} in
++	    amd64)
++		echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
+ 	    *)
+-		echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
++		echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
+ 	esac
+ 	exit ;;
+     i*:CYGWIN*:*)
+ 	echo ${UNAME_MACHINE}-pc-cygwin
+ 	exit ;;
+-    i*:MINGW*:*)
+-	echo ${UNAME_MACHINE}-pc-mingw32
++    *:MINGW64*:*)
++	echo ${UNAME_MACHINE}-pc-mingw64
+ 	exit ;;
+-    i*:MSYS_NT-*:*:*)
++    *:MINGW*:*)
+ 	echo ${UNAME_MACHINE}-pc-mingw32
+ 	exit ;;
++    i*:MSYS*:*)
++	echo ${UNAME_MACHINE}-pc-msys
++	exit ;;
+     i*:windows32*:*)
+-    	# uname -m includes "-pc" on this system.
+-    	echo ${UNAME_MACHINE}-mingw32
++	# uname -m includes "-pc" on this system.
++	echo ${UNAME_MACHINE}-mingw32
+ 	exit ;;
+     i*:PW*:*)
+ 	echo ${UNAME_MACHINE}-pc-pw32
+ 	exit ;;
+-    x86:Interix*:[345]*)
+-	echo i586-pc-interix${UNAME_RELEASE}
+-	exit ;;
+-    EM64T:Interix*:[345]*)
+-	echo x86_64-unknown-interix${UNAME_RELEASE}
+-	exit ;;
++    *:Interix*:*)
++	case ${UNAME_MACHINE} in
++	    x86)
++		echo i586-pc-interix${UNAME_RELEASE}
++		exit ;;
++	    authenticamd | genuineintel | EM64T)
++		echo x86_64-unknown-interix${UNAME_RELEASE}
++		exit ;;
++	    IA64)
++		echo ia64-unknown-interix${UNAME_RELEASE}
++		exit ;;
++	esac ;;
+     [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*)
+ 	echo i${UNAME_MACHINE}-pc-mks
+ 	exit ;;
++    8664:Windows_NT:*)
++	echo x86_64-pc-mks
++	exit ;;
+     i*:Windows_NT*:* | Pentium*:Windows_NT*:*)
+ 	# How do we know it's Interix rather than the generic POSIX subsystem?
+ 	# It also conflicts with pre-2.0 versions of AT&T UWIN. Should we
+@@ -828,17 +866,68 @@ EOF
+     i*86:Minix:*:*)
+ 	echo ${UNAME_MACHINE}-pc-minix
+ 	exit ;;
++    aarch64:Linux:*:*)
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
++	exit ;;
++    aarch64_be:Linux:*:*)
++	UNAME_MACHINE=aarch64_be
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
++	exit ;;
++    alpha:Linux:*:*)
++	case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
++	  EV5)   UNAME_MACHINE=alphaev5 ;;
++	  EV56)  UNAME_MACHINE=alphaev56 ;;
++	  PCA56) UNAME_MACHINE=alphapca56 ;;
++	  PCA57) UNAME_MACHINE=alphapca56 ;;
++	  EV6)   UNAME_MACHINE=alphaev6 ;;
++	  EV67)  UNAME_MACHINE=alphaev67 ;;
++	  EV68*) UNAME_MACHINE=alphaev68 ;;
++	esac
++	objdump --private-headers /bin/sh | grep -q ld.so.1
++	if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
++	echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
++	exit ;;
+     arm*:Linux:*:*)
++	eval $set_cc_for_build
++	if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \
++	    | grep -q __ARM_EABI__
++	then
++	    echo ${UNAME_MACHINE}-unknown-linux-gnu
++	else
++	    if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
++		| grep -q __ARM_PCS_VFP
++	    then
++		echo ${UNAME_MACHINE}-unknown-linux-gnueabi
++	    else
++		echo ${UNAME_MACHINE}-unknown-linux-gnueabihf
++	    fi
++	fi
++	exit ;;
++    avr32*:Linux:*:*)
+ 	echo ${UNAME_MACHINE}-unknown-linux-gnu
+ 	exit ;;
+     cris:Linux:*:*)
+-	echo cris-axis-linux-gnu
++	echo ${UNAME_MACHINE}-axis-linux-gnu
+ 	exit ;;
+     crisv32:Linux:*:*)
+-	echo crisv32-axis-linux-gnu
++	echo ${UNAME_MACHINE}-axis-linux-gnu
+ 	exit ;;
+     frv:Linux:*:*)
+-    	echo frv-unknown-linux-gnu
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
++	exit ;;
++    hexagon:Linux:*:*)
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
++	exit ;;
++    i*86:Linux:*:*)
++	LIBC=gnu
++	eval $set_cc_for_build
++	sed 's/^	//' << EOF >$dummy.c
++	#ifdef __dietlibc__
++	LIBC=dietlibc
++	#endif
++EOF
++	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'`
++	echo "${UNAME_MACHINE}-pc-linux-${LIBC}"
+ 	exit ;;
+     ia64:Linux:*:*)
+ 	echo ${UNAME_MACHINE}-unknown-linux-gnu
+@@ -849,74 +938,33 @@ EOF
+     m68*:Linux:*:*)
+ 	echo ${UNAME_MACHINE}-unknown-linux-gnu
+ 	exit ;;
+-    mips:Linux:*:*)
+-	eval $set_cc_for_build
+-	sed 's/^	//' << EOF >$dummy.c
+-	#undef CPU
+-	#undef mips
+-	#undef mipsel
+-	#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
+-	CPU=mipsel
+-	#else
+-	#if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB)
+-	CPU=mips
+-	#else
+-	CPU=
+-	#endif
+-	#endif
+-EOF
+-	eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n '
+-	    /^CPU/{
+-		s: ::g
+-		p
+-	    }'`"
+-	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; }
+-	;;
+-    mips64:Linux:*:*)
++    mips:Linux:*:* | mips64:Linux:*:*)
+ 	eval $set_cc_for_build
+ 	sed 's/^	//' << EOF >$dummy.c
+ 	#undef CPU
+-	#undef mips64
+-	#undef mips64el
++	#undef ${UNAME_MACHINE}
++	#undef ${UNAME_MACHINE}el
+ 	#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
+-	CPU=mips64el
++	CPU=${UNAME_MACHINE}el
+ 	#else
+ 	#if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB)
+-	CPU=mips64
++	CPU=${UNAME_MACHINE}
+ 	#else
+ 	CPU=
+ 	#endif
+ 	#endif
+ EOF
+-	eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n '
+-	    /^CPU/{
+-		s: ::g
+-		p
+-	    }'`"
++	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
+ 	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; }
+ 	;;
+     or32:Linux:*:*)
+-	echo or32-unknown-linux-gnu
+-	exit ;;
+-    ppc:Linux:*:*)
+-	echo powerpc-unknown-linux-gnu
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
+ 	exit ;;
+-    ppc64:Linux:*:*)
+-	echo powerpc64-unknown-linux-gnu
++    padre:Linux:*:*)
++	echo sparc-unknown-linux-gnu
+ 	exit ;;
+-    alpha:Linux:*:*)
+-	case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
+-	  EV5)   UNAME_MACHINE=alphaev5 ;;
+-	  EV56)  UNAME_MACHINE=alphaev56 ;;
+-	  PCA56) UNAME_MACHINE=alphapca56 ;;
+-	  PCA57) UNAME_MACHINE=alphapca56 ;;
+-	  EV6)   UNAME_MACHINE=alphaev6 ;;
+-	  EV67)  UNAME_MACHINE=alphaev67 ;;
+-	  EV68*) UNAME_MACHINE=alphaev68 ;;
+-        esac
+-	objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null
+-	if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
+-	echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
++    parisc64:Linux:*:* | hppa64:Linux:*:*)
++	echo hppa64-unknown-linux-gnu
+ 	exit ;;
+     parisc:Linux:*:* | hppa:Linux:*:*)
+ 	# Look for CPU level
+@@ -926,14 +974,17 @@ EOF
+ 	  *)    echo hppa-unknown-linux-gnu ;;
+ 	esac
+ 	exit ;;
+-    parisc64:Linux:*:* | hppa64:Linux:*:*)
+-	echo hppa64-unknown-linux-gnu
++    ppc64:Linux:*:*)
++	echo powerpc64-unknown-linux-gnu
++	exit ;;
++    ppc:Linux:*:*)
++	echo powerpc-unknown-linux-gnu
+ 	exit ;;
+     s390:Linux:*:* | s390x:Linux:*:*)
+ 	echo ${UNAME_MACHINE}-ibm-linux
+ 	exit ;;
+     sh64*:Linux:*:*)
+-    	echo ${UNAME_MACHINE}-unknown-linux-gnu
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
+ 	exit ;;
+     sh*:Linux:*:*)
+ 	echo ${UNAME_MACHINE}-unknown-linux-gnu
+@@ -941,75 +992,18 @@ EOF
+     sparc:Linux:*:* | sparc64:Linux:*:*)
+ 	echo ${UNAME_MACHINE}-unknown-linux-gnu
+ 	exit ;;
++    tile*:Linux:*:*)
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
++	exit ;;
+     vax:Linux:*:*)
+ 	echo ${UNAME_MACHINE}-dec-linux-gnu
+ 	exit ;;
+     x86_64:Linux:*:*)
+-	echo x86_64-unknown-linux-gnu
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
++	exit ;;
++    xtensa*:Linux:*:*)
++	echo ${UNAME_MACHINE}-unknown-linux-gnu
+ 	exit ;;
+-    i*86:Linux:*:*)
+-	# The BFD linker knows what the default object file format is, so
+-	# first see if it will tell us. cd to the root directory to prevent
+-	# problems with other programs or directories called `ld' in the path.
+-	# Set LC_ALL=C to ensure ld outputs messages in English.
+-	ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \
+-			 | sed -ne '/supported targets:/!d
+-				    s/[ 	][ 	]*/ /g
+-				    s/.*supported targets: *//
+-				    s/ .*//
+-				    p'`
+-        case "$ld_supported_targets" in
+-	  elf32-i386)
+-		TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu"
+-		;;
+-	  a.out-i386-linux)
+-		echo "${UNAME_MACHINE}-pc-linux-gnuaout"
+-		exit ;;
+-	  coff-i386)
+-		echo "${UNAME_MACHINE}-pc-linux-gnucoff"
+-		exit ;;
+-	  "")
+-		# Either a pre-BFD a.out linker (linux-gnuoldld) or
+-		# one that does not give us useful --help.
+-		echo "${UNAME_MACHINE}-pc-linux-gnuoldld"
+-		exit ;;
+-	esac
+-	# Determine whether the default compiler is a.out or elf
+-	eval $set_cc_for_build
+-	sed 's/^	//' << EOF >$dummy.c
+-	#include 
+-	#ifdef __ELF__
+-	# ifdef __GLIBC__
+-	#  if __GLIBC__ >= 2
+-	LIBC=gnu
+-	#  else
+-	LIBC=gnulibc1
+-	#  endif
+-	# else
+-	LIBC=gnulibc1
+-	# endif
+-	#else
+-	#if defined(__INTEL_COMPILER) || defined(__PGI) || defined(__sun)
+-	LIBC=gnu
+-	#else
+-	LIBC=gnuaout
+-	#endif
+-	#endif
+-	#ifdef __dietlibc__
+-	LIBC=dietlibc
+-	#endif
+-EOF
+-	eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n '
+-	    /^LIBC/{
+-		s: ::g
+-		p
+-	    }'`"
+-	test x"${LIBC}" != x && {
+-		echo "${UNAME_MACHINE}-pc-linux-${LIBC}"
+-		exit
+-	}
+-	test x"${TENTATIVE}" != x && { echo "${TENTATIVE}"; exit; }
+-	;;
+     i*86:DYNIX/ptx:4*:*)
+ 	# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.
+ 	# earlier versions are messed up and put the nodename in both
+@@ -1017,11 +1011,11 @@ EOF
+ 	echo i386-sequent-sysv4
+ 	exit ;;
+     i*86:UNIX_SV:4.2MP:2.*)
+-        # Unixware is an offshoot of SVR4, but it has its own version
+-        # number series starting with 2...
+-        # I am not positive that other SVR4 systems won't match this,
++	# Unixware is an offshoot of SVR4, but it has its own version
++	# number series starting with 2...
++	# I am not positive that other SVR4 systems won't match this,
+ 	# I just have to hope.  -- rms.
+-        # Use sysv4.2uw... so that sysv4* matches it.
++	# Use sysv4.2uw... so that sysv4* matches it.
+ 	echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION}
+ 	exit ;;
+     i*86:OS/2:*:*)
+@@ -1038,7 +1032,7 @@ EOF
+     i*86:syllable:*:*)
+ 	echo ${UNAME_MACHINE}-pc-syllable
+ 	exit ;;
+-    i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*)
++    i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*)
+ 	echo i386-unknown-lynxos${UNAME_RELEASE}
+ 	exit ;;
+     i*86:*DOS:*:*)
+@@ -1053,7 +1047,7 @@ EOF
+ 	fi
+ 	exit ;;
+     i*86:*:5:[678]*)
+-    	# UnixWare 7.x, OpenUNIX and OpenServer 6.
++	# UnixWare 7.x, OpenUNIX and OpenServer 6.
+ 	case `/bin/uname -X | grep "^Machine"` in
+ 	    *486*)	     UNAME_MACHINE=i486 ;;
+ 	    *Pentium)	     UNAME_MACHINE=i586 ;;
+@@ -1081,10 +1075,13 @@ EOF
+ 	exit ;;
+     pc:*:*:*)
+ 	# Left here for compatibility:
+-        # uname -m prints for DJGPP always 'pc', but it prints nothing about
+-        # the processor, so we play safe by assuming i386.
+-	echo i386-pc-msdosdjgpp
+-        exit ;;
++	# uname -m prints for DJGPP always 'pc', but it prints nothing about
++	# the processor, so we play safe by assuming i586.
++	# Note: whatever this is, it MUST be the same as what config.sub
++	# prints for the "djgpp" host, or else GDB configury will decide that
++	# this is a cross-build.
++	echo i586-pc-msdosdjgpp
++	exit ;;
+     Intel:Mach:3*:*)
+ 	echo i386-pc-mach3
+ 	exit ;;
+@@ -1119,8 +1116,18 @@ EOF
+ 	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
+ 	  && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
+     3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+-        /bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+-          && { echo i486-ncr-sysv4; exit; } ;;
++	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
++	  && { echo i486-ncr-sysv4; exit; } ;;
++    NCR*:*:4.2:* | MPRAS*:*:4.2:*)
++	OS_REL='.3'
++	test -r /etc/.relid \
++	    && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
++	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
++	    && { echo i486-ncr-sysv4.3${OS_REL}; exit; }
++	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
++	    && { echo i586-ncr-sysv4.3${OS_REL}; exit; }
++	/bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \
++	    && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
+     m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*)
+ 	echo m68k-unknown-lynxos${UNAME_RELEASE}
+ 	exit ;;
+@@ -1133,7 +1140,7 @@ EOF
+     rs6000:LynxOS:2.*:*)
+ 	echo rs6000-unknown-lynxos${UNAME_RELEASE}
+ 	exit ;;
+-    PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*)
++    PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*)
+ 	echo powerpc-unknown-lynxos${UNAME_RELEASE}
+ 	exit ;;
+     SM[BE]S:UNIX_SV:*:*)
+@@ -1153,10 +1160,10 @@ EOF
+ 		echo ns32k-sni-sysv
+ 	fi
+ 	exit ;;
+-    PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort
+-                      # says 
+-        echo i586-unisys-sysv4
+-        exit ;;
++    PENTIUM:*:4.0*:*)	# Unisys `ClearPath HMP IX 4000' SVR4/MP effort
++			# says 
++	echo i586-unisys-sysv4
++	exit ;;
+     *:UNIX_System_V:4*:FTX*)
+ 	# From Gerald Hewes .
+ 	# How about differentiating between stratus architectures? -djm
+@@ -1182,11 +1189,11 @@ EOF
+ 	exit ;;
+     R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*)
+ 	if [ -d /usr/nec ]; then
+-	        echo mips-nec-sysv${UNAME_RELEASE}
++		echo mips-nec-sysv${UNAME_RELEASE}
+ 	else
+-	        echo mips-unknown-sysv${UNAME_RELEASE}
++		echo mips-unknown-sysv${UNAME_RELEASE}
+ 	fi
+-        exit ;;
++	exit ;;
+     BeBox:BeOS:*:*)	# BeOS running on hardware made by Be, PPC only.
+ 	echo powerpc-be-beos
+ 	exit ;;
+@@ -1196,6 +1203,12 @@ EOF
+     BePC:BeOS:*:*)	# BeOS running on Intel PC compatible.
+ 	echo i586-pc-beos
+ 	exit ;;
++    BePC:Haiku:*:*)	# Haiku running on Intel PC compatible.
++	echo i586-pc-haiku
++	exit ;;
++    x86_64:Haiku:*:*)
++	echo x86_64-unknown-haiku
++	exit ;;
+     SX-4:SUPER-UX:*:*)
+ 	echo sx4-nec-superux${UNAME_RELEASE}
+ 	exit ;;
+@@ -1205,6 +1218,15 @@ EOF
+     SX-6:SUPER-UX:*:*)
+ 	echo sx6-nec-superux${UNAME_RELEASE}
+ 	exit ;;
++    SX-7:SUPER-UX:*:*)
++	echo sx7-nec-superux${UNAME_RELEASE}
++	exit ;;
++    SX-8:SUPER-UX:*:*)
++	echo sx8-nec-superux${UNAME_RELEASE}
++	exit ;;
++    SX-8R:SUPER-UX:*:*)
++	echo sx8r-nec-superux${UNAME_RELEASE}
++	exit ;;
+     Power*:Rhapsody:*:*)
+ 	echo powerpc-apple-rhapsody${UNAME_RELEASE}
+ 	exit ;;
+@@ -1214,6 +1236,16 @@ EOF
+     *:Darwin:*:*)
+ 	UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown
+ 	case $UNAME_PROCESSOR in
++	    i386)
++		eval $set_cc_for_build
++		if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
++		  if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
++		      (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
++		      grep IS_64BIT_ARCH >/dev/null
++		  then
++		      UNAME_PROCESSOR="x86_64"
++		  fi
++		fi ;;
+ 	    unknown) UNAME_PROCESSOR=powerpc ;;
+ 	esac
+ 	echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE}
+@@ -1229,7 +1261,10 @@ EOF
+     *:QNX:*:4*)
+ 	echo i386-pc-qnx
+ 	exit ;;
+-    NSE-?:NONSTOP_KERNEL:*:*)
++    NEO-?:NONSTOP_KERNEL:*:*)
++	echo neo-tandem-nsk${UNAME_RELEASE}
++	exit ;;
++    NSE-*:NONSTOP_KERNEL:*:*)
+ 	echo nse-tandem-nsk${UNAME_RELEASE}
+ 	exit ;;
+     NSR-?:NONSTOP_KERNEL:*:*)
+@@ -1274,13 +1309,13 @@ EOF
+ 	echo pdp10-unknown-its
+ 	exit ;;
+     SEI:*:*:SEIUX)
+-        echo mips-sei-seiux${UNAME_RELEASE}
++	echo mips-sei-seiux${UNAME_RELEASE}
+ 	exit ;;
+     *:DragonFly:*:*)
+ 	echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+ 	exit ;;
+     *:*VMS:*:*)
+-    	UNAME_MACHINE=`(uname -p) 2>/dev/null`
++	UNAME_MACHINE=`(uname -p) 2>/dev/null`
+ 	case "${UNAME_MACHINE}" in
+ 	    A*) echo alpha-dec-vms ; exit ;;
+ 	    I*) echo ia64-dec-vms ; exit ;;
+@@ -1295,11 +1330,14 @@ EOF
+     i*86:rdos:*:*)
+ 	echo ${UNAME_MACHINE}-pc-rdos
+ 	exit ;;
++    i*86:AROS:*:*)
++	echo ${UNAME_MACHINE}-pc-aros
++	exit ;;
++    x86_64:VMkernel:*:*)
++	echo ${UNAME_MACHINE}-unknown-esx
++	exit ;;
+ esac
+ 
+-#echo '(No uname command or uname output not recognized.)' 1>&2
+-#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+-
+ eval $set_cc_for_build
+ cat >$dummy.c <
+   printf ("m68k-sony-newsos%s\n",
+ #ifdef NEWSOS4
+-          "4"
++	"4"
+ #else
+-	  ""
++	""
+ #endif
+-         ); exit (0);
++	); exit (0);
+ #endif
+ #endif
+ 
+@@ -1455,9 +1493,9 @@ This script, last modified $timestamp, has failed to recognize
+ the operating system you are using. It is advised that you
+ download the most up to date version of the config scripts from
+ 
+-  http://savannah.gnu.org/cgi-bin/viewcvs/*checkout*/config/config/config.guess
++  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+ and
+-  http://savannah.gnu.org/cgi-bin/viewcvs/*checkout*/config/config/config.sub
++  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
+ 
+ If the version you run ($0) is already up to date, please
+ send the following data and any information you think might be
+diff --git a/config.sub b/config.sub
+index ad9f395..802a224 100644
+--- a/config.sub
++++ b/config.sub
+@@ -1,43 +1,42 @@
+ #! /bin/sh
+ # Configuration validation subroutine script.
+ #   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
+-#   2000, 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
++#   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
++#   2011, 2012, 2013 Free Software Foundation, Inc.
+ 
+-timestamp='2006-02-23'
++timestamp='2012-12-29'
+ 
+-# This file is (in principle) common to ALL GNU software.
+-# The presence of a machine in this file suggests that SOME GNU software
+-# can handle that machine.  It does not imply ALL GNU software can.
+-#
+-# This file is free software; you can redistribute it and/or modify
+-# it under the terms of the GNU General Public License as published by
+-# the Free Software Foundation; either version 2 of the License, or
++# This file is free software; you can redistribute it and/or modify it
++# under the terms of the GNU General Public License as published by
++# the Free Software Foundation; either version 3 of the License, or
+ # (at your option) any later version.
+ #
+-# This program is distributed in the hope that it will be useful,
+-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-# GNU General Public License for more details.
++# This program is distributed in the hope that it will be useful, but
++# WITHOUT ANY WARRANTY; without even the implied warranty of
++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
++# General Public License for more details.
+ #
+ # You should have received a copy of the GNU General Public License
+-# along with this program; if not, write to the Free Software
+-# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
+-# 02110-1301, USA.
++# along with this program; if not, see .
+ #
+ # As a special exception to the GNU General Public License, if you
+ # distribute this file as part of a program that contains a
+ # configuration script generated by Autoconf, you may include it under
+-# the same distribution terms that you use for the rest of that program.
++# the same distribution terms that you use for the rest of that
++# program.  This Exception is an additional permission under section 7
++# of the GNU General Public License, version 3 ("GPLv3").
+ 
+ 
+-# Please send patches to .  Submit a context
+-# diff and a properly formatted ChangeLog entry.
++# Please send patches with a ChangeLog entry to config-patches@gnu.org.
+ #
+ # Configuration subroutine to validate and canonicalize a configuration type.
+ # Supply the specified configuration type as an argument.
+ # If it is invalid, we print an error message on stderr and exit with code 1.
+ # Otherwise, we print the canonical config type on stdout and succeed.
+ 
++# You can get the latest version of this script from:
++# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
++
+ # This file is supposed to be the same for all GNU packages
+ # and recognize all the CPU types, system types and aliases
+ # that are meaningful with *any* GNU software.
+@@ -71,8 +70,9 @@ Report bugs and patches to ."
+ version="\
+ GNU config.sub ($timestamp)
+ 
+-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005
+-Free Software Foundation, Inc.
++Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
++2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
++2012, 2013 Free Software Foundation, Inc.
+ 
+ This is free software; see the source for copying conditions.  There is NO
+ warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
+@@ -119,12 +119,18 @@ esac
+ # Here we must recognize all the valid KERNEL-OS combinations.
+ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
+ case $maybe_os in
+-  nto-qnx* | linux-gnu* | linux-dietlibc | linux-newlib* | linux-uclibc* | \
+-  uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | \
++  nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
++  linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
++  knetbsd*-gnu* | netbsd*-gnu* | \
++  kopensolaris*-gnu* | \
+   storm-chaos* | os2-emx* | rtmk-nova*)
+     os=-$maybe_os
+     basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
+     ;;
++  android-linux)
++    os=-linux-android
++    basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown
++    ;;
+   *)
+     basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+     if [ $basic_machine != $1 ]
+@@ -147,10 +153,13 @@ case $os in
+ 	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+ 	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+ 	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
+-	-apple | -axis | -knuth | -cray)
++	-apple | -axis | -knuth | -cray | -microblaze*)
+ 		os=
+ 		basic_machine=$1
+ 		;;
++	-bluegene*)
++		os=-cnk
++		;;
+ 	-sim | -cisco | -oki | -wec | -winbond)
+ 		os=
+ 		basic_machine=$1
+@@ -165,10 +174,10 @@ case $os in
+ 		os=-chorusos
+ 		basic_machine=$1
+ 		;;
+- 	-chorusrdb)
+- 		os=-chorusrdb
++	-chorusrdb)
++		os=-chorusrdb
+ 		basic_machine=$1
+- 		;;
++		;;
+ 	-hiux*)
+ 		os=-hiuxwe2
+ 		;;
+@@ -213,6 +222,12 @@ case $os in
+ 	-isc*)
+ 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ 		;;
++	-lynx*178)
++		os=-lynxos178
++		;;
++	-lynx*5)
++		os=-lynxos5
++		;;
+ 	-lynx*)
+ 		os=-lynxos
+ 		;;
+@@ -237,23 +252,34 @@ case $basic_machine in
+ 	# Some are omitted here because they have special meanings below.
+ 	1750a | 580 \
+ 	| a29k \
++	| aarch64 | aarch64_be \
+ 	| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
+ 	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
+ 	| am33_2.0 \
+-	| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \
++	| arc \
++	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
++	| avr | avr32 \
++	| be32 | be64 \
+ 	| bfin \
+ 	| c4x | clipper \
+ 	| d10v | d30v | dlx | dsp16xx \
+-	| fr30 | frv \
++	| epiphany \
++	| fido | fr30 | frv \
+ 	| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
++	| hexagon \
+ 	| i370 | i860 | i960 | ia64 \
+ 	| ip2k | iq2000 \
+-	| m32r | m32rle | m68000 | m68k | m88k | maxq | mb | microblaze | mcore \
++	| le32 | le64 \
++	| lm32 \
++	| m32c | m32r | m32rle | m68000 | m68k | m88k \
++	| maxq | mb | microblaze | microblazeel | mcore | mep | metag \
+ 	| mips | mipsbe | mipseb | mipsel | mipsle \
+ 	| mips16 \
+ 	| mips64 | mips64el \
+-	| mips64vr | mips64vrel \
++	| mips64octeon | mips64octeonel \
+ 	| mips64orion | mips64orionel \
++	| mips64r5900 | mips64r5900el \
++	| mips64vr | mips64vrel \
+ 	| mips64vr4100 | mips64vr4100el \
+ 	| mips64vr4300 | mips64vr4300el \
+ 	| mips64vr5000 | mips64vr5000el \
+@@ -266,31 +292,42 @@ case $basic_machine in
+ 	| mipsisa64sr71k | mipsisa64sr71kel \
+ 	| mipstx39 | mipstx39el \
+ 	| mn10200 | mn10300 \
++	| moxie \
+ 	| mt \
+ 	| msp430 \
++	| nds32 | nds32le | nds32be \
+ 	| nios | nios2 \
+ 	| ns16k | ns32k \
++	| open8 \
+ 	| or32 \
+ 	| pdp10 | pdp11 | pj | pjl \
+-	| powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \
++	| powerpc | powerpc64 | powerpc64le | powerpcle \
+ 	| pyramid \
+-	| sh | sh[1234] | sh[24]a | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \
++	| rl78 | rx \
++	| score \
++	| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
+ 	| sh64 | sh64le \
+-	| sparc | sparc64 | sparc64b | sparc86x | sparclet | sparclite \
+-	| sparcv8 | sparcv9 | sparcv9b \
+-	| strongarm \
+-	| tahoe | thumb | tic4x | tic80 | tron \
+-	| v850 | v850e \
++	| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
++	| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
++	| spu \
++	| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
++	| ubicom32 \
++	| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
+ 	| we32k \
+-	| x86 | xscale | xscalee[bl] | xstormy16 | xtensa \
+-	| z8k)
++	| x86 | xc16x | xstormy16 | xtensa \
++	| z8k | z80)
+ 		basic_machine=$basic_machine-unknown
+ 		;;
+-	m32c)
+-		basic_machine=$basic_machine-unknown
++	c54x)
++		basic_machine=tic54x-unknown
++		;;
++	c55x)
++		basic_machine=tic55x-unknown
+ 		;;
+-	m6811 | m68hc11 | m6812 | m68hc12)
+-		# Motorola 68HC11/12.
++	c6x)
++		basic_machine=tic6x-unknown
++		;;
++	m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip)
+ 		basic_machine=$basic_machine-unknown
+ 		os=-none
+ 		;;
+@@ -300,6 +337,21 @@ case $basic_machine in
+ 		basic_machine=mt-unknown
+ 		;;
+ 
++	strongarm | thumb | xscale)
++		basic_machine=arm-unknown
++		;;
++	xgate)
++		basic_machine=$basic_machine-unknown
++		os=-none
++		;;
++	xscaleeb)
++		basic_machine=armeb-unknown
++		;;
++
++	xscaleel)
++		basic_machine=armel-unknown
++		;;
++
+ 	# We use `pc' rather than `unknown'
+ 	# because (1) that's what they normally are, and
+ 	# (2) the word "unknown" tends to confuse beginning users.
+@@ -314,29 +366,37 @@ case $basic_machine in
+ 	# Recognize the basic CPU types with company name.
+ 	580-* \
+ 	| a29k-* \
++	| aarch64-* | aarch64_be-* \
+ 	| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
+ 	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
+ 	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
+ 	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
+-	| avr-* \
++	| avr-* | avr32-* \
++	| be32-* | be64-* \
+ 	| bfin-* | bs2000-* \
+-	| c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \
++	| c[123]* | c30-* | [cjt]90-* | c4x-* \
+ 	| clipper-* | craynv-* | cydra-* \
+ 	| d10v-* | d30v-* | dlx-* \
+ 	| elxsi-* \
+-	| f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \
++	| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
+ 	| h8300-* | h8500-* \
+ 	| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
++	| hexagon-* \
+ 	| i*86-* | i860-* | i960-* | ia64-* \
+ 	| ip2k-* | iq2000-* \
+-	| m32r-* | m32rle-* \
++	| le32-* | le64-* \
++	| lm32-* \
++	| m32c-* | m32r-* | m32rle-* \
+ 	| m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
+-	| m88110-* | m88k-* | maxq-* | mcore-* \
++	| m88110-* | m88k-* | maxq-* | mcore-* | metag-* \
++	| microblaze-* | microblazeel-* \
+ 	| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
+ 	| mips16-* \
+ 	| mips64-* | mips64el-* \
+-	| mips64vr-* | mips64vrel-* \
++	| mips64octeon-* | mips64octeonel-* \
+ 	| mips64orion-* | mips64orionel-* \
++	| mips64r5900-* | mips64r5900el-* \
++	| mips64vr-* | mips64vrel-* \
+ 	| mips64vr4100-* | mips64vr4100el-* \
+ 	| mips64vr4300-* | mips64vr4300el-* \
+ 	| mips64vr5000-* | mips64vr5000el-* \
+@@ -351,29 +411,36 @@ case $basic_machine in
+ 	| mmix-* \
+ 	| mt-* \
+ 	| msp430-* \
++	| nds32-* | nds32le-* | nds32be-* \
+ 	| nios-* | nios2-* \
+ 	| none-* | np1-* | ns16k-* | ns32k-* \
++	| open8-* \
+ 	| orion-* \
+ 	| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
+-	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \
++	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
+ 	| pyramid-* \
+-	| romp-* | rs6000-* \
+-	| sh-* | sh[1234]-* | sh[24]a-* | sh[23]e-* | sh[34]eb-* | shbe-* \
++	| rl78-* | romp-* | rs6000-* | rx-* \
++	| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
+ 	| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
+-	| sparc-* | sparc64-* | sparc64b-* | sparc86x-* | sparclet-* \
++	| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
+ 	| sparclite-* \
+-	| sparcv8-* | sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \
+-	| tahoe-* | thumb-* \
++	| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \
++	| tahoe-* \
+ 	| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
++	| tile*-* \
+ 	| tron-* \
+-	| v850-* | v850e-* | vax-* \
++	| ubicom32-* \
++	| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
++	| vax-* \
+ 	| we32k-* \
+-	| x86-* | x86_64-* | xps100-* | xscale-* | xscalee[bl]-* \
+-	| xstormy16-* | xtensa-* \
++	| x86-* | x86_64-* | xc16x-* | xps100-* \
++	| xstormy16-* | xtensa*-* \
+ 	| ymp-* \
+-	| z8k-*)
++	| z8k-* | z80-*)
+ 		;;
+-	m32c-*)
++	# Recognize the basic CPU types without company name, with glob match.
++	xtensa*)
++		basic_machine=$basic_machine-unknown
+ 		;;
+ 	# Recognize the various machine names and aliases which stand
+ 	# for a CPU type and a company and sometimes even an OS.
+@@ -391,7 +458,7 @@ case $basic_machine in
+ 		basic_machine=a29k-amd
+ 		os=-udi
+ 		;;
+-    	abacus)
++	abacus)
+ 		basic_machine=abacus-unknown
+ 		;;
+ 	adobe68k)
+@@ -437,6 +504,10 @@ case $basic_machine in
+ 		basic_machine=m68k-apollo
+ 		os=-bsd
+ 		;;
++	aros)
++		basic_machine=i386-pc
++		os=-aros
++		;;
+ 	aux)
+ 		basic_machine=m68k-apple
+ 		os=-aux
+@@ -445,10 +516,35 @@ case $basic_machine in
+ 		basic_machine=ns32k-sequent
+ 		os=-dynix
+ 		;;
++	blackfin)
++		basic_machine=bfin-unknown
++		os=-linux
++		;;
++	blackfin-*)
++		basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'`
++		os=-linux
++		;;
++	bluegene*)
++		basic_machine=powerpc-ibm
++		os=-cnk
++		;;
++	c54x-*)
++		basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'`
++		;;
++	c55x-*)
++		basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'`
++		;;
++	c6x-*)
++		basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'`
++		;;
+ 	c90)
+ 		basic_machine=c90-cray
+ 		os=-unicos
+ 		;;
++	cegcc)
++		basic_machine=arm-unknown
++		os=-cegcc
++		;;
+ 	convex-c1)
+ 		basic_machine=c1-convex
+ 		os=-bsd
+@@ -477,8 +573,8 @@ case $basic_machine in
+ 		basic_machine=craynv-cray
+ 		os=-unicosmp
+ 		;;
+-	cr16c)
+-		basic_machine=cr16c-unknown
++	cr16 | cr16-*)
++		basic_machine=cr16-unknown
+ 		os=-elf
+ 		;;
+ 	crds | unos)
+@@ -516,6 +612,10 @@ case $basic_machine in
+ 		basic_machine=m88k-motorola
+ 		os=-sysv3
+ 		;;
++	dicos)
++		basic_machine=i686-pc
++		os=-dicos
++		;;
+ 	djgpp)
+ 		basic_machine=i586-pc
+ 		os=-msdosdjgpp
+@@ -631,7 +731,6 @@ case $basic_machine in
+ 	i370-ibm* | ibm*)
+ 		basic_machine=i370-ibm
+ 		;;
+-# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+ 	i*86v32)
+ 		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+ 		os=-sysv32
+@@ -670,6 +769,14 @@ case $basic_machine in
+ 		basic_machine=m68k-isi
+ 		os=-sysv
+ 		;;
++	m68knommu)
++		basic_machine=m68k-unknown
++		os=-linux
++		;;
++	m68knommu-*)
++		basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'`
++		os=-linux
++		;;
+ 	m88k-omron*)
+ 		basic_machine=m88k-omron
+ 		;;
+@@ -681,10 +788,21 @@ case $basic_machine in
+ 		basic_machine=ns32k-utek
+ 		os=-sysv
+ 		;;
++	microblaze*)
++		basic_machine=microblaze-xilinx
++		;;
++	mingw64)
++		basic_machine=x86_64-pc
++		os=-mingw64
++		;;
+ 	mingw32)
+ 		basic_machine=i386-pc
+ 		os=-mingw32
+ 		;;
++	mingw32ce)
++		basic_machine=arm-unknown
++		os=-mingw32ce
++		;;
+ 	miniframe)
+ 		basic_machine=m68000-convergent
+ 		;;
+@@ -713,10 +831,18 @@ case $basic_machine in
+ 	ms1-*)
+ 		basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'`
+ 		;;
++	msys)
++		basic_machine=i386-pc
++		os=-msys
++		;;
+ 	mvs)
+ 		basic_machine=i370-ibm
+ 		os=-mvs
+ 		;;
++	nacl)
++		basic_machine=le32-unknown
++		os=-nacl
++		;;
+ 	ncr3000)
+ 		basic_machine=i486-ncr
+ 		os=-sysv4
+@@ -781,6 +907,12 @@ case $basic_machine in
+ 	np1)
+ 		basic_machine=np1-gould
+ 		;;
++	neo-tandem)
++		basic_machine=neo-tandem
++		;;
++	nse-tandem)
++		basic_machine=nse-tandem
++		;;
+ 	nsr-tandem)
+ 		basic_machine=nsr-tandem
+ 		;;
+@@ -811,6 +943,14 @@ case $basic_machine in
+ 		basic_machine=i860-intel
+ 		os=-osf
+ 		;;
++	parisc)
++		basic_machine=hppa-unknown
++		os=-linux
++		;;
++	parisc-*)
++		basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'`
++		os=-linux
++		;;
+ 	pbd)
+ 		basic_machine=sparc-tti
+ 		;;
+@@ -855,9 +995,10 @@ case $basic_machine in
+ 		;;
+ 	power)	basic_machine=power-ibm
+ 		;;
+-	ppc)	basic_machine=powerpc-unknown
++	ppc | ppcbe)	basic_machine=powerpc-unknown
+ 		;;
+-	ppc-*)	basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
++	ppc-* | ppcbe-*)
++		basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+ 		;;
+ 	ppcle | powerpclittle | ppc-le | powerpc-little)
+ 		basic_machine=powerpcle-unknown
+@@ -882,7 +1023,11 @@ case $basic_machine in
+ 		basic_machine=i586-unknown
+ 		os=-pw32
+ 		;;
+-	rdos)
++	rdos | rdos64)
++		basic_machine=x86_64-pc
++		os=-rdos
++		;;
++	rdos32)
+ 		basic_machine=i386-pc
+ 		os=-rdos
+ 		;;
+@@ -912,6 +1057,10 @@ case $basic_machine in
+ 	sb1el)
+ 		basic_machine=mipsisa64sb1el-unknown
+ 		;;
++	sde)
++		basic_machine=mipsisa32-sde
++		os=-elf
++		;;
+ 	sei)
+ 		basic_machine=mips-sei
+ 		os=-seiux
+@@ -923,6 +1072,9 @@ case $basic_machine in
+ 		basic_machine=sh-hitachi
+ 		os=-hms
+ 		;;
++	sh5el)
++		basic_machine=sh5le-unknown
++		;;
+ 	sh64)
+ 		basic_machine=sh64-unknown
+ 		;;
+@@ -944,6 +1096,9 @@ case $basic_machine in
+ 		basic_machine=i860-stratus
+ 		os=-sysv4
+ 		;;
++	strongarm-* | thumb-*)
++		basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'`
++		;;
+ 	sun2)
+ 		basic_machine=m68000-sun
+ 		;;
+@@ -1000,17 +1155,9 @@ case $basic_machine in
+ 		basic_machine=t90-cray
+ 		os=-unicos
+ 		;;
+-	tic54x | c54x*)
+-		basic_machine=tic54x-unknown
+-		os=-coff
+-		;;
+-	tic55x | c55x*)
+-		basic_machine=tic55x-unknown
+-		os=-coff
+-		;;
+-	tic6x | c6x*)
+-		basic_machine=tic6x-unknown
+-		os=-coff
++	tile*)
++		basic_machine=$basic_machine-unknown
++		os=-linux-gnu
+ 		;;
+ 	tx39)
+ 		basic_machine=mipstx39-unknown
+@@ -1079,6 +1226,9 @@ case $basic_machine in
+ 	xps | xps100)
+ 		basic_machine=xps100-honeywell
+ 		;;
++	xscale-* | xscalee[bl]-*)
++		basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'`
++		;;
+ 	ymp)
+ 		basic_machine=ymp-cray
+ 		os=-unicos
+@@ -1087,6 +1237,10 @@ case $basic_machine in
+ 		basic_machine=z8k-unknown
+ 		os=-sim
+ 		;;
++	z80-*-coff)
++		basic_machine=z80-unknown
++		os=-sim
++		;;
+ 	none)
+ 		basic_machine=none-none
+ 		os=-none
+@@ -1125,10 +1279,10 @@ case $basic_machine in
+ 	we32k)
+ 		basic_machine=we32k-att
+ 		;;
+-	sh[1234] | sh[24]a | sh[34]eb | sh[1234]le | sh[23]ele)
++	sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele)
+ 		basic_machine=sh-unknown
+ 		;;
+-	sparc | sparcv8 | sparcv9 | sparcv9b)
++	sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v)
+ 		basic_machine=sparc-sun
+ 		;;
+ 	cydra)
+@@ -1172,9 +1326,12 @@ esac
+ if [ x"$os" != x"" ]
+ then
+ case $os in
+-        # First match some system type aliases
+-        # that might get confused with valid system types.
++	# First match some system type aliases
++	# that might get confused with valid system types.
+ 	# -solaris* is a basic system type, with this one exception.
++	-auroraux)
++		os=-auroraux
++		;;
+ 	-solaris1 | -solaris1.*)
+ 		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+ 		;;
+@@ -1195,21 +1352,23 @@ case $os in
+ 	# Each alternative MUST END IN A *, to match a version number.
+ 	# -sysv* is not here because it comes later, after sysvr4.
+ 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
+-	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
++	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
++	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
++	      | -sym* | -kopensolaris* \
+ 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
+-	      | -aos* \
++	      | -aos* | -aros* \
+ 	      | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
+ 	      | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
+ 	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
+-	      | -openbsd* | -solidbsd* \
++	      | -bitrig* | -openbsd* | -solidbsd* \
+ 	      | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
+ 	      | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
+ 	      | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
+ 	      | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
+-	      | -chorusos* | -chorusrdb* \
+-	      | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
+-	      | -mingw32* | -linux-gnu* | -linux-newlib* | -linux-uclibc* \
++	      | -chorusos* | -chorusrdb* | -cegcc* \
++	      | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
++	      | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
++	      | -linux-newlib* | -linux-musl* | -linux-uclibc* \
+ 	      | -uxpv* | -beos* | -mpeix* | -udk* \
+ 	      | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
+ 	      | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
+@@ -1217,7 +1376,7 @@ case $os in
+ 	      | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
+ 	      | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
+ 	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
+-	      | -skyos* | -haiku* | -rdos*)
++	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*)
+ 	# Remember, each alternative MUST END IN *, to match a version number.
+ 		;;
+ 	-qnx*)
+@@ -1256,7 +1415,7 @@ case $os in
+ 	-opened*)
+ 		os=-openedition
+ 		;;
+-        -os400*)
++	-os400*)
+ 		os=-os400
+ 		;;
+ 	-wince*)
+@@ -1305,7 +1464,7 @@ case $os in
+ 	-sinix*)
+ 		os=-sysv4
+ 		;;
+-        -tpf*)
++	-tpf*)
+ 		os=-tpf
+ 		;;
+ 	-triton*)
+@@ -1347,6 +1506,11 @@ case $os in
+ 	-zvmoe)
+ 		os=-zvmoe
+ 		;;
++	-dicos*)
++		os=-dicos
++		;;
++	-nacl*)
++		;;
+ 	-none)
+ 		;;
+ 	*)
+@@ -1369,6 +1533,12 @@ else
+ # system, and we'll never get to this point.
+ 
+ case $basic_machine in
++	score-*)
++		os=-elf
++		;;
++	spu-*)
++		os=-elf
++		;;
+ 	*-acorn)
+ 		os=-riscix1.2
+ 		;;
+@@ -1378,9 +1548,21 @@ case $basic_machine in
+ 	arm*-semi)
+ 		os=-aout
+ 		;;
+-    c4x-* | tic4x-*)
+-        os=-coff
+-        ;;
++	c4x-* | tic4x-*)
++		os=-coff
++		;;
++	hexagon-*)
++		os=-elf
++		;;
++	tic54x-*)
++		os=-coff
++		;;
++	tic55x-*)
++		os=-coff
++		;;
++	tic6x-*)
++		os=-coff
++		;;
+ 	# This must come before the *-dec entry.
+ 	pdp10-*)
+ 		os=-tops20
+@@ -1399,13 +1581,13 @@ case $basic_machine in
+ 		;;
+ 	m68000-sun)
+ 		os=-sunos3
+-		# This also exists in the configure program, but was not the
+-		# default.
+-		# os=-sunos4
+ 		;;
+ 	m68*-cisco)
+ 		os=-aout
+ 		;;
++	mep-*)
++		os=-elf
++		;;
+ 	mips*-cisco)
+ 		os=-elf
+ 		;;
+@@ -1430,7 +1612,7 @@ case $basic_machine in
+ 	*-ibm)
+ 		os=-aix
+ 		;;
+-    	*-knuth)
++	*-knuth)
+ 		os=-mmixware
+ 		;;
+ 	*-wec)
+@@ -1535,7 +1717,7 @@ case $basic_machine in
+ 			-sunos*)
+ 				vendor=sun
+ 				;;
+-			-aix*)
++			-cnk*|-aix*)
+ 				vendor=ibm
+ 				;;
+ 			-beos*)
+diff --git a/configure b/configure
+index a9e9814..7fd6318 100755
+--- a/configure
++++ b/configure
+@@ -1,4 +1,5 @@
+ #! /bin/sh
++set -- --host=arm-linux-androideabi
+ # Guess values for system-dependent variables and create Makefiles.
+ # Generated by GNU Autoconf 2.69 for Haskell network package 2.3.0.14.
+ #
+diff --git a/include/HsNetworkConfig.h b/include/HsNetworkConfig.h
+index c6e704d..4edc892 100644
+--- a/include/HsNetworkConfig.h
++++ b/include/HsNetworkConfig.h
+@@ -8,7 +8,7 @@
+ #define HAVE_ARPA_INET_H 1
+ 
+ /* Define to 1 if you have a BSDish sendfile(2) implementation. */
+-#define HAVE_BSD_SENDFILE 1
++/* #undef HAVE_BSD_SENDFILE */
+ 
+ /* Define to 1 if you have the declaration of `AI_ADDRCONFIG', and to 0 if you
+    don't. */
+@@ -55,7 +55,7 @@
+ #define HAVE_LIMITS_H 1
+ 
+ /* Define to 1 if you have a Linux sendfile(2) implementation. */
+-/* #undef HAVE_LINUX_SENDFILE */
++#define HAVE_LINUX_SENDFILE 1
+ 
+ /* Define to 1 if you have the  header file. */
+ #define HAVE_MEMORY_H 1
+@@ -91,10 +91,10 @@
+ #define HAVE_STRUCT_MSGHDR_MSG_CONTROL 1
+ 
+ /* Define to 1 if `sa_len' is a member of `struct sockaddr'. */
+-#define HAVE_STRUCT_SOCKADDR_SA_LEN 1
++/* #undef HAVE_STRUCT_SOCKADDR_SA_LEN */
+ 
+ /* Define to 1 if you have both SO_PEERCRED and struct ucred. */
+-/* #undef HAVE_STRUCT_UCRED */
++#define HAVE_STRUCT_UCRED 1
+ 
+ /* Define to 1 if you have the `symlink' function. */
+ #define HAVE_SYMLINK 1
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/network-conduit-0.6.2.2_0001-NoDelay-does-not-work-on-Android.patch b/standalone/android/haskell-patches/network-conduit-0.6.2.2_0001-NoDelay-does-not-work-on-Android.patch
new file mode 100644
index 0000000000..35bafa774f
--- /dev/null
+++ b/standalone/android/haskell-patches/network-conduit-0.6.2.2_0001-NoDelay-does-not-work-on-Android.patch
@@ -0,0 +1,43 @@
+From 3e05f3a3bf886c302fb6d6caa7ee92cf9736b6ad Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:33:45 -0400
+Subject: [PATCH] NoDelay does not work on Android
+
+(I think the other change is no-op)
+---
+ Data/Conduit/Network/Utils.hs |    6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/Data/Conduit/Network/Utils.hs b/Data/Conduit/Network/Utils.hs
+index 32a7286..01ff84e 100644
+--- a/Data/Conduit/Network/Utils.hs
++++ b/Data/Conduit/Network/Utils.hs
+@@ -6,14 +6,14 @@ module Data.Conduit.Network.Utils
+     , getSocket
+     ) where
+ 
+-import Network.Socket (AddrInfo, Socket, SocketType)
++import Network.Socket (Socket, SocketType)
+ import qualified Network.Socket as NS
+ import Data.String (IsString (fromString))
+ import Control.Exception (bracketOnError, IOException)
+ import qualified Control.Exception as E
+ 
+ -- | Attempt to connect to the given host/port using given @SocketType@.
+-getSocket :: String -> Int -> SocketType -> IO (Socket, AddrInfo)
++getSocket :: String -> Int -> SocketType -> IO (Socket, NS.AddrInfo)
+ getSocket host' port' sockettype = do
+     let hints = NS.defaultHints {
+                           NS.addrFlags = [NS.AI_ADDRCONFIG]
+@@ -93,7 +93,7 @@ bindPort p s sockettype = do
+         sockOpts =
+             case sockettype of
+                 NS.Datagram -> [(NS.ReuseAddr,1)]
+-                _           -> [(NS.NoDelay,1), (NS.ReuseAddr,1)]
++                _           -> [(NS.ReuseAddr,1)] -- Android seems to not have NoDelay
+ 
+         theBody addr =
+           bracketOnError
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/persistent-1.1.5.1_0001-disable-TH.patch b/standalone/android/haskell-patches/persistent-1.1.5.1_0001-disable-TH.patch
new file mode 100644
index 0000000000..38cecc5c72
--- /dev/null
+++ b/standalone/android/haskell-patches/persistent-1.1.5.1_0001-disable-TH.patch
@@ -0,0 +1,71 @@
+From 8fddef803ee9191ca15363283b7e4d5af4c70f3a Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:34:10 -0400
+Subject: [PATCH] disable TH
+
+---
+ Database/Persist/GenericSql/Internal.hs |    6 +-----
+ Database/Persist/GenericSql/Raw.hs      |    5 ++---
+ 2 files changed, 3 insertions(+), 8 deletions(-)
+
+diff --git a/Database/Persist/GenericSql/Internal.hs b/Database/Persist/GenericSql/Internal.hs
+index f109887..5273398 100644
+--- a/Database/Persist/GenericSql/Internal.hs
++++ b/Database/Persist/GenericSql/Internal.hs
+@@ -14,7 +14,6 @@ module Database.Persist.GenericSql.Internal
+     , createSqlPool
+     , mkColumns
+     , Column (..)
+-    , logSQL
+     , InsertSqlResult (..)
+     ) where
+ 
+@@ -33,7 +32,7 @@ import Data.Monoid (Monoid, mappend, mconcat)
+ import Database.Persist.EntityDef
+ import qualified Data.Conduit as C
+ import Language.Haskell.TH.Syntax (Q, Exp)
+-import Control.Monad.Logger (logDebugS)
++
+ import Data.Maybe (mapMaybe, listToMaybe)
+ import Data.Int (Int64)
+ 
+@@ -197,6 +196,3 @@ tableColumn t s = go $ entityColumns t
+         | x == s = ColumnDef x y z
+         | otherwise = go rest
+ -}
+-
+-logSQL :: Q Exp
+-logSQL = [|\sql_foo params_foo -> $logDebugS (T.pack "SQL") $ T.pack $ show (sql_foo :: Text) ++ " " ++ show (params_foo :: [PersistValue])|]
+diff --git a/Database/Persist/GenericSql/Raw.hs b/Database/Persist/GenericSql/Raw.hs
+index e4bf9f4..3da8fa0 100644
+--- a/Database/Persist/GenericSql/Raw.hs
++++ b/Database/Persist/GenericSql/Raw.hs
+@@ -26,7 +26,6 @@ import Database.Persist.GenericSql.Internal hiding (execute, withStmt)
+ import Database.Persist.Store (PersistValue)
+ import Data.IORef
+ import Control.Monad.IO.Class
+-import Control.Monad.Logger (logDebugS)
+ import Control.Monad.Trans.Reader
+ import qualified Data.Map as Map
+ import Control.Applicative (Applicative)
+@@ -134,7 +133,7 @@ withStmt :: (MonadSqlPersist m, MonadResource m)
+          -> [PersistValue]
+          -> Source m [PersistValue]
+ withStmt sql vals = do
+-    lift $ $logDebugS (pack "SQL") $ pack $ show sql ++ " " ++ show vals
++    -- lift $  pack $ show sql ++ " " ++ show vals
+     conn <- lift askSqlConn
+     bracketP
+         (getStmt' conn sql)
+@@ -146,7 +145,7 @@ execute x y = liftM (const ()) $ executeCount x y
+ 
+ executeCount :: MonadSqlPersist m => Text -> [PersistValue] -> m Int64
+ executeCount sql vals = do
+-    $logDebugS (pack "SQL") $ pack $ show sql ++ " " ++ show vals
++    -- pack $ show sql ++ " " ++ show vals
+     stmt <- getStmt sql
+     res <- liftIO $ I.execute stmt vals
+     liftIO $ reset stmt
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/primitive-0.5.0.1_0001-disable-i386-opt-stuff-to-allow-cross-compilation.patch b/standalone/android/haskell-patches/primitive-0.5.0.1_0001-disable-i386-opt-stuff-to-allow-cross-compilation.patch
new file mode 100644
index 0000000000..1bd9268718
--- /dev/null
+++ b/standalone/android/haskell-patches/primitive-0.5.0.1_0001-disable-i386-opt-stuff-to-allow-cross-compilation.patch
@@ -0,0 +1,24 @@
+From 5cb5c3dabb213f809b8328b0b4049f7c754e9c77 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:34:32 -0400
+Subject: [PATCH] disable i386 opt stuff to allow cross-compilation
+
+---
+ primitive.cabal |    3 ---
+ 1 file changed, 3 deletions(-)
+
+diff --git a/primitive.cabal b/primitive.cabal
+index 8c4328a..9a6093f 100644
+--- a/primitive.cabal
++++ b/primitive.cabal
+@@ -51,7 +51,4 @@ Library
+   includes: primitive-memops.h
+   c-sources: cbits/primitive-memops.c
+   cc-options: -O3 -ftree-vectorize -fomit-frame-pointer
+-  if arch(i386) || arch(x86_64) {
+-    cc-options: -msse2
+-  }
+ 
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/resourcet-0.4.4_0001-hack-to-build-with-hacked-up-lifted-base-which-is-cu.patch b/standalone/android/haskell-patches/resourcet-0.4.4_0001-hack-to-build-with-hacked-up-lifted-base-which-is-cu.patch
new file mode 100644
index 0000000000..bcf3439fac
--- /dev/null
+++ b/standalone/android/haskell-patches/resourcet-0.4.4_0001-hack-to-build-with-hacked-up-lifted-base-which-is-cu.patch
@@ -0,0 +1,44 @@
+From c10ab80793a21dce0c7516725e1ca3b36a87aa25 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:35:08 -0400
+Subject: [PATCH] hack to build with hacked up lifted-base, which is currently
+ lacking a mask
+
+---
+ Control/Monad/Trans/Resource.hs |    6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/Control/Monad/Trans/Resource.hs b/Control/Monad/Trans/Resource.hs
+index d209dd8..61ab349 100644
+--- a/Control/Monad/Trans/Resource.hs
++++ b/Control/Monad/Trans/Resource.hs
+@@ -5,7 +5,7 @@
+ {-# LANGUAGE TypeFamilies #-}
+ {-# LANGUAGE RankNTypes #-}
+ {-# LANGUAGE CPP #-}
+-{-# LANGUAGE DeriveDataTypeable #-}
++{-# LANGUAGE DeriveDataTypeable, ImpredicativeTypes #-}
+ #if __GLASGOW_HASKELL__ >= 704
+ {-# LANGUAGE ConstraintKinds #-}
+ #endif
+@@ -554,7 +554,7 @@ GOX(Monoid w, Strict.WriterT w)
+ --
+ -- Since 0.3.0
+ resourceForkIO :: MonadBaseControl IO m => ResourceT m () -> ResourceT m ThreadId
+-resourceForkIO (ResourceT f) = ResourceT $ \r -> L.mask $ \restore ->
++resourceForkIO (ResourceT f) = ResourceT $ \r ->
+     -- We need to make sure the counter is incremented before this call
+     -- returns. Otherwise, the parent thread may call runResourceT before
+     -- the child thread increments, and all resources will be freed
+@@ -565,7 +565,7 @@ resourceForkIO (ResourceT f) = ResourceT $ \r -> L.mask $ \restore ->
+         (liftBaseDiscard forkIO $ bracket_
+             (return ())
+             (stateCleanup r)
+-            (restore $ f r))
++            (return ()))
+ 
+ -- | A @Monad@ based on some monad which allows running of some 'IO' actions,
+ -- via unsafe calls. This applies to 'IO' and 'ST', for instance.
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/shakespeare-1.0.3_0001-remove-TH.patch b/standalone/android/haskell-patches/shakespeare-1.0.3_0001-remove-TH.patch
new file mode 100644
index 0000000000..37676dfa70
--- /dev/null
+++ b/standalone/android/haskell-patches/shakespeare-1.0.3_0001-remove-TH.patch
@@ -0,0 +1,194 @@
+From 2e6721d571148cb77fb8c906042f6fa61e660999 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:35:41 -0400
+Subject: [PATCH] remove TH
+
+---
+ Text/Shakespeare.hs      |  109 ----------------------------------------------
+ Text/Shakespeare/Base.hs |   28 ------------
+ 2 files changed, 137 deletions(-)
+
+diff --git a/Text/Shakespeare.hs b/Text/Shakespeare.hs
+index e774e65..d300951 100644
+--- a/Text/Shakespeare.hs
++++ b/Text/Shakespeare.hs
+@@ -12,11 +12,7 @@ module Text.Shakespeare
+     , WrapInsertion (..)
+     , PreConversion (..)
+     , defaultShakespeareSettings
+-    , shakespeare
+-    , shakespeareFile
+-    , shakespeareFileReload
+     -- * low-level
+-    , shakespeareFromString
+     , shakespeareUsedIdentifiers
+     , RenderUrl
+     , VarType
+@@ -133,39 +129,6 @@ defaultShakespeareSettings = ShakespeareSettings {
+   , modifyFinalValue = Nothing
+ }
+ 
+-instance Lift PreConvert where
+-    lift (PreConvert convert ignore comment wrapInsertion) =
+-        [|PreConvert $(lift convert) $(lift ignore) $(lift comment) $(lift wrapInsertion)|]
+-
+-instance Lift WrapInsertion where
+-    lift (WrapInsertion indent sb sep sc e ab ac) =
+-        [|WrapInsertion $(lift indent) $(lift sb) $(lift sep) $(lift sc) $(lift e) $(lift ab) $(lift ac)|]
+-
+-instance Lift PreConversion where
+-    lift (ReadProcess command args) =
+-        [|ReadProcess $(lift command) $(lift args)|]
+-    lift Id = [|Id|]
+-
+-instance Lift ShakespeareSettings where
+-    lift (ShakespeareSettings x1 x2 x3 x4 x5 x6 x7 x8 x9) =
+-        [|ShakespeareSettings
+-            $(lift x1) $(lift x2) $(lift x3)
+-            $(liftExp x4) $(liftExp x5) $(liftExp x6) $(lift x7) $(lift x8) $(liftMExp x9)|]
+-      where
+-        liftExp (VarE n) = [|VarE $(liftName n)|]
+-        liftExp (ConE n) = [|ConE $(liftName n)|]
+-        liftExp _ = error "liftExp only supports VarE and ConE"
+-        liftMExp Nothing = [|Nothing|]
+-        liftMExp (Just e) = [|Just|] `appE` liftExp e
+-        liftName (Name (OccName a) b) = [|Name (OccName $(lift a)) $(liftFlavour b)|]
+-        liftFlavour NameS = [|NameS|]
+-        liftFlavour (NameQ (ModName a)) = [|NameQ (ModName $(lift a))|]
+-        liftFlavour (NameU _) = error "liftFlavour NameU" -- [|NameU $(lift $ fromIntegral a)|]
+-        liftFlavour (NameL _) = error "liftFlavour NameL" -- [|NameU $(lift $ fromIntegral a)|]
+-        liftFlavour (NameG ns (PkgName p) (ModName m)) = [|NameG $(liftNS ns) (PkgName $(lift p)) (ModName $(lift m))|]
+-        liftNS VarName = [|VarName|]
+-        liftNS DataName = [|DataName|]
+-
+ type QueryParameters = [(TS.Text, TS.Text)]
+ type RenderUrl url = (url -> QueryParameters -> TS.Text)
+ type Shakespeare url = RenderUrl url -> Builder
+@@ -300,54 +263,6 @@ pack' = TS.pack
+ {-# NOINLINE pack' #-}
+ #endif
+ 
+-contentsToShakespeare :: ShakespeareSettings -> [Content] -> Q Exp
+-contentsToShakespeare rs a = do
+-    r <- newName "_render"
+-    c <- mapM (contentToBuilder r) a
+-    compiledTemplate <- case c of
+-        -- Make sure we convert this mempty using toBuilder to pin down the
+-        -- type appropriately
+-        []  -> fmap (AppE $ wrap rs) [|mempty|]
+-        [x] -> return x
+-        _   -> do
+-              mc <- [|mconcat|]
+-              return $ mc `AppE` ListE c
+-    fmap (maybe id AppE $ modifyFinalValue rs) $
+-        if justVarInterpolation rs
+-            then return compiledTemplate
+-            else return $ LamE [VarP r] compiledTemplate
+-      where
+-        contentToBuilder :: Name -> Content -> Q Exp
+-        contentToBuilder _ (ContentRaw s') = do
+-            ts <- [|fromText . pack'|]
+-            return $ wrap rs `AppE` (ts `AppE` LitE (StringL s'))
+-        contentToBuilder _ (ContentVar d) =
+-            return $ wrap rs `AppE` (toBuilder rs `AppE` derefToExp [] d)
+-        contentToBuilder r (ContentUrl d) = do
+-            ts <- [|fromText|]
+-            return $ wrap rs `AppE` (ts `AppE` (VarE r `AppE` derefToExp [] d `AppE` ListE []))
+-        contentToBuilder r (ContentUrlParam d) = do
+-            ts <- [|fromText|]
+-            up <- [|\r' (u, p) -> r' u p|]
+-            return $ wrap rs `AppE` (ts `AppE` (up `AppE` VarE r `AppE` derefToExp [] d))
+-        contentToBuilder r (ContentMix d) =
+-            return $ derefToExp [] d `AppE` VarE r
+-
+-shakespeare :: ShakespeareSettings -> QuasiQuoter
+-shakespeare r = QuasiQuoter { quoteExp = shakespeareFromString r }
+-
+-shakespeareFromString :: ShakespeareSettings -> String -> Q Exp
+-shakespeareFromString r str = do
+-    s <- qRunIO $ preFilter r str
+-    contentsToShakespeare r $ contentFromString r s
+-
+-shakespeareFile :: ShakespeareSettings -> FilePath -> Q Exp
+-shakespeareFile r fp = do
+-#ifdef GHC_7_4
+-    qAddDependentFile fp
+-#endif
+-    readFileQ fp >>= shakespeareFromString r
+-
+ data VarType = VTPlain | VTUrl | VTUrlParam | VTMixin
+ 
+ getVars :: Content -> [(Deref, VarType)]
+@@ -367,30 +282,6 @@ data VarExp url = EPlain Builder
+ shakespeareUsedIdentifiers :: ShakespeareSettings -> String -> [(Deref, VarType)]
+ shakespeareUsedIdentifiers settings = concatMap getVars . contentFromString settings
+ 
+-shakespeareFileReload :: ShakespeareSettings -> FilePath -> Q Exp
+-shakespeareFileReload rs fp = do
+-    str <- readFileQ fp
+-    s <- qRunIO $ preFilter rs str
+-    let b = shakespeareUsedIdentifiers rs s
+-    c <- mapM vtToExp b
+-    rt <- [|shakespeareRuntime|]
+-    wrap' <- [|\x -> $(return $ wrap rs) . x|]
+-    r' <- lift rs
+-    return $ wrap' `AppE` (rt `AppE` r' `AppE` (LitE $ StringL fp) `AppE` ListE c)
+-  where
+-    vtToExp :: (Deref, VarType) -> Q Exp
+-    vtToExp (d, vt) = do
+-        d' <- lift d
+-        c' <- c vt
+-        return $ TupE [d', c' `AppE` derefToExp [] d]
+-      where
+-        c :: VarType -> Q Exp
+-        c VTPlain = [|EPlain . $(return $ toBuilder rs)|]
+-        c VTUrl = [|EUrl|]
+-        c VTUrlParam = [|EUrlParam|]
+-        c VTMixin = [|\x -> EMixin $ \r -> $(return $ unwrap rs) $ x r|]
+-
+-
+ shakespeareRuntime :: ShakespeareSettings -> FilePath -> [(Deref, VarExp url)] -> Shakespeare url
+ shakespeareRuntime rs fp cd render' = unsafePerformIO $ do
+     str <- readFileUtf8 fp
+diff --git a/Text/Shakespeare/Base.hs b/Text/Shakespeare/Base.hs
+index 7c96898..ef769b1 100644
+--- a/Text/Shakespeare/Base.hs
++++ b/Text/Shakespeare/Base.hs
+@@ -52,34 +52,6 @@ data Deref = DerefModulesIdent [String] Ident
+            | DerefTuple [Deref]
+     deriving (Show, Eq, Read, Data, Typeable, Ord)
+ 
+-instance Lift Ident where
+-    lift (Ident s) = [|Ident|] `appE` lift s
+-instance Lift Deref where
+-    lift (DerefModulesIdent v s) = do
+-        dl <- [|DerefModulesIdent|]
+-        v' <- lift v
+-        s' <- lift s
+-        return $ dl `AppE` v' `AppE` s'
+-    lift (DerefIdent s) = do
+-        dl <- [|DerefIdent|]
+-        s' <- lift s
+-        return $ dl `AppE` s'
+-    lift (DerefBranch x y) = do
+-        x' <- lift x
+-        y' <- lift y
+-        db <- [|DerefBranch|]
+-        return $ db `AppE` x' `AppE` y'
+-    lift (DerefIntegral i) = [|DerefIntegral|] `appE` lift i
+-    lift (DerefRational r) = do
+-        n <- lift $ numerator r
+-        d <- lift $ denominator r
+-        per <- [|(%) :: Int -> Int -> Ratio Int|]
+-        dr <- [|DerefRational|]
+-        return $ dr `AppE` InfixE (Just n) per (Just d)
+-    lift (DerefString s) = [|DerefString|] `appE` lift s
+-    lift (DerefList x) = [|DerefList $(lift x)|]
+-    lift (DerefTuple x) = [|DerefTuple $(lift x)|]
+-
+ derefParens, derefCurlyBrackets :: UserParser a Deref
+ derefParens        = between (char '(') (char ')') parseDeref
+ derefCurlyBrackets = between (char '{') (char '}') parseDeref
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/shakespeare-css-1.0.2_0001-remove-TH.patch b/standalone/android/haskell-patches/shakespeare-css-1.0.2_0001-remove-TH.patch
new file mode 100644
index 0000000000..3e26372938
--- /dev/null
+++ b/standalone/android/haskell-patches/shakespeare-css-1.0.2_0001-remove-TH.patch
@@ -0,0 +1,260 @@
+From cb77113314702175f066cd801dee5c38d3e26576 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:35:51 -0400
+Subject: [PATCH] remove TH
+
+---
+ Text/Cassius.hs   |   23 ---------------
+ Text/Css.hs       |   84 -----------------------------------------------------
+ Text/CssCommon.hs |    4 ---
+ Text/Lucius.hs    |   30 +------------------
+ 4 files changed, 1 insertion(+), 140 deletions(-)
+
+diff --git a/Text/Cassius.hs b/Text/Cassius.hs
+index ce05374..ae56b0a 100644
+--- a/Text/Cassius.hs
++++ b/Text/Cassius.hs
+@@ -13,10 +13,6 @@ module Text.Cassius
+     , renderCss
+     , renderCssUrl
+       -- * Parsing
+-    , cassius
+-    , cassiusFile
+-    , cassiusFileDebug
+-    , cassiusFileReload
+       -- * ToCss instances
+       -- ** Color
+     , Color (..)
+@@ -27,11 +23,8 @@ module Text.Cassius
+     , AbsoluteUnit (..)
+     , AbsoluteSize (..)
+     , absoluteSize
+-    , EmSize (..)
+-    , ExSize (..)
+     , PercentageSize (..)
+     , percentageSize
+-    , PixelSize (..)
+       -- * Internal
+     , cassiusUsedIdentifiers
+     ) where
+@@ -42,25 +35,9 @@ import Language.Haskell.TH.Quote (QuasiQuoter (..))
+ import Language.Haskell.TH.Syntax
+ import qualified Data.Text.Lazy as TL
+ import Text.CssCommon
+-import Text.Lucius (lucius)
+ import qualified Text.Lucius
+ import Text.IndentToBrace (i2b)
+ 
+-cassius :: QuasiQuoter
+-cassius = QuasiQuoter { quoteExp = quoteExp lucius . i2b }
+-
+-cassiusFile :: FilePath -> Q Exp
+-cassiusFile fp = do
+-#ifdef GHC_7_4
+-    qAddDependentFile fp
+-#endif
+-    contents <- fmap TL.unpack $ qRunIO $ readUtf8File fp
+-    quoteExp cassius contents
+-
+-cassiusFileDebug, cassiusFileReload :: FilePath -> Q Exp
+-cassiusFileDebug = cssFileDebug True [|Text.Lucius.parseTopLevels|] Text.Lucius.parseTopLevels
+-cassiusFileReload = cassiusFileDebug
+-
+ -- | Determine which identifiers are used by the given template, useful for
+ -- creating systems like yesod devel.
+ cassiusUsedIdentifiers :: String -> [(Deref, VarType)]
+diff --git a/Text/Css.hs b/Text/Css.hs
+index 8e6fc09..401a166 100644
+--- a/Text/Css.hs
++++ b/Text/Css.hs
+@@ -108,19 +108,6 @@ cssUsedIdentifiers toi2b parseBlocks s' =
+         (scope, rest') = go rest
+     go' (k, v) = k ++ v
+ 
+-cssFileDebug :: Bool -- ^ perform the indent-to-brace conversion
+-             -> Q Exp -> Parser [TopLevel] -> FilePath -> Q Exp
+-cssFileDebug toi2b parseBlocks' parseBlocks fp = do
+-    s <- fmap TL.unpack $ qRunIO $ readUtf8File fp
+-#ifdef GHC_7_4
+-    qAddDependentFile fp
+-#endif
+-    let vs = cssUsedIdentifiers toi2b parseBlocks s
+-    c <- mapM vtToExp vs
+-    cr <- [|cssRuntime toi2b|]
+-    parseBlocks'' <- parseBlocks'
+-    return $ cr `AppE` parseBlocks'' `AppE` (LitE $ StringL fp) `AppE` ListE c
+-
+ combineSelectors :: Selector -> Selector -> Selector
+ combineSelectors a b = do
+     a' <- a
+@@ -202,17 +189,6 @@ cssRuntime toi2b parseBlocks fp cd render' = unsafePerformIO $ do
+ 
+     addScope scope = map (DerefIdent . Ident *** CDPlain . fromString) scope ++ cd
+ 
+-vtToExp :: (Deref, VarType) -> Q Exp
+-vtToExp (d, vt) = do
+-    d' <- lift d
+-    c' <- c vt
+-    return $ TupE [d', c' `AppE` derefToExp [] d]
+-  where
+-    c :: VarType -> Q Exp
+-    c VTPlain = [|CDPlain . toCss|]
+-    c VTUrl = [|CDUrl|]
+-    c VTUrlParam = [|CDUrlParam|]
+-
+ getVars :: Monad m => [(String, String)] -> Content -> m [(Deref, VarType)]
+ getVars _ ContentRaw{} = return []
+ getVars scope (ContentVar d) =
+@@ -268,68 +244,8 @@ compressBlock (Block x y blocks) =
+     cc (ContentRaw a:ContentRaw b:c) = cc $ ContentRaw (a ++ b) : c
+     cc (a:b) = a : cc b
+ 
+-blockToCss :: Name -> Scope -> Block -> Q Exp
+-blockToCss r scope (Block sel props subblocks) =
+-    [|(:) (Css' $(selectorToBuilder r scope sel) $(listE $ map go props))
+-      . foldr (.) id $(listE $ map subGo subblocks)
+-    |]
+-  where
+-    go (x, y) = tupE [contentsToBuilder r scope x, contentsToBuilder r scope y]
+-    subGo (Block sel' b c) =
+-        blockToCss r scope $ Block sel'' b c
+-      where
+-        sel'' = combineSelectors sel sel'
+-
+-selectorToBuilder :: Name -> Scope -> Selector -> Q Exp
+-selectorToBuilder r scope sels =
+-    contentsToBuilder r scope $ intercalate [ContentRaw ","] sels
+-
+-contentsToBuilder :: Name -> Scope -> [Content] -> Q Exp
+-contentsToBuilder r scope contents =
+-    appE [|mconcat|] $ listE $ map (contentToBuilder r scope) contents
+-
+-contentToBuilder :: Name -> Scope -> Content -> Q Exp
+-contentToBuilder _ _ (ContentRaw x) =
+-    [|fromText . pack|] `appE` litE (StringL x)
+-contentToBuilder _ scope (ContentVar d) =
+-    case d of
+-        DerefIdent (Ident s)
+-            | Just val <- lookup s scope -> [|fromText . pack|] `appE` litE (StringL val)
+-        _ -> [|toCss|] `appE` return (derefToExp [] d)
+-contentToBuilder r _ (ContentUrl u) =
+-    [|fromText|] `appE`
+-        (varE r `appE` return (derefToExp [] u) `appE` listE [])
+-contentToBuilder r _ (ContentUrlParam u) =
+-    [|fromText|] `appE`
+-        ([|uncurry|] `appE` varE r `appE` return (derefToExp [] u))
+-
+ type Scope = [(String, String)]
+ 
+-topLevelsToCassius :: [TopLevel] -> Q Exp
+-topLevelsToCassius a = do
+-    r <- newName "_render"
+-    lamE [varP r] $ appE [|CssNoWhitespace . foldr ($) []|] $ fmap ListE $ go r [] a
+-  where
+-    go _ _ [] = return []
+-    go r scope (TopBlock b:rest) = do
+-        e <- [|(++) $ map Css ($(blockToCss r scope b) [])|]
+-        es <- go r scope rest
+-        return $ e : es
+-    go r scope (TopAtBlock name s b:rest) = do
+-        let s' = contentsToBuilder r scope s
+-        e <- [|(:) $ AtBlock $(lift name) $(s') $(blocksToCassius r scope b)|]
+-        es <- go r scope rest
+-        return $ e : es
+-    go r scope (TopAtDecl dec cs:rest) = do
+-        e <- [|(:) $ AtDecl $(lift dec) $(contentsToBuilder r scope cs)|]
+-        es <- go r scope rest
+-        return $ e : es
+-    go r scope (TopVar k v:rest) = go r ((k, v) : scope) rest
+-
+-blocksToCassius :: Name -> Scope -> [Block] -> Q Exp
+-blocksToCassius r scope a = do
+-    appE [|foldr ($) []|] $ listE $ map (blockToCss r scope) a
+-
+ renderCss :: Css -> TL.Text
+ renderCss css =
+     toLazyText $ mconcat $ map go tops-- FIXME use a foldr
+diff --git a/Text/CssCommon.hs b/Text/CssCommon.hs
+index 719e0a8..8c40e8c 100644
+--- a/Text/CssCommon.hs
++++ b/Text/CssCommon.hs
+@@ -1,4 +1,3 @@
+-{-# LANGUAGE TemplateHaskell #-}
+ {-# LANGUAGE GeneralizedNewtypeDeriving #-}
+ {-# LANGUAGE FlexibleInstances #-}
+ {-# LANGUAGE CPP #-}
+@@ -156,6 +155,3 @@ showSize :: Rational -> String -> String
+ showSize value' unit = printf "%f" value ++ unit
+   where value = fromRational value' :: Double
+ 
+-mkSizeType "EmSize" "em"
+-mkSizeType "ExSize" "ex"
+-mkSizeType "PixelSize" "px"
+diff --git a/Text/Lucius.hs b/Text/Lucius.hs
+index b71614e..a902e1c 100644
+--- a/Text/Lucius.hs
++++ b/Text/Lucius.hs
+@@ -6,12 +6,8 @@
+ {-# OPTIONS_GHC -fno-warn-missing-fields #-}
+ module Text.Lucius
+     ( -- * Parsing
+-      lucius
+-    , luciusFile
+-    , luciusFileDebug
+-    , luciusFileReload
+       -- ** Runtime
+-    , luciusRT
++      luciusRT
+     , luciusRT'
+     , -- * Datatypes
+       Css
+@@ -31,11 +27,8 @@ module Text.Lucius
+     , AbsoluteUnit (..)
+     , AbsoluteSize (..)
+     , absoluteSize
+-    , EmSize (..)
+-    , ExSize (..)
+     , PercentageSize (..)
+     , percentageSize
+-    , PixelSize (..)
+       -- * Internal
+     , parseTopLevels
+     , luciusUsedIdentifiers
+@@ -57,18 +50,6 @@ import Data.Either (partitionEithers)
+ import Data.Monoid (mconcat)
+ import Data.List (isSuffixOf)
+ 
+--- |
+---
+--- >>> renderCss ([lucius|foo{bar:baz}|] undefined)
+--- "foo{bar:baz}"
+-lucius :: QuasiQuoter
+-lucius = QuasiQuoter { quoteExp = luciusFromString }
+-
+-luciusFromString :: String -> Q Exp
+-luciusFromString s =
+-    topLevelsToCassius
+-  $ either (error . show) id $ parse parseTopLevels s s
+-
+ whiteSpace :: Parser ()
+ whiteSpace = many whiteSpace1 >> return ()
+ 
+@@ -179,15 +160,6 @@ parseComment = do
+     _ <- manyTill anyChar $ try $ string "*/"
+     return $ ContentRaw ""
+ 
+-luciusFile :: FilePath -> Q Exp
+-luciusFile fp = do
+-    contents <- fmap TL.unpack $ qRunIO $ readUtf8File fp
+-    luciusFromString contents
+-
+-luciusFileDebug, luciusFileReload :: FilePath -> Q Exp
+-luciusFileDebug = cssFileDebug False [|parseTopLevels|] parseTopLevels
+-luciusFileReload = luciusFileDebug
+-
+ parseTopLevels :: Parser [TopLevel]
+ parseTopLevels =
+     go id
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/shakespeare-i18n-1.0.0.2_0001-remove-TH.patch b/standalone/android/haskell-patches/shakespeare-i18n-1.0.0.2_0001-remove-TH.patch
new file mode 100644
index 0000000000..60528db0dd
--- /dev/null
+++ b/standalone/android/haskell-patches/shakespeare-i18n-1.0.0.2_0001-remove-TH.patch
@@ -0,0 +1,162 @@
+From b128412ecee9677b788abecbbf1fd1edd447eea2 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:35:59 -0400
+Subject: [PATCH] remove TH
+
+---
+ Text/Shakespeare/I18N.hs |  130 +---------------------------------------------
+ 1 file changed, 1 insertion(+), 129 deletions(-)
+
+diff --git a/Text/Shakespeare/I18N.hs b/Text/Shakespeare/I18N.hs
+index 1b486ed..aa5e358 100644
+--- a/Text/Shakespeare/I18N.hs
++++ b/Text/Shakespeare/I18N.hs
+@@ -51,10 +51,7 @@
+ --
+ -- You can also adapt those instructions for use with other systems.
+ module Text.Shakespeare.I18N
+-    ( mkMessage
+-    , mkMessageFor
+-    , mkMessageVariant
+-    , RenderMessage (..)
++    ( RenderMessage (..)
+     , ToMessage (..)
+     , SomeMessage (..)
+     , Lang
+@@ -115,133 +112,8 @@ type Lang = Text
+ --
+ --  3. create a 'RenderMessage' instance
+ --
+-mkMessage :: String   -- ^ base name to use for translation type
+-          -> FilePath -- ^ subdirectory which contains the translation files
+-          -> Lang     -- ^ default translation language
+-          -> Q [Dec]
+-mkMessage dt folder lang =
+-    mkMessageCommon True "Msg" "Message" dt dt folder lang
+ 
+ 
+--- | create 'RenderMessage' instance for an existing data-type
+-mkMessageFor :: String     -- ^ master translation data type
+-             -> String     -- ^ existing type to add translations for
+-             -> FilePath   -- ^ path to translation folder
+-             -> Lang       -- ^ default language
+-             -> Q [Dec]
+-mkMessageFor master dt folder lang = mkMessageCommon False "" "" master dt folder lang
+-
+--- | create an additional set of translations for a type created by `mkMessage`
+-mkMessageVariant :: String     -- ^ master translation data type
+-                 -> String     -- ^ existing type to add translations for
+-                 -> FilePath   -- ^ path to translation folder
+-                 -> Lang       -- ^ default language
+-                 -> Q [Dec]
+-mkMessageVariant master dt folder lang = mkMessageCommon False "Msg" "Message" master dt folder lang
+-
+--- |used by 'mkMessage' and 'mkMessageFor' to generate a 'RenderMessage' and possibly a message data type
+-mkMessageCommon :: Bool      -- ^ generate a new datatype from the constructors found in the .msg files
+-                -> String    -- ^ string to append to constructor names
+-                -> String    -- ^ string to append to datatype name
+-                -> String    -- ^ base name of master datatype
+-                -> String    -- ^ base name of translation datatype
+-                -> FilePath  -- ^ path to translation folder
+-                -> Lang      -- ^ default lang
+-                -> Q [Dec]
+-mkMessageCommon genType prefix postfix master dt folder lang = do
+-    files <- qRunIO $ getDirectoryContents folder
+-    (_files', contents) <- qRunIO $ fmap (unzip . catMaybes) $ mapM (loadLang folder) files
+-#ifdef GHC_7_4
+-    mapM_ qAddDependentFile _files'
+-#endif
+-    sdef <-
+-        case lookup lang contents of
+-            Nothing -> error $ "Did not find main language file: " ++ unpack lang
+-            Just def -> toSDefs def
+-    mapM_ (checkDef sdef) $ map snd contents
+-    let mname = mkName $ dt ++ postfix
+-    c1 <- fmap concat $ mapM (toClauses prefix dt) contents
+-    c2 <- mapM (sToClause prefix dt) sdef
+-    c3 <- defClause
+-    return $
+-     ( if genType 
+-       then ((DataD [] mname [] (map (toCon dt) sdef) []) :)
+-       else id)
+-        [ InstanceD
+-            []
+-            (ConT ''RenderMessage `AppT` (ConT $ mkName master) `AppT` ConT mname)
+-            [ FunD (mkName "renderMessage") $ c1 ++ c2 ++ [c3]
+-            ]
+-        ]
+-
+-toClauses :: String -> String -> (Lang, [Def]) -> Q [Clause]
+-toClauses prefix dt (lang, defs) =
+-    mapM go defs
+-  where
+-    go def = do
+-        a <- newName "lang"
+-        (pat, bod) <- mkBody dt (prefix ++ constr def) (map fst $ vars def) (content def)
+-        guard <- fmap NormalG [|$(return $ VarE a) == pack $(lift $ unpack lang)|]
+-        return $ Clause
+-            [WildP, ConP (mkName ":") [VarP a, WildP], pat]
+-            (GuardedB [(guard, bod)])
+-            []
+-
+-mkBody :: String -- ^ datatype
+-       -> String -- ^ constructor
+-       -> [String] -- ^ variable names
+-       -> [Content]
+-       -> Q (Pat, Exp)
+-mkBody dt cs vs ct = do
+-    vp <- mapM go vs
+-    let pat = RecP (mkName cs) (map (varName dt *** VarP) vp)
+-    let ct' = map (fixVars vp) ct
+-    pack' <- [|Data.Text.pack|]
+-    tomsg <- [|toMessage|]
+-    let ct'' = map (toH pack' tomsg) ct'
+-    mapp <- [|mappend|]
+-    let app a b = InfixE (Just a) mapp (Just b)
+-    e <-
+-        case ct'' of
+-            [] -> [|mempty|]
+-            [x] -> return x
+-            (x:xs) -> return $ foldl' app x xs
+-    return (pat, e)
+-  where
+-    toH pack' _ (Raw s) = pack' `AppE` SigE (LitE (StringL s)) (ConT ''String)
+-    toH _ tomsg (Var d) = tomsg `AppE` derefToExp [] d
+-    go x = do
+-        let y = mkName $ '_' : x
+-        return (x, y)
+-    fixVars vp (Var d) = Var $ fixDeref vp d
+-    fixVars _ (Raw s) = Raw s
+-    fixDeref vp (DerefIdent (Ident i)) = DerefIdent $ Ident $ fixIdent vp i
+-    fixDeref vp (DerefBranch a b) = DerefBranch (fixDeref vp a) (fixDeref vp b)
+-    fixDeref _ d = d
+-    fixIdent vp i =
+-        case lookup i vp of
+-            Nothing -> i
+-            Just y -> nameBase y
+-
+-sToClause :: String -> String -> SDef -> Q Clause
+-sToClause prefix dt sdef = do
+-    (pat, bod) <- mkBody dt (prefix ++ sconstr sdef) (map fst $ svars sdef) (scontent sdef)
+-    return $ Clause
+-        [WildP, ConP (mkName "[]") [], pat]
+-        (NormalB bod)
+-        []
+-
+-defClause :: Q Clause
+-defClause = do
+-    a <- newName "sub"
+-    c <- newName "langs"
+-    d <- newName "msg"
+-    rm <- [|renderMessage|]
+-    return $ Clause
+-        [VarP a, ConP (mkName ":") [WildP, VarP c], VarP d]
+-        (NormalB $ rm `AppE` VarE a `AppE` VarE c `AppE` VarE d)
+-        []
+-
+ toCon :: String -> SDef -> Con
+ toCon dt (SDef c vs _) =
+     RecC (mkName $ "Msg" ++ c) $ map go vs
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/shakespeare-js-1.1.2_0001-remove-TH.patch b/standalone/android/haskell-patches/shakespeare-js-1.1.2_0001-remove-TH.patch
new file mode 100644
index 0000000000..b583d6e0a1
--- /dev/null
+++ b/standalone/android/haskell-patches/shakespeare-js-1.1.2_0001-remove-TH.patch
@@ -0,0 +1,303 @@
+From f3e31696cfb45a528e4b4b6f016dc7101d7cd4fb Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:36:06 -0400
+Subject: [PATCH] remove TH
+
+---
+ Text/Coffee.hs     |   54 -------------------------------------------------
+ Text/Julius.hs     |   53 +-----------------------------------------------
+ Text/Roy.hs        |   54 -------------------------------------------------
+ Text/TypeScript.hs |   57 +---------------------------------------------------
+ 4 files changed, 2 insertions(+), 216 deletions(-)
+
+diff --git a/Text/Coffee.hs b/Text/Coffee.hs
+index 2481936..3f7f9c3 100644
+--- a/Text/Coffee.hs
++++ b/Text/Coffee.hs
+@@ -51,14 +51,6 @@ module Text.Coffee
+       -- ** Template-Reading Functions
+       -- | These QuasiQuoter and Template Haskell methods return values of
+       -- type @'JavascriptUrl' url@. See the Yesod book for details.
+-      coffee
+-    , coffeeFile
+-    , coffeeFileReload
+-    , coffeeFileDebug
+-
+-#ifdef TEST_EXPORT
+-    , coffeeSettings
+-#endif
+     ) where
+ 
+ import Language.Haskell.TH.Quote (QuasiQuoter (..))
+@@ -66,49 +58,3 @@ import Language.Haskell.TH.Syntax
+ import Text.Shakespeare
+ import Text.Julius
+ 
+-coffeeSettings :: Q ShakespeareSettings
+-coffeeSettings = do
+-  jsettings <- javascriptSettings
+-  return $ jsettings { varChar = '%'
+-  , preConversion = Just PreConvert {
+-      preConvert = ReadProcess "coffee" ["-spb"]
+-    , preEscapeIgnoreBalanced = "'\"`"     -- don't insert backtacks for variable already inside strings or backticks.
+-    , preEscapeIgnoreLine = "#"            -- ignore commented lines
+-    , wrapInsertion = Just WrapInsertion { 
+-        wrapInsertionIndent = Just "  "
+-      , wrapInsertionStartBegin = "(("
+-      , wrapInsertionSeparator = ", "
+-      , wrapInsertionStartClose = ") =>"
+-      , wrapInsertionEnd = ")"
+-      , wrapInsertionApplyBegin = "("
+-      , wrapInsertionApplyClose = ")\n"
+-      }
+-    }
+-  }
+-
+--- | Read inline, quasiquoted CoffeeScript.
+-coffee :: QuasiQuoter
+-coffee = QuasiQuoter { quoteExp = \s -> do
+-    rs <- coffeeSettings
+-    quoteExp (shakespeare rs) s
+-    }
+-
+--- | Read in a CoffeeScript template file. This function reads the file once, at
+--- compile time.
+-coffeeFile :: FilePath -> Q Exp
+-coffeeFile fp = do
+-    rs <- coffeeSettings
+-    shakespeareFile rs fp
+-
+--- | Read in a CoffeeScript template file. This impure function uses
+--- unsafePerformIO to re-read the file on every call, allowing for rapid
+--- iteration.
+-coffeeFileReload :: FilePath -> Q Exp
+-coffeeFileReload fp = do
+-    rs <- coffeeSettings
+-    shakespeareFileReload rs fp
+-
+--- | Deprecated synonym for 'coffeeFileReload'
+-coffeeFileDebug :: FilePath -> Q Exp
+-coffeeFileDebug = coffeeFileReload
+-{-# DEPRECATED coffeeFileDebug "Please use coffeeFileReload instead." #-}
+diff --git a/Text/Julius.hs b/Text/Julius.hs
+index 230eac3..b990f73 100644
+--- a/Text/Julius.hs
++++ b/Text/Julius.hs
+@@ -14,17 +14,8 @@ module Text.Julius
+       -- ** Template-Reading Functions
+       -- | These QuasiQuoter and Template Haskell methods return values of
+       -- type @'JavascriptUrl' url@. See the Yesod book for details.
+-      js
+-    , julius
+-    , juliusFile
+-    , jsFile
+-    , juliusFileDebug
+-    , jsFileDebug
+-    , juliusFileReload
+-    , jsFileReload
+-
+       -- * Datatypes
+-    , JavascriptUrl
++      JavascriptUrl
+     , Javascript (..)
+     , RawJavascript (..)
+ 
+@@ -37,7 +28,6 @@ module Text.Julius
+     , renderJavascriptUrl
+ 
+       -- ** internal, used by 'Text.Coffee'
+-    , javascriptSettings
+       -- ** internal
+     , juliusUsedIdentifiers
+     ) where
+@@ -101,47 +91,6 @@ instance RawJS TL.Text where rawJS = RawJavascript . fromLazyText
+ instance RawJS Builder where rawJS = RawJavascript
+ instance RawJS Bool where rawJS = RawJavascript . toJavascript
+ 
+-javascriptSettings :: Q ShakespeareSettings
+-javascriptSettings = do
+-  toJExp <- [|toJavascript|]
+-  wrapExp <- [|Javascript|]
+-  unWrapExp <- [|unJavascript|]
+-  asJavascriptUrl' <- [|asJavascriptUrl|]
+-  return $ defaultShakespeareSettings { toBuilder = toJExp
+-  , wrap = wrapExp
+-  , unwrap = unWrapExp
+-  , modifyFinalValue = Just asJavascriptUrl'
+-  }
+-
+-js, julius :: QuasiQuoter
+-js = QuasiQuoter { quoteExp = \s -> do
+-    rs <- javascriptSettings
+-    quoteExp (shakespeare rs) s
+-    }
+-
+-julius = js
+-
+-jsFile, juliusFile :: FilePath -> Q Exp
+-jsFile fp = do
+-    rs <- javascriptSettings
+-    shakespeareFile rs fp
+-
+-juliusFile = jsFile
+-
+-
+-jsFileReload, juliusFileReload :: FilePath -> Q Exp
+-jsFileReload fp = do
+-    rs <- javascriptSettings
+-    shakespeareFileReload rs fp
+-
+-juliusFileReload = jsFileReload
+-
+-jsFileDebug, juliusFileDebug :: FilePath -> Q Exp
+-juliusFileDebug = jsFileReload
+-{-# DEPRECATED juliusFileDebug "Please use juliusFileReload instead." #-}
+-jsFileDebug = jsFileReload
+-{-# DEPRECATED jsFileDebug "Please use jsFileReload instead." #-}
+-
+ -- | Determine which identifiers are used by the given template, useful for
+ -- creating systems like yesod devel.
+ juliusUsedIdentifiers :: String -> [(Deref, VarType)]
+diff --git a/Text/Roy.hs b/Text/Roy.hs
+index cf09cec..870c9f6 100644
+--- a/Text/Roy.hs
++++ b/Text/Roy.hs
+@@ -23,13 +23,6 @@ module Text.Roy
+       -- ** Template-Reading Functions
+       -- | These QuasiQuoter and Template Haskell methods return values of
+       -- type @'JavascriptUrl' url@. See the Yesod book for details.
+-      roy
+-    , royFile
+-    , royFileReload
+-
+-#ifdef TEST_EXPORT
+-    , roySettings
+-#endif
+     ) where
+ 
+ import Language.Haskell.TH.Quote (QuasiQuoter (..))
+@@ -37,50 +30,3 @@ import Language.Haskell.TH.Syntax
+ import Text.Shakespeare
+ import Text.Julius
+ 
+--- | The Roy language compiles down to Javascript.
+--- We do this compilation once at compile time to avoid needing to do it during the request.
+--- We call this a preConversion because other shakespeare modules like Lucius use Haskell to compile during the request instead rather than a system call.
+-roySettings :: Q ShakespeareSettings
+-roySettings = do
+-  jsettings <- javascriptSettings
+-  return $ jsettings { varChar = '#'
+-  , preConversion = Just PreConvert {
+-      preConvert = ReadProcess "roy" ["--stdio"]
+-    , preEscapeIgnoreBalanced = "'\""
+-    , preEscapeIgnoreLine = "//"
+-    , wrapInsertion = Nothing
+-    {-
+-    Just WrapInsertion { 
+-        wrapInsertionIndent = Just "  "
+-      , wrapInsertionStartBegin = "(\\"
+-      , wrapInsertionSeparator = " "
+-      , wrapInsertionStartClose = " ->\n"
+-      , wrapInsertionEnd = ")"
+-      , wrapInsertionApplyBegin = " "
+-      , wrapInsertionApplyClose = ")\n"
+-      }
+-      -}
+-    }
+-  }
+-
+--- | Read inline, quasiquoted Roy.
+-roy :: QuasiQuoter
+-roy = QuasiQuoter { quoteExp = \s -> do
+-    rs <- roySettings
+-    quoteExp (shakespeare rs) s
+-    }
+-
+--- | Read in a Roy template file. This function reads the file once, at
+--- compile time.
+-royFile :: FilePath -> Q Exp
+-royFile fp = do
+-    rs <- roySettings
+-    shakespeareFile rs fp
+-
+--- | Read in a Roy template file. This impure function uses
+--- unsafePerformIO to re-read the file on every call, allowing for rapid
+--- iteration.
+-royFileReload :: FilePath -> Q Exp
+-royFileReload fp = do
+-    rs <- roySettings
+-    shakespeareFileReload rs fp
+diff --git a/Text/TypeScript.hs b/Text/TypeScript.hs
+index 34bf4bf..30c5388 100644
+--- a/Text/TypeScript.hs
++++ b/Text/TypeScript.hs
+@@ -53,65 +53,10 @@
+ --
+ -- 2. TypeScript: 
+ module Text.TypeScript
+-    ( -- * Functions
+-      -- ** Template-Reading Functions
+-      -- | These QuasiQuoter and Template Haskell methods return values of
+-      -- type @'JavascriptUrl' url@. See the Yesod book for details.
+-      tsc
+-    , typeScriptFile
+-    , typeScriptFileReload
+-
+-#ifdef TEST_EXPORT
+-    , typeScriptSettings
+-#endif
++    (
+     ) where
+ 
+ import Language.Haskell.TH.Quote (QuasiQuoter (..))
+ import Language.Haskell.TH.Syntax
+ import Text.Shakespeare
+ import Text.Julius
+-
+--- | The TypeScript language compiles down to Javascript.
+--- We do this compilation once at compile time to avoid needing to do it during the request.
+--- We call this a preConversion because other shakespeare modules like Lucius use Haskell to compile during the request instead rather than a system call.
+-typeScriptSettings :: Q ShakespeareSettings
+-typeScriptSettings = do
+-  jsettings <- javascriptSettings
+-  return $ jsettings { varChar = '#'
+-  , preConversion = Just PreConvert {
+-      preConvert = ReadProcess "sh" ["-c", "TMP_IN=$(mktemp XXXXXXXXXX.ts); TMP_OUT=$(mktemp XXXXXXXXXX.js); cat /dev/stdin > ${TMP_IN} && tsc --out ${TMP_OUT} ${TMP_IN} && cat ${TMP_OUT}; rm ${TMP_IN} && rm ${TMP_OUT}"]
+-    , preEscapeIgnoreBalanced = "'\""
+-    , preEscapeIgnoreLine = "//"
+-    , wrapInsertion = Just WrapInsertion { 
+-        wrapInsertionIndent = Nothing
+-      , wrapInsertionStartBegin = ";(function("
+-      , wrapInsertionSeparator = ", "
+-      , wrapInsertionStartClose = "){"
+-      , wrapInsertionEnd = "})"
+-      , wrapInsertionApplyBegin = "("
+-      , wrapInsertionApplyClose = ");\n"
+-      }
+-    }
+-  }
+-
+--- | Read inline, quasiquoted TypeScript
+-tsc :: QuasiQuoter
+-tsc = QuasiQuoter { quoteExp = \s -> do
+-    rs <- typeScriptSettings
+-    quoteExp (shakespeare rs) s
+-    }
+-
+--- | Read in a Roy template file. This function reads the file once, at
+--- compile time.
+-typeScriptFile :: FilePath -> Q Exp
+-typeScriptFile fp = do
+-    rs <- typeScriptSettings
+-    shakespeareFile rs fp
+-
+--- | Read in a Roy template file. This impure function uses
+--- unsafePerformIO to re-read the file on every call, allowing for rapid
+--- iteration.
+-typeScriptFileReload :: FilePath -> Q Exp
+-typeScriptFileReload fp = do
+-    rs <- typeScriptSettings
+-    shakespeareFileReload rs fp
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/socks-0.4.2_0001-remove-IPv6-stuff.patch b/standalone/android/haskell-patches/socks-0.4.2_0001-remove-IPv6-stuff.patch
new file mode 100644
index 0000000000..5a343d8759
--- /dev/null
+++ b/standalone/android/haskell-patches/socks-0.4.2_0001-remove-IPv6-stuff.patch
@@ -0,0 +1,107 @@
+From abab0f8202998a3e88c5dc5f67a8245da6c174b3 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:36:20 -0400
+Subject: [PATCH] remove IPv6 stuff
+
+---
+ Network/Socks5.hs         |    1 -
+ Network/Socks5/Command.hs |   16 ++--------------
+ Network/Socks5/Types.hs   |    3 +--
+ Network/Socks5/Wire.hs    |    2 --
+ 4 files changed, 3 insertions(+), 19 deletions(-)
+
+diff --git a/Network/Socks5.hs b/Network/Socks5.hs
+index 67b0060..80efb9c 100644
+--- a/Network/Socks5.hs
++++ b/Network/Socks5.hs
+@@ -54,7 +54,6 @@ socksConnectAddr :: Socket -> SockAddr -> SockAddr -> IO ()
+ socksConnectAddr sock sockserver destaddr = withSocks sock sockserver $ do
+ 	case destaddr of
+ 		SockAddrInet p h      -> socks5ConnectIPV4 sock h p >> return ()
+-		SockAddrInet6 p _ h _ -> socks5ConnectIPV6 sock h p >> return ()
+ 		_                     -> error "unsupported unix sockaddr type"
+ 
+ -- | connect a new socket to the socks server, and connect the stream to a FQDN
+diff --git a/Network/Socks5/Command.hs b/Network/Socks5/Command.hs
+index 2952706..db994c9 100644
+--- a/Network/Socks5/Command.hs
++++ b/Network/Socks5/Command.hs
+@@ -9,9 +9,8 @@
+ --
+ module Network.Socks5.Command
+ 	( socks5Establish
+-	, socks5ConnectIPV4
+-	, socks5ConnectIPV6
+ 	, socks5ConnectDomainName
++	, socks5ConnectIPV4 
+ 	-- * lowlevel interface
+ 	, socks5Rpc
+ 	) where
+@@ -23,7 +22,7 @@ import qualified Data.ByteString as B
+ import qualified Data.ByteString.Char8 as BC
+ import Data.Serialize
+ 
+-import Network.Socket (Socket, PortNumber, HostAddress, HostAddress6)
++import Network.Socket (Socket, PortNumber, HostAddress)
+ import Network.Socket.ByteString
+ 
+ import Network.Socks5.Types
+@@ -46,17 +45,6 @@ socks5ConnectIPV4 socket hostaddr port = onReply <$> socks5Rpc socket request
+ 		onReply (SocksAddrIPV4 h, p) = (h, p)
+ 		onReply _                    = error "ipv4 requested, got something different"
+ 
+-socks5ConnectIPV6 :: Socket -> HostAddress6 -> PortNumber -> IO (HostAddress6, PortNumber)
+-socks5ConnectIPV6 socket hostaddr6 port = onReply <$> socks5Rpc socket request
+-	where
+-		request = SocksRequest
+-			{ requestCommand  = SocksCommandConnect
+-			, requestDstAddr  = SocksAddrIPV6 hostaddr6
+-			, requestDstPort  = fromIntegral port
+-			}
+-		onReply (SocksAddrIPV6 h, p) = (h, p)
+-		onReply _                    = error "ipv6 requested, got something different"
+-
+ -- TODO: FQDN should only be ascii, maybe putting a "fqdn" data type
+ -- in front to make sure and make the BC.pack safe.
+ socks5ConnectDomainName :: Socket -> String -> PortNumber -> IO (SocksAddr, PortNumber)
+diff --git a/Network/Socks5/Types.hs b/Network/Socks5/Types.hs
+index 5dc7d5e..12dea99 100644
+--- a/Network/Socks5/Types.hs
++++ b/Network/Socks5/Types.hs
+@@ -17,7 +17,7 @@ module Network.Socks5.Types
+ import Data.ByteString (ByteString)
+ import Data.Word
+ import Data.Data
+-import Network.Socket (HostAddress, HostAddress6)
++import Network.Socket (HostAddress)
+ import Control.Exception
+ 
+ data SocksCommand =
+@@ -38,7 +38,6 @@ data SocksMethod =
+ data SocksAddr =
+ 	  SocksAddrIPV4 HostAddress
+ 	| SocksAddrDomainName ByteString
+-	| SocksAddrIPV6 HostAddress6
+ 	deriving (Show,Eq)
+ 
+ data SocksReply =
+diff --git a/Network/Socks5/Wire.hs b/Network/Socks5/Wire.hs
+index 2cfed52..d3bd9c5 100644
+--- a/Network/Socks5/Wire.hs
++++ b/Network/Socks5/Wire.hs
+@@ -41,12 +41,10 @@ data SocksResponse = SocksResponse
+ 
+ getAddr 1 = SocksAddrIPV4 <$> getWord32be
+ getAddr 3 = SocksAddrDomainName <$> (getWord8 >>= getByteString . fromIntegral)
+-getAddr 4 = SocksAddrIPV6 <$> (liftM4 (,,,) getWord32le getWord32le getWord32le getWord32le)
+ getAddr n = error ("cannot get unknown socket address type: " ++ show n)
+ 
+ putAddr (SocksAddrIPV4 h)         = putWord8 1 >> putWord32host h
+ putAddr (SocksAddrDomainName b)   = putWord8 3 >> putWord8 (fromIntegral $ B.length b) >> putByteString b
+-putAddr (SocksAddrIPV6 (a,b,c,d)) = putWord8 4 >> mapM_ putWord32host [a,b,c,d]
+ 
+ getSocksRequest 5 = do
+ 	cmd <- toEnum . fromIntegral <$> getWord8
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/split-0.2.1.2_0001-modify-to-build-with-unreleased-ghc.patch b/standalone/android/haskell-patches/split-0.2.1.2_0001-modify-to-build-with-unreleased-ghc.patch
new file mode 100644
index 0000000000..472ccd6785
--- /dev/null
+++ b/standalone/android/haskell-patches/split-0.2.1.2_0001-modify-to-build-with-unreleased-ghc.patch
@@ -0,0 +1,25 @@
+From 2feaef797641587a3da83753ee17d20e712c79cf Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:36:30 -0400
+Subject: [PATCH] modify to build with unreleased ghc
+
+---
+ split.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/split.cabal b/split.cabal
+index 2183c3e..29b9b32 100644
+--- a/split.cabal
++++ b/split.cabal
+@@ -51,7 +51,7 @@ Source-repository head
+ 
+ Library
+   ghc-options:       -Wall
+-  build-depends:     base <4.7
++  build-depends:     base <4.8
+   exposed-modules:   Data.List.Split, Data.List.Split.Internals
+   default-language:  Haskell2010
+   Hs-source-dirs:    src
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/syb-0.3.7_0001-hack-for-cross-compiling.patch b/standalone/android/haskell-patches/syb-0.3.7_0001-hack-for-cross-compiling.patch
new file mode 100644
index 0000000000..e18d6127fe
--- /dev/null
+++ b/standalone/android/haskell-patches/syb-0.3.7_0001-hack-for-cross-compiling.patch
@@ -0,0 +1,25 @@
+From c40fe2c484096c5de4cac8ca14a0ca5d892999f7 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:36:43 -0400
+Subject: [PATCH] hack for cross-compiling
+
+---
+ syb.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/syb.cabal b/syb.cabal
+index 0aee93d..0a645c6 100644
+--- a/syb.cabal
++++ b/syb.cabal
+@@ -17,7 +17,7 @@ description:
+ 
+ category:               Generics
+ stability:              provisional
+-build-type:             Custom
++build-type:             Simple
+ cabal-version:          >= 1.6
+ 
+ extra-source-files:     tests/*.hs,
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/unix-2.6.0.1_0001-remove-stuff-not-available-on-Android.patch b/standalone/android/haskell-patches/unix-2.6.0.1_0001-remove-stuff-not-available-on-Android.patch
new file mode 100644
index 0000000000..ff1da944cf
--- /dev/null
+++ b/standalone/android/haskell-patches/unix-2.6.0.1_0001-remove-stuff-not-available-on-Android.patch
@@ -0,0 +1,91 @@
+From abca378462337ca0eb13a7e4d3073cb96a50d36c Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:37:23 -0400
+Subject: [PATCH] remove stuff not available on Android
+
+---
+ System/Posix/Resource.hsc        |    4 ++++
+ System/Posix/Terminal/Common.hsc |   29 +++--------------------------
+ 2 files changed, 7 insertions(+), 26 deletions(-)
+
+diff --git a/System/Posix/Resource.hsc b/System/Posix/Resource.hsc
+index 6651998..2615b1e 100644
+--- a/System/Posix/Resource.hsc
++++ b/System/Posix/Resource.hsc
+@@ -101,7 +101,9 @@ packResource ResourceTotalMemory   = (#const RLIMIT_AS)
+ #endif
+ 
+ unpackRLimit :: CRLim -> ResourceLimit
++#if 0
+ unpackRLimit (#const RLIM_INFINITY)  = ResourceLimitInfinity
++#endif
+ #ifdef RLIM_SAVED_MAX
+ unpackRLimit (#const RLIM_SAVED_MAX) = ResourceLimitUnknown
+ unpackRLimit (#const RLIM_SAVED_CUR) = ResourceLimitUnknown
+@@ -109,7 +111,9 @@ unpackRLimit (#const RLIM_SAVED_CUR) = ResourceLimitUnknown
+ unpackRLimit other = ResourceLimit (fromIntegral other)
+ 
+ packRLimit :: ResourceLimit -> Bool -> CRLim
++#if 0
+ packRLimit ResourceLimitInfinity _     = (#const RLIM_INFINITY)
++#endif
+ #ifdef RLIM_SAVED_MAX
+ packRLimit ResourceLimitUnknown  True  = (#const RLIM_SAVED_CUR)
+ packRLimit ResourceLimitUnknown  False = (#const RLIM_SAVED_MAX)
+diff --git a/System/Posix/Terminal/Common.hsc b/System/Posix/Terminal/Common.hsc
+index 3a6254d..32a22f2 100644
+--- a/System/Posix/Terminal/Common.hsc
++++ b/System/Posix/Terminal/Common.hsc
+@@ -419,11 +419,7 @@ foreign import ccall unsafe "tcsendbreak"
+ -- | @drainOutput fd@ calls @tcdrain@ to block until all output
+ --   written to @Fd@ @fd@ has been transmitted.
+ drainOutput :: Fd -> IO ()
+-drainOutput (Fd fd) = throwErrnoIfMinus1_ "drainOutput" (c_tcdrain fd)
+-
+-foreign import ccall unsafe "tcdrain"
+-  c_tcdrain :: CInt -> IO CInt
+-
++drainOutput (Fd fd) = error "drainOutput not implemented"
+ 
+ data QueueSelector
+   = InputQueue		-- TCIFLUSH
+@@ -434,16 +430,7 @@ data QueueSelector
+ --   pending input and\/or output for @Fd@ @fd@,
+ --   as indicated by the @QueueSelector@ @queues@.
+ discardData :: Fd -> QueueSelector -> IO ()
+-discardData (Fd fd) queue =
+-  throwErrnoIfMinus1_ "discardData" (c_tcflush fd (queue2Int queue))
+-  where
+-    queue2Int :: QueueSelector -> CInt
+-    queue2Int InputQueue  = (#const TCIFLUSH)
+-    queue2Int OutputQueue = (#const TCOFLUSH)
+-    queue2Int BothQueues  = (#const TCIOFLUSH)
+-
+-foreign import ccall unsafe "tcflush"
+-  c_tcflush :: CInt -> CInt -> IO CInt
++discardData (Fd fd) queue = error "discardData not implemented"
+ 
+ data FlowAction
+   = SuspendOutput	-- ^ TCOOFF
+@@ -455,17 +442,7 @@ data FlowAction
+ --   flow of data on @Fd@ @fd@, as indicated by
+ --   @action@.
+ controlFlow :: Fd -> FlowAction -> IO ()
+-controlFlow (Fd fd) action =
+-  throwErrnoIfMinus1_ "controlFlow" (c_tcflow fd (action2Int action))
+-  where
+-    action2Int :: FlowAction -> CInt
+-    action2Int SuspendOutput = (#const TCOOFF)
+-    action2Int RestartOutput = (#const TCOON)
+-    action2Int TransmitStop  = (#const TCIOFF)
+-    action2Int TransmitStart = (#const TCION)
+-
+-foreign import ccall unsafe "tcflow"
+-  c_tcflow :: CInt -> CInt -> IO CInt
++controlFlow (Fd fd) action = error "controlFlow not implemented"
+ 
+ -- | @getTerminalProcessGroupID fd@ calls @tcgetpgrp@ to
+ --   obtain the @ProcessGroupID@ of the foreground process group 
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/unix-time-0.1.4_0001-hacks-for-android.patch b/standalone/android/haskell-patches/unix-time-0.1.4_0001-hacks-for-android.patch
new file mode 100644
index 0000000000..25f6d6ef57
--- /dev/null
+++ b/standalone/android/haskell-patches/unix-time-0.1.4_0001-hacks-for-android.patch
@@ -0,0 +1,39 @@
+From 8b8a8422a9235b730049de4e6e626821abdc8393 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:37:44 -0400
+Subject: [PATCH] hacks for android
+
+---
+ cbits/conv.c    |    2 +-
+ unix-time.cabal |    2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/cbits/conv.c b/cbits/conv.c
+index 3b6a129..895e6b7 100644
+--- a/cbits/conv.c
++++ b/cbits/conv.c
+@@ -51,7 +51,7 @@ time_t c_parse_unix_time_gmt(char *fmt, char *src) {
+ #else
+     strptime(src, fmt, &dst);
+ #endif
+-    return timegm(&dst);
++    return NULL; /* timegm(&dst); */
+ }
+ 
+ void c_format_unix_time(char *fmt, time_t src, char* dst, int siz) {
+diff --git a/unix-time.cabal b/unix-time.cabal
+index a905d63..98d2495 100644
+--- a/unix-time.cabal
++++ b/unix-time.cabal
+@@ -21,7 +21,7 @@ Library
+                         Data.UnixTime.Types
+                         Data.UnixTime.Sys
+   Build-Depends:        base >= 4 && < 5
+-                      , bytestring
++                      , bytestring (>= 0.10.3.0)
+                       , old-time
+   C-Sources:            cbits/conv.c
+ 
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/vector-0.10.0.1_0001-disable-optimisation-that-breaks-when-cross-compilin.patch b/standalone/android/haskell-patches/vector-0.10.0.1_0001-disable-optimisation-that-breaks-when-cross-compilin.patch
new file mode 100644
index 0000000000..aa50d9c938
--- /dev/null
+++ b/standalone/android/haskell-patches/vector-0.10.0.1_0001-disable-optimisation-that-breaks-when-cross-compilin.patch
@@ -0,0 +1,25 @@
+From 3a4ee8091ba9da44f9f4a04522a5ff45fabe70d9 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:37:56 -0400
+Subject: [PATCH] disable optimisation that breaks when cross-compiling
+
+This needs TH to work actually.
+---
+ Data/Vector/Fusion/Stream/Monadic.hs |    1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/Data/Vector/Fusion/Stream/Monadic.hs b/Data/Vector/Fusion/Stream/Monadic.hs
+index 51fec75..b089b3d 100644
+--- a/Data/Vector/Fusion/Stream/Monadic.hs
++++ b/Data/Vector/Fusion/Stream/Monadic.hs
+@@ -101,7 +101,6 @@ import GHC.Exts ( SpecConstrAnnotation(..) )
+ 
+ data SPEC = SPEC | SPEC2
+ #if __GLASGOW_HASKELL__ >= 700
+-{-# ANN type SPEC ForceSpecConstr #-}
+ #endif
+ 
+ emptyStream :: String
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/wai-extra-1.3.2.1_0001-disable-CGI-module.patch b/standalone/android/haskell-patches/wai-extra-1.3.2.1_0001-disable-CGI-module.patch
new file mode 100644
index 0000000000..7d5d6e2ba2
--- /dev/null
+++ b/standalone/android/haskell-patches/wai-extra-1.3.2.1_0001-disable-CGI-module.patch
@@ -0,0 +1,26 @@
+From dc6d0128e666dcab07ddee56a22a4177ebfc0c7b Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:38:33 -0400
+Subject: [PATCH] disable CGI module
+
+I don't need it and it failed to build.
+---
+ wai-extra.cabal |    2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/wai-extra.cabal b/wai-extra.cabal
+index 9e9f0fc..007dd0f 100644
+--- a/wai-extra.cabal
++++ b/wai-extra.cabal
+@@ -44,7 +44,7 @@ Library
+                    , void                      >= 0.5      && < 0.6
+                    , stringsearch              >= 0.3      && < 0.4
+ 
+-  Exposed-modules:   Network.Wai.Handler.CGI
++  Exposed-modules:
+                      Network.Wai.Middleware.AcceptOverride
+                      Network.Wai.Middleware.Autohead
+                      Network.Wai.Middleware.CleanPath
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/yesod-1.1.8_0001-hacked-up-to-build-on-Android.patch b/standalone/android/haskell-patches/yesod-1.1.8_0001-hacked-up-to-build-on-Android.patch
new file mode 100644
index 0000000000..5a042dc41a
--- /dev/null
+++ b/standalone/android/haskell-patches/yesod-1.1.8_0001-hacked-up-to-build-on-Android.patch
@@ -0,0 +1,157 @@
+From 37abd5d34e18d11ff2961f672cf4491471029684 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:39:18 -0400
+Subject: [PATCH] hacked up to build on Android
+
+removing stuff I don't need and stuff removed from other modules
+---
+ Yesod.hs    |    7 ------
+ yesod.cabal |   77 -----------------------------------------------------------
+ 2 files changed, 84 deletions(-)
+
+diff --git a/Yesod.hs b/Yesod.hs
+index ef9623d..255ab56 100644
+--- a/Yesod.hs
++++ b/Yesod.hs
+@@ -6,7 +6,6 @@ module Yesod
+       module Yesod.Core
+     , module Yesod.Form
+     , module Yesod.Json
+-    , module Yesod.Persist
+       -- * Running your application
+     , warp
+     , warpDebug
+@@ -21,19 +20,14 @@ module Yesod
+     , readIntegral
+       -- * Hamlet library
+       -- ** Hamlet
+-    , hamlet
+-    , xhamlet
+     , HtmlUrl
+     , Html
+     , toHtml
+       -- ** Julius
+-    , julius
+     , JavascriptUrl
+     , renderJavascriptUrl
+     , toJSON
+       -- ** Cassius/Lucius
+-    , cassius
+-    , lucius
+     , CssUrl
+     , renderCssUrl
+     ) where
+@@ -46,7 +40,6 @@ import Text.Julius
+ 
+ import Yesod.Form
+ import Yesod.Json
+-import Yesod.Persist
+ import Control.Monad.IO.Class (liftIO, MonadIO(..))
+ import Control.Monad.Trans.Control (MonadBaseControl)
+ 
+diff --git a/yesod.cabal b/yesod.cabal
+index 741f19a..7566cfb 100644
+--- a/yesod.cabal
++++ b/yesod.cabal
+@@ -13,7 +13,6 @@ description:
+     The Yesod documentation site  has much more information, including on the supporting packages mentioned above.
+ category:        Web, Yesod
+ stability:       Stable
+-cabal-version:   >= 1.6
+ build-type:      Simple
+ homepage:        http://www.yesodweb.com/
+ 
+@@ -28,9 +27,7 @@ extra-source-files:
+ library
+     build-depends:   base                      >= 4.3      && < 5
+                    , yesod-core                >= 1.1.5    && < 1.2
+-                   , yesod-auth                >= 1.1      && < 1.2
+                    , yesod-json                >= 1.1      && < 1.2
+-                   , yesod-persistent          >= 1.1      && < 1.2
+                    , yesod-form                >= 1.1      && < 1.3
+                    , yesod-default             >= 1.1.3    && < 1.2
+                    , monad-control             >= 0.3      && < 0.4
+@@ -48,80 +45,6 @@ library
+     exposed-modules: Yesod
+     ghc-options:     -Wall
+ 
+-executable             yesod-ghc-wrapper
+-    main-is: ghcwrapper.hs
+-    build-depends:
+-                    base                       >= 4         && < 5
+-                  , Cabal
+-
+-executable             yesod-ld-wrapper
+-    main-is: ghcwrapper.hs
+-    cpp-options:     -DLDCMD
+-    build-depends:
+-                    base                       >= 4         && < 5
+-                  , Cabal
+-executable             yesod-ar-wrapper
+-    main-is: ghcwrapper.hs
+-    cpp-options:     -DARCMD
+-    build-depends:
+-                     base                       >= 4         && < 5
+-                   , Cabal
+-
+-executable             yesod
+-    if os(windows)
+-        cpp-options:     -DWINDOWS
+-    build-depends:     base               >= 4.3          && < 5
+-                     , ghc                >= 7.0.3        && < 7.8
+-                     , ghc-paths          >= 0.1
+-                     , parsec             >= 2.1          && < 4
+-                     , text               >= 0.11
+-                     , shakespeare-text   >= 1.0          && < 1.1
+-                     , shakespeare        >= 1.0.2        && < 1.1
+-                     , shakespeare-js     >= 1.0.2        && < 1.2
+-                     , shakespeare-css    >= 1.0.2        && < 1.1
+-                     , bytestring         >= 0.9.1.4
+-                     , time               >= 1.1.4
+-                     , template-haskell
+-                     , directory          >= 1.0
+-                     , Cabal
+-                     , unix-compat        >= 0.2          && < 0.5
+-                     , containers         >= 0.2
+-                     , attoparsec         >= 0.10
+-                     , http-types         >= 0.7
+-                     , blaze-builder      >= 0.2.1.4      && < 0.4
+-                     , filepath           >= 1.1
+-                     , process
+-                     , zlib               >= 0.5          && < 0.6
+-                     , tar                >= 0.4          && < 0.5
+-                     , system-filepath    >= 0.4          && < 0.5
+-                     , system-fileio      >= 0.3          && < 0.4
+-                     , unordered-containers
+-                     , yaml               >= 0.8          && < 0.9
+-                     , optparse-applicative >= 0.4
+-                     , fsnotify           >= 0.0          && < 0.1
+-                     , split              >= 0.2          && < 0.3
+-                     , file-embed
+-                     , conduit            >= 0.5          && < 0.6
+-                     , resourcet          >= 0.3          && < 0.5
+-                     , base64-bytestring
+-                     , lifted-base
+-                     , http-reverse-proxy >= 0.1.1
+-                     , network
+-                     , http-conduit
+-                     , network-conduit
+-                     , project-template   >= 0.1.1
+-
+-    ghc-options:       -Wall -threaded
+-    main-is:           main.hs
+-    other-modules:     Scaffolding.Scaffolder
+-                       Devel
+-                       Build
+-                       GhcBuild
+-                       Keter
+-                       AddHandler
+-                       Paths_yesod
+-                       Options
+-
+ source-repository head
+   type:     git
+   location: https://github.com/yesodweb/yesod
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/yesod-core-1.1.8_0001-remove-TH.patch b/standalone/android/haskell-patches/yesod-core-1.1.8_0001-remove-TH.patch
new file mode 100644
index 0000000000..fd641a1aa3
--- /dev/null
+++ b/standalone/android/haskell-patches/yesod-core-1.1.8_0001-remove-TH.patch
@@ -0,0 +1,476 @@
+From 801f6dea3be43113400e41aabb443456fffcd227 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:39:40 -0400
+Subject: [PATCH 1/2] remove TH
+
+---
+ Yesod/Core.hs           |   10 ----
+ Yesod/Dispatch.hs       |  119 +----------------------------------------------
+ Yesod/Handler.hs        |   27 +----------
+ Yesod/Internal/Cache.hs |    5 --
+ Yesod/Internal/Core.hs  |  119 +++++------------------------------------------
+ Yesod/Widget.hs         |   29 ------------
+ 6 files changed, 13 insertions(+), 296 deletions(-)
+
+diff --git a/Yesod/Core.hs b/Yesod/Core.hs
+index 7268d6c..ce04b7d 100644
+--- a/Yesod/Core.hs
++++ b/Yesod/Core.hs
+@@ -21,16 +21,6 @@ module Yesod.Core
+     , unauthorizedI
+       -- * Logging
+     , LogLevel (..)
+-    , logDebug
+-    , logInfo
+-    , logWarn
+-    , logError
+-    , logOther
+-    , logDebugS
+-    , logInfoS
+-    , logWarnS
+-    , logErrorS
+-    , logOtherS
+       -- * Sessions
+     , SessionBackend (..)
+     , defaultClientSessionBackend
+diff --git a/Yesod/Dispatch.hs b/Yesod/Dispatch.hs
+index 1e19388..dd37475 100644
+--- a/Yesod/Dispatch.hs
++++ b/Yesod/Dispatch.hs
+@@ -6,20 +6,9 @@
+ {-# LANGUAGE MultiParamTypeClasses #-}
+ module Yesod.Dispatch
+     ( -- * Quasi-quoted routing
+-      parseRoutes
+-    , parseRoutesNoCheck
+-    , parseRoutesFile
+-    , parseRoutesFileNoCheck
+-    , mkYesod
+-    , mkYesodSub
+       -- ** More fine-grained
+-    , mkYesodData
+-    , mkYesodSubData
+-    , mkYesodDispatch
+-    , mkYesodSubDispatch
+-    , mkDispatchInstance
+       -- ** Path pieces
+-    , PathPiece (..)
++      PathPiece (..)
+     , PathMultiPiece (..)
+     , Texts
+       -- * Convert to WAI
+@@ -52,117 +41,11 @@ import Data.Monoid (mappend)
+ import qualified Data.ByteString as S
+ import qualified Blaze.ByteString.Builder
+ import Network.HTTP.Types (status301)
+-import Yesod.Routes.TH
+ import Yesod.Content (chooseRep)
+-import Yesod.Routes.Parse
+ import System.Log.FastLogger (Logger)
+ 
+ type Texts = [Text]
+ 
+--- | Generates URL datatype and site function for the given 'Resource's. This
+--- is used for creating sites, /not/ subsites. See 'mkYesodSub' for the latter.
+--- Use 'parseRoutes' to create the 'Resource's.
+-mkYesod :: String -- ^ name of the argument datatype
+-        -> [ResourceTree String]
+-        -> Q [Dec]
+-mkYesod name = fmap (uncurry (++)) . mkYesodGeneral name [] [] False
+-
+--- | Generates URL datatype and site function for the given 'Resource's. This
+--- is used for creating subsites, /not/ sites. See 'mkYesod' for the latter.
+--- Use 'parseRoutes' to create the 'Resource's. In general, a subsite is not
+--- executable by itself, but instead provides functionality to
+--- be embedded in other sites.
+-mkYesodSub :: String -- ^ name of the argument datatype
+-           -> Cxt
+-           -> [ResourceTree String]
+-           -> Q [Dec]
+-mkYesodSub name clazzes =
+-    fmap (uncurry (++)) . mkYesodGeneral name' rest clazzes True
+-  where
+-    (name':rest) = words name
+-
+--- | Sometimes, you will want to declare your routes in one file and define
+--- your handlers elsewhere. For example, this is the only way to break up a
+--- monolithic file into smaller parts. Use this function, paired with
+--- 'mkYesodDispatch', to do just that.
+-mkYesodData :: String -> [ResourceTree String] -> Q [Dec]
+-mkYesodData name res = mkYesodDataGeneral name [] False res
+-
+-mkYesodSubData :: String -> Cxt -> [ResourceTree String] -> Q [Dec]
+-mkYesodSubData name clazzes res = mkYesodDataGeneral name clazzes True res
+-
+-mkYesodDataGeneral :: String -> Cxt -> Bool -> [ResourceTree String] -> Q [Dec]
+-mkYesodDataGeneral name clazzes isSub res = do
+-    let (name':rest) = words name
+-    (x, _) <- mkYesodGeneral name' rest clazzes isSub res
+-    let rname = mkName $ "resources" ++ name
+-    eres <- lift res
+-    let y = [ SigD rname $ ListT `AppT` (ConT ''ResourceTree `AppT` ConT ''String)
+-            , FunD rname [Clause [] (NormalB eres) []]
+-            ]
+-    return $ x ++ y
+-
+--- | See 'mkYesodData'.
+-mkYesodDispatch :: String -> [ResourceTree String] -> Q [Dec]
+-mkYesodDispatch name = fmap snd . mkYesodGeneral name [] [] False
+-
+-mkYesodSubDispatch :: String -> Cxt -> [ResourceTree String] -> Q [Dec]
+-mkYesodSubDispatch name clazzes = fmap snd . mkYesodGeneral name' rest clazzes True 
+-  where (name':rest) = words name
+-
+-mkYesodGeneral :: String                   -- ^ foundation type
+-               -> [String]                 -- ^ arguments for the type
+-               -> Cxt                      -- ^ the type constraints
+-               -> Bool                     -- ^ it this a subsite
+-               -> [ResourceTree String]
+-               -> Q([Dec],[Dec])
+-mkYesodGeneral name args clazzes isSub resS = do
+-     subsite        <- sub
+-     masterTypeSyns <- if isSub then return   [] 
+-                                else sequence [handler, widget]
+-     renderRouteDec <- mkRenderRouteInstance subsite res
+-     dispatchDec    <- mkDispatchInstance context sub master res
+-     return (renderRouteDec ++ masterTypeSyns, dispatchDec)
+-  where sub     = foldl appT subCons subArgs
+-        master  = if isSub then (varT $ mkName "master") else sub
+-        context = if isSub then cxt $ yesod : map return clazzes
+-                           else return []
+-        yesod   = classP ''Yesod [master]
+-        handler = tySynD (mkName "Handler") [] [t| GHandler $master $master    |]
+-        widget  = tySynD (mkName "Widget")  [] [t| GWidget  $master $master () |]
+-        res     = map (fmap parseType) resS
+-        subCons = conT $ mkName name
+-        subArgs = map (varT. mkName) args
+-        
+--- | If the generation of @'YesodDispatch'@ instance require finer
+--- control of the types, contexts etc. using this combinator. You will
+--- hardly need this generality. However, in certain situations, like
+--- when writing library/plugin for yesod, this combinator becomes
+--- handy.
+-mkDispatchInstance :: CxtQ                -- ^ The context
+-                   -> TypeQ               -- ^ The subsite type
+-                   -> TypeQ               -- ^ The master site type
+-                   -> [ResourceTree a]    -- ^ The resource
+-                   -> DecsQ
+-mkDispatchInstance context sub master res = do
+-  logger <- newName "logger"
+-  let loggerE = varE logger
+-      loggerP = VarP logger
+-      yDispatch = conT ''YesodDispatch `appT` sub `appT` master
+-      thisDispatch = do
+-            Clause pat body decs <- mkDispatchClause
+-                                    [|yesodRunner   $loggerE |]
+-                                    [|yesodDispatch $loggerE |]
+-                                    [|fmap chooseRep|]
+-                                    res
+-            return $ FunD 'yesodDispatch
+-                         [ Clause (loggerP:pat)
+-                                  body
+-                                  decs
+-                         ]
+-      in sequence [instanceD context yDispatch [thisDispatch]]
+-        
+-
+ -- | Convert the given argument into a WAI application, executable with any WAI
+ -- handler. This is the same as 'toWaiAppPlain', except it includes two
+ -- middlewares: GZIP compression and autohead. This is the
+diff --git a/Yesod/Handler.hs b/Yesod/Handler.hs
+index 1997bdb..98c915c 100644
+--- a/Yesod/Handler.hs
++++ b/Yesod/Handler.hs
+@@ -42,7 +42,6 @@ module Yesod.Handler
+     , RedirectUrl (..)
+     , redirect
+     , redirectWith
+-    , redirectToPost
+       -- ** Errors
+     , notFound
+     , badMethod
+@@ -100,7 +99,6 @@ module Yesod.Handler
+     , getMessageRender
+       -- * Per-request caching
+     , CacheKey
+-    , mkCacheKey
+     , cacheLookup
+     , cacheInsert
+     , cacheDelete
+@@ -172,7 +170,7 @@ import System.Log.FastLogger
+ import Control.Monad.Logger
+ 
+ import qualified Yesod.Internal.Cache as Cache
+-import Yesod.Internal.Cache (mkCacheKey, CacheKey)
++import Yesod.Internal.Cache (CacheKey)
+ import qualified Data.IORef as I
+ import Control.Exception.Lifted (catch)
+ import Control.Monad.Trans.Control
+@@ -937,29 +935,6 @@ newIdent = do
+     put x { ghsIdent = i' }
+     return $ T.pack $ 'h' : show i'
+ 
+--- | Redirect to a POST resource.
+---
+--- This is not technically a redirect; instead, it returns an HTML page with a
+--- POST form, and some Javascript to automatically submit the form. This can be
+--- useful when you need to post a plain link somewhere that needs to cause
+--- changes on the server.
+-redirectToPost :: RedirectUrl master url => url -> GHandler sub master a
+-redirectToPost url = do
+-    urlText <- toTextUrl url
+-    hamletToRepHtml [hamlet|
+-$newline never
+-$doctype 5
+-
+-
+-    
+-        Redirecting...
+-    <body onload="document.getElementById('form').submit()">
+-        <form id="form" method="post" action=#{urlText}>
+-            <noscript>
+-                <p>Javascript has been disabled; please click on the button below to be redirected.
+-            <input type="submit" value="Continue">
+-|] >>= sendResponse
+-
+ -- | Converts the given Hamlet template into 'Content', which can be used in a
+ -- Yesod 'Response'.
+ hamletToContent :: HtmlUrl (Route master) -> GHandler sub master Content
+diff --git a/Yesod/Internal/Cache.hs b/Yesod/Internal/Cache.hs
+index 4aec0d2..fdef9d7 100644
+--- a/Yesod/Internal/Cache.hs
++++ b/Yesod/Internal/Cache.hs
+@@ -3,7 +3,6 @@
+ module Yesod.Internal.Cache
+     ( Cache
+     , CacheKey
+-    , mkCacheKey
+     , lookup
+     , insert
+     , delete
+@@ -24,10 +23,6 @@ newtype Cache = Cache (Map.IntMap Any)
+ 
+ newtype CacheKey a = CacheKey Int
+ 
+--- | Generate a new 'CacheKey'. Be sure to give a full type signature.
+-mkCacheKey :: Q Exp
+-mkCacheKey = [|CacheKey|] `appE` (LitE . IntegerL . fromIntegral . hashUnique <$> runIO newUnique)
+-
+ lookup :: CacheKey a -> Cache -> Maybe a
+ lookup (CacheKey i) (Cache m) = unsafeCoerce <$> Map.lookup i m
+ 
+diff --git a/Yesod/Internal/Core.hs b/Yesod/Internal/Core.hs
+index c4a9796..90c05fc 100644
+--- a/Yesod/Internal/Core.hs
++++ b/Yesod/Internal/Core.hs
+@@ -44,7 +44,6 @@ module Yesod.Internal.Core
+ 
+ import Yesod.Content
+ import Yesod.Handler hiding (lift, getExpires)
+-import Control.Monad.Logger (logErrorS)
+ 
+ import Yesod.Routes.Class
+ import Data.Time (UTCTime, addUTCTime, getCurrentTime)
+@@ -165,22 +164,7 @@ class RenderRoute a => Yesod a where
+ 
+     -- | Applies some form of layout to the contents of a page.
+     defaultLayout :: GWidget sub a () -> GHandler sub a RepHtml
+-    defaultLayout w = do
+-        p <- widgetToPageContent w
+-        mmsg <- getMessage
+-        hamletToRepHtml [hamlet|
+-$newline never
+-$doctype 5
+-
+-<html>
+-    <head>
+-        <title>#{pageTitle p}
+-        ^{pageHead p}
+-    <body>
+-        $maybe msg <- mmsg
+-            <p .message>#{msg}
+-        ^{pageBody p}
+-|]
++    defaultLayout w = error "defaultLayout not implemented"
+ 
+     -- | Override the rendering function for a particular URL. One use case for
+     -- this is to offload static hosting to a different domain name to avoid
+@@ -521,46 +505,11 @@ applyLayout' title body = fmap chooseRep $ defaultLayout $ do
+ 
+ -- | The default error handler for 'errorHandler'.
+ defaultErrorHandler :: Yesod y => ErrorResponse -> GHandler sub y ChooseRep
+-defaultErrorHandler NotFound = do
+-    r <- waiRequest
+-    let path' = TE.decodeUtf8With TEE.lenientDecode $ W.rawPathInfo r
+-    applyLayout' "Not Found"
+-        [hamlet|
+-$newline never
+-<h1>Not Found
+-<p>#{path'}
+-|]
+-defaultErrorHandler (PermissionDenied msg) =
+-    applyLayout' "Permission Denied"
+-        [hamlet|
+-$newline never
+-<h1>Permission denied
+-<p>#{msg}
+-|]
+-defaultErrorHandler (InvalidArgs ia) =
+-    applyLayout' "Invalid Arguments"
+-        [hamlet|
+-$newline never
+-<h1>Invalid Arguments
+-<ul>
+-    $forall msg <- ia
+-        <li>#{msg}
+-|]
+-defaultErrorHandler (InternalError e) = do
+-    $logErrorS "yesod-core" e
+-    applyLayout' "Internal Server Error"
+-        [hamlet|
+-$newline never
+-<h1>Internal Server Error
+-<pre>#{e}
+-|]
+-defaultErrorHandler (BadMethod m) =
+-    applyLayout' "Bad Method"
+-        [hamlet|
+-$newline never
+-<h1>Method Not Supported
+-<p>Method <code>#{S8.unpack m}</code> not supported
+-|]
++defaultErrorHandler NotFound = error "Not Found"
++defaultErrorHandler (PermissionDenied msg) = error "Permission Denied"
++defaultErrorHandler (InvalidArgs ia) = error "Invalid Arguments"
++defaultErrorHandler (InternalError e) = error "Internal Server Error"
++defaultErrorHandler (BadMethod m) = error "Bad Method"
+ 
+ -- | Return the same URL if the user is authorized to see it.
+ --
+@@ -616,45 +565,10 @@ widgetToPageContent w = do
+     -- modernizr should be at the end of the <head> http://www.modernizr.com/docs/#installing
+     -- the asynchronous loader means your page doesn't have to wait for all the js to load
+     let (mcomplete, asyncScripts) = asyncHelper render scripts jscript jsLoc
+-        regularScriptLoad = [hamlet|
+-$newline never
+-$forall s <- scripts
+-    ^{mkScriptTag s}
+-$maybe j <- jscript
+-    $maybe s <- jsLoc
+-        <script src="#{s}">
+-    $nothing
+-        <script>^{jelper j}
+-|]
+-
+-        headAll = [hamlet|
+-$newline never
+-\^{head'}
+-$forall s <- stylesheets
+-    ^{mkLinkTag s}
+-$forall s <- css
+-    $maybe t <- right $ snd s
+-        $maybe media <- fst s
+-            <link rel=stylesheet media=#{media} href=#{t}>
+-        $nothing
+-            <link rel=stylesheet href=#{t}>
+-    $maybe content <- left $ snd s
+-        $maybe media <- fst s
+-            <style media=#{media}>#{content}
+-        $nothing
+-            <style>#{content}
+-$case jsLoader master
+-  $of BottomOfBody
+-  $of BottomOfHeadAsync asyncJsLoader
+-      ^{asyncJsLoader asyncScripts mcomplete}
+-  $of BottomOfHeadBlocking
+-      ^{regularScriptLoad}
+-|]
+-    let bodyScript = [hamlet|
+-$newline never
+-^{body}
+-^{regularScriptLoad}
+-|]
++        regularScriptLoad = error "TODO"
++
++        headAll = error "TODO"
++    let bodyScript = error "TODO"
+ 
+     return $ PageContent title headAll (case jsLoader master of
+       BottomOfBody -> bodyScript
+@@ -696,18 +610,7 @@ jsonArray = unsafeLazyByteString . encode . Array . Vector.fromList . map String
+ 
+ -- | For use with setting 'jsLoader' to 'BottomOfHeadAsync'
+ loadJsYepnope :: Yesod master => Either Text (Route master) -> [Text] -> Maybe (HtmlUrl (Route master)) -> (HtmlUrl (Route master))
+-loadJsYepnope eyn scripts mcomplete =
+-  [hamlet|
+-$newline never
+-    $maybe yn <- left eyn
+-        <script src=#{yn}>
+-    $maybe yn <- right eyn
+-        <script src=@{yn}>
+-    $maybe complete <- mcomplete
+-        <script>yepnope({load:#{jsonArray scripts},complete:function(){^{complete}}});
+-    $nothing
+-        <script>yepnope({load:#{jsonArray scripts}});
+-|]
++loadJsYepnope eyn scripts mcomplete = error "TODO"
+ 
+ asyncHelper :: (url -> [x] -> Text)
+          -> [Script (url)]
+diff --git a/Yesod/Widget.hs b/Yesod/Widget.hs
+index bd94bd3..bf79150 100644
+--- a/Yesod/Widget.hs
++++ b/Yesod/Widget.hs
+@@ -15,8 +15,6 @@ module Yesod.Widget
+       GWidget
+     , PageContent (..)
+       -- * Special Hamlet quasiquoter/TH for Widgets
+-    , whamlet
+-    , whamletFile
+     , ihamletToRepHtml
+       -- * Convert to Widget
+     , ToWidget (..)
+@@ -54,7 +52,6 @@ module Yesod.Widget
+     , addScriptEither
+       -- * Internal
+     , unGWidget
+-    , whamletFileWithSettings
+     ) where
+ 
+ import Data.Monoid
+@@ -274,32 +271,6 @@ data PageContent url = PageContent
+     , pageBody :: HtmlUrl url
+     }
+ 
+-whamlet :: QuasiQuoter
+-whamlet = NP.hamletWithSettings rules NP.defaultHamletSettings
+-
+-whamletFile :: FilePath -> Q Exp
+-whamletFile = NP.hamletFileWithSettings rules NP.defaultHamletSettings
+-
+-whamletFileWithSettings :: NP.HamletSettings -> FilePath -> Q Exp
+-whamletFileWithSettings = NP.hamletFileWithSettings rules
+-
+-rules :: Q NP.HamletRules
+-rules = do
+-    ah <- [|toWidget|]
+-    let helper qg f = do
+-            x <- newName "urender"
+-            e <- f $ VarE x
+-            let e' = LamE [VarP x] e
+-            g <- qg
+-            bind <- [|(>>=)|]
+-            return $ InfixE (Just g) bind (Just e')
+-    let ur f = do
+-            let env = NP.Env
+-                    (Just $ helper [|liftW getUrlRenderParams|])
+-                    (Just $ helper [|liftM (toHtml .) $ liftW getMessageRender|])
+-            f env
+-    return $ NP.HamletRules ah ur $ \_ b -> return $ ah `AppE` b
+-
+ -- | Wraps the 'Content' generated by 'hamletToContent' in a 'RepHtml'.
+ ihamletToRepHtml :: RenderMessage master message
+                  => HtmlUrlI18n message (Route master)
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/yesod-core-1.1.8_0002-replaced-TH-in-Yesod.Internal.Core.patch b/standalone/android/haskell-patches/yesod-core-1.1.8_0002-replaced-TH-in-Yesod.Internal.Core.patch
new file mode 100644
index 0000000000..e0a3f9b5f3
--- /dev/null
+++ b/standalone/android/haskell-patches/yesod-core-1.1.8_0002-replaced-TH-in-Yesod.Internal.Core.patch
@@ -0,0 +1,264 @@
+From 9ae3db0b3292b53715232fecec3c5e2bf03b89cd Mon Sep 17 00:00:00 2001
+From: Joey Hess <joey@kitenet.net>
+Date: Fri, 1 Mar 2013 01:02:53 -0400
+Subject: [PATCH 2/2] replaced TH in Yesod.Internal.Core
+
+Done by running a build with -ddump-splices and manually pasting in the
+spliced code, and then modifying it until it compiles.
+---
+ Yesod/Internal/Core.hs |  211 +++++++++++++++++++++++++++++++++++++++++++++---
+ 1 file changed, 201 insertions(+), 10 deletions(-)
+
+diff --git a/Yesod/Internal/Core.hs b/Yesod/Internal/Core.hs
+index 90c05fc..b9a0ae8 100644
+--- a/Yesod/Internal/Core.hs
++++ b/Yesod/Internal/Core.hs
+@@ -96,6 +96,9 @@ import System.Log.FastLogger (Logger, mkLogger, loggerDate, LogStr (..), loggerP
+ import Control.Monad.Logger (LogLevel (LevelInfo, LevelOther), LogSource)
+ import System.Log.FastLogger.Date (ZonedDate)
+ import System.IO (stdout)
++import qualified Data.Foldable
++import qualified Text.Blaze.Internal
++import qualified Text.Hamlet
+ 
+ yesodVersion :: String
+ yesodVersion = showVersion Paths_yesod_core.version
+@@ -164,7 +167,28 @@ class RenderRoute a => Yesod a where
+ 
+     -- | Applies some form of layout to the contents of a page.
+     defaultLayout :: GWidget sub a () -> GHandler sub a RepHtml
+-    defaultLayout w = error "defaultLayout not implemented"
++    defaultLayout w = do
++        p <- widgetToPageContent w
++        mmsg <- getMessage
++        hamletToRepHtml $ \ _render_ay88 -> do { id
++                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                   "<!DOCTYPE html>\n<html><head><title>");
++              id (TBH.toHtml (pageTitle p));
++              id ((Text.Blaze.Internal.preEscapedText . T.pack) "");
++              id (pageHead p) _render_ay88;
++              id ((Text.Blaze.Internal.preEscapedText . T.pack) "");
++              Text.Hamlet.maybeH
++                mmsg
++                (\ msg_ay89
++                   -> do { id
++                             ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                "
");
++                           id (TBH.toHtml msg_ay89);
++                           id ((Text.Blaze.Internal.preEscapedText . T.pack) "
") })
++                Nothing;
++              id (pageBody p) _render_ay88;
++              id
++                ((Text.Blaze.Internal.preEscapedText . T.pack) "") }
+ 
+     -- | Override the rendering function for a particular URL. One use case for
+     -- this is to offload static hosting to a different domain name to avoid
+@@ -505,11 +529,45 @@ applyLayout' title body = fmap chooseRep $ defaultLayout $ do
+ 
+ -- | The default error handler for 'errorHandler'.
+ defaultErrorHandler :: Yesod y => ErrorResponse -> GHandler sub y ChooseRep
+-defaultErrorHandler NotFound = error "Not Found"
+-defaultErrorHandler (PermissionDenied msg) = error "Permission Denied"
+-defaultErrorHandler (InvalidArgs ia) = error "Invalid Arguments"
+-defaultErrorHandler (InternalError e) = error "Internal Server Error"
+-defaultErrorHandler (BadMethod m) = error "Bad Method"
++defaultErrorHandler NotFound = do
++    r <- waiRequest
++    let path' = TE.decodeUtf8With TEE.lenientDecode $ W.rawPathInfo r
++    applyLayout' "Not Found" $ \ _render_ayac -> do { id
++                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                   "
Not Found
");
++              id (TBH.toHtml path');
++              id ((Text.Blaze.Internal.preEscapedText . T.pack) "
") }
++defaultErrorHandler (PermissionDenied msg) =
++    applyLayout' "Permission Denied" $  \ _render_ayah -> do { id
++                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                   "
Permission denied
");
++              id (TBH.toHtml msg);
++              id ((Text.Blaze.Internal.preEscapedText . T.pack) "
") }
++defaultErrorHandler (InvalidArgs ia) =
++    applyLayout' "Invalid Arguments" $  \ _render_ayam -> do { id
++                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                   "
Invalid Arguments
    ");
++              Data.Foldable.mapM_
++                (\ msg_ayan
++                   -> do { id ((Text.Blaze.Internal.preEscapedText . T.pack) "
  • ");
++                           id (TBH.toHtml msg_ayan);
++                           id ((Text.Blaze.Internal.preEscapedText . T.pack) "
    • ") })
++                ia;
++              id ((Text.Blaze.Internal.preEscapedText . T.pack) "
") }
++defaultErrorHandler (InternalError e) = do
++    applyLayout' "Internal Server Error" $  \ _render_ayau -> do { id
++                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                   "
Internal Server Error
");
++              id (TBH.toHtml e);
++              id ((Text.Blaze.Internal.preEscapedText . T.pack) "
") }
++defaultErrorHandler (BadMethod m) =
++    applyLayout' "Bad Method" $ \ _render_ayaz -> do { id
++                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                   "
Method Not Supported
Method ");
++              id (TBH.toHtml (S8.unpack m));
++              id
++                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                   " not supported
") }
+ 
+ -- | Return the same URL if the user is authorized to see it.
+ --
+@@ -565,10 +623,99 @@ widgetToPageContent w = do
+     -- modernizr should be at the end of the  http://www.modernizr.com/docs/#installing
+     -- the asynchronous loader means your page doesn't have to wait for all the js to load
+     let (mcomplete, asyncScripts) = asyncHelper render scripts jscript jsLoc
+-        regularScriptLoad = error "TODO"
+-
+-        headAll = error "TODO"
+-    let bodyScript = error "TODO"
++        regularScriptLoad =  \ _render_aybs -> do { Data.Foldable.mapM_
++                (\ s_aybt
++                   -> id (mkScriptTag s_aybt) _render_aybs)
++                scripts;
++              Text.Hamlet.maybeH
++                jscript
++                (\ j_aybu
++                   -> Text.Hamlet.maybeH
++                        jsLoc
++                        (\ s_aybv
++                           -> do { id
++                                     ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                        "") })
++                        (Just
++                           (do { id
++                                   ((Text.Blaze.Internal.preEscapedText . T.pack) "") })))
++                Nothing }
++
++        headAll = \ _render_aybz  -> do
++            { id head' _render_aybz;
++              Data.Foldable.mapM_
++                (\ s_aybA -> id (mkLinkTag s_aybA) _render_aybz)
++                stylesheets;
++              Data.Foldable.mapM_
++                (\ s_aybB
++                   -> do { Text.Hamlet.maybeH
++                             (right (snd s_aybB))
++                             (\ t_aybC
++                                -> Text.Hamlet.maybeH
++                                     (fst s_aybB)
++                                     (\ media_aybD
++                                        -> do { id
++                                                  ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                                     "") })
++                                     (Just
++                                        (do { id
++                                                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                                   "") })))
++                             Nothing;
++                           Text.Hamlet.maybeH
++                             (left (snd s_aybB))
++                             (\ content_aybE
++                                -> Text.Hamlet.maybeH
++                                     (fst s_aybB)
++                                     (\ media_aybF
++                                        -> do { id
++                                                  ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                                     "") })
++                                     (Just
++                                        (do { id
++                                                ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                                   "") })))
++                             Nothing })
++                css;
++              case jsLoader master of
++                BottomOfBody -> return ()
++                BottomOfHeadAsync asyncJsLoader -> id (asyncJsLoader asyncScripts mcomplete) _render_aybz
++                BottomOfHeadBlocking -> id regularScriptLoad _render_aybz
++            }
++
++    let bodyScript = \ _render_aybL -> do {
++              id body _render_aybL;
++              id regularScriptLoad _render_aybL }
+ 
+     return $ PageContent title headAll (case jsLoader master of
+       BottomOfBody -> bodyScript
+@@ -611,6 +758,50 @@ jsonArray = unsafeLazyByteString . encode . Array . Vector.fromList . map String
+ -- | For use with setting 'jsLoader' to 'BottomOfHeadAsync'
+ loadJsYepnope :: Yesod master => Either Text (Route master) -> [Text] -> Maybe (HtmlUrl (Route master)) -> (HtmlUrl (Route master))
+ loadJsYepnope eyn scripts mcomplete = error "TODO"
++{-
++    \ _render_aybU
++      -> do { Text.Hamlet.maybeH
++                (left eyn)
++                (\ yn_aybV
++                   -> do { id
++                             ((Text.Blaze.Internal.preEscapedText . T.pack) "") })
++                Nothing;
++              Text.Hamlet.maybeH
++                (right eyn)
++                (\ yn_aybW
++                   -> do { id
++                             ((Text.Blaze.Internal.preEscapedText . T.pack) "") })
++                Nothing;
++              Text.Hamlet.maybeH
++                mcomplete
++                (\ complete_aybY
++                   -> do { id
++                             ((Text.Blaze.Internal.preEscapedText . T.pack)
++                                "") })
++                (Just
++                   (do { id
++                           ((Text.Blaze.Internal.preEscapedText . T.pack)
++                              "") })) }
++-}
+ 
+ asyncHelper :: (url -> [x] -> Text)
+          -> [Script (url)]
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/yesod-default-1.1.3.2_0001-remove-TH.patch b/standalone/android/haskell-patches/yesod-default-1.1.3.2_0001-remove-TH.patch
new file mode 100644
index 0000000000..e6048ee0a4
--- /dev/null
+++ b/standalone/android/haskell-patches/yesod-default-1.1.3.2_0001-remove-TH.patch
@@ -0,0 +1,102 @@
+From 8ff7908799eb69d440168ff3df1fe3187879df33 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:39:57 -0400
+Subject: [PATCH] remove TH
+
+---
+ Yesod/Default/Util.hs |   61 +------------------------------------------------
+ 1 file changed, 1 insertion(+), 60 deletions(-)
+
+diff --git a/Yesod/Default/Util.hs b/Yesod/Default/Util.hs
+index 578b9bc..178e342 100644
+--- a/Yesod/Default/Util.hs
++++ b/Yesod/Default/Util.hs
+@@ -5,8 +5,6 @@
+ module Yesod.Default.Util
+     ( addStaticContentExternal
+     , globFile
+-    , widgetFileNoReload
+-    , widgetFileReload
+     , TemplateLanguage (..)
+     , defaultTemplateLanguages
+     , WidgetFileSettings
+@@ -21,9 +19,6 @@ import Yesod.Core -- purposely using complete import so that Haddock will see ad
+ import Control.Monad (when, unless)
+ import System.Directory (doesFileExist, createDirectoryIfMissing)
+ import Language.Haskell.TH.Syntax
+-import Text.Lucius (luciusFile, luciusFileReload)
+-import Text.Julius (juliusFile, juliusFileReload)
+-import Text.Cassius (cassiusFile, cassiusFileReload)
+ import Text.Hamlet (HamletSettings, defaultHamletSettings)
+ import Data.Maybe (catMaybes)
+ import Data.Default (Default (def))
+@@ -72,13 +67,7 @@ data TemplateLanguage = TemplateLanguage
+ 
+ defaultTemplateLanguages :: HamletSettings -> [TemplateLanguage]
+ defaultTemplateLanguages hset =
+-    [ TemplateLanguage False "hamlet"  whamletFile' whamletFile'
+-    , TemplateLanguage True  "cassius" cassiusFile  cassiusFileReload
+-    , TemplateLanguage True  "julius"  juliusFile   juliusFileReload
+-    , TemplateLanguage True  "lucius"  luciusFile   luciusFileReload
+-    ]
+-  where
+-    whamletFile' = whamletFileWithSettings hset
++    [ ]
+ 
+ data WidgetFileSettings = WidgetFileSettings
+     { wfsLanguages :: HamletSettings -> [TemplateLanguage]
+@@ -87,51 +76,3 @@ data WidgetFileSettings = WidgetFileSettings
+ 
+ instance Default WidgetFileSettings where
+     def = WidgetFileSettings defaultTemplateLanguages defaultHamletSettings
+-
+-widgetFileNoReload :: WidgetFileSettings -> FilePath -> Q Exp
+-widgetFileNoReload wfs x = combine "widgetFileNoReload" x False $ wfsLanguages wfs $ wfsHamletSettings wfs
+-
+-widgetFileReload :: WidgetFileSettings -> FilePath -> Q Exp
+-widgetFileReload wfs x = combine "widgetFileReload" x True $ wfsLanguages wfs $ wfsHamletSettings wfs
+-
+-combine :: String -> String -> Bool -> [TemplateLanguage] -> Q Exp
+-combine func file isReload tls = do
+-    mexps <- qmexps
+-    case catMaybes mexps of
+-        [] -> error $ concat
+-            [ "Called "
+-            , func
+-            , " on "
+-            , show file
+-            , ", but no template were found."
+-            ]
+-        exps -> return $ DoE $ map NoBindS exps
+-  where
+-    qmexps :: Q [Maybe Exp]
+-    qmexps = mapM go tls
+-
+-    go :: TemplateLanguage -> Q (Maybe Exp)
+-    go tl = whenExists file (tlRequiresToWidget tl) (tlExtension tl) ((if isReload then tlReload else tlNoReload) tl)
+-
+-whenExists :: String
+-           -> Bool -- ^ requires toWidget wrap
+-           -> String -> (FilePath -> Q Exp) -> Q (Maybe Exp)
+-whenExists = warnUnlessExists False
+-
+-warnUnlessExists :: Bool
+-                 -> String
+-                 -> Bool -- ^ requires toWidget wrap
+-                 -> String -> (FilePath -> Q Exp) -> Q (Maybe Exp)
+-warnUnlessExists shouldWarn x wrap glob f = do
+-    let fn = globFile glob x
+-    e <- qRunIO $ doesFileExist fn
+-    when (shouldWarn && not e) $ qRunIO $ putStrLn $ "widget file not found: " ++ fn
+-    if e
+-        then do
+-            ex <- f fn
+-            if wrap
+-                then do
+-                    tw <- [|toWidget|]
+-                    return $ Just $ tw `AppE` ex
+-                else return $ Just ex
+-        else return Nothing
+-- 
+1.7.10.4
+
diff --git a/standalone/android/haskell-patches/yesod-form-1.2.1.3_0001-avoid-TH-hack-job.patch b/standalone/android/haskell-patches/yesod-form-1.2.1.3_0001-avoid-TH-hack-job.patch
new file mode 100644
index 0000000000..c4ab44c302
--- /dev/null
+++ b/standalone/android/haskell-patches/yesod-form-1.2.1.3_0001-avoid-TH-hack-job.patch
@@ -0,0 +1,675 @@
+From c47d263779fba34629130398f1b08be1b8e468f7 Mon Sep 17 00:00:00 2001
+From: Joey Hess 
+Date: Thu, 28 Feb 2013 23:40:05 -0400
+Subject: [PATCH] avoid TH (hack job)
+
+---
+ Yesod/Form/Fields.hs    |   93 ++++++++++++++++++++++++++++---------
+ Yesod/Form/Functions.hs |  118 ++++++++++++++++++++++++++++++++---------------
+ Yesod/Form/Jquery.hs    |   13 ++++--
+ Yesod/Form/MassInput.hs |   18 ++++++--
+ yesod-form.cabal        |    1 -
+ 5 files changed, 173 insertions(+), 70 deletions(-)
+
+diff --git a/Yesod/Form/Fields.hs b/Yesod/Form/Fields.hs
+index adc59de..353c8d0 100644
+--- a/Yesod/Form/Fields.hs
++++ b/Yesod/Form/Fields.hs
+@@ -50,7 +50,7 @@ import Yesod.Form.Types
+ import Yesod.Form.I18n.English
+ import Yesod.Form.Functions (parseHelper)
+ import Yesod.Handler (getMessageRender)
+-import Yesod.Widget (toWidget, whamlet, GWidget)
++import Yesod.Widget (toWidget, GWidget)
+ import Yesod.Message (RenderMessage (renderMessage), SomeMessage (..))
+ import Text.Hamlet
+ import Text.Blaze (ToMarkup (toMarkup), preEscapedToMarkup, unsafeByteString)
+@@ -108,10 +108,12 @@ intField = Field
+             Right (a, "") -> Right a
+             _ -> Left $ MsgInvalidInteger s
+ 
+-    , fieldView = \theId name attrs val isReq -> toWidget [hamlet|
++    , fieldView = \theId name attrs val isReq -> error "intField TH TODO"
++{- toWidget [hamlet|
+ $newline never
+ 
+ |]
++-}
+     , fieldEnctype = UrlEncoded
+     }
+   where
+@@ -125,32 +127,40 @@ doubleField = Field
+             Right (a, "") -> Right a
+             _ -> Left $ MsgInvalidNumber s
+ 
+-    , fieldView = \theId name attrs val isReq -> toWidget [hamlet|
++    , fieldView = \theId name attrs val isReq -> error "doubleField TH TODO"
++{- 
++ - toWidget [hamlet|
+ $newline never
+ 
+ |]
++-}
+     , fieldEnctype = UrlEncoded
+     }
+-  where showVal = either id (pack . show)
++{-
++  where showVal = either id (pack . show)-}
+ 
+ dayField :: RenderMessage master FormMessage => Field sub master Day
+ dayField = Field
+     { fieldParse = parseHelper $ parseDate . unpack
+-    , fieldView = \theId name attrs val isReq -> toWidget [hamlet|
++    , fieldView = \theId name attrs val isReq -> error "dayfield TH TODO"
++{- toWidget [hamlet|
+ $newline never
+ 
+ |]
++-}
+     , fieldEnctype = UrlEncoded
+     }
+-  where showVal = either id (pack . show)
++{-  where showVal = either id (pack . show) -}
+ 
+ timeField :: RenderMessage master FormMessage => Field sub master TimeOfDay
+ timeField = Field
+     { fieldParse = parseHelper parseTime
+-    , fieldView = \theId name attrs val isReq -> toWidget [hamlet|
++    , fieldView = \theId name attrs val isReq -> error "timefield TH TODO"
++{- toWidget [hamlet|
+ $newline never
+ 
+ |]
++-}
+     , fieldEnctype = UrlEncoded
+     }
+   where
+@@ -163,10 +173,12 @@ $newline never
+ htmlField :: RenderMessage master FormMessage => Field sub master Html
+ htmlField = Field
+     { fieldParse = parseHelper $ Right . preEscapedText . sanitizeBalance
+-    , fieldView = \theId name attrs val _isReq -> toWidget [hamlet|
++    , fieldView = \theId name attrs val _isReq -> error "htmlField TH TODO"
++{- toWidget [hamlet|
+ $newline never
+