Merge branch 'master' into sqlite
This commit is contained in:
Joey Hess
commit
02e00fd7ab
37 changed files with 314 additions and 128 deletions
Annex
Assistant
Command
Database
Git
Logs.hsLogs
Remote
Upgrade
doc
devblog
profiling
todo
addunlocked_config_setting
git-annex-cat
making_it_easier_to_smudge_dotfiles
comment_1_73ad5cd7f65c94a1db859d22eb6eece4._commentcomment_2_3912c1194c3f4f0e4c372e7603e0a3e5._comment
optimise_by_converting_Ref_to_ByteString.mdwnoptimise_journal_access.mdwnoptimize_by_converting_String_to_ByteString.mdwn|
|
@ -91,6 +91,8 @@ import Annex.Concurrent
|
|||
import Types.WorkerPool
|
||||
import qualified Utility.RawFilePath as R
|
||||
|
||||
import qualified System.FilePath.ByteString as P
|
||||
|
||||
{- Checks if a given key's content is currently present. -}
|
||||
inAnnex :: Key -> Annex Bool
|
||||
inAnnex key = inAnnexCheck key $ liftIO . R.doesPathExist
|
||||
|
|
@ -742,7 +744,7 @@ listKeys keyloc = do
|
|||
if depth < 2
|
||||
then do
|
||||
contents' <- filterM (present s) contents
|
||||
let keys = mapMaybe (fileKey . takeFileName) contents'
|
||||
let keys = mapMaybe (fileKey . P.takeFileName . toRawFilePath) contents'
|
||||
continue keys []
|
||||
else do
|
||||
let deeper = walk s (depth - 1)
|
||||
|
|
@ -816,7 +818,7 @@ dirKeys dirspec = do
|
|||
contents <- liftIO $ getDirectoryContents dir
|
||||
files <- liftIO $ filterM doesFileExist $
|
||||
map (dir </>) contents
|
||||
return $ mapMaybe (fileKey . takeFileName) files
|
||||
return $ mapMaybe (fileKey . P.takeFileName . toRawFilePath) files
|
||||
, return []
|
||||
)
|
||||
|
||||
|
|
@ -835,7 +837,8 @@ staleKeysPrune dirspec nottransferred = do
|
|||
|
||||
dir <- fromRepo dirspec
|
||||
forM_ dups $ \k ->
|
||||
pruneTmpWorkDirBefore (dir </> keyFile k) (liftIO . removeFile)
|
||||
pruneTmpWorkDirBefore (dir </> fromRawFilePath (keyFile k))
|
||||
(liftIO . removeFile)
|
||||
|
||||
if nottransferred
|
||||
then do
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ checkDiskSpace' need destdir key alreadythere samefilesystem = ifM (Annex.getSta
|
|||
_ -> return True
|
||||
)
|
||||
where
|
||||
dir = maybe (fromRepo gitAnnexDir) return destdir
|
||||
dir = maybe (fromRawFilePath <$> fromRepo gitAnnexDir) return destdir
|
||||
|
||||
needMoreDiskSpace :: Integer -> String
|
||||
needMoreDiskSpace n = "not enough free space, need " ++
|
||||
|
|
|
|||
|
|
@ -253,7 +253,7 @@ parseLinkTargetOrPointerLazy b =
|
|||
{- Parses a symlink target to a Key. -}
|
||||
parseLinkTarget :: S.ByteString -> Maybe Key
|
||||
parseLinkTarget l
|
||||
| isLinkToAnnex l = fileKey' $ snd $ S8.breakEnd pathsep l
|
||||
| isLinkToAnnex l = fileKey $ snd $ S8.breakEnd pathsep l
|
||||
| otherwise = Nothing
|
||||
where
|
||||
pathsep '/' = True
|
||||
|
|
@ -263,9 +263,9 @@ parseLinkTarget l
|
|||
pathsep _ = False
|
||||
|
||||
formatPointer :: Key -> S.ByteString
|
||||
formatPointer k = prefix <> keyFile' k <> nl
|
||||
formatPointer k = prefix <> keyFile k <> nl
|
||||
where
|
||||
prefix = toInternalGitPath $ toRawFilePath (pathSeparator:objectDir)
|
||||
prefix = toInternalGitPath $ P.pathSeparator `S.cons` objectDir'
|
||||
nl = S8.singleton '\n'
|
||||
|
||||
{- Maximum size of a file that could be a pointer to a key.
|
||||
|
|
|
|||
|
|
@ -9,9 +9,7 @@
|
|||
|
||||
module Annex.Locations (
|
||||
keyFile,
|
||||
keyFile',
|
||||
fileKey,
|
||||
fileKey',
|
||||
keyPaths,
|
||||
keyPath,
|
||||
annexDir,
|
||||
|
|
@ -126,19 +124,16 @@ import qualified Utility.RawFilePath as R
|
|||
|
||||
{- The directory git annex uses for local state, relative to the .git
|
||||
- directory -}
|
||||
annexDir :: FilePath
|
||||
annexDir = addTrailingPathSeparator "annex"
|
||||
|
||||
annexDir' :: RawFilePath
|
||||
annexDir' = P.addTrailingPathSeparator "annex"
|
||||
annexDir :: RawFilePath
|
||||
annexDir = P.addTrailingPathSeparator "annex"
|
||||
|
||||
{- The directory git annex uses for locally available object content,
|
||||
- relative to the .git directory -}
|
||||
objectDir :: FilePath
|
||||
objectDir = addTrailingPathSeparator $ annexDir </> "objects"
|
||||
objectDir = fromRawFilePath objectDir'
|
||||
|
||||
objectDir' :: RawFilePath
|
||||
objectDir' = P.addTrailingPathSeparator $ annexDir' P.</> "objects"
|
||||
objectDir' = P.addTrailingPathSeparator $ annexDir P.</> "objects"
|
||||
|
||||
{- Annexed file's possible locations relative to the .git directory.
|
||||
- There are two different possibilities, using different hashes.
|
||||
|
|
@ -262,46 +257,51 @@ gitAnnexInodeCache key r config = do
|
|||
return $ fromRawFilePath loc ++ ".cache"
|
||||
|
||||
gitAnnexInodeSentinal :: Git.Repo -> RawFilePath
|
||||
gitAnnexInodeSentinal r = gitAnnexDir' r P.</> "sentinal"
|
||||
gitAnnexInodeSentinal r = gitAnnexDir r P.</> "sentinal"
|
||||
|
||||
gitAnnexInodeSentinalCache :: Git.Repo -> RawFilePath
|
||||
gitAnnexInodeSentinalCache r = gitAnnexInodeSentinal r <> ".cache"
|
||||
|
||||
{- The annex directory of a repository. -}
|
||||
gitAnnexDir :: Git.Repo -> FilePath
|
||||
gitAnnexDir r = addTrailingPathSeparator $ fromRawFilePath (Git.localGitDir r) </> annexDir
|
||||
|
||||
gitAnnexDir' :: Git.Repo -> RawFilePath
|
||||
gitAnnexDir' r = P.addTrailingPathSeparator $ Git.localGitDir r P.</> annexDir'
|
||||
gitAnnexDir :: Git.Repo -> RawFilePath
|
||||
gitAnnexDir r = P.addTrailingPathSeparator $ Git.localGitDir r P.</> annexDir
|
||||
|
||||
{- The part of the annex directory where file contents are stored. -}
|
||||
gitAnnexObjectDir :: Git.Repo -> FilePath
|
||||
gitAnnexObjectDir r = addTrailingPathSeparator $ fromRawFilePath (Git.localGitDir r) </> objectDir
|
||||
gitAnnexObjectDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ Git.localGitDir r P.</> objectDir'
|
||||
|
||||
{- .git/annex/tmp/ is used for temp files for key's contents -}
|
||||
gitAnnexTmpObjectDir :: Git.Repo -> FilePath
|
||||
gitAnnexTmpObjectDir r = addTrailingPathSeparator $ gitAnnexDir r </> "tmp"
|
||||
gitAnnexTmpObjectDir = fromRawFilePath . gitAnnexTmpObjectDir'
|
||||
|
||||
gitAnnexTmpObjectDir' :: Git.Repo -> RawFilePath
|
||||
gitAnnexTmpObjectDir' r = P.addTrailingPathSeparator $ gitAnnexDir r P.</> "tmp"
|
||||
|
||||
{- .git/annex/othertmp/ is used for other temp files -}
|
||||
gitAnnexTmpOtherDir :: Git.Repo -> FilePath
|
||||
gitAnnexTmpOtherDir r = addTrailingPathSeparator $ gitAnnexDir r </> "othertmp"
|
||||
gitAnnexTmpOtherDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "othertmp"
|
||||
|
||||
{- Lock file for gitAnnexTmpOtherDir. -}
|
||||
gitAnnexTmpOtherLock :: Git.Repo -> FilePath
|
||||
gitAnnexTmpOtherLock r = gitAnnexDir r </> "othertmp.lck"
|
||||
gitAnnexTmpOtherLock r = fromRawFilePath $ gitAnnexDir r P.</> "othertmp.lck"
|
||||
|
||||
{- .git/annex/misctmp/ was used by old versions of git-annex and is still
|
||||
- used during initialization -}
|
||||
gitAnnexTmpOtherDirOld :: Git.Repo -> FilePath
|
||||
gitAnnexTmpOtherDirOld r = addTrailingPathSeparator $ gitAnnexDir r </> "misctmp"
|
||||
gitAnnexTmpOtherDirOld r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "misctmp"
|
||||
|
||||
{- .git/annex/watchtmp/ is used by the watcher and assistant -}
|
||||
gitAnnexTmpWatcherDir :: Git.Repo -> FilePath
|
||||
gitAnnexTmpWatcherDir r = addTrailingPathSeparator $ gitAnnexDir r </> "watchtmp"
|
||||
gitAnnexTmpWatcherDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "watchtmp"
|
||||
|
||||
{- The temp file to use for a given key's content. -}
|
||||
gitAnnexTmpObjectLocation :: Key -> Git.Repo -> FilePath
|
||||
gitAnnexTmpObjectLocation key r = gitAnnexTmpObjectDir r </> keyFile key
|
||||
gitAnnexTmpObjectLocation key r = fromRawFilePath $
|
||||
gitAnnexTmpObjectDir' r P.</> keyFile key
|
||||
|
||||
{- Given a temp file such as gitAnnexTmpObjectLocation, makes a name for a
|
||||
- subdirectory in the same location, that can be used as a work area
|
||||
|
|
@ -318,19 +318,21 @@ gitAnnexTmpWorkDir p =
|
|||
|
||||
{- .git/annex/bad/ is used for bad files found during fsck -}
|
||||
gitAnnexBadDir :: Git.Repo -> FilePath
|
||||
gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r </> "bad"
|
||||
gitAnnexBadDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "bad"
|
||||
|
||||
{- The bad file to use for a given key. -}
|
||||
gitAnnexBadLocation :: Key -> Git.Repo -> FilePath
|
||||
gitAnnexBadLocation key r = gitAnnexBadDir r </> keyFile key
|
||||
gitAnnexBadLocation key r = gitAnnexBadDir r </> fromRawFilePath (keyFile key)
|
||||
|
||||
{- .git/annex/foounused is used to number possibly unused keys -}
|
||||
gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath
|
||||
gitAnnexUnusedLog prefix r = gitAnnexDir r </> (prefix ++ "unused")
|
||||
gitAnnexUnusedLog prefix r =
|
||||
fromRawFilePath (gitAnnexDir r) </> (prefix ++ "unused")
|
||||
|
||||
{- .git/annex/keysdb/ contains a database of information about keys. -}
|
||||
gitAnnexKeysDb :: Git.Repo -> FilePath
|
||||
gitAnnexKeysDb r = gitAnnexDir r </> "keysdb"
|
||||
gitAnnexKeysDb r = fromRawFilePath $ gitAnnexDir r P.</> "keysdb"
|
||||
|
||||
{- Lock file for the keys database. -}
|
||||
gitAnnexKeysDbLock :: Git.Repo -> FilePath
|
||||
|
|
@ -344,7 +346,8 @@ gitAnnexKeysDbIndexCache r = gitAnnexKeysDb r ++ ".cache"
|
|||
{- .git/annex/fsck/uuid/ is used to store information about incremental
|
||||
- fscks. -}
|
||||
gitAnnexFsckDir :: UUID -> Git.Repo -> FilePath
|
||||
gitAnnexFsckDir u r = gitAnnexDir r </> "fsck" </> fromUUID u
|
||||
gitAnnexFsckDir u r = fromRawFilePath $
|
||||
gitAnnexDir r P.</> "fsck" P.</> fromUUID u
|
||||
|
||||
{- used to store information about incremental fscks. -}
|
||||
gitAnnexFsckState :: UUID -> Git.Repo -> FilePath
|
||||
|
|
@ -364,20 +367,21 @@ gitAnnexFsckDbLock u r = gitAnnexFsckDir u r </> "fsck.lck"
|
|||
|
||||
{- .git/annex/fsckresults/uuid is used to store results of git fscks -}
|
||||
gitAnnexFsckResultsLog :: UUID -> Git.Repo -> FilePath
|
||||
gitAnnexFsckResultsLog u r = gitAnnexDir r </> "fsckresults" </> fromUUID u
|
||||
gitAnnexFsckResultsLog u r = fromRawFilePath $
|
||||
gitAnnexDir r P.</> "fsckresults" P.</> fromUUID u
|
||||
|
||||
{- .git/annex/smudge.log is used to log smudges worktree files that need to
|
||||
- be updated. -}
|
||||
gitAnnexSmudgeLog :: Git.Repo -> FilePath
|
||||
gitAnnexSmudgeLog r = gitAnnexDir r </> "smudge.log"
|
||||
gitAnnexSmudgeLog r = fromRawFilePath $ gitAnnexDir r P.</> "smudge.log"
|
||||
|
||||
gitAnnexSmudgeLock :: Git.Repo -> FilePath
|
||||
gitAnnexSmudgeLock r = gitAnnexDir r </> "smudge.lck"
|
||||
gitAnnexSmudgeLock r = fromRawFilePath $ gitAnnexDir r P.</> "smudge.lck"
|
||||
|
||||
{- .git/annex/export/ is used to store information about
|
||||
- exports to special remotes. -}
|
||||
gitAnnexExportDir :: Git.Repo -> FilePath
|
||||
gitAnnexExportDir r = gitAnnexDir r </> "export"
|
||||
gitAnnexExportDir r = fromRawFilePath $ gitAnnexDir r P.</> "export"
|
||||
|
||||
{- Directory containing database used to record export info. -}
|
||||
gitAnnexExportDbDir :: UUID -> Git.Repo -> FilePath
|
||||
|
|
@ -394,7 +398,8 @@ gitAnnexExportUpdateLock u r = gitAnnexExportDbDir u r ++ ".upl"
|
|||
{- Log file used to keep track of files that were in the tree exported to a
|
||||
- remote, but were excluded by its preferred content settings. -}
|
||||
gitAnnexExportExcludeLog :: UUID -> Git.Repo -> FilePath
|
||||
gitAnnexExportExcludeLog u r = gitAnnexDir r </> "export.ex" </> fromUUID u
|
||||
gitAnnexExportExcludeLog u r = fromRawFilePath $
|
||||
gitAnnexDir r P.</> "export.ex" P.</> fromUUID u
|
||||
|
||||
{- Directory containing database used to record remote content ids.
|
||||
-
|
||||
|
|
@ -402,7 +407,7 @@ gitAnnexExportExcludeLog u r = gitAnnexDir r </> "export.ex" </> fromUUID u
|
|||
- need to be rebuilt with a new name.)
|
||||
-}
|
||||
gitAnnexContentIdentifierDbDir :: Git.Repo -> FilePath
|
||||
gitAnnexContentIdentifierDbDir r = gitAnnexDir r </> "cidsdb"
|
||||
gitAnnexContentIdentifierDbDir r = fromRawFilePath $ gitAnnexDir r P.</> "cidsdb"
|
||||
|
||||
{- Lock file for writing to the content id database. -}
|
||||
gitAnnexContentIdentifierLock :: Git.Repo -> FilePath
|
||||
|
|
@ -411,128 +416,137 @@ gitAnnexContentIdentifierLock r = gitAnnexContentIdentifierDbDir r ++ ".lck"
|
|||
{- .git/annex/schedulestate is used to store information about when
|
||||
- scheduled jobs were last run. -}
|
||||
gitAnnexScheduleState :: Git.Repo -> FilePath
|
||||
gitAnnexScheduleState r = gitAnnexDir r </> "schedulestate"
|
||||
gitAnnexScheduleState r = fromRawFilePath $ gitAnnexDir r P.</> "schedulestate"
|
||||
|
||||
{- .git/annex/creds/ is used to store credentials to access some special
|
||||
- remotes. -}
|
||||
gitAnnexCredsDir :: Git.Repo -> FilePath
|
||||
gitAnnexCredsDir r = addTrailingPathSeparator $ gitAnnexDir r </> "creds"
|
||||
gitAnnexCredsDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "creds"
|
||||
|
||||
{- .git/annex/certificate.pem and .git/annex/key.pem are used by the webapp
|
||||
- when HTTPS is enabled -}
|
||||
gitAnnexWebCertificate :: Git.Repo -> FilePath
|
||||
gitAnnexWebCertificate r = gitAnnexDir r </> "certificate.pem"
|
||||
gitAnnexWebCertificate r = fromRawFilePath $ gitAnnexDir r P.</> "certificate.pem"
|
||||
gitAnnexWebPrivKey :: Git.Repo -> FilePath
|
||||
gitAnnexWebPrivKey r = gitAnnexDir r </> "privkey.pem"
|
||||
gitAnnexWebPrivKey r = fromRawFilePath $ gitAnnexDir r P.</> "privkey.pem"
|
||||
|
||||
{- .git/annex/feeds/ is used to record per-key (url) state by importfeeds -}
|
||||
gitAnnexFeedStateDir :: Git.Repo -> FilePath
|
||||
gitAnnexFeedStateDir r = addTrailingPathSeparator $ gitAnnexDir r </> "feedstate"
|
||||
gitAnnexFeedStateDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "feedstate"
|
||||
|
||||
gitAnnexFeedState :: Key -> Git.Repo -> FilePath
|
||||
gitAnnexFeedState k r = gitAnnexFeedStateDir r </> keyFile k
|
||||
gitAnnexFeedState k r = gitAnnexFeedStateDir r </> fromRawFilePath (keyFile k)
|
||||
|
||||
{- .git/annex/merge/ is used as a empty work tree for direct mode merges and
|
||||
- merges in adjusted branches. -}
|
||||
gitAnnexMergeDir :: Git.Repo -> FilePath
|
||||
gitAnnexMergeDir r = addTrailingPathSeparator $ gitAnnexDir r </> "merge"
|
||||
gitAnnexMergeDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "merge"
|
||||
|
||||
{- .git/annex/transfer/ is used to record keys currently
|
||||
- being transferred, and other transfer bookkeeping info. -}
|
||||
gitAnnexTransferDir :: Git.Repo -> FilePath
|
||||
gitAnnexTransferDir r = addTrailingPathSeparator $ gitAnnexDir r </> "transfer"
|
||||
gitAnnexTransferDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "transfer"
|
||||
|
||||
{- .git/annex/journal/ is used to journal changes made to the git-annex
|
||||
- branch -}
|
||||
gitAnnexJournalDir :: Git.Repo -> FilePath
|
||||
gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r </> "journal"
|
||||
gitAnnexJournalDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "journal"
|
||||
|
||||
gitAnnexJournalDir' :: Git.Repo -> RawFilePath
|
||||
gitAnnexJournalDir' r = P.addTrailingPathSeparator $ gitAnnexDir' r P.</> "journal"
|
||||
gitAnnexJournalDir' r = P.addTrailingPathSeparator $ gitAnnexDir r P.</> "journal"
|
||||
|
||||
{- Lock file for the journal. -}
|
||||
gitAnnexJournalLock :: Git.Repo -> FilePath
|
||||
gitAnnexJournalLock r = gitAnnexDir r </> "journal.lck"
|
||||
gitAnnexJournalLock r = fromRawFilePath $ gitAnnexDir r P.</> "journal.lck"
|
||||
|
||||
{- Lock file for flushing a git queue that writes to the git index or
|
||||
- other git state that should only have one writer at a time. -}
|
||||
gitAnnexGitQueueLock :: Git.Repo -> FilePath
|
||||
gitAnnexGitQueueLock r = gitAnnexDir r </> "gitqueue.lck"
|
||||
gitAnnexGitQueueLock r = fromRawFilePath $ gitAnnexDir r P.</> "gitqueue.lck"
|
||||
|
||||
{- Lock file for the pre-commit hook. -}
|
||||
gitAnnexPreCommitLock :: Git.Repo -> FilePath
|
||||
gitAnnexPreCommitLock r = gitAnnexDir r </> "precommit.lck"
|
||||
gitAnnexPreCommitLock r = fromRawFilePath $ gitAnnexDir r P.</> "precommit.lck"
|
||||
|
||||
{- Lock file for direct mode merge. -}
|
||||
gitAnnexMergeLock :: Git.Repo -> FilePath
|
||||
gitAnnexMergeLock r = gitAnnexDir r </> "merge.lck"
|
||||
gitAnnexMergeLock r = fromRawFilePath $ gitAnnexDir r P.</> "merge.lck"
|
||||
|
||||
{- .git/annex/index is used to stage changes to the git-annex branch -}
|
||||
gitAnnexIndex :: Git.Repo -> FilePath
|
||||
gitAnnexIndex r = gitAnnexDir r </> "index"
|
||||
gitAnnexIndex r = fromRawFilePath $ gitAnnexDir r P.</> "index"
|
||||
|
||||
{- Holds the ref of the git-annex branch that the index was last updated to.
|
||||
-
|
||||
- The .lck in the name is a historical accident; this is not used as a
|
||||
- lock. -}
|
||||
gitAnnexIndexStatus :: Git.Repo -> FilePath
|
||||
gitAnnexIndexStatus r = gitAnnexDir r </> "index.lck"
|
||||
gitAnnexIndexStatus r = fromRawFilePath $ gitAnnexDir r P.</> "index.lck"
|
||||
|
||||
{- The index file used to generate a filtered branch view._-}
|
||||
gitAnnexViewIndex :: Git.Repo -> FilePath
|
||||
gitAnnexViewIndex r = gitAnnexDir r </> "viewindex"
|
||||
gitAnnexViewIndex r = fromRawFilePath $ gitAnnexDir r P.</> "viewindex"
|
||||
|
||||
{- File containing a log of recently accessed views. -}
|
||||
gitAnnexViewLog :: Git.Repo -> FilePath
|
||||
gitAnnexViewLog r = gitAnnexDir r </> "viewlog"
|
||||
gitAnnexViewLog r = fromRawFilePath $ gitAnnexDir r P.</> "viewlog"
|
||||
|
||||
{- List of refs that have already been merged into the git-annex branch. -}
|
||||
gitAnnexMergedRefs :: Git.Repo -> FilePath
|
||||
gitAnnexMergedRefs r = gitAnnexDir r </> "mergedrefs"
|
||||
gitAnnexMergedRefs r = fromRawFilePath $ gitAnnexDir r P.</> "mergedrefs"
|
||||
|
||||
{- List of refs that should not be merged into the git-annex branch. -}
|
||||
gitAnnexIgnoredRefs :: Git.Repo -> FilePath
|
||||
gitAnnexIgnoredRefs r = gitAnnexDir r </> "ignoredrefs"
|
||||
gitAnnexIgnoredRefs r = fromRawFilePath $ gitAnnexDir r P.</> "ignoredrefs"
|
||||
|
||||
{- Pid file for daemon mode. -}
|
||||
gitAnnexPidFile :: Git.Repo -> FilePath
|
||||
gitAnnexPidFile r = gitAnnexDir r </> "daemon.pid"
|
||||
gitAnnexPidFile r = fromRawFilePath $ gitAnnexDir r P.</> "daemon.pid"
|
||||
|
||||
{- Pid lock file for pidlock mode -}
|
||||
gitAnnexPidLockFile :: Git.Repo -> FilePath
|
||||
gitAnnexPidLockFile r = gitAnnexDir r </> "pidlock"
|
||||
gitAnnexPidLockFile r = fromRawFilePath $ gitAnnexDir r P.</> "pidlock"
|
||||
|
||||
{- Status file for daemon mode. -}
|
||||
gitAnnexDaemonStatusFile :: Git.Repo -> FilePath
|
||||
gitAnnexDaemonStatusFile r = gitAnnexDir r </> "daemon.status"
|
||||
gitAnnexDaemonStatusFile r = fromRawFilePath $
|
||||
gitAnnexDir r P.</> "daemon.status"
|
||||
|
||||
{- Log file for daemon mode. -}
|
||||
gitAnnexLogFile :: Git.Repo -> FilePath
|
||||
gitAnnexLogFile r = gitAnnexDir r </> "daemon.log"
|
||||
gitAnnexLogFile r = fromRawFilePath $ gitAnnexDir r P.</> "daemon.log"
|
||||
|
||||
{- Log file for fuzz test. -}
|
||||
gitAnnexFuzzTestLogFile :: Git.Repo -> FilePath
|
||||
gitAnnexFuzzTestLogFile r = gitAnnexDir r </> "fuzztest.log"
|
||||
gitAnnexFuzzTestLogFile r = fromRawFilePath $
|
||||
gitAnnexDir r P.</> "fuzztest.log"
|
||||
|
||||
{- Html shim file used to launch the webapp. -}
|
||||
gitAnnexHtmlShim :: Git.Repo -> FilePath
|
||||
gitAnnexHtmlShim r = gitAnnexDir r </> "webapp.html"
|
||||
gitAnnexHtmlShim r = fromRawFilePath $ gitAnnexDir r P.</> "webapp.html"
|
||||
|
||||
{- File containing the url to the webapp. -}
|
||||
gitAnnexUrlFile :: Git.Repo -> FilePath
|
||||
gitAnnexUrlFile r = gitAnnexDir r </> "url"
|
||||
gitAnnexUrlFile r = fromRawFilePath $ gitAnnexDir r P.</> "url"
|
||||
|
||||
{- Temporary file used to edit configuriation from the git-annex branch. -}
|
||||
gitAnnexTmpCfgFile :: Git.Repo -> FilePath
|
||||
gitAnnexTmpCfgFile r = gitAnnexDir r </> "config.tmp"
|
||||
gitAnnexTmpCfgFile r = fromRawFilePath $ gitAnnexDir r P.</> "config.tmp"
|
||||
|
||||
{- .git/annex/ssh/ is used for ssh connection caching -}
|
||||
gitAnnexSshDir :: Git.Repo -> FilePath
|
||||
gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r </> "ssh"
|
||||
gitAnnexSshDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "ssh"
|
||||
|
||||
{- .git/annex/remotes/ is used for remote-specific state. -}
|
||||
gitAnnexRemotesDir :: Git.Repo -> FilePath
|
||||
gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r </> "remotes"
|
||||
gitAnnexRemotesDir r = fromRawFilePath $
|
||||
P.addTrailingPathSeparator $ gitAnnexDir r P.</> "remotes"
|
||||
|
||||
{- This is the base directory name used by the assistant when making
|
||||
- repositories, by default. -}
|
||||
|
|
@ -589,11 +603,8 @@ reSanitizeKeyName = preSanitizeKeyName' True
|
|||
- Changing what this function escapes and how is not a good idea, as it
|
||||
- can cause existing objects to get lost.
|
||||
-}
|
||||
keyFile :: Key -> FilePath
|
||||
keyFile = fromRawFilePath . keyFile'
|
||||
|
||||
keyFile' :: Key -> RawFilePath
|
||||
keyFile' k =
|
||||
keyFile :: Key -> RawFilePath
|
||||
keyFile k =
|
||||
let b = serializeKey' k
|
||||
in if S8.any (`elem` ['&', '%', ':', '/']) b
|
||||
then S8.concatMap esc b
|
||||
|
|
@ -608,11 +619,8 @@ keyFile' k =
|
|||
|
||||
{- Reverses keyFile, converting a filename fragment (ie, the basename of
|
||||
- the symlink target) into a key. -}
|
||||
fileKey :: FilePath -> Maybe Key
|
||||
fileKey = fileKey' . toRawFilePath
|
||||
|
||||
fileKey' :: RawFilePath -> Maybe Key
|
||||
fileKey' = deserializeKey' . S8.intercalate "/" . map go . S8.split '%'
|
||||
fileKey :: RawFilePath -> Maybe Key
|
||||
fileKey = deserializeKey' . S8.intercalate "/" . map go . S8.split '%'
|
||||
where
|
||||
go = S8.concat . unescafterfirst . S8.split '&'
|
||||
unescafterfirst [] = []
|
||||
|
|
@ -634,7 +642,7 @@ fileKey' = deserializeKey' . S8.intercalate "/" . map go . S8.split '%'
|
|||
keyPath :: Key -> Hasher -> RawFilePath
|
||||
keyPath key hasher = hasher key P.</> f P.</> f
|
||||
where
|
||||
f = keyFile' key
|
||||
f = keyFile key
|
||||
|
||||
{- All possibile locations to store a key in a special remote
|
||||
- using different directory hashes.
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ annexFileMode = withShared $ return . go
|
|||
createAnnexDirectory :: FilePath -> Annex ()
|
||||
createAnnexDirectory dir = walk dir [] =<< top
|
||||
where
|
||||
top = parentDir <$> fromRepo gitAnnexDir
|
||||
top = parentDir . fromRawFilePath <$> fromRepo gitAnnexDir
|
||||
walk d below stop
|
||||
| d `equalFilePath` stop = done
|
||||
| otherwise = ifM (liftIO $ doesDirectoryExist d)
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ mkVariant file variant = takeDirectory file
|
|||
-}
|
||||
variantFile :: FilePath -> Key -> FilePath
|
||||
variantFile file key
|
||||
| doubleconflict = mkVariant file (keyFile key)
|
||||
| doubleconflict = mkVariant file (fromRawFilePath (keyFile key))
|
||||
| otherwise = mkVariant file (shortHash $ serializeKey' key)
|
||||
where
|
||||
doubleconflict = variantMarker `isInfixOf` file
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ repairStaleGitLocks r = do
|
|||
repairStaleLocks lockfiles
|
||||
return $ not $ null lockfiles
|
||||
where
|
||||
findgitfiles = dirContentsRecursiveSkipping (== dropTrailingPathSeparator annexDir) True . fromRawFilePath . Git.localGitDir
|
||||
findgitfiles = dirContentsRecursiveSkipping (== dropTrailingPathSeparator (fromRawFilePath annexDir)) True . fromRawFilePath . Git.localGitDir
|
||||
islock f
|
||||
| "gc.pid" `isInfixOf` f = False
|
||||
| ".lock" `isSuffixOf` f = True
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ perform key = next $ do
|
|||
addLink file key Nothing
|
||||
return True
|
||||
where
|
||||
file = "unused." ++ keyFile key
|
||||
file = "unused." ++ fromRawFilePath (keyFile key)
|
||||
|
||||
{- The content is not in the annex, but in another directory, and
|
||||
- it seems better to error out, rather than moving bad/tmp content into
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ performRemote key afile backend numcopies remote =
|
|||
pid <- liftIO getPID
|
||||
t <- fromRepo gitAnnexTmpObjectDir
|
||||
createAnnexDirectory t
|
||||
let tmp = t </> "fsck" ++ show pid ++ "." ++ keyFile key
|
||||
let tmp = t </> "fsck" ++ show pid ++ "." ++ fromRawFilePath (keyFile key)
|
||||
let cleanup = liftIO $ catchIO (removeFile tmp) (const noop)
|
||||
cleanup
|
||||
cleanup `after` a tmp
|
||||
|
|
@ -516,7 +516,7 @@ badContent key = do
|
|||
badContentRemote :: Remote -> FilePath -> Key -> Annex String
|
||||
badContentRemote remote localcopy key = do
|
||||
bad <- fromRepo gitAnnexBadDir
|
||||
let destbad = bad </> keyFile key
|
||||
let destbad = bad </> fromRawFilePath (keyFile key)
|
||||
movedbad <- ifM (inAnnex key <||> liftIO (doesFileExist destbad))
|
||||
( return False
|
||||
, do
|
||||
|
|
|
|||
|
|
@ -454,7 +454,7 @@ disk_size :: Stat
|
|||
disk_size = simpleStat "available local disk space" $
|
||||
calcfree
|
||||
<$> (lift $ annexDiskReserve <$> Annex.getGitConfig)
|
||||
<*> (lift $ inRepo $ getDiskFree . gitAnnexDir)
|
||||
<*> (lift $ inRepo $ getDiskFree . fromRawFilePath . gitAnnexDir)
|
||||
<*> mkSizer
|
||||
where
|
||||
calcfree reserve (Just have) sizer = unwords
|
||||
|
|
@ -674,7 +674,7 @@ staleSize label dirspec = go =<< lift (dirKeys dirspec)
|
|||
keysizes keys = do
|
||||
dir <- lift $ fromRepo dirspec
|
||||
liftIO $ forM keys $ \k -> catchDefaultIO 0 $
|
||||
getFileSize (dir </> keyFile k)
|
||||
getFileSize (dir </> fromRawFilePath (keyFile k))
|
||||
|
||||
aside :: String -> String
|
||||
aside s = " (" ++ s ++ ")"
|
||||
|
|
|
|||
|
|
@ -46,7 +46,9 @@ start = startingNoMessage (ActionItemOther Nothing) $ do
|
|||
umap <- uuidDescMap
|
||||
trustmap <- trustMapLoad
|
||||
|
||||
file <- (</>) <$> fromRepo gitAnnexDir <*> pure "map.dot"
|
||||
file <- (</>)
|
||||
<$> fromRepo (fromRawFilePath . gitAnnexDir)
|
||||
<*> pure "map.dot"
|
||||
|
||||
liftIO $ writeFile file (drawMap rs trustmap umap)
|
||||
next $
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ startCheckIncomplete file _ = giveup $ unlines
|
|||
|
||||
finish :: Annex ()
|
||||
finish = do
|
||||
annexdir <- fromRepo gitAnnexDir
|
||||
annexdir <- fromRawFilePath <$> fromRepo gitAnnexDir
|
||||
annexobjectdir <- fromRepo gitAnnexObjectDir
|
||||
leftovers <- removeUnannexed =<< listKeys InAnnex
|
||||
prepareRemoveAnnexDir annexdir
|
||||
|
|
|
|||
|
|
@ -43,6 +43,9 @@ import Git.Command
|
|||
import Git.Types
|
||||
import Git.Index
|
||||
|
||||
import qualified Data.ByteString as S
|
||||
import qualified System.FilePath.ByteString as P
|
||||
|
||||
{- Runs an action that reads from the database.
|
||||
-
|
||||
- If the database doesn't already exist, it's not created; mempty is
|
||||
|
|
@ -263,7 +266,7 @@ reconcileStaged qh = do
|
|||
-- pointer file. And a pointer file that is replaced with
|
||||
-- a non-pointer file will match this.
|
||||
, Param $ "-G^" ++ fromRawFilePath (toInternalGitPath $
|
||||
toRawFilePath (pathSeparator:objectDir))
|
||||
P.pathSeparator `S.cons` objectDir')
|
||||
-- Don't include files that were deleted, because this only
|
||||
-- wants to update information for files that are present
|
||||
-- in the index.
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import Database.Types
|
|||
import Database.Handle
|
||||
import qualified Database.Queue as H
|
||||
import Utility.InodeCache
|
||||
import Utility.FileSystemEncoding
|
||||
import Git.FilePath
|
||||
|
||||
import Database.Persist.Sql hiding (Key)
|
||||
|
|
|
|||
|
|
@ -70,18 +70,18 @@ diffOpts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
|
|||
doMerge :: HashObjectHandle -> CatFileHandle -> [String] -> Repo -> Streamer
|
||||
doMerge hashhandle ch differ repo streamer = do
|
||||
(diff, cleanup) <- pipeNullSplit (map Param differ) repo
|
||||
go (map decodeBL' diff)
|
||||
go diff
|
||||
void $ cleanup
|
||||
where
|
||||
go [] = noop
|
||||
go (info:file:rest) = mergeFile info file hashhandle ch >>=
|
||||
go (info:file:rest) = mergeFile (decodeBL' info) (L.toStrict file) hashhandle ch >>=
|
||||
maybe (go rest) (\l -> streamer l >> go rest)
|
||||
go (_:[]) = error $ "parse error " ++ show differ
|
||||
|
||||
{- Given an info line from a git raw diff, and the filename, generates
|
||||
- a line suitable for update-index that union merges the two sides of the
|
||||
- diff. -}
|
||||
mergeFile :: String -> FilePath -> HashObjectHandle -> CatFileHandle -> IO (Maybe L.ByteString)
|
||||
mergeFile :: String -> RawFilePath -> HashObjectHandle -> CatFileHandle -> IO (Maybe L.ByteString)
|
||||
mergeFile info file hashhandle h = case filter (/= nullSha) [Ref asha, Ref bsha] of
|
||||
[] -> return Nothing
|
||||
(sha:[]) -> use sha
|
||||
|
|
@ -91,7 +91,7 @@ mergeFile info file hashhandle h = case filter (/= nullSha) [Ref asha, Ref bsha]
|
|||
where
|
||||
[_colonmode, _bmode, asha, bsha, _status] = words info
|
||||
use sha = return $ Just $
|
||||
updateIndexLine sha TreeFile $ asTopFilePath $ toRawFilePath file
|
||||
updateIndexLine sha TreeFile $ asTopFilePath file
|
||||
-- Get file and split into lines to union merge.
|
||||
-- The encoding of the file is assumed to be either ASCII or utf-8;
|
||||
-- in either case it's safe to split on \n
|
||||
|
|
|
|||
18
Logs.hs
18
Logs.hs
|
|
@ -119,18 +119,18 @@ exportLog = "export.log"
|
|||
{- The pathname of the location log file for a given key. -}
|
||||
locationLogFile :: GitConfig -> Key -> RawFilePath
|
||||
locationLogFile config key =
|
||||
branchHashDir config key P.</> keyFile' key <> ".log"
|
||||
branchHashDir config key P.</> keyFile key <> ".log"
|
||||
|
||||
{- The filename of the url log for a given key. -}
|
||||
urlLogFile :: GitConfig -> Key -> RawFilePath
|
||||
urlLogFile config key =
|
||||
branchHashDir config key P.</> keyFile' key <> urlLogExt
|
||||
branchHashDir config key P.</> keyFile key <> urlLogExt
|
||||
|
||||
{- Old versions stored the urls elsewhere. -}
|
||||
oldurlLogs :: GitConfig -> Key -> [RawFilePath]
|
||||
oldurlLogs config key =
|
||||
[ "remote/web" P.</> hdir P.</> serializeKey' key <> ".log"
|
||||
, "remote/web" P.</> hdir P.</> keyFile' key <> ".log"
|
||||
, "remote/web" P.</> hdir P.</> keyFile key <> ".log"
|
||||
]
|
||||
where
|
||||
hdir = branchHashDir config key
|
||||
|
|
@ -145,7 +145,7 @@ isUrlLog file = urlLogExt `S.isSuffixOf` file
|
|||
{- The filename of the remote state log for a given key. -}
|
||||
remoteStateLogFile :: GitConfig -> Key -> RawFilePath
|
||||
remoteStateLogFile config key =
|
||||
(branchHashDir config key P.</> keyFile' key)
|
||||
(branchHashDir config key P.</> keyFile key)
|
||||
<> remoteStateLogExt
|
||||
|
||||
remoteStateLogExt :: S.ByteString
|
||||
|
|
@ -157,7 +157,7 @@ isRemoteStateLog path = remoteStateLogExt `S.isSuffixOf` path
|
|||
{- The filename of the chunk log for a given key. -}
|
||||
chunkLogFile :: GitConfig -> Key -> RawFilePath
|
||||
chunkLogFile config key =
|
||||
(branchHashDir config key P.</> keyFile' key)
|
||||
(branchHashDir config key P.</> keyFile key)
|
||||
<> chunkLogExt
|
||||
|
||||
chunkLogExt :: S.ByteString
|
||||
|
|
@ -169,7 +169,7 @@ isChunkLog path = chunkLogExt `S.isSuffixOf` path
|
|||
{- The filename of the metadata log for a given key. -}
|
||||
metaDataLogFile :: GitConfig -> Key -> RawFilePath
|
||||
metaDataLogFile config key =
|
||||
(branchHashDir config key P.</> keyFile' key)
|
||||
(branchHashDir config key P.</> keyFile key)
|
||||
<> metaDataLogExt
|
||||
|
||||
metaDataLogExt :: S.ByteString
|
||||
|
|
@ -181,7 +181,7 @@ isMetaDataLog path = metaDataLogExt `S.isSuffixOf` path
|
|||
{- The filename of the remote metadata log for a given key. -}
|
||||
remoteMetaDataLogFile :: GitConfig -> Key -> RawFilePath
|
||||
remoteMetaDataLogFile config key =
|
||||
(branchHashDir config key P.</> keyFile' key)
|
||||
(branchHashDir config key P.</> keyFile key)
|
||||
<> remoteMetaDataLogExt
|
||||
|
||||
remoteMetaDataLogExt :: S.ByteString
|
||||
|
|
@ -193,7 +193,7 @@ isRemoteMetaDataLog path = remoteMetaDataLogExt `S.isSuffixOf` path
|
|||
{- The filename of the remote content identifier log for a given key. -}
|
||||
remoteContentIdentifierLogFile :: GitConfig -> Key -> RawFilePath
|
||||
remoteContentIdentifierLogFile config key =
|
||||
(branchHashDir config key P.</> keyFile' key)
|
||||
(branchHashDir config key P.</> keyFile key)
|
||||
<> remoteContentIdentifierExt
|
||||
|
||||
remoteContentIdentifierExt :: S.ByteString
|
||||
|
|
@ -205,7 +205,7 @@ isRemoteContentIdentifierLog path = remoteContentIdentifierExt `S.isSuffixOf` pa
|
|||
{- From an extension and a log filename, get the key that it's a log for. -}
|
||||
extLogFileKey :: S.ByteString -> RawFilePath -> Maybe Key
|
||||
extLogFileKey expectedext path
|
||||
| encodeBS' ext == expectedext = fileKey base
|
||||
| encodeBS' ext == expectedext = fileKey (toRawFilePath base)
|
||||
| otherwise = Nothing
|
||||
where
|
||||
file = takeFileName (fromRawFilePath path)
|
||||
|
|
|
|||
|
|
@ -195,12 +195,12 @@ recordFailedTransfer t info = do
|
|||
transferFile :: Transfer -> Git.Repo -> FilePath
|
||||
transferFile (Transfer direction u kd) r = transferDir direction r
|
||||
</> filter (/= '/') (fromUUID u)
|
||||
</> keyFile (mkKey (const kd))
|
||||
</> fromRawFilePath (keyFile (mkKey (const kd)))
|
||||
|
||||
{- The transfer information file to use to record a failed Transfer -}
|
||||
failedTransferFile :: Transfer -> Git.Repo -> FilePath
|
||||
failedTransferFile (Transfer direction u kd) r = failedTransferDir u direction r
|
||||
</> keyFile (mkKey (const kd))
|
||||
</> fromRawFilePath (keyFile (mkKey (const kd)))
|
||||
|
||||
{- The transfer lock file corresponding to a given transfer info file. -}
|
||||
transferLockFile :: FilePath -> FilePath
|
||||
|
|
@ -215,7 +215,7 @@ parseTransferFile file
|
|||
[direction, u, key] -> Transfer
|
||||
<$> parseDirection direction
|
||||
<*> pure (toUUID u)
|
||||
<*> fmap (fromKey id) (fileKey key)
|
||||
<*> fmap (fromKey id) (fileKey (toRawFilePath key))
|
||||
_ -> Nothing
|
||||
where
|
||||
bits = splitDirectories file
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ downloadTorrentFile u = do
|
|||
createAnnexDirectory (parentDir torrent)
|
||||
if isTorrentMagnetUrl u
|
||||
then withOtherTmp $ \othertmp -> do
|
||||
kf <- keyFile <$> torrentUrlKey u
|
||||
kf <- fromRawFilePath . keyFile <$> torrentUrlKey u
|
||||
let metadir = othertmp </> "torrentmeta" </> kf
|
||||
createAnnexDirectory metadir
|
||||
showOutput
|
||||
|
|
@ -239,7 +239,7 @@ downloadTorrentContent :: Key -> URLString -> FilePath -> Int -> MeterUpdate ->
|
|||
downloadTorrentContent k u dest filenum p = do
|
||||
torrent <- tmpTorrentFile u
|
||||
withOtherTmp $ \othertmp -> do
|
||||
kf <- keyFile <$> torrentUrlKey u
|
||||
kf <- fromRawFilePath . keyFile <$> torrentUrlKey u
|
||||
let downloaddir = othertmp </> "torrent" </> kf
|
||||
createAnnexDirectory downloaddir
|
||||
f <- wantedfile torrent
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ getLocation d k = do
|
|||
{- Directory where the file(s) for a key are stored. -}
|
||||
storeDir :: FilePath -> Key -> FilePath
|
||||
storeDir d k = addTrailingPathSeparator $
|
||||
d </> fromRawFilePath (hashDirLower def k) </> keyFile k
|
||||
d </> fromRawFilePath (hashDirLower def k) </> fromRawFilePath (keyFile k)
|
||||
|
||||
{- Check if there is enough free disk space in the remote's directory to
|
||||
- store the key. Note that the unencrypted key size is checked. -}
|
||||
|
|
@ -164,12 +164,13 @@ store d chunkconfig k b p = liftIO $ do
|
|||
case chunkconfig of
|
||||
LegacyChunks chunksize -> Legacy.store chunksize finalizeStoreGeneric k b p tmpdir destdir
|
||||
_ -> do
|
||||
let tmpf = tmpdir </> keyFile k
|
||||
let tmpf = tmpdir </> kf
|
||||
meteredWriteFile p tmpf b
|
||||
finalizeStoreGeneric tmpdir destdir
|
||||
return True
|
||||
where
|
||||
tmpdir = addTrailingPathSeparator $ d </> "tmp" </> keyFile k
|
||||
tmpdir = addTrailingPathSeparator $ d </> "tmp" </> kf
|
||||
kf = fromRawFilePath (keyFile k)
|
||||
destdir = storeDir d k
|
||||
|
||||
{- Passed a temp directory that contains the files that should be placed
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ store chunksize finalizer k b p = storeHelper finalizer k $ \dests ->
|
|||
retrieve :: (FilePath -> Key -> [FilePath]) -> FilePath -> Preparer Retriever
|
||||
retrieve locations d basek a = withOtherTmp $ \tmpdir -> do
|
||||
showLongNote "This remote uses the deprecated chunksize setting. So this will be quite slow."
|
||||
let tmp = tmpdir </> keyFile basek ++ ".directorylegacy.tmp"
|
||||
let tmp = tmpdir </> fromRawFilePath (keyFile basek) ++ ".directorylegacy.tmp"
|
||||
a $ Just $ byteRetriever $ \k sink -> do
|
||||
liftIO $ void $ withStoredFiles d locations k $ \fs -> do
|
||||
forM_ fs $
|
||||
|
|
|
|||
|
|
@ -351,9 +351,9 @@ store' :: Git.Repo -> Remote -> Remote.Rsync.RsyncOpts -> Storer
|
|||
store' repo r rsyncopts
|
||||
| not $ Git.repoIsUrl repo =
|
||||
byteStorer $ \k b p -> guardUsable repo (return False) $ liftIO $ do
|
||||
let tmpdir = Git.repoLocation repo </> "tmp" </> keyFile k
|
||||
let tmpdir = Git.repoLocation repo </> "tmp" </> fromRawFilePath (keyFile k)
|
||||
void $ tryIO $ createDirectoryIfMissing True tmpdir
|
||||
let tmpf = tmpdir </> keyFile k
|
||||
let tmpf = tmpdir </> fromRawFilePath (keyFile k)
|
||||
meteredWriteFile p tmpf b
|
||||
let destdir = parentDir $ gCryptLocation repo k
|
||||
Remote.Directory.finalizeStoreGeneric tmpdir destdir
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ storeChunks key tmp dest storer recorder finalizer = either onerr return
|
|||
warningIO (show e)
|
||||
return False
|
||||
|
||||
basef = tmp ++ keyFile key
|
||||
basef = tmp ++ fromRawFilePath (keyFile key)
|
||||
tmpdests = map (basef ++ ) chunkStream
|
||||
|
||||
{- Given a list of destinations to use, chunks the data according to the
|
||||
|
|
|
|||
|
|
@ -226,7 +226,7 @@ remove o k = removeGeneric o includes
|
|||
[ parentDir dir
|
||||
, dir
|
||||
-- match content directory and anything in it
|
||||
, dir </> keyFile k </> "***"
|
||||
, dir </> fromRawFilePath (keyFile k) </> "***"
|
||||
]
|
||||
|
||||
{- An empty directory is rsynced to make it delete. Everything is excluded,
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ rsyncUrls :: RsyncOpts -> Key -> [RsyncUrl]
|
|||
rsyncUrls o k = map use dirHashes
|
||||
where
|
||||
use h = rsyncUrl o </> fromRawFilePath (hash h) </> rsyncEscape o (f </> f)
|
||||
f = keyFile k
|
||||
f = fromRawFilePath (keyFile k)
|
||||
#ifndef mingw32_HOST_OS
|
||||
hash h = h def k
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ inLocation d = inDAVLocation (</> d')
|
|||
|
||||
{- The directory where files(s) for a key are stored. -}
|
||||
keyDir :: Key -> DavLocation
|
||||
keyDir k = addTrailingPathSeparator $ hashdir </> keyFile k
|
||||
keyDir k = addTrailingPathSeparator $ hashdir </> fromRawFilePath (keyFile k)
|
||||
where
|
||||
#ifndef mingw32_HOST_OS
|
||||
hashdir = fromRawFilePath $ hashDirLower def k
|
||||
|
|
@ -45,7 +45,7 @@ keyDir k = addTrailingPathSeparator $ hashdir </> keyFile k
|
|||
#endif
|
||||
|
||||
keyLocation :: Key -> DavLocation
|
||||
keyLocation k = keyDir k ++ keyFile k
|
||||
keyLocation k = keyDir k ++ fromRawFilePath (keyFile k)
|
||||
|
||||
{- Paths containing # or ? cannot be represented in an url, so fails on
|
||||
- those. -}
|
||||
|
|
@ -60,7 +60,7 @@ exportLocation l =
|
|||
|
||||
{- Where we store temporary data for a key as it's being uploaded. -}
|
||||
keyTmpLocation :: Key -> DavLocation
|
||||
keyTmpLocation = tmpLocation . keyFile
|
||||
keyTmpLocation = tmpLocation . fromRawFilePath . keyFile
|
||||
|
||||
tmpLocation :: FilePath -> DavLocation
|
||||
tmpLocation f = "git-annex-webdav-tmp-" ++ f
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ upgrade = do
|
|||
showAction "v0 to v1"
|
||||
|
||||
-- do the reorganisation of the key files
|
||||
olddir <- fromRepo gitAnnexDir
|
||||
olddir <- fromRawFilePath <$> fromRepo gitAnnexDir
|
||||
keys <- getKeysPresent0 olddir
|
||||
forM_ keys $ \k -> moveAnnex k $ olddir </> keyFile0 k
|
||||
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ logFile2 = logFile' (hashDirLower def)
|
|||
|
||||
logFile' :: (Key -> RawFilePath) -> Key -> Git.Repo -> String
|
||||
logFile' hasher key repo =
|
||||
gitStateDir repo ++ fromRawFilePath (hasher key) ++ keyFile key ++ ".log"
|
||||
gitStateDir repo ++ fromRawFilePath (hasher key) ++ fromRawFilePath (keyFile key) ++ ".log"
|
||||
|
||||
stateDir :: FilePath
|
||||
stateDir = addTrailingPathSeparator ".git-annex"
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ upgrade automatic = do
|
|||
return True
|
||||
|
||||
gitAnnexKeysDbOld :: Git.Repo -> FilePath
|
||||
gitAnnexKeysDbOld r = gitAnnexDir r </> "keys"
|
||||
gitAnnexKeysDbOld r = fromRawFilePath (gitAnnexDir r) </> "keys"
|
||||
|
||||
gitAnnexKeysDbLockOld :: Git.Repo -> FilePath
|
||||
gitAnnexKeysDbLockOld r = gitAnnexKeysDbOld r ++ ".lck"
|
||||
|
|
@ -55,7 +55,7 @@ gitAnnexKeysDbIndexCacheOld :: Git.Repo -> FilePath
|
|||
gitAnnexKeysDbIndexCacheOld r = gitAnnexKeysDbOld r ++ ".cache"
|
||||
|
||||
gitAnnexContentIdentifierDbDirOld :: Git.Repo -> FilePath
|
||||
gitAnnexContentIdentifierDbDirOld r = gitAnnexDir r </> "cids"
|
||||
gitAnnexContentIdentifierDbDirOld r = fromRawFilePath (gitAnnexDir r) </> "cids"
|
||||
|
||||
gitAnnexContentIdentifierLockOld :: Git.Repo -> FilePath
|
||||
gitAnnexContentIdentifierLockOld r = gitAnnexContentIdentifierDbDirOld r ++ ".lck"
|
||||
|
|
|
|||
15
doc/devblog/day_614__bytestring_wrapping_up.mdwn
Normal file
15
doc/devblog/day_614__bytestring_wrapping_up.mdwn
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
Cut the last release before the switch over to end-to-end ByteString.
|
||||
(Including fixing the rpm repo's index which had not been getting updated.)
|
||||
|
||||
I had left the `bs` branch last week with a known bug, so got that fixed.
|
||||
Also there were some encoding problems on windows with the ByteString
|
||||
filepaths, which needed a new release of filepath-bytestring to clean up.
|
||||
Now I think the `bs` branch is really in a mergeable state.
|
||||
(It's still not tested on Windows at all though.)
|
||||
|
||||
Took the last little while to do some more profiling. Mostly the remaining
|
||||
ByteString conversions barely seem worth doing (1% improvement at most),
|
||||
but [[todo/optimise_journal_access]] seems like it could pay off well.
|
||||
|
||||
Also found time in there somewhere to implement `git annex inprogress
|
||||
--key`
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 7"""
|
||||
date="2019-12-18T19:18:04Z"
|
||||
content="""
|
||||
Updated profiling. git-annex find is now ByteString end-to-end!
|
||||
Note the massive reduction in alloc, and improved runtime.
|
||||
|
||||
Wed Dec 11 14:41 2019 Time and Allocation Profiling Report (Final)
|
||||
|
||||
git-annex +RTS -p -RTS find
|
||||
|
||||
total time = 1.51 secs (1515 ticks @ 1000 us, 1 processor)
|
||||
total alloc = 608,475,328 bytes (excludes profiling overheads)
|
||||
|
||||
COST CENTRE MODULE SRC %time %alloc
|
||||
|
||||
keyFile' Annex.Locations Annex/Locations.hs:(590,1)-(600,30) 8.2 16.6
|
||||
>>=.\.succ' Data.Attoparsec.Internal.Types Data/Attoparsec/Internal/Types.hs:146:13-76 4.7 0.7
|
||||
getAnnexLinkTarget'.probesymlink Annex.Link Annex/Link.hs:79:9-46 4.2 7.6
|
||||
>>=.\ Data.Attoparsec.Internal.Types Data/Attoparsec/Internal/Types.hs:(146,9)-(147,44) 3.9 2.3
|
||||
parseLinkTarget Annex.Link Annex/Link.hs:(255,1)-(263,25) 3.9 11.8
|
||||
doesPathExist Utility.RawFilePath Utility/RawFilePath.hs:30:1-25 3.4 0.6
|
||||
keyFile'.esc Annex.Locations Annex/Locations.hs:(596,9)-(600,30) 3.2 14.7
|
||||
fileKey' Annex.Locations Annex/Locations.hs:(609,1)-(619,41) 3.0 4.7
|
||||
parseLinkTargetOrPointer Annex.Link Annex/Link.hs:(240,1)-(244,25) 2.8 0.2
|
||||
hashUpdates.\.\.\ Crypto.Hash Crypto/Hash.hs:85:48-99 2.5 0.1
|
||||
combineAlways System.FilePath.Posix.ByteString System/FilePath/Posix/../Internal.hs:(698,1)-(704,67) 2.0 3.3
|
||||
getState Annex Annex.hs:(251,1)-(254,27) 2.0 1.1
|
||||
withPtr.makeTrampoline Basement.Block.Base Basement/Block/Base.hs:(401,5)-(404,31) 1.9 1.7
|
||||
withMutablePtrHint Basement.Block.Base Basement/Block/Base.hs:(468,1)-(482,50) 1.8 1.2
|
||||
parseKeyVariety Types.Key Types/Key.hs:(323,1)-(371,42) 1.8 0.0
|
||||
fileKey'.go Annex.Locations Annex/Locations.hs:611:9-55 1.7 2.2
|
||||
isLinkToAnnex Annex.Link Annex/Link.hs:(299,1)-(305,47) 1.7 1.0
|
||||
hashDirMixed Annex.DirHashes Annex/DirHashes.hs:(82,1)-(90,27) 1.7 1.3
|
||||
primitive Basement.Monad Basement/Monad.hs:72:5-18 1.6 0.1
|
||||
withPtr Basement.Block.Base Basement/Block/Base.hs:(395,1)-(404,31) 1.5 1.6
|
||||
mkKeySerialization Types.Key Types/Key.hs:(115,1)-(117,22) 1.1 2.8
|
||||
decimal.step Data.Attoparsec.ByteString.Char8 Data/Attoparsec/ByteString/Char8.hs:448:9-49 0.8 1.2
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 3"""
|
||||
date="2019-12-19T15:29:40Z"
|
||||
content="""
|
||||
Retargeting this todo at something useful post-git-add-kerfluffle,
|
||||
annex.addunlocked could usefully be a pagespec to allow adding some files
|
||||
unlocked and others locked (by git-annex add only, not git add).
|
||||
"true" would be the same as "anything" and false as "nothing".
|
||||
|
||||
---
|
||||
|
||||
It may also then make sense to let it be configured in .gitattributes.
|
||||
Although, the ugliness of setting a pagespec in .gitattributes,
|
||||
as was done for annex.largefiles, coupled with the overhead of needing to
|
||||
query that from git-check-attr for every file, makes me wary.
|
||||
|
||||
(Surprising amount of `git-annex add` time is in querying the
|
||||
annex.largefiles and annex.backend attributes. Setting the former in
|
||||
gitconfig avoids the attribute query and speeds up add of smaller files by
|
||||
2%. Granted I've sped up add (except hashing) by probably 20% this month,
|
||||
and with large files the hashing dominates.)
|
||||
|
||||
The query overhead could maybe be finessed: Since adding a file
|
||||
already queries gitattributes for two other things, a single query could be
|
||||
done for a file and the result cached.
|
||||
|
||||
Letting it be globally configured via `git-annex config` is an alternative
|
||||
that I'm leaning toward.
|
||||
(That would also need some caching, easier to implement and faster
|
||||
since it is not a per-file value as the gitattribute would be.)
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="Ilya_Shlyakhter"
|
||||
avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
|
||||
subject="named pipes as destination files"
|
||||
date="2019-12-18T18:41:57Z"
|
||||
content="""
|
||||
\"getting object content from remotes involve a destination file that is written to\" -- what happens if git-annex makes a named pipe, and passes that as the destination file name to the remote?
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2019-12-19T16:08:09Z"
|
||||
content="""
|
||||
Hmm, it used to be that `git add .` would smudge all dotfiles without that
|
||||
line, but now annex.largefiles has to be configured for it to smudge
|
||||
anything.
|
||||
|
||||
So, this could be dealt with in annex.largefiles. Both `anything` and
|
||||
`include=*` currently match dotfiles. It's kind of weird really that `*`
|
||||
matches dotfiles; it does not in the shell. If `*` did not match dotfiles
|
||||
(and `anything` is just an alias for `include=*`), it would be fairly safe
|
||||
to remove the `.* !filter` line by default. (If annex.largefiles has a
|
||||
content-based setting, and a dotfile is large enough or the right mime type
|
||||
or whatever, it's reasonable to default to smudging it.)
|
||||
|
||||
Then, you could set annex.largfiles to match the dotfiles you want,
|
||||
eg `include=* or include=.mydotfile`. You could put the config in
|
||||
.gitattributes if you want to configure it globally.
|
||||
|
||||
This change to annex.largefiles would also let `git-annex add`
|
||||
stop skipping dotfiles by default; instead annex.largefiles would not match
|
||||
dotfiles unless the user explicitly configured it to, and so the dotfiles
|
||||
would be added as small files, directly to git.
|
||||
|
||||
I like this because it unifies the behaviors of the two ways of adding,
|
||||
and it reduces the complexity, rather than adding more.
|
||||
|
||||
Removing the `.* !filter` line by default
|
||||
would need to be done as part of the v8 upgrade, or a later upgrade.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 2"""
|
||||
date="2019-12-19T17:17:31Z"
|
||||
content="""
|
||||
`*` is not only used in annex.largefiles, but other pagespecs too.
|
||||
Like preferred content:
|
||||
|
||||
exclude=archive/*
|
||||
|
||||
So changing `*` to not match dotfiles would have wide reaching effects,
|
||||
and it's really not good for different versions of git-annex to parse
|
||||
preferred content expressions differently. And it seems too confusing to
|
||||
have `*` match differently in annex.largefiles than in other pagespecs.
|
||||
|
||||
Having a single config that controls both kinds of adds still seems like a
|
||||
good idea, but I don't know what that config should be.
|
||||
annex.largedotfiles?
|
||||
"""]]
|
||||
3
doc/todo/optimise_by_converting_Ref_to_ByteString.mdwn
Normal file
3
doc/todo/optimise_by_converting_Ref_to_ByteString.mdwn
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
Profiling of `git annex find --not --in web` suggests that converting Ref
|
||||
to contain a ByteString, rather than a String, would eliminate a
|
||||
fromRawFilePath that uses about 1% of runtime.
|
||||
21
doc/todo/optimise_journal_access.mdwn
Normal file
21
doc/todo/optimise_journal_access.mdwn
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
Often a command will need to read a number of files from the git-annex
|
||||
branch, and it uses getJournalFile for each to check for any journalled
|
||||
change that has not reached the branch. But typically, the journal is empty
|
||||
and in such a case, that's a lot of time spent trying to open journal files
|
||||
that DNE.
|
||||
|
||||
Profiling eg, `git annex find --in web` shows things called by getJournalFile
|
||||
use around 5% of runtime.
|
||||
|
||||
What if, once at startup, it checked if the journal was entirely empty.
|
||||
If so, it can remember that, and avoid reading journal files.
|
||||
Perhaps paired with staging the journal if it's not empty.
|
||||
|
||||
This could lead to behavior changes in some cases where one command is
|
||||
writing changes and another command used to read them from the journal and
|
||||
may no longer do so. But any such behavior change is of a behavior that
|
||||
used to involve a race; the reader could just as well be ahead of the
|
||||
writer and it would have already behaved as it would after the change.
|
||||
|
||||
But: When a process writes to the journal, it will need to update its state
|
||||
to remember it's no longer empty. --[[Joey]]
|
||||
|
|
@ -9,7 +9,7 @@ Benchmarking `git-annex find`, speedups range from 28-66%. The files fly by
|
|||
much more snappily. Other commands likely also speed up, but do more work
|
||||
than find so the improvement is not as large.
|
||||
|
||||
The `bs` branch is in a mergeable state now.
|
||||
The `bs` branch is in a mergeable state now. [[done]]
|
||||
|
||||
Stuff not entirely finished:
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue