diff --git a/Annex.hs b/Annex.hs index 9eb4c5f391..b35836ffb3 100644 --- a/Annex.hs +++ b/Annex.hs @@ -147,7 +147,7 @@ data AnnexState = AnnexState , activeremotes :: MVar (M.Map (Types.Remote.RemoteA Annex) Integer) , keysdbhandle :: Maybe Keys.DbHandle , cachedcurrentbranch :: (Maybe (Maybe Git.Branch, Maybe Adjustment)) - , cachedgitenv :: Maybe [(String, String)] + , cachedgitenv :: Maybe (FilePath, [(String, String)]) , urloptions :: Maybe UrlOptions } diff --git a/Annex/AutoMerge.hs b/Annex/AutoMerge.hs index 766e5274ae..f537081d71 100644 --- a/Annex/AutoMerge.hs +++ b/Annex/AutoMerge.hs @@ -104,7 +104,7 @@ autoMergeFrom branch currbranch mergeconfig canresolvemerge commitmode = do -} resolveMerge :: Maybe Git.Ref -> Git.Ref -> Bool -> Annex Bool resolveMerge us them inoverlay = do - top <- if inoverlay + top <- toRawFilePath <$> if inoverlay then pure "." else fromRepo Git.repoPath (fs, cleanup) <- inRepo (LsFiles.unmerged [top]) @@ -122,7 +122,7 @@ resolveMerge us them inoverlay = do unless (null deleted) $ Annex.Queue.addCommand "rm" [Param "--quiet", Param "-f", Param "--"] - deleted + (map fromRawFilePath deleted) void $ liftIO cleanup2 when merged $ do @@ -169,7 +169,7 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do -- Neither side is annexed file; cannot resolve. (Nothing, Nothing) -> return ([], Nothing) where - file = LsFiles.unmergedFile u + file = fromRawFilePath $ LsFiles.unmergedFile u getkey select = case select (LsFiles.unmergedSha u) of @@ -202,20 +202,20 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do makesymlink key dest = do l <- calcRepo $ gitAnnexLink dest key unless inoverlay $ replacewithsymlink dest l - dest' <- stagefile dest + dest' <- toRawFilePath <$> stagefile dest stageSymlink dest' =<< hashSymlink l replacewithsymlink dest link = withworktree dest $ \f -> - replaceFile f $ makeGitLink link + replaceFile f $ makeGitLink link . toRawFilePath makepointer key dest destmode = do unless inoverlay $ unlessM (reuseOldFile unstagedmap key file dest) $ linkFromAnnex key dest destmode >>= \case LinkAnnexFailed -> liftIO $ - writePointerFile dest key destmode + writePointerFile (toRawFilePath dest) key destmode _ -> noop - dest' <- stagefile dest + dest' <- toRawFilePath <$> stagefile dest stagePointerFile dest' destmode =<< hashPointerFile key unless inoverlay $ Database.Keys.addAssociatedFile key @@ -239,7 +239,7 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do Nothing -> noop Just sha -> do link <- catSymLinkTarget sha - replacewithsymlink item link + replacewithsymlink item (fromRawFilePath link) -- And when grafting in anything else vs a symlink, -- the work tree already contains what we want. (_, Just TreeSymlink) -> noop @@ -290,8 +290,8 @@ cleanConflictCruft resolvedks resolvedfs unstagedmap = do matchesresolved is i f | S.member f fs || S.member (conflictCruftBase f) fs = anyM id [ pure (S.member i is) - , inks <$> isAnnexLink f - , inks <$> liftIO (isPointerFile f) + , inks <$> isAnnexLink (toRawFilePath f) + , inks <$> liftIO (isPointerFile (toRawFilePath f)) ] | otherwise = return False @@ -328,13 +328,14 @@ commitResolvedMerge commitmode = inRepo $ Git.Branch.commitCommand commitmode type InodeMap = M.Map InodeCacheKey FilePath -inodeMap :: Annex ([FilePath], IO Bool) -> Annex InodeMap +inodeMap :: Annex ([RawFilePath], IO Bool) -> Annex InodeMap inodeMap getfiles = do (fs, cleanup) <- getfiles fsis <- forM fs $ \f -> do - mi <- withTSDelta (liftIO . genInodeCache f) + let f' = fromRawFilePath f + mi <- withTSDelta (liftIO . genInodeCache f') return $ case mi of Nothing -> Nothing - Just i -> Just (inodeCacheToKey Strongly i, f) + Just i -> Just (inodeCacheToKey Strongly i, f') void $ liftIO cleanup return $ M.fromList $ catMaybes fsis diff --git a/Annex/Branch.hs b/Annex/Branch.hs index faf11ce05a..c39807f61e 100644 --- a/Annex/Branch.hs +++ b/Annex/Branch.hs @@ -215,7 +215,7 @@ updateTo' pairs = do - content is returned. - - Returns an empty string if the file doesn't exist yet. -} -get :: FilePath -> Annex L.ByteString +get :: RawFilePath -> Annex L.ByteString get file = do update getLocal file @@ -224,21 +224,21 @@ get file = do - reflect changes in remotes. - (Changing the value this returns, and then merging is always the - same as using get, and then changing its value.) -} -getLocal :: FilePath -> Annex L.ByteString +getLocal :: RawFilePath -> Annex L.ByteString getLocal file = go =<< getJournalFileStale file where go (Just journalcontent) = return journalcontent go Nothing = getRef fullname file {- Gets the content of a file as staged in the branch's index. -} -getStaged :: FilePath -> Annex L.ByteString +getStaged :: RawFilePath -> Annex L.ByteString getStaged = getRef indexref where -- This makes git cat-file be run with ":file", -- so it looks at the index. indexref = Ref "" -getHistorical :: RefDate -> FilePath -> Annex L.ByteString +getHistorical :: RefDate -> RawFilePath -> Annex L.ByteString getHistorical date file = -- This check avoids some ugly error messages when the reflog -- is empty. @@ -247,7 +247,7 @@ getHistorical date file = , getRef (Git.Ref.dateRef fullname date) file ) -getRef :: Ref -> FilePath -> Annex L.ByteString +getRef :: Ref -> RawFilePath -> Annex L.ByteString getRef ref file = withIndex $ catFile ref file {- Applies a function to modify the content of a file. @@ -255,11 +255,11 @@ getRef ref file = withIndex $ catFile ref file - Note that this does not cause the branch to be merged, it only - modifes the current content of the file on the branch. -} -change :: Journalable content => FilePath -> (L.ByteString -> content) -> Annex () +change :: Journalable content => RawFilePath -> (L.ByteString -> content) -> Annex () change file f = lockJournal $ \jl -> f <$> getLocal file >>= set jl file {- Applies a function which can modify the content of a file, or not. -} -maybeChange :: Journalable content => FilePath -> (L.ByteString -> Maybe content) -> Annex () +maybeChange :: Journalable content => RawFilePath -> (L.ByteString -> Maybe content) -> Annex () maybeChange file f = lockJournal $ \jl -> do v <- getLocal file case f v of @@ -269,7 +269,7 @@ maybeChange file f = lockJournal $ \jl -> do _ -> noop {- Records new content of a file into the journal -} -set :: Journalable content => JournalLocked -> FilePath -> content -> Annex () +set :: Journalable content => JournalLocked -> RawFilePath -> content -> Annex () set = setJournalFile {- Commit message used when making a commit of whatever data has changed @@ -353,23 +353,23 @@ commitIndex' jl branchref message basemessage retrynum parents = do {- Lists all files on the branch. including ones in the journal - that have not been committed yet. There may be duplicates in the list. -} -files :: Annex [FilePath] +files :: Annex [RawFilePath] files = do update -- ++ forces the content of the first list to be buffered in memory, -- so use getJournalledFilesStale which should be much smaller most -- of the time. branchFiles will stream as the list is consumed. (++) - <$> getJournalledFilesStale + <$> (map toRawFilePath <$> getJournalledFilesStale) <*> branchFiles {- Files in the branch, not including any from journalled changes, - and without updating the branch. -} -branchFiles :: Annex [FilePath] +branchFiles :: Annex [RawFilePath] branchFiles = withIndex $ inRepo branchFiles' -branchFiles' :: Git.Repo -> IO [FilePath] -branchFiles' = Git.Command.pipeNullSplitZombie +branchFiles' :: Git.Repo -> IO [RawFilePath] +branchFiles' = Git.Command.pipeNullSplitZombie' (lsTreeParams Git.LsTree.LsTreeRecursive fullname [Param "--name-only"]) {- Populates the branch's index file with the current branch contents. @@ -593,14 +593,14 @@ performTransitionsLocked jl ts neednewlocalbranch transitionedrefs = do if L.null content' then do Annex.Queue.addUpdateIndex - =<< inRepo (Git.UpdateIndex.unstageFile file) + =<< inRepo (Git.UpdateIndex.unstageFile (fromRawFilePath file)) -- File is deleted; can't run any other -- transitions on it. return () else do sha <- hashBlob content' Annex.Queue.addUpdateIndex $ Git.UpdateIndex.pureStreamer $ - Git.UpdateIndex.updateIndexLine sha TreeFile (asTopFilePath file) + Git.UpdateIndex.updateIndexLine sha TreeFile (asTopFilePath (fromRawFilePath file)) apply rest file content' checkBranchDifferences :: Git.Ref -> Annex () diff --git a/Annex/Branch/Transitions.hs b/Annex/Branch/Transitions.hs index 1ed2e4d505..a360919890 100644 --- a/Annex/Branch/Transitions.hs +++ b/Annex/Branch/Transitions.hs @@ -34,7 +34,7 @@ data FileTransition = ChangeFile Builder | PreserveFile -type TransitionCalculator = TrustMap -> M.Map UUID RemoteConfig -> FilePath -> L.ByteString -> FileTransition +type TransitionCalculator = TrustMap -> M.Map UUID RemoteConfig -> RawFilePath -> L.ByteString -> FileTransition getTransitionCalculator :: Transition -> Maybe TransitionCalculator getTransitionCalculator ForgetGitHistory = Nothing diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs index 149fde4475..2037693e91 100644 --- a/Annex/CatFile.hs +++ b/Annex/CatFile.hs @@ -39,12 +39,12 @@ import Annex.Link import Annex.CurrentBranch import Types.AdjustedBranch -catFile :: Git.Branch -> FilePath -> Annex L.ByteString +catFile :: Git.Branch -> RawFilePath -> Annex L.ByteString catFile branch file = do h <- catFileHandle liftIO $ Git.CatFile.catFile h branch file -catFileDetails :: Git.Branch -> FilePath -> Annex (Maybe (L.ByteString, Sha, ObjectType)) +catFileDetails :: Git.Branch -> RawFilePath -> Annex (Maybe (L.ByteString, Sha, ObjectType)) catFileDetails branch file = do h <- catFileHandle liftIO $ Git.CatFile.catFileDetails h branch file @@ -109,8 +109,8 @@ catKey ref = go =<< catObjectMetaData ref go _ = return Nothing {- Gets a symlink target. -} -catSymLinkTarget :: Sha -> Annex String -catSymLinkTarget sha = fromInternalGitPath . decodeBL <$> get +catSymLinkTarget :: Sha -> Annex RawFilePath +catSymLinkTarget sha = fromInternalGitPath . L.toStrict <$> get where -- Avoid buffering the whole file content, which might be large. -- 8192 is enough if it really is a symlink. @@ -137,24 +137,24 @@ catSymLinkTarget sha = fromInternalGitPath . decodeBL <$> get - - So, this gets info from the index, unless running as a daemon. -} -catKeyFile :: FilePath -> Annex (Maybe Key) +catKeyFile :: RawFilePath -> Annex (Maybe Key) catKeyFile f = ifM (Annex.getState Annex.daemon) ( catKeyFileHEAD f , catKey $ Git.Ref.fileRef f ) -catKeyFileHEAD :: FilePath -> Annex (Maybe Key) +catKeyFileHEAD :: RawFilePath -> Annex (Maybe Key) catKeyFileHEAD f = catKey $ Git.Ref.fileFromRef Git.Ref.headRef f {- Look in the original branch from whence an adjusted branch is based - to find the file. But only when the adjustment hides some files. -} -catKeyFileHidden :: FilePath -> CurrBranch -> Annex (Maybe Key) +catKeyFileHidden :: RawFilePath -> CurrBranch -> Annex (Maybe Key) catKeyFileHidden = hiddenCat catKey -catObjectMetaDataHidden :: FilePath -> CurrBranch -> Annex (Maybe (Integer, ObjectType)) +catObjectMetaDataHidden :: RawFilePath -> CurrBranch -> Annex (Maybe (Integer, ObjectType)) catObjectMetaDataHidden = hiddenCat catObjectMetaData -hiddenCat :: (Ref -> Annex (Maybe a)) -> FilePath -> CurrBranch -> Annex (Maybe a) +hiddenCat :: (Ref -> Annex (Maybe a)) -> RawFilePath -> CurrBranch -> Annex (Maybe a) hiddenCat a f (Just origbranch, Just adj) | adjustmentHidesFiles adj = a (Git.Ref.fileFromRef origbranch f) hiddenCat _ _ _ = return Nothing diff --git a/Annex/Content.hs b/Annex/Content.hs index 43fc3238c6..b3752c6ba9 100644 --- a/Annex/Content.hs +++ b/Annex/Content.hs @@ -329,7 +329,7 @@ getViaTmpFromDisk rsp v key action = checkallowed $ do checkallowed a = case rsp of RetrievalAllKeysSecure -> a RetrievalVerifiableKeysSecure - | isVerifiable (keyVariety key) -> a + | isVerifiable (fromKey keyVariety key) -> a | otherwise -> ifM (annexAllowUnverifiedDownloads <$> Annex.getGitConfig) ( a , warnUnverifiableInsecure key >> return False @@ -353,7 +353,7 @@ verifyKeyContent :: RetrievalSecurityPolicy -> VerifyConfig -> Verification -> K verifyKeyContent rsp v verification k f = case (rsp, verification) of (_, Verified) -> return True (RetrievalVerifiableKeysSecure, _) - | isVerifiable (keyVariety k) -> verify + | isVerifiable (fromKey keyVariety k) -> verify | otherwise -> ifM (annexAllowUnverifiedDownloads <$> Annex.getGitConfig) ( verify , warnUnverifiableInsecure k >> return False @@ -365,12 +365,12 @@ verifyKeyContent rsp v verification k f = case (rsp, verification) of (_, MustVerify) -> verify where verify = enteringStage VerifyStage $ verifysize <&&> verifycontent - verifysize = case keySize k of + verifysize = case fromKey keySize k of Nothing -> return True Just size -> do size' <- liftIO $ catchDefaultIO 0 $ getFileSize f return (size' == size) - verifycontent = case Types.Backend.verifyKeyContent =<< Backend.maybeLookupBackendVariety (keyVariety k) of + verifycontent = case Types.Backend.verifyKeyContent =<< Backend.maybeLookupBackendVariety (fromKey keyVariety k) of Nothing -> return True Just verifier -> verifier k f @@ -382,7 +382,7 @@ warnUnverifiableInsecure k = warning $ unwords , "this safety check.)" ] where - kv = decodeBS (formatKeyVariety (keyVariety k)) + kv = decodeBS (formatKeyVariety (fromKey keyVariety k)) data VerifyConfig = AlwaysVerify | NoVerify | RemoteVerify Remote | DefaultVerify @@ -483,17 +483,17 @@ moveAnnex key src = ifM (checkSecureHashes key) fs <- map (`fromTopFilePath` g) <$> Database.Keys.getAssociatedFiles key unless (null fs) $ do - ics <- mapM (populatePointerFile (Restage True) key dest) fs + ics <- mapM (populatePointerFile (Restage True) key (toRawFilePath dest) . toRawFilePath) fs Database.Keys.storeInodeCaches' key [dest] (catMaybes ics) ) alreadyhave = liftIO $ removeFile src checkSecureHashes :: Key -> Annex Bool checkSecureHashes key - | cryptographicallySecure (keyVariety key) = return True + | cryptographicallySecure (fromKey keyVariety key) = return True | otherwise = ifM (annexSecureHashesOnly <$> Annex.getGitConfig) ( do - warning $ "annex.securehashesonly blocked adding " ++ decodeBS (formatKeyVariety (keyVariety key)) ++ " key to annex objects" + warning $ "annex.securehashesonly blocked adding " ++ decodeBS (formatKeyVariety (fromKey keyVariety key)) ++ " key to annex objects" return False , return True ) @@ -650,7 +650,7 @@ removeAnnex (ContentRemovalLock key) = withObjectLoc key $ \file -> -- Check associated pointer file for modifications, and reset if -- it's unmodified. resetpointer file = ifM (isUnmodified key file) - ( depopulatePointerFile key file + ( depopulatePointerFile key (toRawFilePath file) -- Modified file, so leave it alone. -- If it was a hard link to the annex object, -- that object might have been frozen as part of the diff --git a/Annex/Content/LowLevel.hs b/Annex/Content/LowLevel.hs index 599ff7d4ff..546e647def 100644 --- a/Annex/Content/LowLevel.hs +++ b/Annex/Content/LowLevel.hs @@ -100,7 +100,7 @@ preserveGitMode _ _ = return True - when doing concurrent downloads. -} checkDiskSpace :: Maybe FilePath -> Key -> Integer -> Bool -> Annex Bool -checkDiskSpace destdir key = checkDiskSpace' (fromMaybe 1 (keySize key)) destdir key +checkDiskSpace destdir key = checkDiskSpace' (fromMaybe 1 (fromKey keySize key)) destdir key {- Allows specifying the size of the key, if it's known, which is useful - as not all keys know their size. -} diff --git a/Annex/Content/PointerFile.hs b/Annex/Content/PointerFile.hs index 2ed0db5ab9..59825a9d70 100644 --- a/Annex/Content/PointerFile.hs +++ b/Annex/Content/PointerFile.hs @@ -30,16 +30,17 @@ import Utility.Touch - - Returns an InodeCache if it populated the pointer file. -} -populatePointerFile :: Restage -> Key -> FilePath -> FilePath -> Annex (Maybe InodeCache) +populatePointerFile :: Restage -> Key -> RawFilePath -> RawFilePath -> Annex (Maybe InodeCache) populatePointerFile restage k obj f = go =<< liftIO (isPointerFile f) where go (Just k') | k == k' = do - destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus f - liftIO $ nukeFile f - (ic, populated) <- replaceFile f $ \tmp -> do - ok <- linkOrCopy k obj tmp destmode >>= \case + let f' = fromRawFilePath f + destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus f' + liftIO $ nukeFile f' + (ic, populated) <- replaceFile f' $ \tmp -> do + ok <- linkOrCopy k (fromRawFilePath obj) tmp destmode >>= \case Just _ -> thawContent tmp >> return True - Nothing -> liftIO (writePointerFile tmp k destmode) >> return False + Nothing -> liftIO (writePointerFile (toRawFilePath tmp) k destmode) >> return False ic <- withTSDelta (liftIO . genInodeCache tmp) return (ic, ok) maybe noop (restagePointerFile restage f) ic @@ -51,14 +52,15 @@ populatePointerFile restage k obj f = go =<< liftIO (isPointerFile f) {- Removes the content from a pointer file, replacing it with a pointer. - - Does not check if the pointer file is modified. -} -depopulatePointerFile :: Key -> FilePath -> Annex () +depopulatePointerFile :: Key -> RawFilePath -> Annex () depopulatePointerFile key file = do - st <- liftIO $ catchMaybeIO $ getFileStatus file + let file' = fromRawFilePath file + st <- liftIO $ catchMaybeIO $ getFileStatus file' let mode = fmap fileMode st - secureErase file - liftIO $ nukeFile file - ic <- replaceFile file $ \tmp -> do - liftIO $ writePointerFile tmp key mode + secureErase file' + liftIO $ nukeFile file' + ic <- replaceFile file' $ \tmp -> do + liftIO $ writePointerFile (toRawFilePath tmp) key mode #if ! defined(mingw32_HOST_OS) -- Don't advance mtime; this avoids unncessary re-smudging -- by git in some cases. diff --git a/Annex/Difference.hs b/Annex/Difference.hs index be621dc6fc..4d13c7211c 100644 --- a/Annex/Difference.hs +++ b/Annex/Difference.hs @@ -54,5 +54,5 @@ setDifferences = do else return ds ) forM_ (listDifferences ds') $ \d -> - setConfig (ConfigKey $ differenceConfigKey d) (differenceConfigVal d) + setConfig (differenceConfigKey d) (differenceConfigVal d) recordDifferences ds' u diff --git a/Annex/DirHashes.hs b/Annex/DirHashes.hs index 76e8bf7981..1fb0073826 100644 --- a/Annex/DirHashes.hs +++ b/Annex/DirHashes.hs @@ -65,14 +65,14 @@ hashDirs (HashLevels 1) sz s = addTrailingPathSeparator $ take sz s hashDirs _ sz s = addTrailingPathSeparator $ take sz s drop sz s hashDirLower :: HashLevels -> Hasher -hashDirLower n k = hashDirs n 3 $ take 6 $ show $ md5 $ serializeKey' $ nonChunkKey k +hashDirLower n k = hashDirs n 3 $ take 6 $ show $ md5s $ serializeKey' $ nonChunkKey k {- This was originally using Data.Hash.MD5 from MissingH. This new version - is faster, but ugly as it has to replicate the 4 Word32's that produced. -} hashDirMixed :: HashLevels -> Hasher hashDirMixed n k = hashDirs n 2 $ take 4 $ concatMap display_32bits_as_dir $ encodeWord32 $ map fromIntegral $ Data.ByteArray.unpack $ - Utility.Hash.md5 $ serializeKey' $ nonChunkKey k + Utility.Hash.md5s $ serializeKey' $ nonChunkKey k where encodeWord32 (b1:b2:b3:b4:rest) = (shiftL b4 24 .|. shiftL b3 16 .|. shiftL b2 8 .|. b1) diff --git a/Annex/Drop.hs b/Annex/Drop.hs index 96f0eb7e30..f2489e5482 100644 --- a/Annex/Drop.hs +++ b/Annex/Drop.hs @@ -49,7 +49,8 @@ type Reason = String handleDropsFrom :: [UUID] -> [Remote] -> Reason -> Bool -> Key -> AssociatedFile -> [VerifiedCopy] -> (CommandStart -> CommandCleanup) -> Annex () handleDropsFrom locs rs reason fromhere key afile preverified runner = do g <- Annex.gitRepo - l <- map (`fromTopFilePath` g) <$> Database.Keys.getAssociatedFiles key + l <- map toRawFilePath . map (`fromTopFilePath` g) + <$> Database.Keys.getAssociatedFiles key let fs = case afile of AssociatedFile (Just f) -> nub (f : l) AssociatedFile Nothing -> l @@ -62,7 +63,7 @@ handleDropsFrom locs rs reason fromhere key afile preverified runner = do (untrusted, have) <- trustPartition UnTrusted locs numcopies <- if null fs then getNumCopies - else maximum <$> mapM getFileNumCopies fs + else maximum <$> mapM (getFileNumCopies . fromRawFilePath) fs return (NumCopies (length have), numcopies, S.fromList untrusted) {- Check that we have enough copies still to drop the content. @@ -107,7 +108,7 @@ handleDropsFrom locs rs reason fromhere key afile preverified runner = do [ "dropped" , case afile of AssociatedFile Nothing -> serializeKey key - AssociatedFile (Just af) -> af + AssociatedFile (Just af) -> fromRawFilePath af , "(from " ++ maybe "here" show u ++ ")" , "(copies now " ++ show (fromNumCopies have - 1) ++ ")" , ": " ++ reason diff --git a/Annex/Environment.hs b/Annex/Environment.hs index ea9eda0339..3da7ce980b 100644 --- a/Annex/Environment.hs +++ b/Annex/Environment.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Annex.Environment where import Annex.Common @@ -45,6 +47,6 @@ ensureCommit a = either retry return =<< tryNonAsync a where retry _ = do name <- liftIO $ either (const "unknown") id <$> myUserName - setConfig (ConfigKey "user.name") name - setConfig (ConfigKey "user.email") name + setConfig "user.name" name + setConfig "user.email" name a diff --git a/Annex/Export.hs b/Annex/Export.hs index a72d5d0dbb..16786476de 100644 --- a/Annex/Export.hs +++ b/Annex/Export.hs @@ -33,7 +33,7 @@ exportKey :: Git.Sha -> Annex ExportKey exportKey sha = mk <$> catKey sha where mk (Just k) = AnnexKey k - mk Nothing = GitKey $ Key + mk Nothing = GitKey $ mkKey $ \k -> k { keyName = encodeBS $ Git.fromRef sha , keyVariety = SHA1Key (HasExt False) , keySize = Nothing diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs index b41a4a421f..05e6e7f761 100644 --- a/Annex/FileMatcher.hs +++ b/Annex/FileMatcher.hs @@ -54,7 +54,7 @@ checkFileMatcher' getmatcher file notconfigured = do matcher <- getmatcher file checkMatcher matcher Nothing afile S.empty notconfigured d where - afile = AssociatedFile (Just file) + afile = AssociatedFile (Just (toRawFilePath file)) -- checkMatcher will never use this, because afile is provided. d = return True @@ -62,7 +62,7 @@ checkMatcher :: FileMatcher Annex -> Maybe Key -> AssociatedFile -> AssumeNotPre checkMatcher matcher mkey afile notpresent notconfigured d | isEmpty matcher = notconfigured | otherwise = case (mkey, afile) of - (_, AssociatedFile (Just file)) -> go =<< fileMatchInfo file + (_, AssociatedFile (Just file)) -> go =<< fileMatchInfo (fromRawFilePath file) (Just key, _) -> go (MatchingKey key afile) _ -> d where diff --git a/Annex/Fixup.hs b/Annex/Fixup.hs index d167086b09..547458c08f 100644 --- a/Annex/Fixup.hs +++ b/Annex/Fixup.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Annex.Fixup where import Git.Types @@ -53,7 +55,7 @@ fixupDirect r@(Repo { location = l@(Local { gitdir = d, worktree = Nothing }) }) { location = l { worktree = Just (parentDir d) } , gitGlobalOpts = gitGlobalOpts r ++ [ Param "-c" - , Param $ coreBare ++ "=" ++ boolConfig False + , Param $ fromConfigKey coreBare ++ "=" ++ boolConfig False ] } fixupDirect r = r diff --git a/Annex/GitOverlay.hs b/Annex/GitOverlay.hs index de355217d2..0b3e9c2b88 100644 --- a/Annex/GitOverlay.hs +++ b/Annex/GitOverlay.hs @@ -14,7 +14,6 @@ import Git import Git.Types import Git.Index import Git.Env -import Utility.Env import qualified Annex import qualified Annex.Queue @@ -23,28 +22,29 @@ withIndexFile :: FilePath -> Annex a -> Annex a withIndexFile f a = do f' <- liftIO $ indexEnvVal f withAltRepo - (usecachedgitenv $ \g -> liftIO $ addGitEnv g indexEnv f') + (usecachedgitenv f' $ \g -> addGitEnv g indexEnv f') (\g g' -> g' { gitEnv = gitEnv g }) a where -- This is an optimisation. Since withIndexFile is run repeatedly, - -- and addGitEnv uses the slow getEnvironment when gitEnv is Nothing, - -- we cache the environment the first time, and reuse it in - -- subsequent calls. + -- typically with the same file, and addGitEnv uses the slow + -- getEnvironment when gitEnv is Nothing, and has to do a + -- nontrivial amount of work, we cache the modified environment + -- the first time, and reuse it in subsequent calls for the same + -- index file. -- -- (This could be done at another level; eg when creating the -- Git object in the first place, but it's more efficient to let - -- the enviroment be inherited in all calls to git where it + -- the environment be inherited in all calls to git where it -- does not need to be modified.) - usecachedgitenv m g = case gitEnv g of - Just _ -> m g - Nothing -> do - e <- Annex.withState $ \s -> case Annex.cachedgitenv s of - Nothing -> do - e <- getEnvironment - return (s { Annex.cachedgitenv = Just e }, e) - Just e -> return (s, e) - m (g { gitEnv = Just e }) + usecachedgitenv f' m g = case gitEnv g of + Just _ -> liftIO $ m g + Nothing -> Annex.withState $ \s -> case Annex.cachedgitenv s of + Just (cachedf, cachede) | f' == cachedf -> + return (s, g { gitEnv = Just cachede }) + _ -> do + g' <- m g + return (s { Annex.cachedgitenv = (,) <$> Just f' <*> gitEnv g' }, g') {- Runs an action using a different git work tree. - diff --git a/Annex/Import.hs b/Annex/Import.hs index 9a939937af..8291cd51bf 100644 --- a/Annex/Import.hs +++ b/Annex/Import.hs @@ -264,7 +264,7 @@ buildImportTrees basetree msubdir importable = History graftTree' importtree subdir basetree repo hdl mktreeitem (loc, k) = do - let lf = fromImportLocation loc + let lf = fromRawFilePath (fromImportLocation loc) let treepath = asTopFilePath lf let topf = asTopFilePath $ maybe lf (\sd -> getTopFilePath sd lf) msubdir @@ -327,7 +327,7 @@ downloadImport remote importtreeconfig importablecontents = do (k:_) -> return $ Left $ Just (loc, k) [] -> do job <- liftIO $ newEmptyTMVarIO - let ai = ActionItemOther (Just (fromImportLocation loc)) + let ai = ActionItemOther (Just (fromRawFilePath (fromImportLocation loc))) let downloadaction = starting ("import " ++ Remote.name remote) ai $ do when oldversion $ showNote "old version" @@ -377,9 +377,9 @@ downloadImport remote importtreeconfig importablecontents = do fmap fst <$> genKey ks nullMeterUpdate backend locworktreefilename loc = asTopFilePath $ case importtreeconfig of - ImportTree -> fromImportLocation loc + ImportTree -> fromRawFilePath (fromImportLocation loc) ImportSubTree subdir _ -> - getTopFilePath subdir fromImportLocation loc + getTopFilePath subdir fromRawFilePath (fromImportLocation loc) getcidkey cidmap db cid = liftIO $ CIDDb.getContentIdentifierKeys db rs cid >>= \case @@ -398,7 +398,7 @@ downloadImport remote importtreeconfig importablecontents = do {- Temporary key used for import of a ContentIdentifier while downloading - content, before generating its real key. -} importKey :: ContentIdentifier -> Integer -> Key -importKey (ContentIdentifier cid) size = stubKey +importKey (ContentIdentifier cid) size = mkKey $ \k -> k { keyName = cid , keyVariety = OtherKey "CID" , keySize = Just size @@ -450,7 +450,7 @@ wantImport :: FileMatcher Annex -> ImportLocation -> ByteSize -> Annex Bool wantImport matcher loc sz = checkMatcher' matcher mi mempty where mi = MatchingInfo $ ProvidedInfo - { providedFilePath = Right $ fromImportLocation loc + { providedFilePath = Right $ fromRawFilePath $ fromImportLocation loc , providedKey = unavail "key" , providedFileSize = Right sz , providedMimeType = unavail "mime" @@ -503,4 +503,4 @@ listImportableContents r = fmap removegitspecial , importableHistory = map removegitspecial (importableHistory ic) } - gitspecial l = ".git" `elem` Posix.splitDirectories (fromImportLocation l) + gitspecial l = ".git" `elem` Posix.splitDirectories (fromRawFilePath (fromImportLocation l)) diff --git a/Annex/Ingest.hs b/Annex/Ingest.hs index 1406c4007c..5d5636b2e5 100644 --- a/Annex/Ingest.hs +++ b/Annex/Ingest.hs @@ -136,7 +136,7 @@ ingestAdd' meterupdate ld@(Just (LockedDown cfg source)) mk = do then addLink f k mic else do mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus (contentLocation source) - stagePointerFile f mode =<< hashPointerFile k + stagePointerFile (toRawFilePath f) mode =<< hashPointerFile k return (Just k) {- Ingests a locked down file into the annex. Does not update the working @@ -187,7 +187,7 @@ ingest' preferredbackend meterupdate (Just (LockedDown cfg source)) mk restage = gounlocked _ _ _ = failure "failed statting file" success k mcache s = do - genMetaData k (keyFilename source) s + genMetaData k (toRawFilePath (keyFilename source)) s return (Just k, mcache) failure msg = do @@ -208,13 +208,13 @@ finishIngestUnlocked' key source restage = do {- Copy to any other locations using the same key. -} populateAssociatedFiles :: Key -> KeySource -> Restage -> Annex () populateAssociatedFiles key source restage = do - obj <- calcRepo (gitAnnexLocation key) + obj <- toRawFilePath <$> calcRepo (gitAnnexLocation key) g <- Annex.gitRepo ingestedf <- flip fromTopFilePath g <$> inRepo (toTopFilePath (keyFilename source)) afs <- map (`fromTopFilePath` g) <$> Database.Keys.getAssociatedFiles key forM_ (filter (/= ingestedf) afs) $ - populatePointerFile restage key obj + populatePointerFile restage key obj . toRawFilePath cleanCruft :: KeySource -> Annex () cleanCruft source = when (contentLocation source /= keyFilename source) $ @@ -264,7 +264,7 @@ restoreFile file key e = do makeLink :: FilePath -> Key -> Maybe InodeCache -> Annex String makeLink file key mcache = flip catchNonAsync (restoreFile file key) $ do l <- calcRepo $ gitAnnexLink file key - replaceFile file $ makeAnnexLink l + replaceFile file $ makeAnnexLink l . toRawFilePath -- touch symlink to have same time as the original file, -- as provided in the InodeCache @@ -291,7 +291,7 @@ addLink file key mcache = ifM (coreSymlinks <$> Annex.getGitConfig) Annex.Queue.addCommand "add" (ps++[Param "--"]) [file] , do l <- makeLink file key mcache - addAnnexLink l file + addAnnexLink l (toRawFilePath file) ) {- Parameters to pass to git add, forcing addition of ignored files. -} @@ -329,7 +329,7 @@ addAnnexedFile file key mtmp = ifM addUnlocked (pure Nothing) (\tmp -> liftIO $ catchMaybeIO $ fileMode <$> getFileStatus tmp) mtmp - stagePointerFile file mode =<< hashPointerFile key + stagePointerFile (toRawFilePath file) mode =<< hashPointerFile key Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath file) case mtmp of Just tmp -> ifM (moveAnnex key tmp) @@ -349,6 +349,6 @@ addAnnexedFile file key mtmp = ifM addUnlocked where linkunlocked mode = linkFromAnnex key file mode >>= \case LinkAnnexFailed -> liftIO $ - writePointerFile file key mode + writePointerFile (toRawFilePath file) key mode _ -> return () - writepointer mode = liftIO $ writePointerFile file key mode + writepointer mode = liftIO $ writePointerFile (toRawFilePath file) key mode diff --git a/Annex/Init.hs b/Annex/Init.hs index a762bf690c..3accd18ff3 100644 --- a/Annex/Init.hs +++ b/Annex/Init.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} module Annex.Init ( ensureInitialized, @@ -22,6 +23,7 @@ import qualified Annex import qualified Git import qualified Git.Config import qualified Git.Objects +import Git.Types (fromConfigValue) import qualified Annex.Branch import Logs.UUID import Logs.Trust.Basic @@ -204,7 +206,7 @@ checkCrippledFileSystem = whenM probeCrippledFileSystem $ do - filesystem. -} whenM (coreSymlinks <$> Annex.getGitConfig) $ do warning "Disabling core.symlinks." - setConfig (ConfigKey "core.symlinks") + setConfig "core.symlinks" (Git.Config.boolConfig False) probeLockSupport :: Annex Bool @@ -274,5 +276,5 @@ initSharedClone True = do - affect it. -} propigateSecureHashesOnly :: Annex () propigateSecureHashesOnly = - maybe noop (setConfig (ConfigKey "annex.securehashesonly")) + maybe noop (setConfig "annex.securehashesonly" . fromConfigValue) =<< getGlobalConfig "annex.securehashesonly" diff --git a/Annex/Journal.hs b/Annex/Journal.hs index 917d638aa8..e7e624f354 100644 --- a/Annex/Journal.hs +++ b/Annex/Journal.hs @@ -44,18 +44,18 @@ instance Journalable Builder where - getJournalFileStale to always return a consistent journal file - content, although possibly not the most current one. -} -setJournalFile :: Journalable content => JournalLocked -> FilePath -> content -> Annex () +setJournalFile :: Journalable content => JournalLocked -> RawFilePath -> content -> Annex () setJournalFile _jl file content = withOtherTmp $ \tmp -> do createAnnexDirectory =<< fromRepo gitAnnexJournalDir -- journal file is written atomically - jfile <- fromRepo $ journalFile file + jfile <- fromRepo $ journalFile $ fromRawFilePath file let tmpfile = tmp takeFileName jfile liftIO $ do withFile tmpfile WriteMode $ \h -> writeJournalHandle h content moveFile tmpfile jfile {- Gets any journalled content for a file in the branch. -} -getJournalFile :: JournalLocked -> FilePath -> Annex (Maybe L.ByteString) +getJournalFile :: JournalLocked -> RawFilePath -> Annex (Maybe L.ByteString) getJournalFile _jl = getJournalFileStale {- Without locking, this is not guaranteed to be the most recent @@ -69,9 +69,9 @@ getJournalFile _jl = getJournalFileStale - concurrency or other issues with a lazy read, and the minor loss of - laziness doesn't matter much, as the files are not very large. -} -getJournalFileStale :: FilePath -> Annex (Maybe L.ByteString) +getJournalFileStale :: RawFilePath -> Annex (Maybe L.ByteString) getJournalFileStale file = inRepo $ \g -> catchMaybeIO $ - L.fromStrict <$> S.readFile (journalFile file g) + L.fromStrict <$> S.readFile (journalFile (fromRawFilePath file) g) {- List of existing journal files, but without locking, may miss new ones - just being added, or may have false positives if the journal is staged diff --git a/Annex/Link.hs b/Annex/Link.hs index 00c2d68d9e..b012b7d933 100644 --- a/Annex/Link.hs +++ b/Annex/Link.hs @@ -43,7 +43,7 @@ import qualified Data.ByteString.Lazy as L type LinkTarget = String {- Checks if a file is a link to a key. -} -isAnnexLink :: FilePath -> Annex (Maybe Key) +isAnnexLink :: RawFilePath -> Annex (Maybe Key) isAnnexLink file = maybe Nothing parseLinkTargetOrPointer <$> getAnnexLinkTarget file {- Gets the link target of a symlink. @@ -54,13 +54,13 @@ isAnnexLink file = maybe Nothing parseLinkTargetOrPointer <$> getAnnexLinkTarget - Returns Nothing if the file is not a symlink, or not a link to annex - content. -} -getAnnexLinkTarget :: FilePath -> Annex (Maybe S.ByteString) +getAnnexLinkTarget :: RawFilePath -> Annex (Maybe S.ByteString) getAnnexLinkTarget f = getAnnexLinkTarget' f =<< (coreSymlinks <$> Annex.getGitConfig) {- Pass False to force looking inside file, for when git checks out - symlinks as plain files. -} -getAnnexLinkTarget' :: FilePath -> Bool -> Annex (Maybe S.ByteString) +getAnnexLinkTarget' :: RawFilePath -> Bool -> Annex (Maybe S.ByteString) getAnnexLinkTarget' file coresymlinks = if coresymlinks then check probesymlink $ return Nothing @@ -75,9 +75,9 @@ getAnnexLinkTarget' file coresymlinks = if coresymlinks | otherwise -> return Nothing Nothing -> fallback - probesymlink = R.readSymbolicLink $ toRawFilePath file + probesymlink = R.readSymbolicLink file - probefilecontent = withFile file ReadMode $ \h -> do + probefilecontent = withFile (fromRawFilePath file) ReadMode $ \h -> do s <- S.hGet h unpaddedMaxPointerSz -- If we got the full amount, the file is too large -- to be a symlink target. @@ -92,7 +92,7 @@ getAnnexLinkTarget' file coresymlinks = if coresymlinks then mempty else s -makeAnnexLink :: LinkTarget -> FilePath -> Annex () +makeAnnexLink :: LinkTarget -> RawFilePath -> Annex () makeAnnexLink = makeGitLink {- Creates a link on disk. @@ -102,48 +102,48 @@ makeAnnexLink = makeGitLink - it's staged as such, so use addAnnexLink when adding a new file or - modified link to git. -} -makeGitLink :: LinkTarget -> FilePath -> Annex () +makeGitLink :: LinkTarget -> RawFilePath -> Annex () makeGitLink linktarget file = ifM (coreSymlinks <$> Annex.getGitConfig) ( liftIO $ do - void $ tryIO $ removeFile file - createSymbolicLink linktarget file - , liftIO $ writeFile file linktarget + void $ tryIO $ removeFile (fromRawFilePath file) + createSymbolicLink linktarget (fromRawFilePath file) + , liftIO $ writeFile (fromRawFilePath file) linktarget ) {- Creates a link on disk, and additionally stages it in git. -} -addAnnexLink :: LinkTarget -> FilePath -> Annex () +addAnnexLink :: LinkTarget -> RawFilePath -> Annex () addAnnexLink linktarget file = do makeAnnexLink linktarget file stageSymlink file =<< hashSymlink linktarget {- Injects a symlink target into git, returning its Sha. -} hashSymlink :: LinkTarget -> Annex Sha -hashSymlink linktarget = hashBlob $ toRawFilePath $ toInternalGitPath linktarget +hashSymlink = hashBlob . toInternalGitPath . toRawFilePath {- Stages a symlink to an annexed object, using a Sha of its target. -} -stageSymlink :: FilePath -> Sha -> Annex () +stageSymlink :: RawFilePath -> Sha -> Annex () stageSymlink file sha = Annex.Queue.addUpdateIndex =<< - inRepo (Git.UpdateIndex.stageSymlink file sha) + inRepo (Git.UpdateIndex.stageSymlink (fromRawFilePath file) sha) {- Injects a pointer file content into git, returning its Sha. -} hashPointerFile :: Key -> Annex Sha hashPointerFile key = hashBlob $ formatPointer key {- Stages a pointer file, using a Sha of its content -} -stagePointerFile :: FilePath -> Maybe FileMode -> Sha -> Annex () +stagePointerFile :: RawFilePath -> Maybe FileMode -> Sha -> Annex () stagePointerFile file mode sha = Annex.Queue.addUpdateIndex =<< - inRepo (Git.UpdateIndex.stageFile sha treeitemtype file) + inRepo (Git.UpdateIndex.stageFile sha treeitemtype $ fromRawFilePath file) where treeitemtype | maybe False isExecutable mode = TreeExecutable | otherwise = TreeFile -writePointerFile :: FilePath -> Key -> Maybe FileMode -> IO () +writePointerFile :: RawFilePath -> Key -> Maybe FileMode -> IO () writePointerFile file k mode = do - S.writeFile file (formatPointer k) - maybe noop (setFileMode file) mode + S.writeFile (fromRawFilePath file) (formatPointer k) + maybe noop (setFileMode $ fromRawFilePath file) mode newtype Restage = Restage Bool @@ -172,17 +172,17 @@ newtype Restage = Restage Bool - the worktree file is changed by something else before git update-index - gets to look at it. -} -restagePointerFile :: Restage -> FilePath -> InodeCache -> Annex () +restagePointerFile :: Restage -> RawFilePath -> InodeCache -> Annex () restagePointerFile (Restage False) f _ = - toplevelWarning True $ unableToRestage (Just f) + toplevelWarning True $ unableToRestage $ Just $ fromRawFilePath f restagePointerFile (Restage True) f orig = withTSDelta $ \tsd -> do -- update-index is documented as picky about "./file" and it -- fails on "../../repo/path/file" when cwd is not in the repo -- being acted on. Avoid these problems with an absolute path. - absf <- liftIO $ absPath f + absf <- liftIO $ absPath $ fromRawFilePath f Annex.Queue.addInternalAction runner [(absf, isunmodified tsd)] where - isunmodified tsd = genInodeCache f tsd >>= return . \case + isunmodified tsd = genInodeCache' f tsd >>= return . \case Nothing -> False Just new -> compareStrong orig new @@ -264,7 +264,7 @@ parseLinkTarget l formatPointer :: Key -> S.ByteString formatPointer k = prefix <> keyFile' k <> nl where - prefix = toRawFilePath $ toInternalGitPath (pathSeparator:objectDir) + prefix = toInternalGitPath $ toRawFilePath (pathSeparator:objectDir) nl = S8.singleton '\n' {- Maximum size of a file that could be a pointer to a key. @@ -283,8 +283,8 @@ unpaddedMaxPointerSz = 8192 {- Checks if a worktree file is a pointer to a key. - - Unlocked files whose content is present are not detected by this. -} -isPointerFile :: FilePath -> IO (Maybe Key) -isPointerFile f = catchDefaultIO Nothing $ withFile f ReadMode $ \h -> +isPointerFile :: RawFilePath -> IO (Maybe Key) +isPointerFile f = catchDefaultIO Nothing $ withFile (fromRawFilePath f) ReadMode $ \h -> parseLinkTargetOrPointer <$> S.hGet h unpaddedMaxPointerSz {- Checks a symlink target or pointer file first line to see if it diff --git a/Annex/Locations.hs b/Annex/Locations.hs index f29c67f576..88080a647b 100644 --- a/Annex/Locations.hs +++ b/Annex/Locations.hs @@ -95,7 +95,6 @@ module Annex.Locations ( import Data.Char import Data.Default import qualified Data.ByteString.Char8 as S8 -import qualified Data.ByteString.Lazy as L import Common import Key @@ -195,7 +194,8 @@ gitAnnexLink file key r config = do let absfile = absNormPathUnix currdir file let gitdir = getgitdir currdir loc <- gitAnnexLocation' key r config False False (\_ -> return True) gitdir - toInternalGitPath <$> relPathDirToFile (parentDir absfile) loc + fromRawFilePath . toInternalGitPath . toRawFilePath + <$> relPathDirToFile (parentDir absfile) loc where getgitdir currdir {- This special case is for git submodules on filesystems not @@ -204,8 +204,10 @@ gitAnnexLink file key r config = do | not (coreSymlinks config) && needsSubmoduleFixup r = absNormPathUnix currdir $ Git.repoPath r ".git" | otherwise = Git.localGitDir r - absNormPathUnix d p = toInternalGitPath $ - absPathFrom (toInternalGitPath d) (toInternalGitPath p) + absNormPathUnix d p = fromRawFilePath $ toInternalGitPath $ toRawFilePath $ + absPathFrom + (fromRawFilePath $ toInternalGitPath $ toRawFilePath d) + (fromRawFilePath $ toInternalGitPath $ toRawFilePath p) {- Calculates a symlink target as would be used in a typical git - repository, with .git in the top of the work tree. -} @@ -569,8 +571,8 @@ keyFile = fromRawFilePath . keyFile' keyFile' :: Key -> RawFilePath keyFile' k = - let b = L.toStrict (serializeKey' k) - in if any (`S8.elem` b) ['&', '%', ':', '/'] + let b = serializeKey' k + in if S8.any (`elem` ['&', '%', ':', '/']) b then S8.concatMap esc b else b where @@ -580,6 +582,7 @@ keyFile' k = esc '/' = "%" esc c = S8.singleton c + {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Maybe Key diff --git a/Annex/MetaData.hs b/Annex/MetaData.hs index 4b355dbb72..4e0a541af9 100644 --- a/Annex/MetaData.hs +++ b/Annex/MetaData.hs @@ -37,7 +37,7 @@ import Data.Time.Clock.POSIX - - Also, can generate new metadata, if configured to do so. -} -genMetaData :: Key -> FilePath -> FileStatus -> Annex () +genMetaData :: Key -> RawFilePath -> FileStatus -> Annex () genMetaData key file status = do catKeyFileHEAD file >>= \case Nothing -> noop @@ -53,8 +53,8 @@ genMetaData key file status = do where mtime = posixSecondsToUTCTime $ realToFrac $ modificationTime status warncopied = warning $ - "Copied metadata from old version of " ++ file ++ " to new version. " ++ - "If you don't want this copied metadata, run: git annex metadata --remove-all " ++ file + "Copied metadata from old version of " ++ fromRawFilePath file ++ " to new version. " ++ + "If you don't want this copied metadata, run: git annex metadata --remove-all " ++ fromRawFilePath file -- If the only fields copied were date metadata, and they'll -- be overwritten with the current mtime, no need to warn about -- copying. diff --git a/Annex/Notification.hs b/Annex/Notification.hs index 29b8fc9828..186676cd3e 100644 --- a/Annex/Notification.hs +++ b/Annex/Notification.hs @@ -60,7 +60,7 @@ notifyDrop (AssociatedFile (Just f)) ok = do wanted <- Annex.getState Annex.desktopnotify when (notifyFinish wanted) $ liftIO $ do client <- DBus.Client.connectSession - void $ Notify.notify client (droppedNote ok f) + void $ Notify.notify client (droppedNote ok (fromRawFilePath f)) #else notifyDrop (AssociatedFile (Just _)) _ = noop #endif diff --git a/Annex/NumCopies.hs b/Annex/NumCopies.hs index 0072614674..0b9b9b7096 100644 --- a/Annex/NumCopies.hs +++ b/Annex/NumCopies.hs @@ -72,7 +72,7 @@ getFileNumCopies f = fromSources getAssociatedFileNumCopies :: AssociatedFile -> Annex NumCopies getAssociatedFileNumCopies (AssociatedFile afile) = - maybe getNumCopies getFileNumCopies afile + maybe getNumCopies getFileNumCopies (fromRawFilePath <$> afile) {- This is the globally visible numcopies value for a file. So it does - not include local configuration in the git config or command line diff --git a/Annex/SpecialRemote.hs b/Annex/SpecialRemote.hs index 37e0e2129a..f27c405d6b 100644 --- a/Annex/SpecialRemote.hs +++ b/Annex/SpecialRemote.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Annex.SpecialRemote ( module Annex.SpecialRemote, module Annex.SpecialRemote.Config diff --git a/Annex/Transfer.hs b/Annex/Transfer.hs index 909bd22dae..193adf857a 100644 --- a/Annex/Transfer.hs +++ b/Annex/Transfer.hs @@ -40,15 +40,15 @@ import Data.Ord upload :: Observable v => UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> NotifyWitness -> Annex v upload u key f d a _witness = guardHaveUUID u $ - runTransfer (Transfer Upload u key) f d a + runTransfer (Transfer Upload u (fromKey id key)) f d a alwaysUpload :: Observable v => UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> NotifyWitness -> Annex v alwaysUpload u key f d a _witness = guardHaveUUID u $ - alwaysRunTransfer (Transfer Upload u key) f d a + alwaysRunTransfer (Transfer Upload u (fromKey id key)) f d a download :: Observable v => UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> NotifyWitness -> Annex v download u key f d a _witness = guardHaveUUID u $ - runTransfer (Transfer Download u key) f d a + runTransfer (Transfer Download u (fromKey id key)) f d a guardHaveUUID :: Observable v => UUID -> Annex v -> Annex v guardHaveUUID u a @@ -185,7 +185,7 @@ checkSecureHashes t a , a ) where - variety = keyVariety (transferKey t) + variety = fromKey keyVariety (transferKey t) type RetryDecider = Annex (TransferInfo -> TransferInfo -> Annex Bool) diff --git a/Annex/UUID.hs b/Annex/UUID.hs index 98d0c5219b..f3fc4c8acf 100644 --- a/Annex/UUID.hs +++ b/Annex/UUID.hs @@ -11,7 +11,10 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Annex.UUID ( + configkeyUUID, getUUID, getRepoUUID, getUncachedUUID, @@ -32,6 +35,7 @@ import Annex.Common import qualified Annex import qualified Git import qualified Git.Config +import Git.Types import Config import qualified Data.UUID as U @@ -39,8 +43,8 @@ import qualified Data.UUID.V4 as U4 import qualified Data.UUID.V5 as U5 import Data.String -configkey :: ConfigKey -configkey = annexConfig "uuid" +configkeyUUID :: ConfigKey +configkeyUUID = annexConfig "uuid" {- Generates a random UUID, that does not include the MAC address. -} genUUID :: IO UUID @@ -81,20 +85,16 @@ getRepoUUID r = do removeRepoUUID :: Annex () removeRepoUUID = do - unsetConfig configkey + unsetConfig configkeyUUID storeUUID NoUUID getUncachedUUID :: Git.Repo -> UUID -getUncachedUUID = toUUID . Git.Config.get key "" - where - (ConfigKey key) = configkey +getUncachedUUID = toUUID . Git.Config.get configkeyUUID "" -- Does the repo's config have a key for the UUID? -- True even when the key has no value. isUUIDConfigured :: Git.Repo -> Bool -isUUIDConfigured = isJust . Git.Config.getMaybe key - where - (ConfigKey key) = configkey +isUUIDConfigured = isJust . Git.Config.getMaybe configkeyUUID {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -104,7 +104,7 @@ prepUUID = whenM ((==) NoUUID <$> getUUID) $ storeUUID :: UUID -> Annex () storeUUID u = do Annex.changeGitConfig $ \c -> c { annexUUID = u } - storeUUIDIn configkey u + storeUUIDIn configkeyUUID u storeUUIDIn :: ConfigKey -> UUID -> Annex () storeUUIDIn configfield = setConfig configfield . fromUUID @@ -112,7 +112,7 @@ storeUUIDIn configfield = setConfig configfield . fromUUID {- Only sets the configkey in the Repo; does not change .git/config -} setUUID :: Git.Repo -> UUID -> IO Git.Repo setUUID r u = do - let s = show configkey ++ "=" ++ fromUUID u + let s = encodeBS' $ show configkeyUUID ++ "=" ++ fromUUID u Git.Config.store s r -- Dummy uuid for the whole web. Do not alter. diff --git a/Annex/VariantFile.hs b/Annex/VariantFile.hs index 58a6b4b634..65f989ebae 100644 --- a/Annex/VariantFile.hs +++ b/Annex/VariantFile.hs @@ -10,7 +10,7 @@ module Annex.VariantFile where import Annex.Common import Utility.Hash -import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S variantMarker :: String variantMarker = ".variant-" @@ -41,5 +41,5 @@ variantFile file key where doubleconflict = variantMarker `isInfixOf` file -shortHash :: L.ByteString -> String -shortHash = take 4 . show . md5 +shortHash :: S.ByteString -> String +shortHash = take 4 . show . md5s diff --git a/Annex/Version.hs b/Annex/Version.hs index ca987bf208..df9ac1514c 100644 --- a/Annex/Version.hs +++ b/Annex/Version.hs @@ -6,11 +6,13 @@ -} {-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} module Annex.Version where import Annex.Common import Config +import Git.Types import Types.RepoVersion import qualified Annex diff --git a/Annex/View.hs b/Annex/View.hs index 412cca8e0e..d20bbb8caa 100644 --- a/Annex/View.hs +++ b/Annex/View.hs @@ -343,11 +343,11 @@ narrowView = applyView' viewedFileReuse getViewedFileMetaData applyView' :: MkViewedFile -> (FilePath -> MetaData) -> View -> Annex Git.Branch applyView' mkviewedfile getfilemetadata view = do top <- fromRepo Git.repoPath - (l, clean) <- inRepo $ Git.LsFiles.stagedDetails [top] + (l, clean) <- inRepo $ Git.LsFiles.stagedDetails [toRawFilePath top] liftIO . nukeFile =<< fromRepo gitAnnexViewIndex uh <- withViewIndex $ inRepo Git.UpdateIndex.startUpdateIndex forM_ l $ \(f, sha, mode) -> do - topf <- inRepo (toTopFilePath f) + topf <- inRepo (toTopFilePath $ fromRawFilePath f) go uh topf sha (toTreeItemType =<< mode) =<< lookupFile f liftIO $ do void $ stopUpdateIndex uh diff --git a/Annex/WorkTree.hs b/Annex/WorkTree.hs index 8d797ccfe6..08015c4df5 100644 --- a/Annex/WorkTree.hs +++ b/Annex/WorkTree.hs @@ -32,35 +32,35 @@ import Config - When in an adjusted branch that may have hidden the file, looks for a - pointer to a key in the original branch. -} -lookupFile :: FilePath -> Annex (Maybe Key) +lookupFile :: RawFilePath -> Annex (Maybe Key) lookupFile = lookupFile' catkeyfile where catkeyfile file = - ifM (liftIO $ doesFileExist file) + ifM (liftIO $ doesFileExist $ fromRawFilePath file) ( catKeyFile file , catKeyFileHidden file =<< getCurrentBranch ) -lookupFileNotHidden :: FilePath -> Annex (Maybe Key) +lookupFileNotHidden :: RawFilePath -> Annex (Maybe Key) lookupFileNotHidden = lookupFile' catkeyfile where catkeyfile file = - ifM (liftIO $ doesFileExist file) + ifM (liftIO $ doesFileExist $ fromRawFilePath file) ( catKeyFile file , return Nothing ) -lookupFile' :: (FilePath -> Annex (Maybe Key)) -> FilePath -> Annex (Maybe Key) +lookupFile' :: (RawFilePath -> Annex (Maybe Key)) -> RawFilePath -> Annex (Maybe Key) lookupFile' catkeyfile file = isAnnexLink file >>= \case Just key -> return (Just key) Nothing -> catkeyfile file {- Modifies an action to only act on files that are already annexed, - and passes the key on to it. -} -whenAnnexed :: (FilePath -> Key -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a) +whenAnnexed :: (RawFilePath -> Key -> Annex (Maybe a)) -> RawFilePath -> Annex (Maybe a) whenAnnexed a file = ifAnnexed file (a file) (return Nothing) -ifAnnexed :: FilePath -> (Key -> Annex a) -> Annex a -> Annex a +ifAnnexed :: RawFilePath -> (Key -> Annex a) -> Annex a -> Annex a ifAnnexed file yes no = maybe no yes =<< lookupFile file {- Find all unlocked files and update the keys database for them. @@ -95,7 +95,7 @@ scanUnlockedFiles = whenM (inRepo Git.Ref.headExists <&&> not <$> isBareRepo) $ liftIO . Database.Keys.SQL.addAssociatedFileFast k tf whenM (inAnnex k) $ do f <- fromRepo $ fromTopFilePath tf - liftIO (isPointerFile f) >>= \case + liftIO (isPointerFile (toRawFilePath f)) >>= \case Just k' | k' == k -> do destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus f ic <- replaceFile f $ \tmp -> @@ -104,7 +104,7 @@ scanUnlockedFiles = whenM (inRepo Git.Ref.headExists <&&> not <$> isBareRepo) $ withTSDelta (liftIO . genInodeCache tmp) LinkAnnexNoop -> return Nothing LinkAnnexFailed -> liftIO $ do - writePointerFile tmp k destmode + writePointerFile (toRawFilePath tmp) k destmode return Nothing - maybe noop (restagePointerFile (Restage True) f) ic + maybe noop (restagePointerFile (Restage True) (toRawFilePath f)) ic _ -> noop diff --git a/Assistant/DeleteRemote.hs b/Assistant/DeleteRemote.hs index 99560d7afc..c7cf807831 100644 --- a/Assistant/DeleteRemote.hs +++ b/Assistant/DeleteRemote.hs @@ -64,7 +64,7 @@ removableRemote urlrenderer uuid = do where queueremaining r k = queueTransferWhenSmall "remaining object in unwanted remote" - (AssociatedFile Nothing) (Transfer Download uuid k) r + (AssociatedFile Nothing) (Transfer Download uuid (fromKey id k)) r {- Scanning for keys can take a long time; do not tie up - the Annex monad while doing it, so other threads continue to - run. -} diff --git a/Assistant/MakeRemote.hs b/Assistant/MakeRemote.hs index 99d68ab82d..ba4df37f97 100644 --- a/Assistant/MakeRemote.hs +++ b/Assistant/MakeRemote.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Assistant.MakeRemote where import Assistant.Common diff --git a/Assistant/MakeRepo.hs b/Assistant/MakeRepo.hs index 67e83ef5cd..f1dac121d2 100644 --- a/Assistant/MakeRepo.hs +++ b/Assistant/MakeRepo.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Assistant.MakeRepo where import Assistant.WebApp.Common diff --git a/Assistant/Sync.hs b/Assistant/Sync.hs index c528cf565f..4a90b09943 100644 --- a/Assistant/Sync.hs +++ b/Assistant/Sync.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Assistant.Sync where import Assistant.Common diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs index 09fac0b311..5ed49166bb 100644 --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -286,7 +286,7 @@ handleAdds lockdowndir havelsof delayadd cs = returnWhen (null incomplete) $ do ks = keySource ld doadd = sanitycheck ks $ do (mkey, _mcache) <- liftAnnex $ do - showStart "add" $ keyFilename ks + showStart "add" $ toRawFilePath $ keyFilename ks ingest nullMeterUpdate (Just $ LockedDown lockdownconfig ks) Nothing maybe (failedingest change) (done change $ keyFilename ks) mkey add _ _ = return Nothing @@ -325,7 +325,7 @@ handleAdds lockdowndir havelsof delayadd cs = returnWhen (null incomplete) $ do removedKeysMap :: InodeComparisonType -> [Change] -> Annex (M.Map InodeCacheKey Key) removedKeysMap ct l = do mks <- forM (filter isRmChange l) $ \c -> - catKeyFile $ changeFile c + catKeyFile $ toRawFilePath $ changeFile c M.fromList . concat <$> mapM mkpairs (catMaybes mks) where mkpairs k = map (\c -> (inodeCacheToKey ct c, k)) <$> @@ -339,7 +339,7 @@ handleAdds lockdowndir havelsof delayadd cs = returnWhen (null incomplete) $ do done change file key = liftAnnex $ do logStatus key InfoPresent mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file - stagePointerFile file mode =<< hashPointerFile key + stagePointerFile (toRawFilePath file) mode =<< hashPointerFile key showEndOk return $ Just $ finishedChange change key @@ -457,5 +457,5 @@ checkChangeContent change@(Change { changeInfo = i }) = handleDrops "file renamed" present k af [] where f = changeFile change - af = AssociatedFile (Just f) + af = AssociatedFile (Just (toRawFilePath f)) checkChangeContent _ = noop diff --git a/Assistant/Threads/ConfigMonitor.hs b/Assistant/Threads/ConfigMonitor.hs index cbfd8c823b..cabda5d259 100644 --- a/Assistant/Threads/ConfigMonitor.hs +++ b/Assistant/Threads/ConfigMonitor.hs @@ -44,7 +44,8 @@ configMonitorThread = namedThread "ConfigMonitor" $ loop =<< getConfigs when (old /= new) $ do let changedconfigs = new `S.difference` old debug $ "reloading config" : - map fst (S.toList changedconfigs) + map (fromRawFilePath . fst) + (S.toList changedconfigs) reloadConfigs new {- Record a commit to get this config - change pushed out to remotes. -} @@ -53,10 +54,10 @@ configMonitorThread = namedThread "ConfigMonitor" $ loop =<< getConfigs loop new {- Config files, and their checksums. -} -type Configs = S.Set (FilePath, Sha) +type Configs = S.Set (RawFilePath, Sha) {- All git-annex's config files, and actions to run when they change. -} -configFilesActions :: [(FilePath, Assistant ())] +configFilesActions :: [(RawFilePath, Assistant ())] configFilesActions = [ (uuidLog, void $ liftAnnex uuidDescMapLoad) , (remoteLog, void $ liftAnnex remoteListRefresh) @@ -89,5 +90,5 @@ getConfigs :: Assistant Configs getConfigs = S.fromList . map extract <$> liftAnnex (inRepo $ LsTree.lsTreeFiles Annex.Branch.fullname files) where - files = map fst configFilesActions - extract treeitem = (getTopFilePath $ LsTree.file treeitem, LsTree.sha treeitem) + files = map (fromRawFilePath . fst) configFilesActions + extract treeitem = (toRawFilePath $ getTopFilePath $ LsTree.file treeitem, LsTree.sha treeitem) diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs index bc65d9aa6f..28beacb2ea 100644 --- a/Assistant/Threads/SanityChecker.hs +++ b/Assistant/Threads/SanityChecker.hs @@ -155,10 +155,11 @@ dailyCheck urlrenderer = do (unstaged, cleanup) <- liftIO $ Git.LsFiles.notInRepo False ["."] g now <- liftIO getPOSIXTime forM_ unstaged $ \file -> do - ms <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file + let file' = fromRawFilePath file + ms <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus file' case ms of Just s | toonew (statusChangeTime s) now -> noop - | isSymbolicLink s -> addsymlink file ms + | isSymbolicLink s -> addsymlink file' ms _ -> noop liftIO $ void cleanup diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs index c6fc97fad0..71d7dd0462 100644 --- a/Assistant/Threads/TransferScanner.hs +++ b/Assistant/Threads/TransferScanner.hs @@ -186,7 +186,7 @@ genTransfer :: Direction -> Bool -> Key -> S.Set UUID -> Remote -> Maybe (Remote genTransfer direction want key slocs r | direction == Upload && Remote.readonly r = Nothing | S.member (Remote.uuid r) slocs == want = Just - (r, Transfer direction (Remote.uuid r) key) + (r, Transfer direction (Remote.uuid r) (fromKey id key)) | otherwise = Nothing remoteHas :: Remote -> Key -> Annex Bool diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index 67c986301b..5322998644 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -136,10 +136,12 @@ startupScan scanner = do -- Notice any files that were deleted before -- watching was started. top <- liftAnnex $ fromRepo Git.repoPath - (fs, cleanup) <- liftAnnex $ inRepo $ LsFiles.deleted [top] + (fs, cleanup) <- liftAnnex $ inRepo $ LsFiles.deleted + [toRawFilePath top] forM_ fs $ \f -> do - liftAnnex $ onDel' f - maybe noop recordChange =<< madeChange f RmChange + let f' = fromRawFilePath f + liftAnnex $ onDel' f' + maybe noop recordChange =<< madeChange f' RmChange void $ liftIO cleanup liftAnnex $ showAction "started" @@ -206,7 +208,7 @@ shouldRestage ds = scanComplete ds || forceRestage ds onAddUnlocked :: Bool -> GetFileMatcher -> Handler onAddUnlocked symlinkssupported matcher f fs = do - mk <- liftIO $ isPointerFile f + mk <- liftIO $ isPointerFile $ toRawFilePath f case mk of Nothing -> onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkssupported matcher f fs Just k -> addlink f k @@ -228,7 +230,7 @@ onAddUnlocked symlinkssupported matcher f fs = do logStatus oldkey InfoMissing addlink file key = do mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file - liftAnnex $ stagePointerFile file mode =<< hashPointerFile key + liftAnnex $ stagePointerFile (toRawFilePath file) mode =<< hashPointerFile key madeChange file $ LinkChange (Just key) onAddUnlocked' @@ -240,7 +242,7 @@ onAddUnlocked' -> GetFileMatcher -> Handler onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkssupported matcher file fs = do - v <- liftAnnex $ catKeyFile file + v <- liftAnnex $ catKeyFile (toRawFilePath file) case (v, fs) of (Just key, Just filestatus) -> ifM (liftAnnex $ samefilestatus key file filestatus) @@ -270,7 +272,8 @@ onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkss guardSymlinkStandin mk a | symlinkssupported = a | otherwise = do - linktarget <- liftAnnex $ getAnnexLinkTarget file + linktarget <- liftAnnex $ getAnnexLinkTarget $ + toRawFilePath file case linktarget of Nothing -> a Just lt -> do @@ -287,7 +290,7 @@ onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkss onAddSymlink :: Handler onAddSymlink file filestatus = unlessIgnored file $ do linktarget <- liftIO (catchMaybeIO $ readSymbolicLink file) - kv <- liftAnnex (lookupFile file) + kv <- liftAnnex (lookupFile (toRawFilePath file)) onAddSymlink' linktarget kv file filestatus onAddSymlink' :: Maybe String -> Maybe Key -> Handler @@ -299,7 +302,7 @@ onAddSymlink' linktarget mk file filestatus = go mk then ensurestaged (Just link) =<< getDaemonStatus else do liftAnnex $ replaceFile file $ - makeAnnexLink link + makeAnnexLink link . toRawFilePath addLink file link (Just key) -- other symlink, not git-annex go Nothing = ensurestaged linktarget =<< getDaemonStatus @@ -332,8 +335,8 @@ addLink file link mk = do case v of Just (currlink, sha, _type) | s2w8 link == L.unpack currlink -> - stageSymlink file sha - _ -> stageSymlink file =<< hashSymlink link + stageSymlink (toRawFilePath file) sha + _ -> stageSymlink (toRawFilePath file) =<< hashSymlink link madeChange file $ LinkChange mk onDel :: Handler @@ -349,7 +352,7 @@ onDel' file = do Annex.Queue.addUpdateIndex =<< inRepo (Git.UpdateIndex.unstageFile file) where - withkey a = maybe noop a =<< catKeyFile file + withkey a = maybe noop a =<< catKeyFile (toRawFilePath file) {- A directory has been deleted, or moved, so tell git to remove anything - that was inside it from its cache. Since it could reappear at any time, @@ -360,14 +363,15 @@ onDel' file = do onDelDir :: Handler onDelDir dir _ = do debug ["directory deleted", dir] - (fs, clean) <- liftAnnex $ inRepo $ LsFiles.deleted [dir] + (fs, clean) <- liftAnnex $ inRepo $ LsFiles.deleted [toRawFilePath dir] + let fs' = map fromRawFilePath fs - liftAnnex $ mapM_ onDel' fs + liftAnnex $ mapM_ onDel' fs' -- Get the events queued up as fast as possible, so the -- committer sees them all in one block. now <- liftIO getCurrentTime - recordChanges $ map (\f -> Change now f RmChange) fs + recordChanges $ map (\f -> Change now f RmChange) fs' void $ liftIO clean noChange diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index 43c1cf29b9..c852615ed1 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -96,7 +96,7 @@ queueTransfersMatching matching reason schedule k f direction inset s r = S.member (Remote.uuid r) s gentransfer r = Transfer { transferDirection = direction - , transferKey = k + , transferKeyData = fromKey id k , transferUUID = Remote.uuid r } defer @@ -129,7 +129,7 @@ queueDeferredDownloads reason schedule = do where gentransfer r = Transfer { transferDirection = Download - , transferKey = k + , transferKeyData = fromKey id k , transferUUID = Remote.uuid r } diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs index 6dacefbf45..5b555548e7 100644 --- a/Assistant/TransferSlots.hs +++ b/Assistant/TransferSlots.hs @@ -161,7 +161,7 @@ genTransfer t info = case transferRemote info of AssociatedFile Nothing -> noop AssociatedFile (Just af) -> void $ addAlert $ makeAlertFiller True $ - transferFileAlert direction True af + transferFileAlert direction True (fromRawFilePath af) unless isdownload $ handleDrops ("object uploaded to " ++ show remote) diff --git a/Assistant/Unused.hs b/Assistant/Unused.hs index d9c17e7a2c..da73f77abd 100644 --- a/Assistant/Unused.hs +++ b/Assistant/Unused.hs @@ -62,7 +62,7 @@ describeUnused' whenbig = liftAnnex $ go =<< readUnusedLog "" tenthused Nothing _ = False tenthused (Just disksize) used = used >= disksize `div` 10 - sumkeysize s k = s + fromMaybe 0 (keySize k) + sumkeysize s k = s + fromMaybe 0 (fromKey keySize k) forpath a = inRepo $ liftIO . a . Git.repoPath diff --git a/Assistant/Upgrade.hs b/Assistant/Upgrade.hs index e46ac86ced..0ea52f3158 100644 --- a/Assistant/Upgrade.hs +++ b/Assistant/Upgrade.hs @@ -25,7 +25,6 @@ import Annex.Content import Annex.UUID import qualified Backend import qualified Types.Backend -import qualified Types.Key import Assistant.TransferQueue import Assistant.TransferSlots import Remote (remoteFromUUID) @@ -88,16 +87,16 @@ startDistributionDownload d = go =<< liftIO . newVersionLocation d =<< liftIO ol hook <- asIO1 $ distributionDownloadComplete d dest cleanup modifyDaemonStatus_ $ \s -> s { transferHook = M.insert k hook (transferHook s) } - maybe noop (queueTransfer "upgrade" Next (AssociatedFile (Just f)) t) + maybe noop (queueTransfer "upgrade" Next (AssociatedFile (Just (toRawFilePath f))) t) =<< liftAnnex (remoteFromUUID webUUID) startTransfer t - k = distributionKey d + k = mkKey $ const $ distributionKey d u = distributionUrl d f = takeFileName u ++ " (for upgrade)" t = Transfer { transferDirection = Download , transferUUID = webUUID - , transferKey = k + , transferKeyData = fromKey id k } cleanup = liftAnnex $ do lockContentForRemoval k removeAnnex @@ -117,8 +116,8 @@ distributionDownloadComplete d dest cleanup t =<< liftAnnex (withObjectLoc k fsckit) | otherwise = cleanup where - k = distributionKey d - fsckit f = case Backend.maybeLookupBackendVariety (Types.Key.keyVariety k) of + k = mkKey $ const $ distributionKey d + fsckit f = case Backend.maybeLookupBackendVariety (fromKey keyVariety k) of Nothing -> return $ Just f Just b -> case Types.Backend.verifyKeyContent b of Nothing -> return $ Just f diff --git a/Assistant/WebApp/Configurators/Edit.hs b/Assistant/WebApp/Configurators/Edit.hs index 1237f22339..b711761a42 100644 --- a/Assistant/WebApp/Configurators/Edit.hs +++ b/Assistant/WebApp/Configurators/Edit.hs @@ -101,11 +101,12 @@ setRepoConfig uuid mremote oldc newc = do - there's not. Special remotes don't normally - have that, and don't use it. Temporarily add - it if it's missing. -} - let remotefetch = "remote." ++ T.unpack (repoName oldc) ++ ".fetch" + let remotefetch = Git.ConfigKey $ encodeBS' $ + "remote." ++ T.unpack (repoName oldc) ++ ".fetch" needfetch <- isNothing <$> fromRepo (Git.Config.getMaybe remotefetch) when needfetch $ inRepo $ Git.Command.run - [Param "config", Param remotefetch, Param ""] + [Param "config", Param (Git.fromConfigKey remotefetch), Param ""] inRepo $ Git.Command.run [ Param "remote" , Param "rename" diff --git a/Assistant/WebApp/Configurators/Local.hs b/Assistant/WebApp/Configurators/Local.hs index eb52be0093..faf3cde57e 100644 --- a/Assistant/WebApp/Configurators/Local.hs +++ b/Assistant/WebApp/Configurators/Local.hs @@ -336,7 +336,7 @@ getFinishAddDriveR drive = go isnew <- liftIO $ makeRepo dir True {- Removable drives are not reliable media, so enable fsync. -} liftIO $ inDir dir $ - setConfig (ConfigKey "core.fsyncobjectfiles") + setConfig "core.fsyncobjectfiles" (Git.Config.boolConfig True) (u, r) <- a isnew when isnew $ diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs index b140e99dcc..9ed76bef48 100644 --- a/Assistant/WebApp/Configurators/Ssh.hs +++ b/Assistant/WebApp/Configurators/Ssh.hs @@ -20,7 +20,7 @@ import Types.StandardGroups import Utility.UserInfo import Utility.Gpg import Types.Remote (RemoteConfig) -import Git.Types (RemoteName, fromRef) +import Git.Types (RemoteName, fromRef, fromConfigKey) import qualified Remote.GCrypt as GCrypt import qualified Annex import qualified Git.Command @@ -317,7 +317,8 @@ testServer sshinput@(SshInput { inputHostname = Just hn }) = do else T.pack $ "Failed to ssh to the server. Transcript: " ++ s finduuid (k, v) | k == "annex.uuid" = Just $ toUUID v - | k == GCrypt.coreGCryptId = Just $ genUUIDInNameSpace gCryptNameSpace v + | k == fromConfigKey GCrypt.coreGCryptId = + Just $ genUUIDInNameSpace gCryptNameSpace v | otherwise = Nothing checkcommand c = "if which " ++ c ++ "; then " ++ report c ++ "; fi" diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs index 09a1e5f047..6b9d8787cb 100644 --- a/Assistant/WebApp/DashBoard.hs +++ b/Assistant/WebApp/DashBoard.hs @@ -45,7 +45,7 @@ transfersDisplay = do transferPaused info || isNothing (startedTime info) desc transfer info = case associatedFile info of AssociatedFile Nothing -> serializeKey $ transferKey transfer - AssociatedFile (Just af) -> af + AssociatedFile (Just af) -> fromRawFilePath af {- Simplifies a list of transfers, avoiding display of redundant - equivilant transfers. -} diff --git a/Backend.hs b/Backend.hs index 2b2962ff90..9a0abf7290 100644 --- a/Backend.hs +++ b/Backend.hs @@ -59,16 +59,18 @@ genKey source meterupdate preferredbackend = do Just k -> Just (makesane k, b) where -- keyNames should not contain newline characters. - makesane k = k { keyName = S8.map fixbadchar (keyName k) } + makesane k = alterKey k $ \d -> d + { keyName = S8.map fixbadchar (fromKey keyName k) + } fixbadchar c | c == '\n' = '_' | otherwise = c getBackend :: FilePath -> Key -> Annex (Maybe Backend) -getBackend file k = case maybeLookupBackendVariety (keyVariety k) of +getBackend file k = case maybeLookupBackendVariety (fromKey keyVariety k) of Just backend -> return $ Just backend Nothing -> do - warning $ "skipping " ++ file ++ " (unknown backend " ++ decodeBS (formatKeyVariety (keyVariety k)) ++ ")" + warning $ "skipping " ++ file ++ " (unknown backend " ++ decodeBS (formatKeyVariety (fromKey keyVariety k)) ++ ")" return Nothing {- Looks up the backend that should be used for a file. @@ -95,4 +97,4 @@ varietyMap = M.fromList $ zip (map B.backendVariety list) list isStableKey :: Key -> Bool isStableKey k = maybe False (`B.isStableKey` k) - (maybeLookupBackendVariety (keyVariety k)) + (maybeLookupBackendVariety (fromKey keyVariety k)) diff --git a/Backend/Hash.hs b/Backend/Hash.hs index 6cac6e3718..aec60f0cfe 100644 --- a/Backend/Hash.hs +++ b/Backend/Hash.hs @@ -91,7 +91,7 @@ keyValue hash source meterupdate = do let file = contentLocation source filesize <- liftIO $ getFileSize file s <- hashFile hash file meterupdate - return $ Just $ stubKey + return $ Just $ mkKey $ \k -> k { keyName = encodeBS s , keyVariety = hashKeyVariety hash (HasExt False) , keySize = Just filesize @@ -105,8 +105,8 @@ keyValueE hash source meterupdate = addE k = do maxlen <- annexMaxExtensionLength <$> Annex.getGitConfig let ext = selectExtension maxlen (keyFilename source) - return $ Just $ k - { keyName = keyName k <> encodeBS ext + return $ Just $ alterKey k $ \d -> d + { keyName = keyName d <> encodeBS ext , keyVariety = hashKeyVariety hash (HasExt True) } @@ -169,7 +169,7 @@ needsUpgrade :: Key -> Bool needsUpgrade key = or [ "\\" `S8.isPrefixOf` keyHash key , any (not . validInExtension) (decodeBS $ snd $ splitKeyNameExtension key) - , not (hasExt (keyVariety key)) && keyHash key /= keyName key + , not (hasExt (fromKey keyVariety key)) && keyHash key /= fromKey keyName key ] trivialMigrate :: Key -> Backend -> AssociatedFile -> Annex (Maybe Key) @@ -179,30 +179,31 @@ trivialMigrate oldkey newbackend afile = trivialMigrate' oldkey newbackend afile trivialMigrate' :: Key -> Backend -> AssociatedFile -> Maybe Int -> Maybe Key trivialMigrate' oldkey newbackend afile maxextlen {- Fast migration from hashE to hash backend. -} - | migratable && hasExt oldvariety = Just $ oldkey + | migratable && hasExt oldvariety = Just $ alterKey oldkey $ \d -> d { keyName = keyHash oldkey , keyVariety = newvariety } {- Fast migration from hash to hashE backend. -} | migratable && hasExt newvariety = case afile of AssociatedFile Nothing -> Nothing - AssociatedFile (Just file) -> Just $ oldkey + AssociatedFile (Just file) -> Just $ alterKey oldkey $ \d -> d { keyName = keyHash oldkey - <> encodeBS (selectExtension maxextlen file) + <> encodeBS' (selectExtension maxextlen (fromRawFilePath file)) , keyVariety = newvariety } {- Upgrade to fix bad previous migration that created a - non-extension preserving key, with an extension - in its keyName. -} | newvariety == oldvariety && not (hasExt oldvariety) && - keyHash oldkey /= keyName oldkey = Just $ oldkey - { keyName = keyHash oldkey - } + keyHash oldkey /= fromKey keyName oldkey = + Just $ alterKey oldkey $ \d -> d + { keyName = keyHash oldkey + } | otherwise = Nothing where migratable = oldvariety /= newvariety && sameExceptExt oldvariety newvariety - oldvariety = keyVariety oldkey + oldvariety = fromKey keyVariety oldkey newvariety = backendVariety newbackend hashFile :: Hash -> FilePath -> MeterUpdate -> Annex String @@ -294,5 +295,7 @@ testKeyBackend = let b = genBackendE (SHA2Hash (HashSize 256)) in b { getKey = \ks p -> (fmap addE) <$> getKey b ks p } where - addE k = k { keyName = keyName k <> longext } + addE k = alterKey k $ \d -> d + { keyName = keyName d <> longext + } longext = ".this-is-a-test-key" diff --git a/Backend/URL.hs b/Backend/URL.hs index aad6c87db8..7e6313dc1e 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -32,7 +32,7 @@ backend = Backend {- Every unique url has a corresponding key. -} fromUrl :: String -> Maybe Integer -> Key -fromUrl url size = stubKey +fromUrl url size = mkKey $ \k -> k { keyName = genKeyName url , keyVariety = URLKey , keySize = size diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 5455951d9e..cd6be25fb1 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -39,7 +39,7 @@ keyValue source _ = do stat <- liftIO $ getFileStatus f sz <- liftIO $ getFileSize' f stat relf <- getTopFilePath <$> inRepo (toTopFilePath $ keyFilename source) - return $ Just $ stubKey + return $ Just $ mkKey $ \k -> k { keyName = genKeyName relf , keyVariety = WORMKey , keySize = Just sz @@ -48,14 +48,14 @@ keyValue source _ = do {- Old WORM keys could contain spaces, and can be upgraded to remove them. -} needsUpgrade :: Key -> Bool -needsUpgrade key = ' ' `S8.elem` keyName key +needsUpgrade key = ' ' `S8.elem` fromKey keyName key removeSpaces :: Key -> Backend -> AssociatedFile -> Annex (Maybe Key) removeSpaces oldkey newbackend _ - | migratable = return $ Just $ oldkey - { keyName = encodeBS $ reSanitizeKeyName $ decodeBS $ keyName oldkey } + | migratable = return $ Just $ alterKey oldkey $ \d -> d + { keyName = encodeBS $ reSanitizeKeyName $ decodeBS $ keyName d } | otherwise = return Nothing where migratable = oldvariety == newvariety - oldvariety = keyVariety oldkey + oldvariety = fromKey keyVariety oldkey newvariety = backendVariety newbackend diff --git a/CHANGELOG b/CHANGELOG index 67ba098423..d50472a4bb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -18,6 +18,14 @@ git-annex (8.20191107) UNRELEASED; urgency=medium git-annex (7.20191115) UNRELEASED; urgency=medium + * Sped up many git-annex commands that operate on many files, by + using ByteStrings. Some commands like find got up to 60% faster. + * Sped up many git-annex commands that operate on many files, by + avoiding reserialization of keys. + find got 7% faster; whereis 3% faster; and git-annex get when + all files are already present got 5% faster + * Sped up many git-annex commands that query the git-annex branch. + In particular whereis got 1.5% faster. * Stop displaying rsync progress, and use git-annex's own progress display for local-to-local repo transfers. * git-lfs: The url provided to initremote/enableremote will now be diff --git a/COPYRIGHT b/COPYRIGHT index a2324d7c58..858d7f0b74 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -29,6 +29,11 @@ Copyright: 2018 Joey Hess 2013 Michael Snoyman License: Expat +Files: Utility/Attoparsec.hs +Copyright: 2019 Joey Hess + 2007-2015 Bryan O'Sullivan +License: BSD-3-clause + Files: Utility/GitLFS.hs Copyright: © 2019 Joey Hess License: AGPL-3+ @@ -112,7 +117,35 @@ License: BSD-2-clause LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - + +License: BSD-3-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + . + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + . + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + . + 3. Neither the name of the author nor the names of his contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + . + THIS SOFTWARE IS PROVIDED BY THE CONTRIBUTORS ``AS IS'' AND ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + License: Expat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs index 117a876edb..fc25b3c817 100644 --- a/CmdLine/GitAnnex/Options.hs +++ b/CmdLine/GitAnnex/Options.hs @@ -92,7 +92,7 @@ gitAnnexGlobalOptions = commonGlobalOptions ++ where setnumcopies n = Annex.changeState $ \s -> s { Annex.forcenumcopies = Just $ NumCopies n } setuseragent v = Annex.changeState $ \s -> s { Annex.useragent = Just v } - setgitconfig v = Annex.adjustGitRepo $ \r -> Git.Config.store v $ + setgitconfig v = Annex.adjustGitRepo $ \r -> Git.Config.store (encodeBS' v) $ r { gitGlobalOpts = gitGlobalOpts r ++ [Param "-c", Param v] } setdesktopnotify v = Annex.changeState $ \s -> s { Annex.desktopnotify = Annex.desktopnotify s <> v } diff --git a/CmdLine/GitAnnexShell/Fields.hs b/CmdLine/GitAnnexShell/Fields.hs index c5c0118a43..639adf3477 100644 --- a/CmdLine/GitAnnexShell/Fields.hs +++ b/CmdLine/GitAnnexShell/Fields.hs @@ -30,7 +30,7 @@ remoteUUID = Field "remoteuuid" $ associatedFile :: Field associatedFile = Field "associatedfile" $ \f -> -- is the file a safe relative filename? - not (absoluteGitPath f) && not ("../" `isPrefixOf` f) + not (absoluteGitPath (toRawFilePath f)) && not ("../" `isPrefixOf` f) direct :: Field direct = Field "direct" $ \f -> f == "1" diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs index 4107e9dcd6..68ee9efc02 100644 --- a/CmdLine/Seek.hs +++ b/CmdLine/Seek.hs @@ -34,11 +34,11 @@ import Annex.Content import Annex.InodeSentinal import qualified Database.Keys -withFilesInGit :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesInGit :: (RawFilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek withFilesInGit a l = seekActions $ prepFiltered a $ seekHelper LsFiles.inRepo l -withFilesInGitNonRecursive :: String -> (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesInGitNonRecursive :: String -> (RawFilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek withFilesInGitNonRecursive needforce a l = ifM (Annex.getState Annex.force) ( withFilesInGit a l , if null l @@ -48,7 +48,7 @@ withFilesInGitNonRecursive needforce a l = ifM (Annex.getState Annex.force) where getfiles c [] = return (reverse c) getfiles c ((WorkTreeItem p):ps) = do - (fs, cleanup) <- inRepo $ LsFiles.inRepo [p] + (fs, cleanup) <- inRepo $ LsFiles.inRepo [toRawFilePath p] case fs of [f] -> do void $ liftIO $ cleanup @@ -58,11 +58,11 @@ withFilesInGitNonRecursive needforce a l = ifM (Annex.getState Annex.force) getfiles c ps _ -> giveup needforce -withFilesNotInGit :: Bool -> (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesNotInGit :: Bool -> (RawFilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek withFilesNotInGit skipdotfiles a l | skipdotfiles = do {- dotfiles are not acted on unless explicitly listed -} - files <- filter (not . dotfile) <$> + files <- filter (not . dotfile . fromRawFilePath) <$> seekunless (null ps && not (null l)) ps dotfiles <- seekunless (null dotps) dotps go (files++dotfiles) @@ -74,9 +74,9 @@ withFilesNotInGit skipdotfiles a l force <- Annex.getState Annex.force g <- gitRepo liftIO $ Git.Command.leaveZombie - <$> LsFiles.notInRepo force (map (\(WorkTreeItem f) -> f) l') g + <$> LsFiles.notInRepo force (map (\(WorkTreeItem f) -> toRawFilePath f) l') g go fs = seekActions $ prepFiltered a $ - return $ concat $ segmentPaths (map (\(WorkTreeItem f) -> f) l) fs + return $ concat $ segmentPaths (map (\(WorkTreeItem f) -> toRawFilePath f) l) fs withPathContents :: ((FilePath, FilePath) -> CommandSeek) -> CmdParams -> CommandSeek withPathContents a params = do @@ -110,30 +110,30 @@ withPairs a params = seekActions $ return $ map a $ pairs [] params pairs c (x:y:xs) = pairs ((x,y):c) xs pairs _ _ = giveup "expected pairs" -withFilesToBeCommitted :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesToBeCommitted :: (RawFilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek withFilesToBeCommitted a l = seekActions $ prepFiltered a $ seekHelper LsFiles.stagedNotDeleted l -isOldUnlocked :: FilePath -> Annex Bool +isOldUnlocked :: RawFilePath -> Annex Bool isOldUnlocked f = liftIO (notSymlink f) <&&> (isJust <$> catKeyFile f <||> isJust <$> catKeyFileHEAD f) {- unlocked pointer files that are staged, and whose content has not been - modified-} -withUnmodifiedUnlockedPointers :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withUnmodifiedUnlockedPointers :: (RawFilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek withUnmodifiedUnlockedPointers a l = seekActions $ prepFiltered a unlockedfiles where unlockedfiles = filterM isUnmodifiedUnlocked =<< seekHelper LsFiles.typeChangedStaged l -isUnmodifiedUnlocked :: FilePath -> Annex Bool +isUnmodifiedUnlocked :: RawFilePath -> Annex Bool isUnmodifiedUnlocked f = catKeyFile f >>= \case Nothing -> return False - Just k -> sameInodeCache f =<< Database.Keys.getInodeCaches k + Just k -> sameInodeCache (fromRawFilePath f) =<< Database.Keys.getInodeCaches k {- Finds files that may be modified. -} -withFilesMaybeModified :: (FilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek +withFilesMaybeModified :: (RawFilePath -> CommandSeek) -> [WorkTreeItem] -> CommandSeek withFilesMaybeModified a params = seekActions $ prepFiltered a $ seekHelper LsFiles.modified params @@ -169,7 +169,7 @@ withKeyOptions ko auto keyaction = withKeyOptions' ko auto mkkeyaction return $ \v@(k, ai) -> let i = case ai of ActionItemBranchFilePath (BranchFilePath _ topf) _ -> - MatchingKey k (AssociatedFile $ Just $ getTopFilePath topf) + MatchingKey k (AssociatedFile $ Just $ toRawFilePath $ getTopFilePath topf) _ -> MatchingKey k (AssociatedFile Nothing) in whenM (matcher i) $ keyaction v @@ -225,20 +225,22 @@ withKeyOptions' ko auto mkkeyaction fallbackaction params = do forM_ ts $ \(t, i) -> keyaction (transferKey t, mkActionItem (t, i)) -prepFiltered :: (FilePath -> CommandSeek) -> Annex [FilePath] -> Annex [CommandSeek] +prepFiltered :: (RawFilePath -> CommandSeek) -> Annex [RawFilePath] -> Annex [CommandSeek] prepFiltered a fs = do matcher <- Limit.getMatcher map (process matcher) <$> fs where - process matcher f = whenM (matcher $ MatchingFile $ FileInfo f f) $ a f + process matcher f = + let f' = fromRawFilePath f + in whenM (matcher $ MatchingFile $ FileInfo f' f') $ a f seekActions :: Annex [CommandSeek] -> Annex () seekActions gen = sequence_ =<< gen -seekHelper :: ([FilePath] -> Git.Repo -> IO ([FilePath], IO Bool)) -> [WorkTreeItem] -> Annex [FilePath] +seekHelper :: ([RawFilePath] -> Git.Repo -> IO ([RawFilePath], IO Bool)) -> [WorkTreeItem] -> Annex [RawFilePath] seekHelper a l = inRepo $ \g -> concat . concat <$> forM (segmentXargsOrdered l') - (runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g)) + (runSegmentPaths (\fs -> Git.Command.leaveZombie <$> a fs g) . map toRawFilePath) where l' = map (\(WorkTreeItem f) -> f) l @@ -264,14 +266,14 @@ workTreeItems' (AllowHidden allowhidden) ps = do unlessM (exists p <||> hidden currbranch p) $ do toplevelWarning False (p ++ " not found") Annex.incError - return (map WorkTreeItem ps) + return (map (WorkTreeItem) ps) where exists p = isJust <$> liftIO (catchMaybeIO $ getSymbolicLinkStatus p) hidden currbranch p | allowhidden = do f <- liftIO $ relPathCwdToFile p - isJust <$> catObjectMetaDataHidden f currbranch + isJust <$> catObjectMetaDataHidden (toRawFilePath f) currbranch | otherwise = return False -notSymlink :: FilePath -> IO Bool -notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f +notSymlink :: RawFilePath -> IO Bool +notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus (fromRawFilePath f) diff --git a/Command/Add.hs b/Command/Add.hs index 200f66e768..0ebe42d735 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -50,7 +50,7 @@ optParser desc = AddOptions seek :: AddOptions -> CommandSeek seek o = startConcurrency commandStages $ do matcher <- largeFilesMatcher - let gofile file = ifM (checkFileMatcher matcher file <||> Annex.getState Annex.force) + let gofile file = ifM (checkFileMatcher matcher (fromRawFilePath file) <||> Annex.getState Annex.force) ( start file , ifM (annexAddSmallFiles <$> Annex.getGitConfig) ( startSmall file @@ -61,7 +61,7 @@ seek o = startConcurrency commandStages $ do Batch fmt | updateOnly o -> giveup "--update --batch is not supported" - | otherwise -> batchFilesMatching fmt gofile + | otherwise -> batchFilesMatching fmt (gofile . toRawFilePath) NoBatch -> do l <- workTreeItems (addThese o) let go a = a (commandAction . gofile) l @@ -71,28 +71,28 @@ seek o = startConcurrency commandStages $ do go withUnmodifiedUnlockedPointers {- Pass file off to git-add. -} -startSmall :: FilePath -> CommandStart +startSmall :: RawFilePath -> CommandStart startSmall file = starting "add" (ActionItemWorkTreeFile file) $ next $ addSmall file -addSmall :: FilePath -> Annex Bool +addSmall :: RawFilePath -> Annex Bool addSmall file = do showNote "non-large file; adding content to git repository" addFile file -addFile :: FilePath -> Annex Bool +addFile :: RawFilePath -> Annex Bool addFile file = do ps <- forceParams - Annex.Queue.addCommand "add" (ps++[Param "--"]) [file] + Annex.Queue.addCommand "add" (ps++[Param "--"]) [fromRawFilePath file] return True -start :: FilePath -> CommandStart +start :: RawFilePath -> CommandStart start file = do mk <- liftIO $ isPointerFile file maybe go fixuppointer mk where go = ifAnnexed file addpresent add - add = liftIO (catchMaybeIO $ getSymbolicLinkStatus file) >>= \case + add = liftIO (catchMaybeIO $ getSymbolicLinkStatus (fromRawFilePath file)) >>= \case Nothing -> stop Just s | not (isRegularFile s) && not (isSymbolicLink s) -> stop @@ -102,28 +102,28 @@ start file = do then next $ addFile file else perform file addpresent key = - liftIO (catchMaybeIO $ getSymbolicLinkStatus file) >>= \case + liftIO (catchMaybeIO $ getSymbolicLinkStatus $ fromRawFilePath file) >>= \case Just s | isSymbolicLink s -> fixuplink key _ -> add fixuplink key = starting "add" (ActionItemWorkTreeFile file) $ do -- the annexed symlink is present but not yet added to git - liftIO $ removeFile file - addLink file key Nothing + liftIO $ removeFile (fromRawFilePath file) + addLink (fromRawFilePath file) key Nothing next $ cleanup key =<< inAnnex key fixuppointer key = starting "add" (ActionItemWorkTreeFile file) $ do -- the pointer file is present, but not yet added to git - Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath file) + Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath (fromRawFilePath file)) next $ addFile file -perform :: FilePath -> CommandPerform +perform :: RawFilePath -> CommandPerform perform file = withOtherTmp $ \tmpdir -> do lockingfile <- not <$> addUnlocked let cfg = LockDownConfig { lockingFile = lockingfile , hardlinkFileTmpDir = Just tmpdir } - ld <- lockDown cfg file + ld <- lockDown cfg (fromRawFilePath file) let sizer = keySource <$> ld v <- metered Nothing sizer $ \_meter meterupdate -> ingestAdd meterupdate ld diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs index aafa764919..1d814037e5 100644 --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -156,13 +156,13 @@ startRemote r o file uri sz = do performRemote r o uri file' sz performRemote :: Remote -> AddUrlOptions -> URLString -> FilePath -> Maybe Integer -> CommandPerform -performRemote r o uri file sz = ifAnnexed file adduri geturi +performRemote r o uri file sz = ifAnnexed (toRawFilePath file) adduri geturi where loguri = setDownloader uri OtherDownloader adduri = addUrlChecked o loguri file (Remote.uuid r) checkexistssize checkexistssize key = return $ case sz of Nothing -> (True, True, loguri) - Just n -> (True, n == fromMaybe n (keySize key), loguri) + Just n -> (True, n == fromMaybe n (fromKey keySize key), loguri) geturi = next $ isJust <$> downloadRemoteFile r (downloadOptions o) uri file sz downloadRemoteFile :: Remote -> DownloadOptions -> URLString -> FilePath -> Maybe Integer -> Annex (Maybe Key) @@ -180,7 +180,7 @@ downloadRemoteFile r o uri file sz = checkCanAdd file $ do setTempUrl urlkey loguri let downloader = \dest p -> fst <$> Remote.retrieveKeyFile r urlkey - (AssociatedFile (Just file)) dest p + (AssociatedFile (Just (toRawFilePath file))) dest p ret <- downloadWith downloader urlkey (Remote.uuid r) loguri file removeTempUrl urlkey return ret @@ -212,13 +212,13 @@ startWeb o urlstring = go $ fromMaybe bad $ parseURI urlstring performWeb o urlstring file urlinfo performWeb :: AddUrlOptions -> URLString -> FilePath -> Url.UrlInfo -> CommandPerform -performWeb o url file urlinfo = ifAnnexed file addurl geturl +performWeb o url file urlinfo = ifAnnexed (toRawFilePath file) addurl geturl where geturl = next $ isJust <$> addUrlFile (downloadOptions o) url urlinfo file addurl = addUrlChecked o url file webUUID $ \k -> ifM (pure (not (rawOption (downloadOptions o))) <&&> youtubeDlSupported url) ( return (True, True, setDownloader url YoutubeDownloader) - , return (Url.urlExists urlinfo, Url.urlSize urlinfo == keySize k, url) + , return (Url.urlExists urlinfo, Url.urlSize urlinfo == fromKey keySize k, url) ) {- Check that the url exists, and has the same size as the key, @@ -258,7 +258,7 @@ addUrlFile o url urlinfo file = downloadWeb :: DownloadOptions -> URLString -> Url.UrlInfo -> FilePath -> Annex (Maybe Key) downloadWeb o url urlinfo file = - go =<< downloadWith' downloader urlkey webUUID url (AssociatedFile (Just file)) + go =<< downloadWith' downloader urlkey webUUID url (AssociatedFile (Just (toRawFilePath file))) where urlkey = addSizeUrlKey urlinfo $ Backend.URL.fromUrl url Nothing downloader f p = downloadUrl urlkey p [url] f @@ -278,7 +278,7 @@ downloadWeb o url urlinfo file = -- first, and check if that is already an annexed file, -- to avoid unnecessary work in that case. | otherwise = youtubeDlFileNameHtmlOnly url >>= \case - Right dest -> ifAnnexed dest + Right dest -> ifAnnexed (toRawFilePath dest) (alreadyannexed dest) (dl dest) Left _ -> normalfinish tmp @@ -345,7 +345,7 @@ downloadWith :: (FilePath -> MeterUpdate -> Annex Bool) -> Key -> UUID -> URLStr downloadWith downloader dummykey u url file = go =<< downloadWith' downloader dummykey u url afile where - afile = AssociatedFile (Just file) + afile = AssociatedFile (Just (toRawFilePath file)) go Nothing = return Nothing go (Just tmp) = finishDownloadWith tmp u url file @@ -379,7 +379,9 @@ finishDownloadWith tmp u url file = do {- Adds the url size to the Key. -} addSizeUrlKey :: Url.UrlInfo -> Key -> Key -addSizeUrlKey urlinfo key = key { keySize = Url.urlSize urlinfo } +addSizeUrlKey urlinfo key = alterKey key $ \d -> d + { keySize = Url.urlSize urlinfo + } {- Adds worktree file to the repository. -} addWorkTree :: UUID -> URLString -> FilePath -> Key -> Maybe FilePath -> Annex () @@ -399,7 +401,7 @@ addWorkTree u url file key mtmp = case mtmp of -- than the work tree file. liftIO $ renameFile file tmp go - else void $ Command.Add.addSmall file + else void $ Command.Add.addSmall (toRawFilePath file) where go = do maybeShowJSON $ JSONChunk [("key", serializeKey key)] diff --git a/Command/Config.hs b/Command/Config.hs index 15ab85daeb..6764ca5e92 100644 --- a/Command/Config.hs +++ b/Command/Config.hs @@ -10,6 +10,9 @@ module Command.Config where import Command import Logs.Config import Config +import Git.Types (ConfigKey(..), fromConfigValue) + +import qualified Data.ByteString as S cmd :: Command cmd = noMessages $ command "config" SectionSetup @@ -17,9 +20,9 @@ cmd = noMessages $ command "config" SectionSetup paramNothing (seek <$$> optParser) data Action - = SetConfig ConfigName ConfigValue - | GetConfig ConfigName - | UnsetConfig ConfigName + = SetConfig ConfigKey ConfigValue + | GetConfig ConfigKey + | UnsetConfig ConfigKey type Name = String type Value = String @@ -48,19 +51,19 @@ optParser _ = setconfig <|> getconfig <|> unsetconfig ) seek :: Action -> CommandSeek -seek (SetConfig name val) = commandAction $ - startingUsualMessages name (ActionItemOther (Just val)) $ do - setGlobalConfig name val - setConfig (ConfigKey name) val +seek (SetConfig ck@(ConfigKey name) val) = commandAction $ + startingUsualMessages (decodeBS' name) (ActionItemOther (Just (fromConfigValue val))) $ do + setGlobalConfig ck val + setConfig ck (fromConfigValue val) next $ return True -seek (UnsetConfig name) = commandAction $ - startingUsualMessages name (ActionItemOther (Just "unset")) $do - unsetGlobalConfig name - unsetConfig (ConfigKey name) +seek (UnsetConfig ck@(ConfigKey name)) = commandAction $ + startingUsualMessages (decodeBS' name) (ActionItemOther (Just "unset")) $do + unsetGlobalConfig ck + unsetConfig ck next $ return True -seek (GetConfig name) = commandAction $ +seek (GetConfig ck) = commandAction $ startingCustomOutput (ActionItemOther Nothing) $ do - getGlobalConfig name >>= \case + getGlobalConfig ck >>= \case Nothing -> return () - Just v -> liftIO $ putStrLn v + Just (ConfigValue v) -> liftIO $ S.putStrLn v next $ return True diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs index 53b89c3489..bb33f7102b 100644 --- a/Command/ConfigList.hs +++ b/Command/ConfigList.hs @@ -12,6 +12,7 @@ import Annex.UUID import Annex.Init import qualified Annex.Branch import qualified Git.Config +import Git.Types import Remote.GCrypt (coreGCryptId) import qualified CmdLine.GitAnnexShell.Fields as Fields import CmdLine.GitAnnexShell.Checks @@ -28,11 +29,12 @@ seek = withNothing (commandAction start) start :: CommandStart start = do u <- findOrGenUUID - showConfig "annex.uuid" $ fromUUID u - showConfig coreGCryptId =<< fromRepo (Git.Config.get coreGCryptId "") + showConfig configkeyUUID $ fromUUID u + showConfig coreGCryptId . fromConfigValue + =<< fromRepo (Git.Config.get coreGCryptId mempty) stop where - showConfig k v = liftIO $ putStrLn $ k ++ "=" ++ v + showConfig k v = liftIO $ putStrLn $ fromConfigKey k ++ "=" ++ v {- The repository may not yet have a UUID; automatically initialize it - when there's a git-annex branch available or if the autoinit field was diff --git a/Command/Copy.hs b/Command/Copy.hs index 91fec7fef6..ba7c83bf47 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -47,7 +47,7 @@ seek :: CopyOptions -> CommandSeek seek o = startConcurrency commandStages $ do let go = whenAnnexed $ start o case batchOption o of - Batch fmt -> batchFilesMatching fmt go + Batch fmt -> batchFilesMatching fmt (go . toRawFilePath) NoBatch -> withKeyOptions (keyOptions o) (autoMode o) (commandAction . Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever) @@ -57,12 +57,12 @@ seek o = startConcurrency commandStages $ do {- A copy is just a move that does not delete the source file. - However, auto mode avoids unnecessary copies, and avoids getting or - sending non-preferred content. -} -start :: CopyOptions -> FilePath -> Key -> CommandStart +start :: CopyOptions -> RawFilePath -> Key -> CommandStart start o file key = stopUnless shouldCopy $ Command.Move.start (fromToOptions o) Command.Move.RemoveNever file key where shouldCopy - | autoMode o = want <||> numCopiesCheck file key (<) + | autoMode o = want <||> numCopiesCheck (fromRawFilePath file) key (<) | otherwise = return True want = case fromToOptions o of Right (ToRemote dest) -> diff --git a/Command/DiffDriver.hs b/Command/DiffDriver.hs index f4251c0929..ecc05ca093 100644 --- a/Command/DiffDriver.hs +++ b/Command/DiffDriver.hs @@ -85,9 +85,9 @@ fixupReq req@(Req {}) = check rOldFile rOldMode (\r f -> r { rOldFile = f }) req >>= check rNewFile rNewMode (\r f -> r { rNewFile = f }) where - check getfile getmode setfile r = case readTreeItemType (getmode r) of + check getfile getmode setfile r = case readTreeItemType (encodeBS' (getmode r)) of Just TreeSymlink -> do - v <- getAnnexLinkTarget' (getfile r) False + v <- getAnnexLinkTarget' (toRawFilePath (getfile r)) False case parseLinkTargetOrPointer =<< v of Nothing -> return r Just k -> withObjectLoc k (pure . setfile r) diff --git a/Command/Drop.hs b/Command/Drop.hs index 5d0d6179c7..9b8c4710ec 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -54,7 +54,7 @@ parseDropFromOption = parseRemoteOption <$> strOption seek :: DropOptions -> CommandSeek seek o = startConcurrency transferStages $ case batchOption o of - Batch fmt -> batchFilesMatching fmt go + Batch fmt -> batchFilesMatching fmt (go . toRawFilePath) NoBatch -> withKeyOptions (keyOptions o) (autoMode o) (commandAction . startKeys o) (withFilesInGit (commandAction . go)) @@ -62,7 +62,7 @@ seek o = startConcurrency transferStages $ where go = whenAnnexed $ start o -start :: DropOptions -> FilePath -> Key -> CommandStart +start :: DropOptions -> RawFilePath -> Key -> CommandStart start o file key = start' o key afile ai where afile = AssociatedFile (Just file) diff --git a/Command/EnableRemote.hs b/Command/EnableRemote.hs index 8cf86ea5ed..f43ab68f8b 100644 --- a/Command/EnableRemote.hs +++ b/Command/EnableRemote.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Command.EnableRemote where import Command diff --git a/Command/ExamineKey.hs b/Command/ExamineKey.hs index 9cb8defb9c..040fd15f32 100644 --- a/Command/ExamineKey.hs +++ b/Command/ExamineKey.hs @@ -22,5 +22,5 @@ cmd = noCommit $ noMessages $ dontCheck repoExists $ run :: Maybe Utility.Format.Format -> String -> Annex Bool run format p = do let k = fromMaybe (giveup "bad key") $ deserializeKey p - showFormatted format (serializeKey k) (keyVars k) + showFormatted format (serializeKey' k) (keyVars k) return True diff --git a/Command/Export.hs b/Command/Export.hs index 1163f5bad2..77ebc009f9 100644 --- a/Command/Export.hs +++ b/Command/Export.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE TupleSections, BangPatterns #-} +{-# LANGUAGE OverloadedStrings #-} module Command.Export where @@ -70,7 +71,7 @@ optParser _ = ExportOptions -- To handle renames which swap files, the exported file is first renamed -- to a stable temporary name based on the key. exportTempName :: ExportKey -> ExportLocation -exportTempName ek = mkExportLocation $ +exportTempName ek = mkExportLocation $ toRawFilePath $ ".git-annex-tmp-content-" ++ serializeKey (asKey (ek)) seek :: ExportOptions -> CommandSeek @@ -258,9 +259,9 @@ startExport r db cvar allfilledvar ti = do performExport r db ek af (Git.LsTree.sha ti) loc allfilledvar ) where - loc = mkExportLocation f + loc = mkExportLocation (toRawFilePath f) f = getTopFilePath (Git.LsTree.file ti) - af = AssociatedFile (Just f) + af = AssociatedFile (Just (toRawFilePath f)) notrecordedpresent ek = (||) <$> liftIO (notElem loc <$> getExportedLocation db (asKey ek)) -- If content was removed from the remote, the export db @@ -316,14 +317,14 @@ startUnexport r db f shas = do else starting ("unexport " ++ name r) (ActionItemOther (Just f')) $ performUnexport r db eks loc where - loc = mkExportLocation f' + loc = mkExportLocation (toRawFilePath f') f' = getTopFilePath f startUnexport' :: Remote -> ExportHandle -> TopFilePath -> ExportKey -> CommandStart startUnexport' r db f ek = starting ("unexport " ++ name r) (ActionItemOther (Just f')) $ performUnexport r db [ek] loc where - loc = mkExportLocation f' + loc = mkExportLocation (toRawFilePath f') f' = getTopFilePath f -- Unlike a usual drop from a repository, this does not check that @@ -363,19 +364,19 @@ startRecoverIncomplete r db sha oldf | otherwise = do ek <- exportKey sha let loc = exportTempName ek - starting ("unexport " ++ name r) (ActionItemOther (Just (fromExportLocation loc))) $ do + starting ("unexport " ++ name r) (ActionItemOther (Just (fromRawFilePath (fromExportLocation loc)))) $ do liftIO $ removeExportedLocation db (asKey ek) oldloc performUnexport r db [ek] loc where - oldloc = mkExportLocation oldf' + oldloc = mkExportLocation (toRawFilePath oldf') oldf' = getTopFilePath oldf startMoveToTempName :: Remote -> ExportHandle -> TopFilePath -> ExportKey -> CommandStart startMoveToTempName r db f ek = starting ("rename " ++ name r) - (ActionItemOther $ Just $ f' ++ " -> " ++ fromExportLocation tmploc) + (ActionItemOther $ Just $ f' ++ " -> " ++ fromRawFilePath (fromExportLocation tmploc)) (performRename r db ek loc tmploc) where - loc = mkExportLocation f' + loc = mkExportLocation (toRawFilePath f') f' = getTopFilePath f tmploc = exportTempName ek @@ -383,10 +384,10 @@ startMoveFromTempName :: Remote -> ExportHandle -> ExportKey -> TopFilePath -> C startMoveFromTempName r db ek f = do let tmploc = exportTempName ek stopUnless (liftIO $ elem tmploc <$> getExportedLocation db (asKey ek)) $ - starting ("rename " ++ name r) (ActionItemOther (Just (fromExportLocation tmploc ++ " -> " ++ f'))) $ + starting ("rename " ++ name r) (ActionItemOther (Just (fromRawFilePath (fromExportLocation tmploc) ++ " -> " ++ f'))) $ performRename r db ek tmploc loc where - loc = mkExportLocation f' + loc = mkExportLocation (toRawFilePath f') f' = getTopFilePath f performRename :: Remote -> ExportHandle -> ExportKey -> ExportLocation -> ExportLocation -> CommandPerform @@ -468,7 +469,7 @@ filterPreferredContent r tree = logExportExcluded (uuid r) $ \logwriter -> do -- Match filename relative to the -- top of the tree. let af = AssociatedFile $ Just $ - getTopFilePath topf + toRawFilePath $ getTopFilePath topf let mi = MatchingKey k af ifM (checkMatcher' matcher mi mempty) ( return (Just ti) diff --git a/Command/Find.hs b/Command/Find.hs index dd16e31d01..9ed9583c6b 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -9,6 +9,8 @@ module Command.Find where import Data.Default import qualified Data.Map as M +import qualified Data.ByteString as S +import qualified Data.ByteString.Char8 as S8 import Command import Annex.Content @@ -57,29 +59,29 @@ seek o = case batchOption o of (commandAction . startKeys o) (withFilesInGit (commandAction . go)) =<< workTreeItems (findThese o) - Batch fmt -> batchFilesMatching fmt go + Batch fmt -> batchFilesMatching fmt (go . toRawFilePath) where go = whenAnnexed $ start o -- only files inAnnex are shown, unless the user has requested -- others via a limit -start :: FindOptions -> FilePath -> Key -> CommandStart +start :: FindOptions -> RawFilePath -> Key -> CommandStart start o file key = stopUnless (limited <||> inAnnex key) $ startingCustomOutput key $ do - showFormatted (formatOption o) file $ ("file", file) : keyVars key + showFormatted (formatOption o) file $ ("file", fromRawFilePath file) : keyVars key next $ return True startKeys :: FindOptions -> (Key, ActionItem) -> CommandStart startKeys o (key, ActionItemBranchFilePath (BranchFilePath _ topf) _) = - start o (getTopFilePath topf) key + start o (toRawFilePath (getTopFilePath topf)) key startKeys _ _ = stop -showFormatted :: Maybe Utility.Format.Format -> String -> [(String, String)] -> Annex () +showFormatted :: Maybe Utility.Format.Format -> S.ByteString -> [(String, String)] -> Annex () showFormatted format unformatted vars = unlessM (showFullJSON $ JSONChunk vars) $ case format of - Nothing -> liftIO $ putStrLn unformatted + Nothing -> liftIO $ S8.putStrLn unformatted Just formatter -> liftIO $ putStr $ Utility.Format.format formatter $ M.fromList vars @@ -87,14 +89,14 @@ showFormatted format unformatted vars = keyVars :: Key -> [(String, String)] keyVars key = [ ("key", serializeKey key) - , ("backend", decodeBS $ formatKeyVariety $ keyVariety key) + , ("backend", decodeBS $ formatKeyVariety $ fromKey keyVariety key) , ("bytesize", size show) , ("humansize", size $ roughSize storageUnits True) - , ("keyname", decodeBS $ keyName key) + , ("keyname", decodeBS $ fromKey keyName key) , ("hashdirlower", hashDirLower def key) , ("hashdirmixed", hashDirMixed def key) - , ("mtime", whenavail show $ keyMtime key) + , ("mtime", whenavail show $ fromKey keyMtime key) ] where - size c = whenavail c $ keySize key + size c = whenavail c $ fromKey keySize key whenavail = maybe "unknown" diff --git a/Command/Fix.hs b/Command/Fix.hs index c3f818b01b..52e076f30b 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -17,6 +17,7 @@ import Annex.Content import Annex.Perms import qualified Annex.Queue import qualified Database.Keys +import qualified Utility.RawFilePath as R #if ! defined(mingw32_HOST_OS) import Utility.Touch @@ -37,13 +38,14 @@ seek ps = unlessM crippledFileSystem $ do data FixWhat = FixSymlinks | FixAll -start :: FixWhat -> FilePath -> Key -> CommandStart +start :: FixWhat -> RawFilePath -> Key -> CommandStart start fixwhat file key = do - currlink <- liftIO $ catchMaybeIO $ readSymbolicLink file - wantlink <- calcRepo $ gitAnnexLink file key + currlink <- liftIO $ catchMaybeIO $ R.readSymbolicLink file + wantlink <- calcRepo $ gitAnnexLink (fromRawFilePath file) key case currlink of Just l - | l /= wantlink -> fixby $ fixSymlink file wantlink + | l /= toRawFilePath wantlink -> fixby $ + fixSymlink (fromRawFilePath file) wantlink | otherwise -> stop Nothing -> case fixwhat of FixAll -> fixthin @@ -52,9 +54,9 @@ start fixwhat file key = do fixby = starting "fix" (mkActionItem (key, file)) fixthin = do obj <- calcRepo $ gitAnnexLocation key - stopUnless (isUnmodified key file <&&> isUnmodified key obj) $ do + stopUnless (isUnmodified key (fromRawFilePath file) <&&> isUnmodified key obj) $ do thin <- annexThin <$> Annex.getGitConfig - fs <- liftIO $ catchMaybeIO $ getFileStatus file + fs <- liftIO $ catchMaybeIO $ R.getFileStatus file os <- liftIO $ catchMaybeIO $ getFileStatus obj case (linkCount <$> fs, linkCount <$> os, thin) of (Just 1, Just 1, True) -> @@ -63,21 +65,21 @@ start fixwhat file key = do fixby $ breakHardLink file key obj _ -> stop -breakHardLink :: FilePath -> Key -> FilePath -> CommandPerform +breakHardLink :: RawFilePath -> Key -> FilePath -> CommandPerform breakHardLink file key obj = do - replaceFile file $ \tmp -> do - mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + replaceFile (fromRawFilePath file) $ \tmp -> do + mode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus file unlessM (checkedCopyFile key obj tmp mode) $ error "unable to break hard link" thawContent tmp modifyContent obj $ freezeContent obj - Database.Keys.storeInodeCaches key [file] + Database.Keys.storeInodeCaches key [fromRawFilePath file] next $ return True -makeHardLink :: FilePath -> Key -> CommandPerform +makeHardLink :: RawFilePath -> Key -> CommandPerform makeHardLink file key = do - replaceFile file $ \tmp -> do - mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + replaceFile (fromRawFilePath file) $ \tmp -> do + mode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus file linkFromAnnex key tmp mode >>= \case LinkAnnexFailed -> error "unable to make hard link" _ -> noop diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 7f1ef6f474..f3e7487272 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -49,19 +49,19 @@ seekBatch fmt = batchInput fmt parse commandAction parse s = let (keyname, file) = separate (== ' ') s in if not (null keyname) && not (null file) - then Right $ go file (mkKey keyname) + then Right $ go file (keyOpt keyname) else Left "Expected pairs of key and filename" - go file key = starting "fromkey" (mkActionItem (key, file)) $ + go file key = starting "fromkey" (mkActionItem (key, toRawFilePath file)) $ perform key file start :: Bool -> (String, FilePath) -> CommandStart start force (keyname, file) = do - let key = mkKey keyname + let key = keyOpt keyname unless force $ do inbackend <- inAnnex key unless inbackend $ giveup $ "key ("++ keyname ++") is not present in backend (use --force to override this sanity check)" - starting "fromkey" (mkActionItem (key, file)) $ + starting "fromkey" (mkActionItem (key, toRawFilePath file)) $ perform key file -- From user input to a Key. @@ -71,8 +71,8 @@ start force (keyname, file) = do -- For example, "WORM--a:a" parses as an uri. To disambiguate, check -- the uri scheme, to see if it looks like the prefix of a key. This relies -- on key backend names never containing a ':'. -mkKey :: String -> Key -mkKey s = case parseURI s of +keyOpt :: String -> Key +keyOpt s = case parseURI s of Just u | not (isKeyPrefix (uriScheme u)) -> Backend.URL.fromUrl s Nothing _ -> case deserializeKey s of @@ -80,7 +80,7 @@ mkKey s = case parseURI s of Nothing -> giveup $ "bad key/url " ++ s perform :: Key -> FilePath -> CommandPerform -perform key file = lookupFileNotHidden file >>= \case +perform key file = lookupFileNotHidden (toRawFilePath file) >>= \case Nothing -> ifM (liftIO $ doesFileExist file) ( hasothercontent , do diff --git a/Command/Fsck.hs b/Command/Fsck.hs index bed59a52c7..256bdfa894 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -35,6 +35,7 @@ import qualified Database.Fsck as FsckDb import Types.CleanupActions import Types.Key import Types.ActionItem +import qualified Utility.RawFilePath as R import Data.Time.Clock.POSIX import System.Posix.Types (EpochTime) @@ -102,11 +103,11 @@ checkDeadRepo u = whenM ((==) DeadTrusted <$> lookupTrust u) $ earlyWarning "Warning: Fscking a repository that is currently marked as dead." -start :: Maybe Remote -> Incremental -> FilePath -> Key -> CommandStart -start from inc file key = Backend.getBackend file key >>= \case +start :: Maybe Remote -> Incremental -> RawFilePath -> Key -> CommandStart +start from inc file key = Backend.getBackend (fromRawFilePath file) key >>= \case Nothing -> stop Just backend -> do - numcopies <- getFileNumCopies file + numcopies <- getFileNumCopies (fromRawFilePath file) case from of Nothing -> go $ perform key file backend numcopies Just r -> go $ performRemote key afile backend numcopies r @@ -114,9 +115,9 @@ start from inc file key = Backend.getBackend file key >>= \case go = runFsck inc (mkActionItem (key, afile)) key afile = AssociatedFile (Just file) -perform :: Key -> FilePath -> Backend -> NumCopies -> Annex Bool +perform :: Key -> RawFilePath -> Backend -> NumCopies -> Annex Bool perform key file backend numcopies = do - keystatus <- getKeyFileStatus key file + keystatus <- getKeyFileStatus key (fromRawFilePath file) check -- order matters [ fixLink key file @@ -182,7 +183,7 @@ performRemote key afile backend numcopies remote = startKey :: Maybe Remote -> Incremental -> (Key, ActionItem) -> NumCopies -> CommandStart startKey from inc (key, ai) numcopies = - case Backend.maybeLookupBackendVariety (keyVariety key) of + case Backend.maybeLookupBackendVariety (fromKey keyVariety key) of Nothing -> stop Just backend -> runFsck inc ai key $ case from of @@ -203,18 +204,18 @@ check :: [Annex Bool] -> Annex Bool check cs = and <$> sequence cs {- Checks that symlinks points correctly to the annexed content. -} -fixLink :: Key -> FilePath -> Annex Bool +fixLink :: Key -> RawFilePath -> Annex Bool fixLink key file = do - want <- calcRepo $ gitAnnexLink file key + want <- calcRepo $ gitAnnexLink (fromRawFilePath file) key have <- getAnnexLinkTarget file maybe noop (go want) have return True where go want have - | want /= fromInternalGitPath (fromRawFilePath have) = do + | want /= fromRawFilePath (fromInternalGitPath have) = do showNote "fixing link" - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ removeFile file + liftIO $ createDirectoryIfMissing True (parentDir (fromRawFilePath file)) + liftIO $ removeFile (fromRawFilePath file) addAnnexLink want file | otherwise = noop @@ -244,9 +245,9 @@ verifyLocationLog key keystatus ai = do - insecure hash is present. This should only be able to happen - if the repository already contained the content before the - config was set. -} - when (present && not (cryptographicallySecure (keyVariety key))) $ + when (present && not (cryptographicallySecure (fromKey keyVariety key))) $ whenM (annexSecureHashesOnly <$> Annex.getGitConfig) $ - warning $ "** Despite annex.securehashesonly being set, " ++ obj ++ " has content present in the annex using an insecure " ++ decodeBS (formatKeyVariety (keyVariety key)) ++ " key" + warning $ "** Despite annex.securehashesonly being set, " ++ obj ++ " has content present in the annex using an insecure " ++ decodeBS (formatKeyVariety (fromKey keyVariety key)) ++ " key" verifyLocationLog' key ai present u (logChange key u) @@ -267,7 +268,7 @@ verifyLocationLog' key ai present u updatestatus = do fix InfoMissing warning $ "** Based on the location log, " ++ - actionItemDesc ai ++ + decodeBS' (actionItemDesc ai) ++ "\n** was expected to be present, " ++ "but its content is missing." return False @@ -302,23 +303,23 @@ verifyRequiredContent key ai@(ActionItemAssociatedFile afile _) = do missingrequired <- Remote.prettyPrintUUIDs "missingrequired" missinglocs warning $ "** Required content " ++ - actionItemDesc ai ++ + decodeBS' (actionItemDesc ai) ++ " is missing from these repositories:\n" ++ missingrequired return False verifyRequiredContent _ _ = return True {- Verifies the associated file records. -} -verifyAssociatedFiles :: Key -> KeyStatus -> FilePath -> Annex Bool +verifyAssociatedFiles :: Key -> KeyStatus -> RawFilePath -> Annex Bool verifyAssociatedFiles key keystatus file = do when (isKeyUnlockedThin keystatus) $ do - f <- inRepo $ toTopFilePath file + f <- inRepo $ toTopFilePath $ fromRawFilePath file afs <- Database.Keys.getAssociatedFiles key unless (getTopFilePath f `elem` map getTopFilePath afs) $ Database.Keys.addAssociatedFile key f return True -verifyWorkTree :: Key -> FilePath -> Annex Bool +verifyWorkTree :: Key -> RawFilePath -> Annex Bool verifyWorkTree key file = do {- Make sure that a pointer file is replaced with its content, - when the content is available. -} @@ -326,8 +327,8 @@ verifyWorkTree key file = do case mk of Just k | k == key -> whenM (inAnnex key) $ do showNote "fixing worktree content" - replaceFile file $ \tmp -> do - mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + replaceFile (fromRawFilePath file) $ \tmp -> do + mode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus file ifM (annexThin <$> Annex.getGitConfig) ( void $ linkFromAnnex key tmp mode , do @@ -335,7 +336,7 @@ verifyWorkTree key file = do void $ checkedCopyFile key obj tmp mode thawContent tmp ) - Database.Keys.storeInodeCaches key [file] + Database.Keys.storeInodeCaches key [fromRawFilePath file] _ -> return () return True @@ -362,7 +363,7 @@ checkKeySizeRemote key remote ai localcopy = checkKeySizeOr (badContentRemote remote localcopy) key localcopy ai checkKeySizeOr :: (Key -> Annex String) -> Key -> FilePath -> ActionItem -> Annex Bool -checkKeySizeOr bad key file ai = case keySize key of +checkKeySizeOr bad key file ai = case fromKey keySize key of Nothing -> return True Just size -> do size' <- liftIO $ getFileSize file @@ -375,7 +376,7 @@ checkKeySizeOr bad key file ai = case keySize key of badsize a b = do msg <- bad key warning $ concat - [ actionItemDesc ai + [ decodeBS' (actionItemDesc ai) , ": Bad file size (" , compareSizes storageUnits True a b , "); " @@ -393,11 +394,11 @@ checkKeyUpgrade backend key ai (AssociatedFile (Just file)) = case Types.Backend.canUpgradeKey backend of Just a | a key -> do warning $ concat - [ actionItemDesc ai + [ decodeBS' (actionItemDesc ai) , ": Can be upgraded to an improved key format. " , "You can do so by running: git annex migrate --backend=" - , decodeBS (formatKeyVariety (keyVariety key)) ++ " " - , file + , decodeBS (formatKeyVariety (fromKey keyVariety key)) ++ " " + , decodeBS' file ] return True _ -> return True @@ -448,7 +449,7 @@ checkBackendOr' bad backend key file ai postcheck = unless ok $ do msg <- bad key warning $ concat - [ actionItemDesc ai + [ decodeBS' (actionItemDesc ai) , ": Bad file content; " , msg ] @@ -460,7 +461,7 @@ checkKeyNumCopies :: Key -> AssociatedFile -> NumCopies -> Annex Bool checkKeyNumCopies key afile numcopies = do let (desc, hasafile) = case afile of AssociatedFile Nothing -> (serializeKey key, False) - AssociatedFile (Just af) -> (af, True) + AssociatedFile (Just af) -> (fromRawFilePath af, True) locs <- loggedLocations key (untrustedlocations, otherlocations) <- trustPartition UnTrusted locs (deadlocations, safelocations) <- trustPartition DeadTrusted otherlocations @@ -680,7 +681,7 @@ getKeyFileStatus key file = do s <- getKeyStatus key case s of KeyUnlockedThin -> catchDefaultIO KeyUnlockedThin $ - ifM (isJust <$> isAnnexLink file) + ifM (isJust <$> isAnnexLink (toRawFilePath file)) ( return KeyLockedThin , return KeyUnlockedThin ) diff --git a/Command/FuzzTest.hs b/Command/FuzzTest.hs index 5e0812516e..ba232f3167 100644 --- a/Command/FuzzTest.hs +++ b/Command/FuzzTest.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Command.FuzzTest where import Command @@ -13,6 +15,7 @@ import qualified Git.Config import Config import Utility.ThreadScheduler import Utility.DiskFree +import Git.Types (fromConfigKey) import Data.Time.Clock import System.Random (getStdRandom, random, randomR) @@ -32,25 +35,23 @@ start :: CommandStart start = do guardTest logf <- fromRepo gitAnnexFuzzTestLogFile - showStart "fuzztest" logf + showStart "fuzztest" (toRawFilePath logf) logh <- liftIO $ openFile logf WriteMode void $ forever $ fuzz logh stop guardTest :: Annex () -guardTest = unlessM (fromMaybe False . Git.Config.isTrue <$> getConfig key "") $ +guardTest = unlessM (fromMaybe False . Git.Config.isTrue' <$> getConfig key mempty) $ giveup $ unlines [ "Running fuzz tests *writes* to and *deletes* files in" , "this repository, and pushes those changes to other" , "repositories! This is a developer tool, not something" , "to play with." , "" - , "Refusing to run fuzz tests, since " ++ keyname ++ " is not set!" + , "Refusing to run fuzz tests, since " ++ fromConfigKey key ++ " is not set!" ] where key = annexConfig "eat-my-repository" - (ConfigKey keyname) = key - fuzz :: Handle -> Annex () fuzz logh = do diff --git a/Command/Get.hs b/Command/Get.hs index ba4dcb2010..e3bf47cb59 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -42,19 +42,19 @@ seek o = startConcurrency transferStages $ do from <- maybe (pure Nothing) (Just <$$> getParsed) (getFrom o) let go = whenAnnexed $ start o from case batchOption o of - Batch fmt -> batchFilesMatching fmt go + Batch fmt -> batchFilesMatching fmt (go . toRawFilePath) NoBatch -> withKeyOptions (keyOptions o) (autoMode o) (commandAction . startKeys from) (withFilesInGit (commandAction . go)) =<< workTreeItems (getFiles o) -start :: GetOptions -> Maybe Remote -> FilePath -> Key -> CommandStart +start :: GetOptions -> Maybe Remote -> RawFilePath -> Key -> CommandStart start o from file key = start' expensivecheck from key afile ai where afile = AssociatedFile (Just file) ai = mkActionItem (key, afile) expensivecheck - | autoMode o = numCopiesCheck file key (<) + | autoMode o = numCopiesCheck (fromRawFilePath file) key (<) <||> wantGet False (Just key) afile | otherwise = return True diff --git a/Command/Import.hs b/Command/Import.hs index 0488ef4cb7..58c1b40f93 100644 --- a/Command/Import.hs +++ b/Command/Import.hs @@ -117,7 +117,7 @@ seek o@(RemoteImportOptions {}) = startConcurrency commandStages $ do startLocal :: GetFileMatcher -> DuplicateMode -> (FilePath, FilePath) -> CommandStart startLocal largematcher mode (srcfile, destfile) = ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile) - ( starting "import" (ActionItemWorkTreeFile destfile) + ( starting "import" (ActionItemWorkTreeFile (toRawFilePath destfile)) pickaction , stop ) @@ -202,7 +202,7 @@ startLocal largematcher mode (srcfile, destfile) = >>= maybe stop (\addedk -> next $ Command.Add.cleanup addedk True) - , next $ Command.Add.addSmall destfile + , next $ Command.Add.addSmall $ toRawFilePath destfile ) notoverwriting why = do warning $ "not overwriting existing " ++ destfile ++ " " ++ why diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs index 2eca658649..dc4fb8749c 100644 --- a/Command/ImportFeed.hs +++ b/Command/ImportFeed.hs @@ -67,7 +67,7 @@ seek o = do getFeed :: ImportFeedOptions -> Cache -> URLString -> CommandSeek getFeed opts cache url = do - showStart "importfeed" url + showStart' "importfeed" (Just url) downloadFeed url >>= \case Nothing -> showEndResult =<< feedProblem url "downloading the feed failed" @@ -222,7 +222,7 @@ performDownload opts cache todownload = case location todownload of case dest of Nothing -> return True Just f -> do - showStart "addurl" url + showStart' "addurl" (Just url) ks <- getter f if null ks then do @@ -244,7 +244,7 @@ performDownload opts cache todownload = case location todownload of - to be re-downloaded. -} makeunique url n file = ifM alreadyexists ( ifM forced - ( ifAnnexed f checksameurl tryanother + ( ifAnnexed (toRawFilePath f) checksameurl tryanother , tryanother ) , return $ Just f diff --git a/Command/Info.hs b/Command/Info.hs index 23a8fc2899..a0099ca06d 100644 --- a/Command/Info.hs +++ b/Command/Info.hs @@ -50,23 +50,23 @@ import qualified Command.Unused type Stat = StatState (Maybe (String, StatState String)) -- data about a set of keys -data KeyData = KeyData +data KeyInfo = KeyInfo { countKeys :: Integer , sizeKeys :: Integer , unknownSizeKeys :: Integer , backendsKeys :: M.Map KeyVariety Integer } -instance Sem.Semigroup KeyData where - a <> b = KeyData +instance Sem.Semigroup KeyInfo where + a <> b = KeyInfo { countKeys = countKeys a + countKeys b , sizeKeys = sizeKeys a + sizeKeys b , unknownSizeKeys = unknownSizeKeys a + unknownSizeKeys b , backendsKeys = backendsKeys a <> backendsKeys b } -instance Monoid KeyData where - mempty = KeyData 0 0 0 M.empty +instance Monoid KeyInfo where + mempty = KeyInfo 0 0 0 M.empty data NumCopiesStats = NumCopiesStats { numCopiesVarianceMap :: M.Map Variance Integer @@ -82,9 +82,9 @@ instance Show Variance where -- cached info that multiple Stats use data StatInfo = StatInfo - { presentData :: Maybe KeyData - , referencedData :: Maybe KeyData - , repoData :: M.Map UUID KeyData + { presentData :: Maybe KeyInfo + , referencedData :: Maybe KeyInfo + , repoData :: M.Map UUID KeyInfo , numCopiesStats :: Maybe NumCopiesStats , infoOptions :: InfoOptions } @@ -152,7 +152,7 @@ itemInfo o p = ifM (isdir p) v' <- Remote.nameToUUID' p case v' of Right u -> uuidInfo o u - Left _ -> ifAnnexed p + Left _ -> ifAnnexed (toRawFilePath p) (fileInfo o p) (treeishInfo o p) ) @@ -161,7 +161,7 @@ itemInfo o p = ifM (isdir p) noInfo :: String -> Annex () noInfo s = do - showStart "info" s + showStart "info" (encodeBS' s) showNote $ "not a directory or an annexed file or a treeish or a remote or a uuid" showEndFail @@ -311,8 +311,8 @@ showStat :: Stat -> StatState () showStat s = maybe noop calc =<< s where calc (desc, a) = do - (lift . showHeader) desc - lift . showRaw =<< a + (lift . showHeader . encodeBS') desc + lift . showRaw . encodeBS' =<< a repo_list :: TrustLevel -> Stat repo_list level = stat n $ nojson $ lift $ do @@ -435,7 +435,7 @@ transfer_list = stat desc $ nojson $ lift $ do desc = "transfers in progress" line uuidmap t i = unwords [ formatDirection (transferDirection t) ++ "ing" - , actionItemDesc $ mkActionItem + , fromRawFilePath $ actionItemDesc $ mkActionItem (transferKey t, associatedFile i) , if transferDirection t == Upload then "to" else "from" , maybe (fromUUID $ transferUUID t) Remote.name $ @@ -444,7 +444,7 @@ transfer_list = stat desc $ nojson $ lift $ do jsonify t i = object $ map (\(k, v) -> (packString k, v)) $ [ ("transfer", toJSON' (formatDirection (transferDirection t))) , ("key", toJSON' (transferKey t)) - , ("file", toJSON' afile) + , ("file", toJSON' (fromRawFilePath <$> afile)) , ("remote", toJSON' (fromUUID (transferUUID t) :: String)) ] where @@ -512,7 +512,7 @@ reposizes_total :: Stat reposizes_total = simpleStat "combined size of repositories containing these files" $ showSizeKeys . mconcat . M.elems =<< cachedRepoData -cachedPresentData :: StatState KeyData +cachedPresentData :: StatState KeyInfo cachedPresentData = do s <- get case presentData s of @@ -522,7 +522,7 @@ cachedPresentData = do put s { presentData = Just v } return v -cachedRemoteData :: UUID -> StatState KeyData +cachedRemoteData :: UUID -> StatState KeyInfo cachedRemoteData u = do s <- get case M.lookup u (repoData s) of @@ -531,19 +531,19 @@ cachedRemoteData u = do let combinedata d uk = finishCheck uk >>= \case Nothing -> return d Just k -> return $ addKey k d - v <- lift $ foldM combinedata emptyKeyData + v <- lift $ foldM combinedata emptyKeyInfo =<< loggedKeysFor' u put s { repoData = M.insert u v (repoData s) } return v -cachedReferencedData :: StatState KeyData +cachedReferencedData :: StatState KeyInfo cachedReferencedData = do s <- get case referencedData s of Just v -> return v Nothing -> do !v <- lift $ Command.Unused.withKeysReferenced - emptyKeyData addKey + emptyKeyInfo addKey put s { referencedData = Just v } return v @@ -552,7 +552,7 @@ cachedNumCopiesStats :: StatState (Maybe NumCopiesStats) cachedNumCopiesStats = numCopiesStats <$> get -- currently only available for directory info -cachedRepoData :: StatState (M.Map UUID KeyData) +cachedRepoData :: StatState (M.Map UUID KeyInfo) cachedRepoData = repoData <$> get getDirStatInfo :: InfoOptions -> FilePath -> Annex StatInfo @@ -564,9 +564,9 @@ getDirStatInfo o dir = do (update matcher fast) return $ StatInfo (Just presentdata) (Just referenceddata) repodata (Just numcopiesstats) o where - initial = (emptyKeyData, emptyKeyData, emptyNumCopiesStats, M.empty) + initial = (emptyKeyInfo, emptyKeyInfo, emptyNumCopiesStats, M.empty) update matcher fast key file vs@(presentdata, referenceddata, numcopiesstats, repodata) = - ifM (matcher $ MatchingFile $ FileInfo file file) + ifM (matcher $ MatchingFile $ FileInfo file' file') ( do !presentdata' <- ifM (inAnnex key) ( return $ addKey key presentdata @@ -577,11 +577,13 @@ getDirStatInfo o dir = do then return (numcopiesstats, repodata) else do locs <- Remote.keyLocations key - nc <- updateNumCopiesStats file numcopiesstats locs + nc <- updateNumCopiesStats file' numcopiesstats locs return (nc, updateRepoData key locs repodata) return $! (presentdata', referenceddata', numcopiesstats', repodata') , return vs ) + where + file' = fromRawFilePath file getTreeStatInfo :: InfoOptions -> Git.Ref -> Annex (Maybe StatInfo) getTreeStatInfo o r = do @@ -594,7 +596,7 @@ getTreeStatInfo o r = do , return Nothing ) where - initial = (emptyKeyData, emptyKeyData, M.empty) + initial = (emptyKeyInfo, emptyKeyInfo, M.empty) go _ [] vs = return vs go fast (l:ls) vs@(presentdata, referenceddata, repodata) = do mk <- catKey (LsTree.sha l) @@ -613,33 +615,33 @@ getTreeStatInfo o r = do return (updateRepoData key locs repodata) go fast ls $! (presentdata', referenceddata', repodata') -emptyKeyData :: KeyData -emptyKeyData = KeyData 0 0 0 M.empty +emptyKeyInfo :: KeyInfo +emptyKeyInfo = KeyInfo 0 0 0 M.empty emptyNumCopiesStats :: NumCopiesStats emptyNumCopiesStats = NumCopiesStats M.empty -foldKeys :: [Key] -> KeyData -foldKeys = foldl' (flip addKey) emptyKeyData +foldKeys :: [Key] -> KeyInfo +foldKeys = foldl' (flip addKey) emptyKeyInfo -addKey :: Key -> KeyData -> KeyData -addKey key (KeyData count size unknownsize backends) = - KeyData count' size' unknownsize' backends' +addKey :: Key -> KeyInfo -> KeyInfo +addKey key (KeyInfo count size unknownsize backends) = + KeyInfo count' size' unknownsize' backends' where {- All calculations strict to avoid thunks when repeatedly - applied to many keys. -} !count' = count + 1 - !backends' = M.insertWith (+) (keyVariety key) 1 backends + !backends' = M.insertWith (+) (fromKey keyVariety key) 1 backends !size' = maybe size (+ size) ks !unknownsize' = maybe (unknownsize + 1) (const unknownsize) ks - ks = keySize key + ks = fromKey keySize key -updateRepoData :: Key -> [UUID] -> M.Map UUID KeyData -> M.Map UUID KeyData +updateRepoData :: Key -> [UUID] -> M.Map UUID KeyInfo -> M.Map UUID KeyInfo updateRepoData key locs m = m' where !m' = M.unionWith (\_old new -> new) m $ M.fromList $ zip locs (map update locs) - update loc = addKey key (fromMaybe emptyKeyData $ M.lookup loc m) + update loc = addKey key (fromMaybe emptyKeyInfo $ M.lookup loc m) updateNumCopiesStats :: FilePath -> NumCopiesStats -> [UUID] -> Annex NumCopiesStats updateNumCopiesStats file (NumCopiesStats m) locs = do @@ -649,7 +651,7 @@ updateNumCopiesStats file (NumCopiesStats m) locs = do let !ret = NumCopiesStats m' return ret -showSizeKeys :: KeyData -> StatState String +showSizeKeys :: KeyInfo -> StatState String showSizeKeys d = do sizer <- mkSizer return $ total sizer ++ missingnote diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs index 00ba46dc90..09aee869dc 100644 --- a/Command/InitRemote.hs +++ b/Command/InitRemote.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Command.InitRemote where import qualified Data.Map as M diff --git a/Command/Inprogress.hs b/Command/Inprogress.hs index e571fa8d3b..45d68da745 100644 --- a/Command/Inprogress.hs +++ b/Command/Inprogress.hs @@ -42,7 +42,7 @@ seek o = do (commandAction . (whenAnnexed (start s))) =<< workTreeItems (inprogressFiles o) -start :: S.Set Key -> FilePath -> Key -> CommandStart +start :: S.Set Key -> RawFilePath -> Key -> CommandStart start s _file k | S.member k s = start' k | otherwise = stop diff --git a/Command/List.hs b/Command/List.hs index ae9e6a70f1..7b41a304ec 100644 --- a/Command/List.hs +++ b/Command/List.hs @@ -72,7 +72,7 @@ getList o printHeader :: [(UUID, RemoteName, TrustLevel)] -> Annex () printHeader l = liftIO $ putStrLn $ lheader $ map (\(_, n, t) -> (n, t)) l -start :: [(UUID, RemoteName, TrustLevel)] -> FilePath -> Key -> CommandStart +start :: [(UUID, RemoteName, TrustLevel)] -> RawFilePath -> Key -> CommandStart start l file key = do ls <- S.fromList <$> keyLocations key liftIO $ putStrLn $ format (map (\(u, _, t) -> (t, S.member u ls)) l) file @@ -88,8 +88,8 @@ lheader remotes = unlines (zipWith formatheader [0..] remotes) ++ pipes (length trust UnTrusted = " (untrusted)" trust _ = "" -format :: [(TrustLevel, Present)] -> FilePath -> String -format remotes file = thereMap ++ " " ++ file +format :: [(TrustLevel, Present)] -> RawFilePath -> String +format remotes file = thereMap ++ " " ++ fromRawFilePath file where thereMap = concatMap there remotes there (UnTrusted, True) = "x" diff --git a/Command/Lock.hs b/Command/Lock.hs index 2f2eab21b4..24dd6810ed 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -32,7 +32,7 @@ seek ps = do l <- workTreeItems ps withFilesInGit (commandAction . (whenAnnexed startNew)) l -startNew :: FilePath -> Key -> CommandStart +startNew :: RawFilePath -> Key -> CommandStart startNew file key = ifM (isJust <$> isAnnexLink file) ( stop , starting "lock" (mkActionItem (key, file)) $ @@ -43,7 +43,7 @@ startNew file key = ifM (isJust <$> isAnnexLink file) | key' == key = cont | otherwise = errorModified go Nothing = - ifM (isUnmodified key file) + ifM (isUnmodified key (fromRawFilePath file)) ( cont , ifM (Annex.getState Annex.force) ( cont @@ -52,11 +52,11 @@ startNew file key = ifM (isJust <$> isAnnexLink file) ) cont = performNew file key -performNew :: FilePath -> Key -> CommandPerform +performNew :: RawFilePath -> Key -> CommandPerform performNew file key = do lockdown =<< calcRepo (gitAnnexLocation key) - addLink file key - =<< withTSDelta (liftIO . genInodeCache file) + addLink (fromRawFilePath file) key + =<< withTSDelta (liftIO . genInodeCache' file) next $ cleanupNew file key where lockdown obj = do @@ -70,7 +70,7 @@ performNew file key = do -- It's ok if the file is hard linked to obj, but if some other -- associated file is, we need to break that link to lock down obj. breakhardlink obj = whenM (catchBoolIO $ (> 1) . linkCount <$> liftIO (getFileStatus obj)) $ do - mfc <- withTSDelta (liftIO . genInodeCache file) + mfc <- withTSDelta (liftIO . genInodeCache' file) unlessM (sameInodeCache obj (maybeToList mfc)) $ do modifyContent obj $ replaceFile obj $ \tmp -> do unlessM (checkedCopyFile key obj tmp Nothing) $ @@ -92,21 +92,21 @@ performNew file key = do lostcontent = logStatus key InfoMissing -cleanupNew :: FilePath -> Key -> CommandCleanup +cleanupNew :: RawFilePath -> Key -> CommandCleanup cleanupNew file key = do - Database.Keys.removeAssociatedFile key =<< inRepo (toTopFilePath file) + Database.Keys.removeAssociatedFile key =<< inRepo (toTopFilePath (fromRawFilePath file)) return True -startOld :: FilePath -> CommandStart +startOld :: RawFilePath -> CommandStart startOld file = do unlessM (Annex.getState Annex.force) errorModified starting "lock" (ActionItemWorkTreeFile file) $ performOld file -performOld :: FilePath -> CommandPerform +performOld :: RawFilePath -> CommandPerform performOld file = do - Annex.Queue.addCommand "checkout" [Param "--"] [file] + Annex.Queue.addCommand "checkout" [Param "--"] [fromRawFilePath file] next $ return True errorModified :: a diff --git a/Command/Log.hs b/Command/Log.hs index 554afa947a..19ededcc02 100644 --- a/Command/Log.hs +++ b/Command/Log.hs @@ -92,10 +92,10 @@ seek o = do ([], True) -> commandAction (startAll o outputter) (_, True) -> giveup "Cannot specify both files and --all" -start :: LogOptions -> (FilePath -> Outputter) -> FilePath -> Key -> CommandStart +start :: LogOptions -> (FilePath -> Outputter) -> RawFilePath -> Key -> CommandStart start o outputter file key = do (changes, cleanup) <- getKeyLog key (passthruOptions o) - showLogIncremental (outputter file) changes + showLogIncremental (outputter (fromRawFilePath file)) changes void $ liftIO cleanup stop @@ -201,7 +201,7 @@ getKeyLog key os = do top <- fromRepo Git.repoPath p <- liftIO $ relPathCwdToFile top config <- Annex.getGitConfig - let logfile = p locationLogFile config key + let logfile = p fromRawFilePath (locationLogFile config key) getGitLog [logfile] (Param "--remove-empty" : os) {- Streams the git log for all git-annex branch changes. -} @@ -220,7 +220,7 @@ getGitLog fs os = do [ Param $ Git.fromRef Annex.Branch.fullname , Param "--" ] ++ map Param fs - return (parseGitRawLog ls, cleanup) + return (parseGitRawLog (map decodeBL' ls), cleanup) -- Parses chunked git log --raw output, which looks something like: -- @@ -250,7 +250,7 @@ parseGitRawLog = parse epoch (tss, cl') -> (parseTimeStamp tss, cl') mrc = do (old, new) <- parseRawChangeLine cl - key <- locationLogFileKey c2 + key <- locationLogFileKey (toRawFilePath c2) return $ RefChange { changetime = ts , oldref = old diff --git a/Command/LookupKey.hs b/Command/LookupKey.hs index 11fa0c9461..1525046f2d 100644 --- a/Command/LookupKey.hs +++ b/Command/LookupKey.hs @@ -29,11 +29,12 @@ run _ file = seekSingleGitFile file >>= \case -- To support absolute filenames, pass through git ls-files. -- But, this plumbing command does not recurse through directories. -seekSingleGitFile :: FilePath -> Annex (Maybe FilePath) +seekSingleGitFile :: FilePath -> Annex (Maybe RawFilePath) seekSingleGitFile file = do - (l, cleanup) <- inRepo (Git.LsFiles.inRepo [file]) + (l, cleanup) <- inRepo (Git.LsFiles.inRepo [toRawFilePath file]) r <- case l of - (f:[]) | takeFileName f == takeFileName file -> return (Just f) + (f:[]) | takeFileName (fromRawFilePath f) == takeFileName file -> + return (Just f) _ -> return Nothing void $ liftIO cleanup return r diff --git a/Command/MatchExpression.hs b/Command/MatchExpression.hs index 77ceb0662e..3e79a0387d 100644 --- a/Command/MatchExpression.hs +++ b/Command/MatchExpression.hs @@ -67,7 +67,7 @@ optParser desc = MatchExpressionOptions missingdata datadesc = bail $ "cannot match this expression without " ++ datadesc ++ " data" -- When a key is provided, make its size also be provided. addkeysize p = case providedKey p of - Right k -> case keySize k of + Right k -> case fromKey keySize k of Just sz -> p { providedFileSize = Right sz } Nothing -> p Left _ -> p diff --git a/Command/MetaData.hs b/Command/MetaData.hs index d1c7e50607..e0b86e5302 100644 --- a/Command/MetaData.hs +++ b/Command/MetaData.hs @@ -92,7 +92,7 @@ seek o = case batchOption o of ) _ -> giveup "--batch is currently only supported in --json mode" -start :: VectorClock -> MetaDataOptions -> FilePath -> Key -> CommandStart +start :: VectorClock -> MetaDataOptions -> RawFilePath -> Key -> CommandStart start c o file k = startKeys c o (k, mkActionItem (k, afile)) where afile = AssociatedFile (Just file) @@ -147,7 +147,7 @@ instance FromJSON MetaDataFields where fieldsField :: T.Text fieldsField = T.pack "fields" -parseJSONInput :: String -> Either String (Either FilePath Key, MetaData) +parseJSONInput :: String -> Either String (Either RawFilePath Key, MetaData) parseJSONInput i = do v <- eitherDecode (BU.fromString i) let m = case itemAdded v of @@ -155,16 +155,16 @@ parseJSONInput i = do Just (MetaDataFields m') -> m' case (itemKey v, itemFile v) of (Just k, _) -> Right (Right k, m) - (Nothing, Just f) -> Right (Left f, m) + (Nothing, Just f) -> Right (Left (toRawFilePath f), m) (Nothing, Nothing) -> Left "JSON input is missing either file or key" -startBatch :: (Either FilePath Key, MetaData) -> CommandStart +startBatch :: (Either RawFilePath Key, MetaData) -> CommandStart startBatch (i, (MetaData m)) = case i of Left f -> do mk <- lookupFile f case mk of Just k -> go k (mkActionItem (k, AssociatedFile (Just f))) - Nothing -> giveup $ "not an annexed file: " ++ f + Nothing -> giveup $ "not an annexed file: " ++ fromRawFilePath f Right k -> go k (mkActionItem k) where go k ai = starting "metadata" ai $ do diff --git a/Command/Migrate.hs b/Command/Migrate.hs index ca656e028d..0f964bb749 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -28,16 +28,16 @@ cmd = withGlobalOptions [annexedMatchingOptions] $ seek :: CmdParams -> CommandSeek seek = withFilesInGit (commandAction . (whenAnnexed start)) <=< workTreeItems -start :: FilePath -> Key -> CommandStart +start :: RawFilePath -> Key -> CommandStart start file key = do forced <- Annex.getState Annex.force - v <- Backend.getBackend file key + v <- Backend.getBackend (fromRawFilePath file) key case v of Nothing -> stop Just oldbackend -> do exists <- inAnnex key newbackend <- maybe defaultBackend return - =<< chooseBackend file + =<< chooseBackend (fromRawFilePath file) if (newbackend /= oldbackend || upgradableKey oldbackend key || forced) && exists then starting "migrate" (mkActionItem (key, file)) $ perform file key oldbackend newbackend @@ -50,7 +50,7 @@ start file key = do - - Something has changed in the backend, such as a bug fix. -} upgradableKey :: Backend -> Key -> Bool -upgradableKey backend key = isNothing (keySize key) || backendupgradable +upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable where backendupgradable = maybe False (\a -> a key) (canUpgradeKey backend) @@ -63,7 +63,7 @@ upgradableKey backend key = isNothing (keySize key) || backendupgradable - data cannot get corrupted after the fsck but before the new key is - generated. -} -perform :: FilePath -> Key -> Backend -> Backend -> CommandPerform +perform :: RawFilePath -> Key -> Backend -> Backend -> CommandPerform perform file oldkey oldbackend newbackend = go =<< genkey (fastMigrate oldbackend) where go Nothing = stop @@ -85,7 +85,7 @@ perform file oldkey oldbackend newbackend = go =<< genkey (fastMigrate oldbacken genkey Nothing = do content <- calcRepo $ gitAnnexLocation oldkey let source = KeySource - { keyFilename = file + { keyFilename = fromRawFilePath file , contentLocation = content , inodeCache = Nothing } diff --git a/Command/Mirror.hs b/Command/Mirror.hs index be7b7c5920..ecfff8fdba 100644 --- a/Command/Mirror.hs +++ b/Command/Mirror.hs @@ -47,7 +47,7 @@ seek o = startConcurrency transferStages $ (withFilesInGit (commandAction . (whenAnnexed $ start o))) =<< workTreeItems (mirrorFiles o) -start :: MirrorOptions -> FilePath -> Key -> CommandStart +start :: MirrorOptions -> RawFilePath -> Key -> CommandStart start o file k = startKey o afile (k, ai) where afile = AssociatedFile (Just file) @@ -75,4 +75,4 @@ startKey o afile (key, ai) = case fromToOptions o of where getnumcopies = case afile of AssociatedFile Nothing -> getNumCopies - AssociatedFile (Just af) -> getFileNumCopies af + AssociatedFile (Just af) -> getFileNumCopies (fromRawFilePath af) diff --git a/Command/Move.hs b/Command/Move.hs index a5f6e9a025..68bc419e20 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -57,13 +57,13 @@ seek :: MoveOptions -> CommandSeek seek o = startConcurrency transferStages $ do let go = whenAnnexed $ start (fromToOptions o) (removeWhen o) case batchOption o of - Batch fmt -> batchFilesMatching fmt go + Batch fmt -> batchFilesMatching fmt (go . toRawFilePath) NoBatch -> withKeyOptions (keyOptions o) False (commandAction . startKey (fromToOptions o) (removeWhen o)) (withFilesInGit (commandAction . go)) =<< workTreeItems (moveFiles o) -start :: FromToHereOptions -> RemoveWhen -> FilePath -> Key -> CommandStart +start :: FromToHereOptions -> RemoveWhen -> RawFilePath -> Key -> CommandStart start fromto removewhen f k = start' fromto removewhen afile k ai where afile = AssociatedFile (Just f) diff --git a/Command/Multicast.hs b/Command/Multicast.hs index 6c6d2c418b..97966984a1 100644 --- a/Command/Multicast.hs +++ b/Command/Multicast.hs @@ -137,7 +137,7 @@ send ups fs = do mk <- lookupFile f case mk of Nothing -> noop - Just k -> withObjectLoc k (addlist f) + Just k -> withObjectLoc k (addlist (fromRawFilePath f)) liftIO $ hClose h serverkey <- uftpKey diff --git a/Command/P2P.hs b/Command/P2P.hs index ae86f59076..e1896c7a3f 100644 --- a/Command/P2P.hs +++ b/Command/P2P.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Command.P2P where import Command diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 8c366ec14b..ad39953e3c 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -53,11 +53,11 @@ seek ps = lockPreCommitHook $ do (removeViewMetaData v) addViewMetaData :: View -> ViewedFile -> Key -> CommandStart -addViewMetaData v f k = starting "metadata" (mkActionItem (k, f)) $ +addViewMetaData v f k = starting "metadata" (mkActionItem (k, toRawFilePath f)) $ next $ changeMetaData k $ fromView v f removeViewMetaData :: View -> ViewedFile -> Key -> CommandStart -removeViewMetaData v f k = starting "metadata" (mkActionItem (k, f)) $ +removeViewMetaData v f k = starting "metadata" (mkActionItem (k, toRawFilePath f)) $ next $ changeMetaData k $ unsetMetaData $ fromView v f changeMetaData :: Key -> MetaData -> CommandCleanup diff --git a/Command/ReKey.hs b/Command/ReKey.hs index 6670298ae5..6e0678c2cc 100644 --- a/Command/ReKey.hs +++ b/Command/ReKey.hs @@ -19,6 +19,7 @@ import Git.FilePath import qualified Database.Keys import Annex.InodeSentinal import Utility.InodeCache +import qualified Utility.RawFilePath as R cmd :: Command cmd = command "rekey" SectionPlumbing @@ -38,13 +39,13 @@ optParser desc = ReKeyOptions -- Split on the last space, since a FilePath can contain whitespace, -- but a Key very rarely does. -batchParser :: String -> Either String (FilePath, Key) +batchParser :: String -> Either String (RawFilePath, Key) batchParser s = case separate (== ' ') (reverse s) of (rk, rf) | null rk || null rf -> Left "Expected: \"file key\"" | otherwise -> case deserializeKey (reverse rk) of Nothing -> Left "bad key" - Just k -> Right (reverse rf, k) + Just k -> Right (toRawFilePath (reverse rf), k) seek :: ReKeyOptions -> CommandSeek seek o = case batchOption o of @@ -52,9 +53,9 @@ seek o = case batchOption o of NoBatch -> withPairs (commandAction . start . parsekey) (reKeyThese o) where parsekey (file, skey) = - (file, fromMaybe (giveup "bad key") (deserializeKey skey)) + (toRawFilePath file, fromMaybe (giveup "bad key") (deserializeKey skey)) -start :: (FilePath, Key) -> CommandStart +start :: (RawFilePath, Key) -> CommandStart start (file, newkey) = ifAnnexed file go stop where go oldkey @@ -62,19 +63,19 @@ start (file, newkey) = ifAnnexed file go stop | otherwise = starting "rekey" (ActionItemWorkTreeFile file) $ perform file oldkey newkey -perform :: FilePath -> Key -> Key -> CommandPerform +perform :: RawFilePath -> Key -> Key -> CommandPerform perform file oldkey newkey = do ifM (inAnnex oldkey) ( unlessM (linkKey file oldkey newkey) $ giveup "failed creating link from old to new key" , unlessM (Annex.getState Annex.force) $ - giveup $ file ++ " is not available (use --force to override)" + giveup $ fromRawFilePath file ++ " is not available (use --force to override)" ) next $ cleanup file oldkey newkey {- Make a hard link to the old key content (when supported), - to avoid wasting disk space. -} -linkKey :: FilePath -> Key -> Key -> Annex Bool +linkKey :: RawFilePath -> Key -> Key -> Annex Bool linkKey file oldkey newkey = ifM (isJust <$> isAnnexLink file) {- If the object file is already hardlinked to elsewhere, a hard - link won't be made by getViaTmpFromDisk, but a copy instead. @@ -89,40 +90,40 @@ linkKey file oldkey newkey = ifM (isJust <$> isAnnexLink file) - it's hard linked to the old key, that link must be broken. -} oldobj <- calcRepo (gitAnnexLocation oldkey) v <- tryNonAsync $ do - st <- liftIO $ getFileStatus file + st <- liftIO $ R.getFileStatus file when (linkCount st > 1) $ do freezeContent oldobj - replaceFile file $ \tmp -> do + replaceFile (fromRawFilePath file) $ \tmp -> do unlessM (checkedCopyFile oldkey oldobj tmp Nothing) $ error "can't lock old key" thawContent tmp - ic <- withTSDelta (liftIO . genInodeCache file) + ic <- withTSDelta (liftIO . genInodeCache' file) case v of Left e -> do warning (show e) return False Right () -> do - r <- linkToAnnex newkey file ic + r <- linkToAnnex newkey (fromRawFilePath file) ic return $ case r of LinkAnnexFailed -> False LinkAnnexOk -> True LinkAnnexNoop -> True ) -cleanup :: FilePath -> Key -> Key -> CommandCleanup +cleanup :: RawFilePath -> Key -> Key -> CommandCleanup cleanup file oldkey newkey = do ifM (isJust <$> isAnnexLink file) ( do -- Update symlink to use the new key. - liftIO $ removeFile file - addLink file newkey Nothing + liftIO $ removeFile (fromRawFilePath file) + addLink (fromRawFilePath file) newkey Nothing , do - mode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus file + mode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus file liftIO $ whenM (isJust <$> isPointerFile file) $ writePointerFile file newkey mode stagePointerFile file mode =<< hashPointerFile newkey Database.Keys.removeAssociatedFile oldkey - =<< inRepo (toTopFilePath file) + =<< inRepo (toTopFilePath (fromRawFilePath file)) ) whenM (inAnnex newkey) $ logStatus newkey InfoPresent diff --git a/Command/RegisterUrl.hs b/Command/RegisterUrl.hs index d13889a9b7..7fdd2836f6 100644 --- a/Command/RegisterUrl.hs +++ b/Command/RegisterUrl.hs @@ -11,7 +11,7 @@ module Command.RegisterUrl where import Command import Logs.Web -import Command.FromKey (mkKey) +import Command.FromKey (keyOpt) import qualified Remote cmd :: Command @@ -41,7 +41,7 @@ seek o = case (batchOption o, keyUrlPairs o) of start :: [String] -> CommandStart start (keyname:url:[]) = starting "registerurl" (ActionItemOther (Just url)) $ do - let key = mkKey keyname + let key = keyOpt keyname perform key url start _ = giveup "specify a key and an url" @@ -55,7 +55,7 @@ massAdd fmt = go True =<< map (separate (== ' ')) <$> batchLines fmt where go status [] = next $ return status go status ((keyname,u):rest) | not (null keyname) && not (null u) = do - let key = mkKey keyname + let key = keyOpt keyname ok <- perform' key u let !status' = status && ok go status' rest diff --git a/Command/Reinject.hs b/Command/Reinject.hs index df975531ce..d33817debf 100644 --- a/Command/Reinject.hs +++ b/Command/Reinject.hs @@ -42,7 +42,7 @@ seek os startSrcDest :: [FilePath] -> CommandStart startSrcDest (src:dest:[]) | src == dest = stop - | otherwise = notAnnexed src $ ifAnnexed dest go stop + | otherwise = notAnnexed src $ ifAnnexed (toRawFilePath dest) go stop where go key = starting "reinject" (ActionItemOther (Just src)) $ ifM (verifyKeyContent RetrievalAllKeysSecure DefaultVerify UnVerified key src) @@ -65,7 +65,7 @@ startKnown src = notAnnexed src $ ) notAnnexed :: FilePath -> CommandStart -> CommandStart -notAnnexed src = ifAnnexed src $ +notAnnexed src = ifAnnexed (toRawFilePath src) $ giveup $ "cannot used annexed file as src: " ++ src perform :: FilePath -> Key -> CommandPerform diff --git a/Command/RmUrl.hs b/Command/RmUrl.hs index 3d8d8ca2df..04c3165ce5 100644 --- a/Command/RmUrl.hs +++ b/Command/RmUrl.hs @@ -42,9 +42,11 @@ batchParser s = case separate (== ' ') (reverse s) of | otherwise -> Right (reverse rf, reverse ru) start :: (FilePath, URLString) -> CommandStart -start (file, url) = flip whenAnnexed file $ \_ key -> - starting "rmurl" (mkActionItem (key, AssociatedFile (Just file))) $ +start (file, url) = flip whenAnnexed file' $ \_ key -> + starting "rmurl" (mkActionItem (key, AssociatedFile (Just file'))) $ next $ cleanup url key + where + file' = toRawFilePath file cleanup :: String -> Key -> CommandCleanup cleanup url key = do diff --git a/Command/SendKey.hs b/Command/SendKey.hs index 36959a8ae4..aa7aa092f7 100644 --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -46,10 +46,11 @@ start key = do fieldTransfer :: Direction -> Key -> (MeterUpdate -> Annex Bool) -> CommandStart fieldTransfer direction key a = do liftIO $ debugM "fieldTransfer" "transfer start" - afile <- AssociatedFile <$> Fields.getField Fields.associatedFile + afile <- AssociatedFile . (fmap toRawFilePath) + <$> Fields.getField Fields.associatedFile ok <- maybe (a $ const noop) -- Using noRetry here because we're the sender. - (\u -> runner (Transfer direction (toUUID u) key) afile noRetry a) + (\u -> runner (Transfer direction (toUUID u) (fromKey id key)) afile noRetry a) =<< Fields.getField Fields.remoteUUID liftIO $ debugM "fieldTransfer" "transfer done" liftIO $ exitBool ok diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 1cf7fb14e2..703679494d 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -21,11 +21,11 @@ seek = withWords (commandAction . start) start :: [String] -> CommandStart start (keyname:file:[]) = starting "setkey" (ActionItemOther (Just file)) $ - perform file (mkKey keyname) + perform file (keyOpt keyname) start _ = giveup "specify a key and a content file" -mkKey :: String -> Key -mkKey = fromMaybe (giveup "bad key") . deserializeKey +keyOpt :: String -> Key +keyOpt = fromMaybe (giveup "bad key") . deserializeKey perform :: FilePath -> Key -> CommandPerform perform file key = do diff --git a/Command/Smudge.hs b/Command/Smudge.hs index 90c55e3cc1..30e2f2d168 100644 --- a/Command/Smudge.hs +++ b/Command/Smudge.hs @@ -86,9 +86,9 @@ clean file = do ( liftIO $ L.hPut stdout b , case parseLinkTargetOrPointerLazy b of Just k -> do - getMoveRaceRecovery k file + getMoveRaceRecovery k (toRawFilePath file) liftIO $ L.hPut stdout b - Nothing -> go b =<< catKeyFile file + Nothing -> go b =<< catKeyFile (toRawFilePath file) ) stop where @@ -119,7 +119,7 @@ clean file = do -- Look up the backend that was used for this file -- before, so that when git re-cleans a file its -- backend does not change. - let oldbackend = maybe Nothing (maybeLookupBackendVariety . keyVariety) oldkey + let oldbackend = maybe Nothing (maybeLookupBackendVariety . fromKey keyVariety) oldkey -- Can't restage associated files because git add -- runs this and has the index locked. let norestage = Restage False @@ -187,10 +187,10 @@ emitPointer = S.putStr . formatPointer -- This also handles the case where a copy of a pointer file is made, -- then git-annex gets the content, and later git add is run on -- the pointer copy. It will then be populated with the content. -getMoveRaceRecovery :: Key -> FilePath -> Annex () +getMoveRaceRecovery :: Key -> RawFilePath -> Annex () getMoveRaceRecovery k file = void $ tryNonAsync $ whenM (inAnnex k) $ do - obj <- calcRepo (gitAnnexLocation k) + obj <- toRawFilePath <$> calcRepo (gitAnnexLocation k) -- Cannot restage because git add is running and has -- the index locked. populatePointerFile (Restage False) k obj file >>= \case @@ -204,11 +204,11 @@ update = do updateSmudged :: Restage -> Annex () updateSmudged restage = streamSmudged $ \k topf -> do - f <- fromRepo $ fromTopFilePath topf + f <- toRawFilePath <$> fromRepo (fromTopFilePath topf) whenM (inAnnex k) $ do - obj <- calcRepo (gitAnnexLocation k) + obj <- toRawFilePath <$> calcRepo (gitAnnexLocation k) unlessM (isJust <$> populatePointerFile restage k obj f) $ liftIO (isPointerFile f) >>= \case Just k' | k' == k -> toplevelWarning False $ - "unable to populate worktree file " ++ f + "unable to populate worktree file " ++ fromRawFilePath f _ -> noop diff --git a/Command/Sync.hs b/Command/Sync.hs index d35986c0f3..880b1dbbc0 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -7,6 +7,7 @@ -} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE OverloadedStrings #-} module Command.Sync ( cmd, diff --git a/Command/TestRemote.hs b/Command/TestRemote.hs index 0e7403f19b..292697a781 100644 --- a/Command/TestRemote.hs +++ b/Command/TestRemote.hs @@ -107,14 +107,14 @@ perform rs unavailrs exportr ks = do next $ cleanup rs ks ok where desc r' k = intercalate "; " $ map unwords - [ [ "key size", show (keySize k) ] + [ [ "key size", show (fromKey keySize k) ] , [ show (getChunkConfig (Remote.config r')) ] , ["encryption", fromMaybe "none" (M.lookup "encryption" (Remote.config r'))] ] descexport k1 k2 = intercalate "; " $ map unwords [ [ "exporttree=yes" ] - , [ "key1 size", show (keySize k1) ] - , [ "key2 size", show (keySize k2) ] + , [ "key1 size", show (fromKey keySize k1) ] + , [ "key2 size", show (fromKey keySize k2) ] ] adjustChunkSize :: Remote -> Int -> Annex (Maybe Remote) @@ -199,7 +199,7 @@ test st r k = catMaybes Annex.eval st (Annex.setOutput QuietOutput >> a) @? "failed" present b = check ("present " ++ show b) $ (== Right b) <$> Remote.hasKey r k - fsck = case maybeLookupBackendVariety (keyVariety k) of + fsck = case maybeLookupBackendVariety (fromKey keyVariety k) of Nothing -> return True Just b -> case Backend.verifyKeyContent b of Nothing -> return True @@ -236,7 +236,7 @@ testExportTree st (Just _) ea k1 k2 = ] where testexportdirectory = "testremote-export" - testexportlocation = mkExportLocation (testexportdirectory "location") + testexportlocation = mkExportLocation (toRawFilePath (testexportdirectory "location")) check desc a = testCase desc $ Annex.eval st (Annex.setOutput QuietOutput >> a) @? "failed" storeexport k = do @@ -252,7 +252,7 @@ testExportTree st (Just _) ea k1 k2 = removeexport k = Remote.removeExport ea k testexportlocation removeexportdirectory = case Remote.removeExportDirectory ea of Nothing -> return True - Just a -> a (mkExportDirectory testexportdirectory) + Just a -> a (mkExportDirectory (toRawFilePath testexportdirectory)) testUnavailable :: Annex.AnnexState -> Remote -> Key -> [TestTree] testUnavailable st r k = @@ -326,7 +326,7 @@ randKey sz = withTmpFile "randkey" $ \f h -> do return k getReadonlyKey :: Remote -> FilePath -> Annex Key -getReadonlyKey r f = lookupFile f >>= \case +getReadonlyKey r f = lookupFile (toRawFilePath f) >>= \case Nothing -> giveup $ f ++ " is not an annexed file" Just k -> do unlessM (inAnnex k) $ diff --git a/Command/TransferInfo.hs b/Command/TransferInfo.hs index abcf8c6b14..402f1ef8ec 100644 --- a/Command/TransferInfo.hs +++ b/Command/TransferInfo.hs @@ -41,13 +41,14 @@ start (k:[]) = do case deserializeKey k of Nothing -> error "bad key" (Just key) -> whenM (inAnnex key) $ do - afile <- AssociatedFile <$> Fields.getField Fields.associatedFile + afile <- AssociatedFile . (fmap toRawFilePath) + <$> Fields.getField Fields.associatedFile u <- maybe (error "missing remoteuuid") toUUID <$> Fields.getField Fields.remoteUUID let t = Transfer { transferDirection = Upload , transferUUID = u - , transferKey = key + , transferKeyData = fromKey id key } tinfo <- liftIO $ startTransferInfo afile (update, tfile, createtfile, _) <- mkProgressUpdater t tinfo diff --git a/Command/TransferKeys.hs b/Command/TransferKeys.hs index 2d3cbaef49..9fa233fb90 100644 --- a/Command/TransferKeys.hs +++ b/Command/TransferKeys.hs @@ -116,10 +116,10 @@ instance TCSerialized Direction where deserialize _ = Nothing instance TCSerialized AssociatedFile where - serialize (AssociatedFile (Just f)) = f + serialize (AssociatedFile (Just f)) = fromRawFilePath f serialize (AssociatedFile Nothing) = "" deserialize "" = Just (AssociatedFile Nothing) - deserialize f = Just (AssociatedFile (Just f)) + deserialize f = Just (AssociatedFile (Just (toRawFilePath f))) instance TCSerialized RemoteName where serialize n = n diff --git a/Command/Unannex.hs b/Command/Unannex.hs index cbb8cb5214..7610b56176 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -25,10 +25,10 @@ cmd = withGlobalOptions [annexedMatchingOptions] $ seek :: CmdParams -> CommandSeek seek ps = (withFilesInGit $ commandAction . whenAnnexed start) =<< workTreeItems ps -start :: FilePath -> Key -> CommandStart +start :: RawFilePath -> Key -> CommandStart start file key = stopUnless (inAnnex key) $ starting "unannex" (mkActionItem (key, file)) $ - perform file key + perform (fromRawFilePath file) key perform :: FilePath -> Key -> CommandPerform perform file key = do diff --git a/Command/Undo.hs b/Command/Undo.hs index 8a1939394e..fd4b3b263d 100644 --- a/Command/Undo.hs +++ b/Command/Undo.hs @@ -27,9 +27,9 @@ seek :: CmdParams -> CommandSeek seek ps = do -- Safety first; avoid any undo that would touch files that are not -- in the index. - (fs, cleanup) <- inRepo $ LsFiles.notInRepo False ps + (fs, cleanup) <- inRepo $ LsFiles.notInRepo False (map toRawFilePath ps) unless (null fs) $ - giveup $ "Cannot undo changes to files that are not checked into git: " ++ unwords fs + giveup $ "Cannot undo changes to files that are not checked into git: " ++ unwords (map fromRawFilePath fs) void $ liftIO $ cleanup -- Committing staged changes before undo allows later diff --git a/Command/Uninit.hs b/Command/Uninit.hs index 3f2a45c10f..1e4ebdf2dc 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -34,14 +34,14 @@ check = do whenM ((/=) <$> liftIO (absPath top) <*> liftIO (absPath currdir)) $ giveup "can only run uninit from the top of the git repository" where - current_branch = Git.Ref . Prelude.head . lines <$> revhead + current_branch = Git.Ref . Prelude.head . lines . decodeBS' <$> revhead revhead = inRepo $ Git.Command.pipeReadStrict [Param "rev-parse", Param "--abbrev-ref", Param "HEAD"] seek :: CmdParams -> CommandSeek seek ps = do l <- workTreeItems ps - withFilesNotInGit False (commandAction . whenAnnexed startCheckIncomplete) l + withFilesNotInGit False (commandAction . whenAnnexed (startCheckIncomplete . fromRawFilePath)) l Annex.changeState $ \s -> s { Annex.fast = True } withFilesInGit (commandAction . whenAnnexed Command.Unannex.start) l finish diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 2fc605c6de..443ac46e3c 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -14,6 +14,7 @@ import Annex.Link import Annex.ReplaceFile import Git.FilePath import qualified Database.Keys +import qualified Utility.RawFilePath as R cmd :: Command cmd = mkcmd "unlock" "unlock files for modification" @@ -31,17 +32,17 @@ seek ps = withFilesInGit (commandAction . whenAnnexed start) =<< workTreeItems p {- Before v6, the unlock subcommand replaces the symlink with a copy of - the file's content. In v6 and above, it converts the file from a symlink - to a pointer. -} -start :: FilePath -> Key -> CommandStart +start :: RawFilePath -> Key -> CommandStart start file key = ifM (isJust <$> isAnnexLink file) ( starting "unlock" (mkActionItem (key, AssociatedFile (Just file))) $ perform file key , stop ) -perform :: FilePath -> Key -> CommandPerform +perform :: RawFilePath -> Key -> CommandPerform perform dest key = do - destmode <- liftIO $ catchMaybeIO $ fileMode <$> getFileStatus dest - replaceFile dest $ \tmp -> + destmode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus dest + replaceFile (fromRawFilePath dest) $ \tmp -> ifM (inAnnex key) ( do r <- linkFromAnnex key tmp destmode @@ -49,12 +50,12 @@ perform dest key = do LinkAnnexOk -> return () LinkAnnexNoop -> return () LinkAnnexFailed -> error "unlock failed" - , liftIO $ writePointerFile tmp key destmode + , liftIO $ writePointerFile (toRawFilePath tmp) key destmode ) next $ cleanup dest key destmode -cleanup :: FilePath -> Key -> Maybe FileMode -> CommandCleanup +cleanup :: RawFilePath -> Key -> Maybe FileMode -> CommandCleanup cleanup dest key destmode = do stagePointerFile dest destmode =<< hashPointerFile key - Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath dest) + Database.Keys.addAssociatedFile key =<< inRepo (toTopFilePath (fromRawFilePath dest)) return True diff --git a/Command/Unused.hs b/Command/Unused.hs index 95f953395d..345111ec81 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -192,10 +192,10 @@ withKeysReferencedM a = withKeysReferenced' Nothing () calla calla k _ _ = a k {- Folds an action over keys and files referenced in a particular directory. -} -withKeysFilesReferencedIn :: FilePath -> v -> (Key -> FilePath -> v -> Annex v) -> Annex v +withKeysFilesReferencedIn :: FilePath -> v -> (Key -> RawFilePath -> v -> Annex v) -> Annex v withKeysFilesReferencedIn = withKeysReferenced' . Just -withKeysReferenced' :: Maybe FilePath -> v -> (Key -> FilePath -> v -> Annex v) -> Annex v +withKeysReferenced' :: Maybe FilePath -> v -> (Key -> RawFilePath -> v -> Annex v) -> Annex v withKeysReferenced' mdir initial a = do (files, clean) <- getfiles r <- go initial files @@ -207,9 +207,9 @@ withKeysReferenced' mdir initial a = do ( return ([], return True) , do top <- fromRepo Git.repoPath - inRepo $ LsFiles.allFiles [top] + inRepo $ LsFiles.allFiles [toRawFilePath top] ) - Just dir -> inRepo $ LsFiles.inRepo [dir] + Just dir -> inRepo $ LsFiles.inRepo [toRawFilePath dir] go v [] = return v go v (f:fs) = do mk <- lookupFile f @@ -221,7 +221,8 @@ withKeysReferenced' mdir initial a = do withKeysReferencedDiffGitRefs :: RefSpec -> (Key -> Annex ()) -> Annex () withKeysReferencedDiffGitRefs refspec a = do - rs <- relevantrefs <$> inRepo (Git.Command.pipeReadStrict [Param "show-ref"]) + rs <- relevantrefs . decodeBS' + <$> inRepo (Git.Command.pipeReadStrict [Param "show-ref"]) shaHead <- maybe (return Nothing) (inRepo . Git.Ref.sha) =<< inRepo Git.Branch.currentUnsafe let haveHead = any (\(shaRef, _) -> Just shaRef == shaHead) rs diff --git a/Command/Vicfg.hs b/Command/Vicfg.hs index 4e842f4ea7..70bccac542 100644 --- a/Command/Vicfg.hs +++ b/Command/Vicfg.hs @@ -31,6 +31,7 @@ import Types.StandardGroups import Types.ScheduledActivity import Types.NumCopies import Remote +import Git.Types (ConfigKey(..), fromConfigKey, fromConfigValue) cmd :: Command cmd = command "vicfg" SectionSetup "edit configuration in git-annex branch" @@ -70,7 +71,7 @@ data Cfg = Cfg , cfgRequiredContentMap :: M.Map UUID PreferredContentExpression , cfgGroupPreferredContentMap :: M.Map Group PreferredContentExpression , cfgScheduleMap :: M.Map UUID [ScheduledActivity] - , cfgGlobalConfigs :: M.Map ConfigName ConfigValue + , cfgGlobalConfigs :: M.Map ConfigKey ConfigValue , cfgNumCopies :: Maybe NumCopies } @@ -218,9 +219,9 @@ genCfg cfg descs = unlines $ intercalate [""] [ com "Other global configuration" ] (\(s, g) -> gline g s) - (\g -> gline g "") + (\g -> gline g mempty) where - gline g val = [ unwords ["config", g, "=", val] ] + gline k v = [ unwords ["config", fromConfigKey k, "=", fromConfigValue v] ] line setting u val = [ com $ "(for " ++ fromUUIDDesc (fromMaybe mempty (M.lookup u descs)) ++ ")" @@ -308,7 +309,7 @@ parseCfg defcfg = go [] defcfg . lines let m = M.insert u l (cfgScheduleMap cfg) in Right $ cfg { cfgScheduleMap = m } | setting == "config" = - let m = M.insert f val (cfgGlobalConfigs cfg) + let m = M.insert (ConfigKey (encodeBS' f)) (ConfigValue (encodeBS' val)) (cfgGlobalConfigs cfg) in Right $ cfg { cfgGlobalConfigs = m } | setting == "numcopies" = case readish val of Nothing -> Left "parse error (expected an integer)" diff --git a/Command/View.hs b/Command/View.hs index 88b9a4866d..58e7a8c8b0 100644 --- a/Command/View.hs +++ b/Command/View.hs @@ -101,7 +101,8 @@ checkoutViewBranch view mkbranch = do - removed.) -} top <- liftIO . absPath =<< fromRepo Git.repoPath (l, cleanup) <- inRepo $ - LsFiles.notInRepoIncludingEmptyDirectories False [top] + LsFiles.notInRepoIncludingEmptyDirectories False + [toRawFilePath top] forM_ l (removeemptydir top) liftIO $ void cleanup unlessM (liftIO $ doesDirectoryExist here) $ do @@ -109,7 +110,7 @@ checkoutViewBranch view mkbranch = do return ok where removeemptydir top d = do - p <- inRepo $ toTopFilePath d + p <- inRepo $ toTopFilePath $ fromRawFilePath d liftIO $ tryIO $ removeDirectory (top getTopFilePath p) cwdmissing top = unlines [ "This view does not include the subdirectory you are currently in." diff --git a/Command/WebApp.hs b/Command/WebApp.hs index 3f8002d68b..95bd8af9d4 100644 --- a/Command/WebApp.hs +++ b/Command/WebApp.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} module Command.WebApp where @@ -22,6 +23,7 @@ import Utility.Daemon (checkDaemon) import Utility.UserInfo import Annex.Init import qualified Git +import Git.Types (fromConfigValue) import qualified Git.Config import qualified Git.CurrentRepo import qualified Annex @@ -229,7 +231,7 @@ openBrowser' mcmd htmlshim realurl outh errh = {- web.browser is a generic git config setting for a web browser program -} webBrowser :: Git.Repo -> Maybe FilePath -webBrowser = Git.Config.getMaybe "web.browser" +webBrowser = fmap fromConfigValue <$> Git.Config.getMaybe "web.browser" fileUrl :: FilePath -> String fileUrl file = "file://" ++ file diff --git a/Command/Whereis.hs b/Command/Whereis.hs index c5010473c4..1946cfbdf6 100644 --- a/Command/Whereis.hs +++ b/Command/Whereis.hs @@ -40,14 +40,14 @@ seek o = do m <- remoteMap id let go = whenAnnexed $ start m case batchOption o of - Batch fmt -> batchFilesMatching fmt go + Batch fmt -> batchFilesMatching fmt (go . toRawFilePath) NoBatch -> withKeyOptions (keyOptions o) False (commandAction . startKeys m) (withFilesInGit (commandAction . go)) =<< workTreeItems (whereisFiles o) -start :: M.Map UUID Remote -> FilePath -> Key -> CommandStart +start :: M.Map UUID Remote -> RawFilePath -> Key -> CommandStart start remotemap file key = startKeys remotemap (key, mkActionItem (key, afile)) where afile = AssociatedFile (Just file) diff --git a/Config.hs b/Config.hs index cbd82e50f7..68c657aa47 100644 --- a/Config.hs +++ b/Config.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-} +{-# LANGUAGE OverloadedStrings #-} module Config where @@ -22,25 +23,26 @@ import qualified Types.Remote as Remote import qualified Annex.SpecialRemote.Config as SpecialRemote import qualified Data.Map as M +import qualified Data.ByteString as S -type UnqualifiedConfigKey = String -data ConfigKey = ConfigKey String - -instance Show ConfigKey where - show (ConfigKey s) = s +type UnqualifiedConfigKey = S.ByteString {- Looks up a setting in git config. This is not as efficient as using the - GitConfig type. -} -getConfig :: ConfigKey -> String -> Annex String -getConfig (ConfigKey key) d = fromRepo $ Git.Config.get key d +getConfig :: ConfigKey -> ConfigValue -> Annex ConfigValue +getConfig key d = fromRepo $ Git.Config.get key d -getConfigMaybe :: ConfigKey -> Annex (Maybe String) -getConfigMaybe (ConfigKey key) = fromRepo $ Git.Config.getMaybe key +getConfigMaybe :: ConfigKey -> Annex (Maybe ConfigValue) +getConfigMaybe key = fromRepo $ Git.Config.getMaybe key {- Changes a git config setting in both internal state and .git/config -} setConfig :: ConfigKey -> String -> Annex () setConfig (ConfigKey key) value = do - inRepo $ Git.Command.run [Param "config", Param key, Param value] + inRepo $ Git.Command.run + [ Param "config" + , Param (decodeBS' key) + , Param value + ] reloadConfig reloadConfig :: Annex () @@ -48,7 +50,7 @@ reloadConfig = Annex.changeGitRepo =<< inRepo Git.Config.reRead {- Unsets a git config setting. (Leaves it in state.) -} unsetConfig :: ConfigKey -> Annex () -unsetConfig (ConfigKey key) = void $ inRepo $ Git.Config.unset key +unsetConfig key = void $ inRepo $ Git.Config.unset key class RemoteNameable r where getRemoteName :: r -> RemoteName @@ -68,11 +70,11 @@ instance RemoteNameable Remote.RemoteConfig where {- A per-remote config setting in git config. -} remoteConfig :: RemoteNameable r => r -> UnqualifiedConfigKey -> ConfigKey remoteConfig r key = ConfigKey $ - "remote." ++ getRemoteName r ++ ".annex-" ++ key + "remote." <> encodeBS' (getRemoteName r) <> ".annex-" <> key {- A global annex setting in git config. -} annexConfig :: UnqualifiedConfigKey -> ConfigKey -annexConfig key = ConfigKey $ "annex." ++ key +annexConfig key = ConfigKey ("annex." <> key) {- Calculates cost for a remote. Either the specific default, or as configured - by remote..annex-cost, or if remote..annex-cost-command diff --git a/Config/Smudge.hs b/Config/Smudge.hs index b81db28139..68e39c4b8d 100644 --- a/Config/Smudge.hs +++ b/Config/Smudge.hs @@ -5,12 +5,15 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Config.Smudge where import Annex.Common import qualified Annex import qualified Git import qualified Git.Command +import Git.Types import Config configureSmudgeFilter :: Annex () diff --git a/Crypto.hs b/Crypto.hs index 8dd4e3d4f1..08aef47cd5 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -161,7 +161,7 @@ type EncKey = Key -> Key - reversable, nor does it need to be the same type of encryption used - on content. It does need to be repeatable. -} encryptKey :: Mac -> Cipher -> EncKey -encryptKey mac c k = stubKey +encryptKey mac c k = mkKey $ \d -> d { keyName = encodeBS (macWithCipher mac c (serializeKey k)) , keyVariety = OtherKey $ encryptedBackendNamePrefix <> encodeBS (showMac mac) @@ -171,7 +171,7 @@ encryptedBackendNamePrefix :: S.ByteString encryptedBackendNamePrefix = "GPG" isEncKey :: Key -> Bool -isEncKey k = case keyVariety k of +isEncKey k = case fromKey keyVariety k of OtherKey s -> encryptedBackendNamePrefix `S.isPrefixOf` s _ -> False diff --git a/Database/Benchmark.hs b/Database/Benchmark.hs index 7f7aa3d11e..6683b8446e 100644 --- a/Database/Benchmark.hs +++ b/Database/Benchmark.hs @@ -86,7 +86,7 @@ populateAssociatedFiles h num = do H.flushDbQueue h keyN :: Integer -> Key -keyN n = stubKey +keyN n = mkKey $ \k -> k { keyName = B8.pack $ "key" ++ show n , keyVariety = OtherKey "BENCH" } diff --git a/Database/ContentIdentifier.hs b/Database/ContentIdentifier.hs index 522b80973e..b821578283 100644 --- a/Database/ContentIdentifier.hs +++ b/Database/ContentIdentifier.hs @@ -147,7 +147,7 @@ updateFromLog db (oldtree, currtree) = do recordAnnexBranchTree db currtree flushDbQueue db where - go ti = case extLogFileKey remoteContentIdentifierExt (getTopFilePath (DiffTree.file ti)) of + go ti = case extLogFileKey remoteContentIdentifierExt (toRawFilePath (getTopFilePath (DiffTree.file ti))) of Nothing -> return () Just k -> do l <- Log.getContentIdentifiers k diff --git a/Database/Export.hs b/Database/Export.hs index 2d403c52c0..241e5d0774 100644 --- a/Database/Export.hs +++ b/Database/Export.hs @@ -130,26 +130,26 @@ addExportedLocation :: ExportHandle -> Key -> ExportLocation -> IO () addExportedLocation h k el = queueDb h $ do void $ insertUnique $ Exported k ef let edirs = map - (\ed -> ExportedDirectory (toSFilePath (fromExportDirectory ed)) ef) + (\ed -> ExportedDirectory (toSFilePath (fromRawFilePath (fromExportDirectory ed))) ef) (exportDirectories el) putMany edirs where - ef = toSFilePath (fromExportLocation el) + ef = SFilePath (fromExportLocation el) removeExportedLocation :: ExportHandle -> Key -> ExportLocation -> IO () removeExportedLocation h k el = queueDb h $ do deleteWhere [ExportedKey ==. k, ExportedFile ==. ef] - let subdirs = map (toSFilePath . fromExportDirectory) + let subdirs = map (SFilePath . fromExportDirectory) (exportDirectories el) deleteWhere [ExportedDirectoryFile ==. ef, ExportedDirectorySubdir <-. subdirs] where - ef = toSFilePath (fromExportLocation el) + ef = SFilePath (fromExportLocation el) {- Note that this does not see recently queued changes. -} getExportedLocation :: ExportHandle -> Key -> IO [ExportLocation] getExportedLocation (ExportHandle h _) k = H.queryDbQueue h $ do l <- selectList [ExportedKey ==. k] [] - return $ map (mkExportLocation . fromSFilePath . exportedFile . entityVal) l + return $ map (mkExportLocation . (\(SFilePath f) -> f) . exportedFile . entityVal) l {- Note that this does not see recently queued changes. -} isExportDirectoryEmpty :: ExportHandle -> ExportDirectory -> IO Bool @@ -157,13 +157,13 @@ isExportDirectoryEmpty (ExportHandle h _) d = H.queryDbQueue h $ do l <- selectList [ExportedDirectorySubdir ==. ed] [] return $ null l where - ed = toSFilePath $ fromExportDirectory d + ed = SFilePath $ fromExportDirectory d {- Get locations in the export that might contain a key. -} getExportTree :: ExportHandle -> Key -> IO [ExportLocation] getExportTree (ExportHandle h _) k = H.queryDbQueue h $ do l <- selectList [ExportTreeKey ==. k] [] - return $ map (mkExportLocation . fromSFilePath . exportTreeFile . entityVal) l + return $ map (mkExportLocation . (\(SFilePath f) -> f) . exportTreeFile . entityVal) l {- Get keys that might be currently exported to a location. - @@ -174,19 +174,19 @@ getExportTreeKey (ExportHandle h _) el = H.queryDbQueue h $ do map (exportTreeKey . entityVal) <$> selectList [ExportTreeFile ==. ef] [] where - ef = toSFilePath (fromExportLocation el) + ef = SFilePath (fromExportLocation el) addExportTree :: ExportHandle -> Key -> ExportLocation -> IO () addExportTree h k loc = queueDb h $ void $ insertUnique $ ExportTree k ef where - ef = toSFilePath (fromExportLocation loc) + ef = SFilePath (fromExportLocation loc) removeExportTree :: ExportHandle -> Key -> ExportLocation -> IO () removeExportTree h k loc = queueDb h $ deleteWhere [ExportTreeKey ==. k, ExportTreeFile ==. ef] where - ef = toSFilePath (fromExportLocation loc) + ef = SFilePath (fromExportLocation loc) -- An action that is passed the old and new values that were exported, -- and updates state. @@ -211,7 +211,7 @@ mkExportDiffUpdater removeold addnew h srcek dstek i = do Nothing -> return () Just k -> liftIO $ addnew h (asKey k) loc where - loc = mkExportLocation $ getTopFilePath $ Git.DiffTree.file i + loc = mkExportLocation $ toRawFilePath $ getTopFilePath $ Git.DiffTree.file i runExportDiffUpdater :: ExportDiffUpdater -> ExportHandle -> Sha -> Sha -> Annex () runExportDiffUpdater updater h old new = do diff --git a/Database/Keys.hs b/Database/Keys.hs index f1dfcaf879..4f0da81fc9 100644 --- a/Database/Keys.hs +++ b/Database/Keys.hs @@ -235,7 +235,7 @@ reconcileStaged qh = do where go cur indexcache = do (l, cleanup) <- inRepo $ pipeNullSplit diff - changed <- procdiff l False + changed <- procdiff (map decodeBL' l) False void $ liftIO cleanup -- Flush database changes immediately -- so other processes can see them. @@ -262,7 +262,8 @@ reconcileStaged qh = do -- perfect. A file could start with this and not be a -- pointer file. And a pointer file that is replaced with -- a non-pointer file will match this. - , Param $ "-G^" ++ toInternalGitPath (pathSeparator:objectDir) + , Param $ "-G^" ++ fromRawFilePath (toInternalGitPath $ + toRawFilePath (pathSeparator:objectDir)) -- Don't include files that were deleted, because this only -- wants to update information for files that are present -- in the index. @@ -277,7 +278,7 @@ reconcileStaged qh = do procdiff (info:file:rest) changed = case words info of ((':':_srcmode):dstmode:_srcsha:dstsha:_change:[]) -- Only want files, not symlinks - | dstmode /= fmtTreeItemType TreeSymlink -> do + | dstmode /= decodeBS' (fmtTreeItemType TreeSymlink) -> do maybe noop (reconcile (asTopFilePath file)) =<< catKey (Ref dstsha) procdiff rest True @@ -292,11 +293,11 @@ reconcileStaged qh = do caches <- liftIO $ SQL.getInodeCaches key (SQL.ReadHandle qh) keyloc <- calcRepo (gitAnnexLocation key) keypopulated <- sameInodeCache keyloc caches - p <- fromRepo $ fromTopFilePath file - filepopulated <- sameInodeCache p caches + p <- fromRepo $ toRawFilePath . fromTopFilePath file + filepopulated <- sameInodeCache (fromRawFilePath p) caches case (keypopulated, filepopulated) of (True, False) -> - populatePointerFile (Restage True) key keyloc p >>= \case + populatePointerFile (Restage True) key (toRawFilePath keyloc) p >>= \case Nothing -> return () Just ic -> liftIO $ SQL.addInodeCaches key [ic] (SQL.WriteHandle qh) diff --git a/Database/Types.hs b/Database/Types.hs index 66acc4dbf4..9f41ebc551 100644 --- a/Database/Types.hs +++ b/Database/Types.hs @@ -18,7 +18,6 @@ module Database.Types ( import Database.Persist.Class hiding (Key) import Database.Persist.Sql hiding (Key) import qualified Data.ByteString as S -import qualified Data.ByteString.Lazy as L import qualified Data.Text as T import qualified Data.Attoparsec.ByteString as A import System.PosixCompat.Types @@ -35,7 +34,7 @@ import Types.UUID import Types.Import instance PersistField Key where - toPersistValue = toPersistValue . L.toStrict . serializeKey' + toPersistValue = toPersistValue . serializeKey' fromPersistValue b = fromPersistValue b >>= parse where parse = either (Left . T.pack) Right . A.parseOnly keyParser diff --git a/Git/AutoCorrect.hs b/Git/AutoCorrect.hs index c7b0fd2995..06823a182f 100644 --- a/Git/AutoCorrect.hs +++ b/Git/AutoCorrect.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Git.AutoCorrect where import Common @@ -44,7 +46,7 @@ fuzzymatches input showchoice choices = fst $ unzip $ -} prepare :: String -> (c -> String) -> [c] -> Maybe Repo -> IO () prepare input showmatch matches r = - case readish . Git.Config.get "help.autocorrect" "0" =<< r of + case readish . fromConfigValue . Git.Config.get "help.autocorrect" "0" =<< r of Just n | n == 0 -> list | n < 0 -> warn Nothing diff --git a/Git/Branch.hs b/Git/Branch.hs index 2de6f9e0fd..21103b65f6 100644 --- a/Git/Branch.hs +++ b/Git/Branch.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE BangPatterns #-} +{-# LANGUAGE OverloadedStrings #-} module Git.Branch where @@ -16,6 +17,8 @@ import Git.Command import qualified Git.Config import qualified Git.Ref +import qualified Data.ByteString as B + {- The currently checked out branch. - - In a just initialized git repo before the first commit, @@ -29,19 +32,19 @@ current r = do case v of Nothing -> return Nothing Just branch -> - ifM (null <$> pipeReadStrict [Param "show-ref", Param $ fromRef branch] r) + ifM (B.null <$> pipeReadStrict [Param "show-ref", Param $ fromRef branch] r) ( return Nothing , return v ) {- The current branch, which may not really exist yet. -} currentUnsafe :: Repo -> IO (Maybe Branch) -currentUnsafe r = parse . firstLine +currentUnsafe r = parse . firstLine' <$> pipeReadStrict [Param "symbolic-ref", Param "-q", Param $ fromRef Git.Ref.headRef] r where - parse l - | null l = Nothing - | otherwise = Just $ Git.Ref l + parse b + | B.null b = Nothing + | otherwise = Just $ Git.Ref $ decodeBS b {- Checks if the second branch has any commits not present on the first - branch. -} @@ -53,7 +56,8 @@ changed origbranch newbranch repo where changed' :: Branch -> Branch -> [CommandParam] -> Repo -> IO String -changed' origbranch newbranch extraps repo = pipeReadStrict ps repo +changed' origbranch newbranch extraps repo = + decodeBS <$> pipeReadStrict ps repo where ps = [ Param "log" @@ -72,7 +76,7 @@ changedCommits origbranch newbranch extraps repo = - - This requires there to be a path from the old to the new. -} fastForwardable :: Ref -> Ref -> Repo -> IO Bool -fastForwardable old new repo = not . null <$> +fastForwardable old new repo = not . B.null <$> pipeReadStrict [ Param "log" , Param $ fromRef old ++ ".." ++ fromRef new @@ -132,8 +136,8 @@ applyCommitMode commitmode ps applyCommitModeForCommitTree :: CommitMode -> [CommandParam] -> Repo -> [CommandParam] applyCommitModeForCommitTree commitmode ps r | commitmode == ManualCommit = - case (Git.Config.getMaybe "commit.gpgsign" r) of - Just s | Git.Config.isTrue s == Just True -> + case Git.Config.getMaybe "commit.gpgsign" r of + Just s | Git.Config.isTrue' s == Just True -> Param "-S":ps _ -> ps' | otherwise = ps' @@ -160,7 +164,7 @@ commitCommand' runner commitmode ps = runner $ commit :: CommitMode -> Bool -> String -> Branch -> [Ref] -> Repo -> IO (Maybe Sha) commit commitmode allowempty message branch parentrefs repo = do tree <- getSha "write-tree" $ - pipeReadStrict [Param "write-tree"] repo + decodeBS' <$> pipeReadStrict [Param "write-tree"] repo ifM (cancommit tree) ( do sha <- commitTree commitmode message parentrefs tree repo diff --git a/Git/CatFile.hs b/Git/CatFile.hs index 49b89454c9..732c18a643 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -66,13 +66,13 @@ catFileStop h = do CoProcess.stop (checkFileProcess h) {- Reads a file from a specified branch. -} -catFile :: CatFileHandle -> Branch -> FilePath -> IO L.ByteString +catFile :: CatFileHandle -> Branch -> RawFilePath -> IO L.ByteString catFile h branch file = catObject h $ Ref $ - fromRef branch ++ ":" ++ toInternalGitPath file + fromRef branch ++ ":" ++ fromRawFilePath (toInternalGitPath file) -catFileDetails :: CatFileHandle -> Branch -> FilePath -> IO (Maybe (L.ByteString, Sha, ObjectType)) +catFileDetails :: CatFileHandle -> Branch -> RawFilePath -> IO (Maybe (L.ByteString, Sha, ObjectType)) catFileDetails h branch file = catObjectDetails h $ Ref $ - fromRef branch ++ ":" ++ toInternalGitPath file + fromRef branch ++ ":" ++ fromRawFilePath (toInternalGitPath file) {- Uses a running git cat-file read the content of an object. - Objects that do not exist will have "" returned. -} @@ -148,7 +148,7 @@ parseResp object l | otherwise = case words l of [sha, objtype, size] | length sha == shaSize -> - case (readObjectType objtype, reads size) of + case (readObjectType (encodeBS objtype), reads size) of (Just t, [(bytes, "")]) -> Just $ ParsedResp (Ref sha) bytes t _ -> Nothing @@ -185,7 +185,7 @@ querySize r repo = maybe Nothing (readMaybe . takeWhile (/= '\n')) <$> querySingle (Param "-s") r repo hGetContentsStrict queryObjectType :: Ref -> Repo -> IO (Maybe ObjectType) -queryObjectType r repo = maybe Nothing (readObjectType . takeWhile (/= '\n')) +queryObjectType r repo = maybe Nothing (readObjectType . encodeBS . takeWhile (/= '\n')) <$> querySingle (Param "-t") r repo hGetContentsStrict queryContent :: Ref -> Repo -> IO (Maybe L.ByteString) diff --git a/Git/Command.hs b/Git/Command.hs index 3101acd79e..c2477529cf 100644 --- a/Git/Command.hs +++ b/Git/Command.hs @@ -14,6 +14,9 @@ import Git import Git.Types import qualified Utility.CoProcess as CoProcess +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S + {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: [CommandParam] -> Repo -> [CommandParam] gitCommandLine params r@(Repo { location = l@(Local { } ) }) = @@ -50,10 +53,10 @@ runQuiet params repo = withQuietOutput createProcessSuccess $ - read, that will wait on the command, and - return True if it succeeded. Failure to wait will result in zombies. -} -pipeReadLazy :: [CommandParam] -> Repo -> IO (String, IO Bool) +pipeReadLazy :: [CommandParam] -> Repo -> IO (L.ByteString, IO Bool) pipeReadLazy params repo = assertLocal repo $ do (_, Just h, _, pid) <- createProcess p { std_out = CreatePipe } - c <- hGetContents h + c <- L.hGetContents h return (c, checkSuccessProcess pid) where p = gitCreateProcess params repo @@ -62,8 +65,8 @@ pipeReadLazy params repo = assertLocal repo $ do - - Nonzero exit status is ignored. -} -pipeReadStrict :: [CommandParam] -> Repo -> IO String -pipeReadStrict = pipeReadStrict' hGetContentsStrict +pipeReadStrict :: [CommandParam] -> Repo -> IO S.ByteString +pipeReadStrict = pipeReadStrict' S.hGetContents {- The reader action must be strict. -} pipeReadStrict' :: (Handle -> IO a) -> [CommandParam] -> Repo -> IO a @@ -93,23 +96,30 @@ pipeWrite params repo = assertLocal repo $ {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it. -} -pipeNullSplit :: [CommandParam] -> Repo -> IO ([String], IO Bool) +pipeNullSplit :: [CommandParam] -> Repo -> IO ([L.ByteString], IO Bool) pipeNullSplit params repo = do (s, cleanup) <- pipeReadLazy params repo - return (filter (not . null) $ splitc sep s, cleanup) - where - sep = '\0' + return (filter (not . L.null) $ L.split 0 s, cleanup) -pipeNullSplitStrict :: [CommandParam] -> Repo -> IO [String] +{- Reads lazily, but copies each part to a strict ByteString for + - convenience. + -} +pipeNullSplit' :: [CommandParam] -> Repo -> IO ([S.ByteString], IO Bool) +pipeNullSplit' params repo = do + (s, cleanup) <- pipeNullSplit params repo + return (map L.toStrict s, cleanup) + +pipeNullSplitStrict :: [CommandParam] -> Repo -> IO [S.ByteString] pipeNullSplitStrict params repo = do s <- pipeReadStrict params repo - return $ filter (not . null) $ splitc sep s - where - sep = '\0' + return $ filter (not . S.null) $ S.split 0 s -pipeNullSplitZombie :: [CommandParam] -> Repo -> IO [String] +pipeNullSplitZombie :: [CommandParam] -> Repo -> IO [L.ByteString] pipeNullSplitZombie params repo = leaveZombie <$> pipeNullSplit params repo +pipeNullSplitZombie' :: [CommandParam] -> Repo -> IO [S.ByteString] +pipeNullSplitZombie' params repo = leaveZombie <$> pipeNullSplit' params repo + {- Doesn't run the cleanup action. A zombie results. -} leaveZombie :: (a, IO Bool) -> a leaveZombie = fst diff --git a/Git/Config.hs b/Git/Config.hs index 9ebd4bd0f5..5276e46835 100644 --- a/Git/Config.hs +++ b/Git/Config.hs @@ -1,13 +1,17 @@ {- git repository configuration handling - - - Copyright 2010-2012 Joey Hess + - Copyright 2010-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Git.Config where import qualified Data.Map as M +import qualified Data.ByteString as S +import qualified Data.ByteString.Char8 as S8 import Data.Char import Common @@ -17,16 +21,16 @@ import qualified Git.Command import qualified Git.Construct import Utility.UserInfo -{- Returns a single git config setting, or a default value if not set. -} -get :: String -> String -> Repo -> String -get key defaultValue repo = M.findWithDefault defaultValue key (config repo) +{- Returns a single git config setting, or a fallback value if not set. -} +get :: ConfigKey -> ConfigValue -> Repo -> ConfigValue +get key fallback repo = M.findWithDefault fallback key (config repo) -{- Returns a list with each line of a multiline config setting. -} -getList :: String -> Repo -> [String] +{- Returns a list of values. -} +getList :: ConfigKey -> Repo -> [ConfigValue] getList key repo = M.findWithDefault [] key (fullconfig repo) {- Returns a single git config setting, if set. -} -getMaybe :: String -> Repo -> Maybe String +getMaybe :: ConfigKey -> Repo -> Maybe ConfigValue getMaybe key repo = M.lookup key (config repo) {- Runs git config and populates a repo with its config. @@ -79,14 +83,14 @@ global = do {- Reads git config from a handle and populates a repo with it. -} hRead :: Repo -> Handle -> IO Repo hRead repo h = do - val <- hGetContentsStrict h + val <- S.hGetContents h store val repo {- Stores a git config into a Repo, returning the new version of the Repo. - The git config may be multiple lines, or a single line. - Config settings can be updated incrementally. -} -store :: String -> Repo -> IO Repo +store :: S.ByteString -> Repo -> IO Repo store s repo = do let c = parse s updateLocation $ repo @@ -96,7 +100,7 @@ store s repo = do {- Stores a single config setting in a Repo, returning the new version of - the Repo. Config settings can be updated incrementally. -} -store' :: String -> String -> Repo -> Repo +store' :: ConfigKey -> ConfigValue -> Repo -> Repo store' k v repo = repo { config = M.singleton k v `M.union` config repo , fullconfig = M.unionWith (++) (M.singleton k [v]) (fullconfig repo) @@ -124,52 +128,66 @@ updateLocation' :: Repo -> RepoLocation -> IO Repo updateLocation' r l = do l' <- case getMaybe "core.worktree" r of Nothing -> return l - Just d -> do + Just (ConfigValue d) -> do {- core.worktree is relative to the gitdir -} top <- absPath $ gitdir l - return $ l { worktree = Just $ absPathFrom top d } + let p = absPathFrom top (fromRawFilePath d) + return $ l { worktree = Just p } return $ r { location = l' } {- Parses git config --list or git config --null --list output into a - config map. -} -parse :: String -> M.Map String [String] -parse [] = M.empty +parse :: S.ByteString -> M.Map ConfigKey [ConfigValue] parse s - -- --list output will have an = in the first line - | all ('=' `elem`) (take 1 ls) = sep '=' ls + | S.null s = M.empty + -- --list output will have a '=' in the first line + -- (The first line of --null --list output is the name of a key, + -- which is assumed to never contain '='.) + | S.elem eq firstline = sep eq $ S.split nl s -- --null --list output separates keys from values with newlines - | otherwise = sep '\n' $ splitc '\0' s + | otherwise = sep nl $ S.split 0 s where - ls = lines s - sep c = M.fromListWith (++) . map (\(k,v) -> (k, [v])) . - map (separate (== c)) + nl = fromIntegral (ord '\n') + eq = fromIntegral (ord '=') + firstline = S.takeWhile (/= nl) s + + sep c = M.fromListWith (++) + . map (\(k,v) -> (ConfigKey k, [ConfigValue (S.drop 1 v)])) + . map (S.break (== c)) {- Checks if a string from git config is a true value. -} isTrue :: String -> Maybe Bool -isTrue s +isTrue = isTrue' . ConfigValue . encodeBS' + +isTrue' :: ConfigValue -> Maybe Bool +isTrue' (ConfigValue s) | s' == "true" = Just True | s' == "false" = Just False | otherwise = Nothing where - s' = map toLower s + s' = S8.map toLower s boolConfig :: Bool -> String boolConfig True = "true" boolConfig False = "false" -isBare :: Repo -> Bool -isBare r = fromMaybe False $ isTrue =<< getMaybe coreBare r +boolConfig' :: Bool -> S.ByteString +boolConfig' True = "true" +boolConfig' False = "false" -coreBare :: String +isBare :: Repo -> Bool +isBare r = fromMaybe False $ isTrue' =<< getMaybe coreBare r + +coreBare :: ConfigKey coreBare = "core.bare" {- Runs a command to get the configuration of a repo, - and returns a repo populated with the configuration, as well as the raw - output of the command. -} -fromPipe :: Repo -> String -> [CommandParam] -> IO (Either SomeException (Repo, String)) +fromPipe :: Repo -> String -> [CommandParam] -> IO (Either SomeException (Repo, S.ByteString)) fromPipe r cmd params = try $ withHandle StdoutHandle createProcessSuccess p $ \h -> do - val <- hGetContentsStrict h + val <- S.hGetContents h r' <- store val r return (r', val) where @@ -177,7 +195,7 @@ fromPipe r cmd params = try $ {- Reads git config from a specified file and returns the repo populated - with the configuration. -} -fromFile :: Repo -> FilePath -> IO (Either SomeException (Repo, String)) +fromFile :: Repo -> FilePath -> IO (Either SomeException (Repo, S.ByteString)) fromFile r f = fromPipe r "git" [ Param "config" , Param "--file" @@ -187,13 +205,13 @@ fromFile r f = fromPipe r "git" {- Changes a git config setting in the specified config file. - (Creates the file if it does not already exist.) -} -changeFile :: FilePath -> String -> String -> IO Bool -changeFile f k v = boolSystem "git" +changeFile :: FilePath -> ConfigKey -> S.ByteString -> IO Bool +changeFile f (ConfigKey k) v = boolSystem "git" [ Param "config" , Param "--file" , File f - , Param k - , Param v + , Param (decodeBS' k) + , Param (decodeBS' v) ] {- Unsets a git config setting, in both the git repo, @@ -202,10 +220,10 @@ changeFile f k v = boolSystem "git" - If unsetting the config fails, including in a read-only repo, or - when the config is not set, returns Nothing. -} -unset :: String -> Repo -> IO (Maybe Repo) -unset k r = ifM (Git.Command.runBool ps r) - ( return $ Just $ r { config = M.delete k (config r) } +unset :: ConfigKey -> Repo -> IO (Maybe Repo) +unset ck@(ConfigKey k) r = ifM (Git.Command.runBool ps r) + ( return $ Just $ r { config = M.delete ck (config r) } , return Nothing ) where - ps = [Param "config", Param "--unset-all", Param k] + ps = [Param "config", Param "--unset-all", Param (decodeBS' k)] diff --git a/Git/ConfigTypes.hs b/Git/ConfigTypes.hs index 2e262c643a..f01a2cef40 100644 --- a/Git/ConfigTypes.hs +++ b/Git/ConfigTypes.hs @@ -5,12 +5,16 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Git.ConfigTypes where import Data.Char +import qualified Data.ByteString.Char8 as S8 import Common import Git +import Git.Types import qualified Git.Config data SharedRepository = UnShared | GroupShared | AllShared | UmaskShared Int @@ -18,23 +22,27 @@ data SharedRepository = UnShared | GroupShared | AllShared | UmaskShared Int getSharedRepository :: Repo -> SharedRepository getSharedRepository r = - case map toLower $ Git.Config.get "core.sharedrepository" "" r of - "1" -> GroupShared - "2" -> AllShared - "group" -> GroupShared - "true" -> GroupShared - "all" -> AllShared - "world" -> AllShared - "everybody" -> AllShared - v -> maybe UnShared UmaskShared (readish v) + case Git.Config.getMaybe "core.sharedrepository" r of + Nothing -> UnShared + Just (ConfigValue v) -> case S8.map toLower v of + "1" -> GroupShared + "2" -> AllShared + "group" -> GroupShared + "true" -> GroupShared + "all" -> AllShared + "world" -> AllShared + "everybody" -> AllShared + _ -> maybe UnShared UmaskShared (readish (decodeBS' v)) data DenyCurrentBranch = UpdateInstead | RefusePush | WarnPush | IgnorePush deriving (Eq) getDenyCurrentBranch :: Repo -> DenyCurrentBranch -getDenyCurrentBranch r = - case map toLower $ Git.Config.get "receive.denycurrentbranch" "" r of - "updateinstead" -> UpdateInstead - "warn" -> WarnPush - "ignore" -> IgnorePush - _ -> RefusePush +getDenyCurrentBranch r = + case Git.Config.getMaybe "receive.denycurrentbranch" r of + Just (ConfigValue v) -> case S8.map toLower v of + "updateinstead" -> UpdateInstead + "warn" -> WarnPush + "ignore" -> IgnorePush + _ -> RefusePush + Nothing -> RefusePush diff --git a/Git/Construct.hs b/Git/Construct.hs index d032c59c39..7a58a5d444 100644 --- a/Git/Construct.hs +++ b/Git/Construct.hs @@ -58,7 +58,7 @@ fromPath dir = fromAbsPath =<< absPath dir - specified. -} fromAbsPath :: FilePath -> IO Repo fromAbsPath dir - | absoluteGitPath dir = hunt + | absoluteGitPath (encodeBS dir) = hunt | otherwise = error $ "internal error, " ++ dir ++ " is not absolute" where @@ -128,7 +128,7 @@ fromRemotes repo = mapM construct remotepairs filterconfig f = filter f $ M.toList $ config repo filterkeys f = filterconfig (\(k,_) -> f k) remotepairs = filterkeys isRemoteKey - construct (k,v) = remoteNamedFromKey k $ fromRemoteLocation v repo + construct (k,v) = remoteNamedFromKey k (fromRemoteLocation (fromConfigValue v) repo) {- Sets the name of a remote when constructing the Repo to represent it. -} remoteNamed :: String -> IO Repo -> IO Repo @@ -138,7 +138,7 @@ remoteNamed n constructor = do {- Sets the name of a remote based on the git config key, such as - "remote.foo.url". -} -remoteNamedFromKey :: String -> IO Repo -> IO Repo +remoteNamedFromKey :: ConfigKey -> IO Repo -> IO Repo remoteNamedFromKey = remoteNamed . remoteKeyToRemoteName {- Constructs a new Repo for one of a Repo's remotes using a given diff --git a/Git/DiffTree.hs b/Git/DiffTree.hs index 0aad4db188..f6c5c60955 100644 --- a/Git/DiffTree.hs +++ b/Git/DiffTree.hs @@ -89,7 +89,7 @@ commitDiff ref = getdiff (Param "show") getdiff :: CommandParam -> [CommandParam] -> Repo -> IO ([DiffTreeItem], IO Bool) getdiff command params repo = do (diff, cleanup) <- pipeNullSplit ps repo - return (parseDiffRaw diff, cleanup) + return (parseDiffRaw (map decodeBL diff), cleanup) where ps = command : @@ -113,7 +113,7 @@ parseDiffRaw l = go l , srcsha = fromMaybe (error "bad srcsha") $ extractSha ssha , dstsha = fromMaybe (error "bad dstsha") $ extractSha dsha , status = s - , file = asTopFilePath $ fromInternalGitPath $ Git.Filename.decode f + , file = asTopFilePath $ fromRawFilePath $ fromInternalGitPath $ Git.Filename.decode $ toRawFilePath f } where readmode = fst . Prelude.head . readOct diff --git a/Git/FilePath.hs b/Git/FilePath.hs index f0c3b69ed7..bb80df4815 100644 --- a/Git/FilePath.hs +++ b/Git/FilePath.hs @@ -12,6 +12,7 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE OverloadedStrings #-} module Git.FilePath ( TopFilePath, @@ -33,8 +34,9 @@ import Git import qualified System.FilePath.Posix import GHC.Generics import Control.DeepSeq +import qualified Data.ByteString as S -{- A FilePath, relative to the top of the git repository. -} +{- A RawFilePath, relative to the top of the git repository. -} newtype TopFilePath = TopFilePath { getTopFilePath :: FilePath } deriving (Show, Eq, Ord, Generic) @@ -45,8 +47,9 @@ data BranchFilePath = BranchFilePath Ref TopFilePath deriving (Show, Eq, Ord) {- Git uses the branch:file form to refer to a BranchFilePath -} -descBranchFilePath :: BranchFilePath -> String -descBranchFilePath (BranchFilePath b f) = fromRef b ++ ':' : getTopFilePath f +descBranchFilePath :: BranchFilePath -> S.ByteString +descBranchFilePath (BranchFilePath b f) = + encodeBS' (fromRef b) <> ":" <> toRawFilePath (getTopFilePath f) {- Path to a TopFilePath, within the provided git repo. -} fromTopFilePath :: TopFilePath -> Git.Repo -> FilePath @@ -68,25 +71,25 @@ asTopFilePath file = TopFilePath file - despite Windows using '\'. - -} -type InternalGitPath = String +type InternalGitPath = RawFilePath -toInternalGitPath :: FilePath -> InternalGitPath +toInternalGitPath :: RawFilePath -> InternalGitPath #ifndef mingw32_HOST_OS toInternalGitPath = id #else -toInternalGitPath = replace "\\" "/" +toInternalGitPath = encodeBS . replace "\\" "/" . decodeBS #endif -fromInternalGitPath :: InternalGitPath -> FilePath +fromInternalGitPath :: InternalGitPath -> RawFilePath #ifndef mingw32_HOST_OS fromInternalGitPath = id #else -fromInternalGitPath = replace "/" "\\" +fromInternalGitPath = encodeBS . replace "/" "\\" . decodeBS #endif {- isAbsolute on Windows does not think "/foo" or "\foo" is absolute, - so try posix paths. -} -absoluteGitPath :: FilePath -> Bool -absoluteGitPath p = isAbsolute p || - System.FilePath.Posix.isAbsolute (toInternalGitPath p) +absoluteGitPath :: RawFilePath -> Bool +absoluteGitPath p = isAbsolute (decodeBS p) || + System.FilePath.Posix.isAbsolute (decodeBS (toInternalGitPath p)) diff --git a/Git/Filename.hs b/Git/Filename.hs index 40a449c9c9..eda4a4d907 100644 --- a/Git/Filename.hs +++ b/Git/Filename.hs @@ -12,23 +12,44 @@ import Common import Utility.Format (decode_c, encode_c) import Data.Char +import Data.Word +import qualified Data.ByteString as S -decode :: String -> FilePath -decode [] = [] -decode f@(c:s) - -- encoded strings will be inside double quotes - | c == '"' && end s == ['"'] = decode_c $ beginning s - | otherwise = f +-- encoded filenames will be inside double quotes +decode :: S.ByteString -> RawFilePath +decode b = case S.uncons b of + Nothing -> b + Just (h, t) + | h /= q -> b + | otherwise -> case S.unsnoc t of + Nothing -> b + Just (i, l) + | l /= q -> b + | otherwise -> + encodeBS $ decode_c $ decodeBS i + where + q :: Word8 + q = fromIntegral (ord '"') {- Should not need to use this, except for testing decode. -} -encode :: FilePath -> String -encode s = "\"" ++ encode_c s ++ "\"" +encode :: RawFilePath -> S.ByteString +encode s = encodeBS $ "\"" ++ encode_c (decodeBS s) ++ "\"" -{- For quickcheck. - - - - See comment on Utility.Format.prop_encode_c_decode_c_roundtrip for - - why this only tests chars < 256 -} -prop_encode_decode_roundtrip :: String -> Bool -prop_encode_decode_roundtrip s = s' == decode (encode s') +prop_encode_decode_roundtrip :: FilePath -> Bool +prop_encode_decode_roundtrip s = s' == + fromRawFilePath (decode (encode (toRawFilePath s'))) where - s' = filter (\c -> ord c < 256) s + s' = nonul (nohigh s) + -- Encoding and then decoding roundtrips only when + -- the string does not contain high unicode, because eg, + -- both "\12345" and "\227\128\185" are encoded to + -- "\343\200\271". + -- + -- This property papers over the problem, by only + -- testing chars < 256. + nohigh = filter (\c -> ord c < 256) + -- A String can contain a NUL, but toRawFilePath + -- truncates on the NUL, which is generally fine + -- because unix filenames cannot contain NUL. + -- So the encoding only roundtrips when there is no nul. + nonul = filter (/= '\NUL') diff --git a/Git/GCrypt.hs b/Git/GCrypt.hs index f19b77a98b..7a70a2eaf8 100644 --- a/Git/GCrypt.hs +++ b/Git/GCrypt.hs @@ -7,6 +7,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Git.GCrypt where import Common @@ -16,6 +18,8 @@ import qualified Git.Config as Config import qualified Git.Command as Command import Utility.Gpg +import qualified Data.ByteString as S + urlScheme :: String urlScheme = "gcrypt:" @@ -75,9 +79,9 @@ type GCryptId = String - which is stored in the repository (in encrypted form) - and cached in a per-remote gcrypt-id configuration setting. -} remoteRepoId :: Repo -> Maybe RemoteName -> Maybe GCryptId -remoteRepoId = getRemoteConfig "gcrypt-id" +remoteRepoId r n = fromConfigValue <$> getRemoteConfig "gcrypt-id" r n -getRemoteConfig :: String -> Repo -> Maybe RemoteName -> Maybe String +getRemoteConfig :: S.ByteString -> Repo -> Maybe RemoteName -> Maybe ConfigValue getRemoteConfig field repo remotename = do n <- remotename Config.getMaybe (remoteConfigKey field n) repo @@ -92,18 +96,19 @@ getParticiantList globalconfigrepo repo remotename = KeyIds $ parse $ firstJust ] where defaultkey = "gcrypt.participants" - parse (Just "simple") = [] - parse (Just l) = words l + parse (Just (ConfigValue "simple")) = [] + parse (Just (ConfigValue b)) = words (decodeBS' b) parse Nothing = [] -remoteParticipantConfigKey :: RemoteName -> String +remoteParticipantConfigKey :: RemoteName -> ConfigKey remoteParticipantConfigKey = remoteConfigKey "gcrypt-participants" -remotePublishParticipantConfigKey :: RemoteName -> String +remotePublishParticipantConfigKey :: RemoteName -> ConfigKey remotePublishParticipantConfigKey = remoteConfigKey "gcrypt-publish-participants" -remoteSigningKey :: RemoteName -> String +remoteSigningKey :: RemoteName -> ConfigKey remoteSigningKey = remoteConfigKey "gcrypt-signingkey" -remoteConfigKey :: String -> RemoteName -> String -remoteConfigKey key remotename = "remote." ++ remotename ++ "." ++ key +remoteConfigKey :: S.ByteString -> RemoteName -> ConfigKey +remoteConfigKey key remotename = ConfigKey $ + "remote." <> encodeBS' remotename <> "." <> key diff --git a/Git/HashObject.hs b/Git/HashObject.hs index 2453cc1c5f..605e6d504c 100644 --- a/Git/HashObject.hs +++ b/Git/HashObject.hs @@ -73,4 +73,4 @@ hashObject' objtype writer repo = getSha subcmd $ pipeWriteRead (map Param params) (Just writer) repo where subcmd = "hash-object" - params = [subcmd, "-t", show objtype, "-w", "--stdin", "--no-filters"] + params = [subcmd, "-t", decodeBS (fmtObjectType objtype), "-w", "--stdin", "--no-filters"] diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs index f5f74a22a5..2d010ed27e 100644 --- a/Git/LsFiles.hs +++ b/Git/LsFiles.hs @@ -38,37 +38,40 @@ import Utility.TimeStamp import Numeric import System.Posix.Types import qualified Data.Map as M +import qualified Data.ByteString.Lazy as L {- Scans for files that are checked into git's index at the specified locations. -} -inRepo :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +inRepo :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) inRepo = inRepo' [] -inRepo' :: [CommandParam] -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) -inRepo' ps l = pipeNullSplit $ - Param "ls-files" : - Param "--cached" : - Param "-z" : - ps ++ - (Param "--" : map File l) +inRepo' :: [CommandParam] -> [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) +inRepo' ps l repo = pipeNullSplit' params repo + where + params = + Param "ls-files" : + Param "--cached" : + Param "-z" : + ps ++ + (Param "--" : map (File . fromRawFilePath) l) {- Files that are checked into the index or have been committed to a - branch. -} -inRepoOrBranch :: Branch -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +inRepoOrBranch :: Branch -> [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) inRepoOrBranch (Ref b) = inRepo' [Param $ "--with-tree=" ++ b] {- Scans for files at the specified locations that are not checked into git. -} -notInRepo :: Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +notInRepo :: Bool -> [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) notInRepo = notInRepo' [] -notInRepo' :: [CommandParam] -> Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) -notInRepo' ps include_ignored l repo = pipeNullSplit params repo +notInRepo' :: [CommandParam] -> Bool -> [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) +notInRepo' ps include_ignored l repo = pipeNullSplit' params repo where params = concat [ [ Param "ls-files", Param "--others"] , ps , exclude , [ Param "-z", Param "--" ] - , map File l + , map (File . fromRawFilePath) l ] exclude | include_ignored = [] @@ -76,48 +79,48 @@ notInRepo' ps include_ignored l repo = pipeNullSplit params repo {- Scans for files at the specified locations that are not checked into - git. Empty directories are included in the result. -} -notInRepoIncludingEmptyDirectories :: Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +notInRepoIncludingEmptyDirectories :: Bool -> [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) notInRepoIncludingEmptyDirectories = notInRepo' [Param "--directory"] {- Finds all files in the specified locations, whether checked into git or - not. -} -allFiles :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) -allFiles l = pipeNullSplit $ +allFiles :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) +allFiles l = pipeNullSplit' $ Param "ls-files" : Param "--cached" : Param "--others" : Param "-z" : Param "--" : - map File l + map (File . fromRawFilePath) l {- Returns a list of files in the specified locations that have been - deleted. -} -deleted :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) -deleted l repo = pipeNullSplit params repo +deleted :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) +deleted l repo = pipeNullSplit' params repo where params = Param "ls-files" : Param "--deleted" : Param "-z" : Param "--" : - map File l + map (File . fromRawFilePath) l {- Returns a list of files in the specified locations that have been - modified. -} -modified :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) -modified l repo = pipeNullSplit params repo +modified :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) +modified l repo = pipeNullSplit' params repo where params = Param "ls-files" : Param "--modified" : Param "-z" : Param "--" : - map File l + map (File . fromRawFilePath) l {- Files that have been modified or are not checked into git (and are not - ignored). -} -modifiedOthers :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) -modifiedOthers l repo = pipeNullSplit params repo +modifiedOthers :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) +modifiedOthers l repo = pipeNullSplit' params repo where params = Param "ls-files" : @@ -126,69 +129,69 @@ modifiedOthers l repo = pipeNullSplit params repo Param "--exclude-standard" : Param "-z" : Param "--" : - map File l + map (File . fromRawFilePath) l {- Returns a list of all files that are staged for commit. -} -staged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +staged :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) staged = staged' [] {- Returns a list of the files, staged for commit, that are being added, - moved, or changed (but not deleted), from the specified locations. -} -stagedNotDeleted :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +stagedNotDeleted :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) stagedNotDeleted = staged' [Param "--diff-filter=ACMRT"] -staged' :: [CommandParam] -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) -staged' ps l = pipeNullSplit $ prefix ++ ps ++ suffix +staged' :: [CommandParam] -> [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) +staged' ps l repo = pipeNullSplit' (prefix ++ ps ++ suffix) repo where prefix = [Param "diff", Param "--cached", Param "--name-only", Param "-z"] - suffix = Param "--" : map File l + suffix = Param "--" : map (File . fromRawFilePath) l -type StagedDetails = (FilePath, Maybe Sha, Maybe FileMode) +type StagedDetails = (RawFilePath, Maybe Sha, Maybe FileMode) {- Returns details about files that are staged in the index, - as well as files not yet in git. Skips ignored files. -} -stagedOthersDetails :: [FilePath] -> Repo -> IO ([StagedDetails], IO Bool) +stagedOthersDetails :: [RawFilePath] -> Repo -> IO ([StagedDetails], IO Bool) stagedOthersDetails = stagedDetails' [Param "--others", Param "--exclude-standard"] {- Returns details about all files that are staged in the index. -} -stagedDetails :: [FilePath] -> Repo -> IO ([StagedDetails], IO Bool) +stagedDetails :: [RawFilePath] -> Repo -> IO ([StagedDetails], IO Bool) stagedDetails = stagedDetails' [] {- Gets details about staged files, including the Sha of their staged - contents. -} -stagedDetails' :: [CommandParam] -> [FilePath] -> Repo -> IO ([StagedDetails], IO Bool) +stagedDetails' :: [CommandParam] -> [RawFilePath] -> Repo -> IO ([StagedDetails], IO Bool) stagedDetails' ps l repo = do (ls, cleanup) <- pipeNullSplit params repo return (map parse ls, cleanup) where params = Param "ls-files" : Param "--stage" : Param "-z" : ps ++ - Param "--" : map File l + Param "--" : map (File . fromRawFilePath) l parse s - | null file = (s, Nothing, Nothing) - | otherwise = (file, extractSha $ take shaSize rest, readmode mode) + | null file = (L.toStrict s, Nothing, Nothing) + | otherwise = (toRawFilePath file, extractSha $ take shaSize rest, readmode mode) where - (metadata, file) = separate (== '\t') s + (metadata, file) = separate (== '\t') (decodeBL' s) (mode, rest) = separate (== ' ') metadata readmode = fst <$$> headMaybe . readOct {- Returns a list of the files in the specified locations that are staged - for commit, and whose type has changed. -} -typeChangedStaged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChangedStaged :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) typeChangedStaged = typeChanged' [Param "--cached"] {- Returns a list of the files in the specified locations whose type has - changed. Files only staged for commit will not be included. -} -typeChanged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChanged :: [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) typeChanged = typeChanged' [] -typeChanged' :: [CommandParam] -> [FilePath] -> Repo -> IO ([FilePath], IO Bool) +typeChanged' :: [CommandParam] -> [RawFilePath] -> Repo -> IO ([RawFilePath], IO Bool) typeChanged' ps l repo = do (fs, cleanup) <- pipeNullSplit (prefix ++ ps ++ suffix) repo -- git diff returns filenames relative to the top of the git repo; -- convert to filenames relative to the cwd, like git ls-files. top <- absPath (repoPath repo) currdir <- getCurrentDirectory - return (map (\f -> relPathDirToFileAbs currdir $ top f) fs, cleanup) + return (map (\f -> toRawFilePath (relPathDirToFileAbs currdir $ top decodeBL' f)) fs, cleanup) where prefix = [ Param "diff" @@ -196,7 +199,7 @@ typeChanged' ps l repo = do , Param "--diff-filter=T" , Param "-z" ] - suffix = Param "--" : (if null l then [File "."] else map File l) + suffix = Param "--" : (if null l then [File "."] else map (File . fromRawFilePath) l) {- A item in conflict has two possible values. - Either can be Nothing, when that side deleted the file. -} @@ -206,7 +209,7 @@ data Conflicting v = Conflicting } deriving (Show) data Unmerged = Unmerged - { unmergedFile :: FilePath + { unmergedFile :: RawFilePath , unmergedTreeItemType :: Conflicting TreeItemType , unmergedSha :: Conflicting Sha } @@ -221,21 +224,21 @@ data Unmerged = Unmerged - 3 = them - If a line is omitted, that side removed the file. -} -unmerged :: [FilePath] -> Repo -> IO ([Unmerged], IO Bool) +unmerged :: [RawFilePath] -> Repo -> IO ([Unmerged], IO Bool) unmerged l repo = do (fs, cleanup) <- pipeNullSplit params repo - return (reduceUnmerged [] $ catMaybes $ map parseUnmerged fs, cleanup) + return (reduceUnmerged [] $ catMaybes $ map (parseUnmerged . decodeBL') fs, cleanup) where params = Param "ls-files" : Param "--unmerged" : Param "-z" : Param "--" : - map File l + map (File . fromRawFilePath) l data InternalUnmerged = InternalUnmerged { isus :: Bool - , ifile :: FilePath + , ifile :: RawFilePath , itreeitemtype :: Maybe TreeItemType , isha :: Maybe Sha } @@ -249,9 +252,9 @@ parseUnmerged s if stage /= 2 && stage /= 3 then Nothing else do - treeitemtype <- readTreeItemType rawtreeitemtype + treeitemtype <- readTreeItemType (encodeBS rawtreeitemtype) sha <- extractSha rawsha - return $ InternalUnmerged (stage == 2) file + return $ InternalUnmerged (stage == 2) (toRawFilePath file) (Just treeitemtype) (Just sha) _ -> Nothing where @@ -285,10 +288,10 @@ reduceUnmerged c (i:is) = reduceUnmerged (new:c) rest - Note that this uses a --debug option whose output could change at some - point in the future. If the output is not as expected, will use Nothing. -} -inodeCaches :: [FilePath] -> Repo -> IO ([(FilePath, Maybe InodeCache)], IO Bool) +inodeCaches :: [RawFilePath] -> Repo -> IO ([(FilePath, Maybe InodeCache)], IO Bool) inodeCaches locs repo = do (ls, cleanup) <- pipeNullSplit params repo - return (parse Nothing ls, cleanup) + return (parse Nothing (map decodeBL ls), cleanup) where params = Param "ls-files" : @@ -296,7 +299,7 @@ inodeCaches locs repo = do Param "-z" : Param "--debug" : Param "--" : - map File locs + map (File . fromRawFilePath) locs parse Nothing (f:ls) = parse (Just f) ls parse (Just f) (s:[]) = diff --git a/Git/LsTree.hs b/Git/LsTree.hs index 8ca805402b..0196d21a1f 100644 --- a/Git/LsTree.hs +++ b/Git/LsTree.hs @@ -24,14 +24,19 @@ import Git.Command import Git.Sha import Git.FilePath import qualified Git.Filename +import Utility.Attoparsec import Numeric -import Data.Char +import Data.Either import System.Posix.Types +import qualified Data.ByteString as S +import qualified Data.ByteString.Lazy as L +import qualified Data.Attoparsec.ByteString.Lazy as A +import qualified Data.Attoparsec.ByteString.Char8 as A8 data TreeItem = TreeItem { mode :: FileMode - , typeobj :: String + , typeobj :: S.ByteString , sha :: Ref , file :: TopFilePath } deriving Show @@ -45,7 +50,7 @@ lsTree = lsTree' [] lsTree' :: [CommandParam] -> LsTreeMode -> Ref -> Repo -> IO ([TreeItem], IO Bool) lsTree' ps lsmode t repo = do (l, cleanup) <- pipeNullSplit (lsTreeParams lsmode t ps) repo - return (map parseLsTree l, cleanup) + return (rights (map parseLsTree l), cleanup) lsTreeParams :: LsTreeMode -> Ref -> [CommandParam] -> [CommandParam] lsTreeParams lsmode r ps = @@ -63,7 +68,8 @@ lsTreeParams lsmode r ps = {- Lists specified files in a tree. -} lsTreeFiles :: Ref -> [FilePath] -> Repo -> IO [TreeItem] -lsTreeFiles t fs repo = map parseLsTree <$> pipeNullSplitStrict ps repo +lsTreeFiles t fs repo = rights . map (parseLsTree . L.fromStrict) + <$> pipeNullSplitStrict ps repo where ps = [ Param "ls-tree" @@ -73,30 +79,34 @@ lsTreeFiles t fs repo = map parseLsTree <$> pipeNullSplitStrict ps repo , File $ fromRef t ] ++ map File fs +parseLsTree :: L.ByteString -> Either String TreeItem +parseLsTree b = case A.parse parserLsTree b of + A.Done _ r -> Right r + A.Fail _ _ err -> Left err + {- Parses a line of ls-tree output, in format: - mode SP type SP sha TAB file - - (The --long format is not currently supported.) -} -parseLsTree :: String -> TreeItem -parseLsTree l = TreeItem - { mode = smode - , typeobj = t - , sha = Ref s - , file = sfile - } - where - (m, past_m) = splitAt 7 l -- mode is 6 bytes - (!t, past_t) = separate isSpace past_m - (!s, past_s) = splitAt shaSize past_t - !f = drop 1 past_s - !smode = fst $ Prelude.head $ readOct m - !sfile = asTopFilePath $ Git.Filename.decode f +parserLsTree :: A.Parser TreeItem +parserLsTree = TreeItem + -- mode + <$> octal + <* A8.char ' ' + -- type + <*> A.takeTill (== 32) + <* A8.char ' ' + -- sha + <*> (Ref . decodeBS' <$> A.take shaSize) + <* A8.char '\t' + -- file + <*> (asTopFilePath . decodeBS' . Git.Filename.decode <$> A.takeByteString) {- Inverse of parseLsTree -} formatLsTree :: TreeItem -> String formatLsTree ti = unwords [ showOct (mode ti) "" - , typeobj ti + , decodeBS (typeobj ti) , fromRef (sha ti) , getTopFilePath (file ti) ] diff --git a/Git/Ref.hs b/Git/Ref.hs index 964dbafb08..8c8511ae04 100644 --- a/Git/Ref.hs +++ b/Git/Ref.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Git.Ref where import Common @@ -13,7 +15,8 @@ import Git.Command import Git.Sha import Git.Types -import Data.Char (chr) +import Data.Char (chr, ord) +import qualified Data.ByteString as S headRef :: Ref headRef = Ref "HEAD" @@ -62,8 +65,8 @@ branchRef = underBase "refs/heads" - Prefixing the file with ./ makes this work even if in a subdirectory - of a repo. -} -fileRef :: FilePath -> Ref -fileRef f = Ref $ ":./" ++ f +fileRef :: RawFilePath -> Ref +fileRef f = Ref $ ":./" ++ fromRawFilePath f {- Converts a Ref to refer to the content of the Ref on a given date. -} dateRef :: Ref -> RefDate -> Ref @@ -71,7 +74,7 @@ dateRef (Ref r) (RefDate d) = Ref $ r ++ "@" ++ d {- A Ref that can be used to refer to a file in the repository as it - appears in a given Ref. -} -fileFromRef :: Ref -> FilePath -> Ref +fileFromRef :: Ref -> RawFilePath -> Ref fileFromRef (Ref r) f = let (Ref fr) = fileRef f in Ref (r ++ fr) {- Checks if a ref exists. -} @@ -88,8 +91,10 @@ file ref repo = localGitDir repo fromRef ref - that was just created. -} headExists :: Repo -> IO Bool headExists repo = do - ls <- lines <$> pipeReadStrict [Param "show-ref", Param "--head"] repo - return $ any (" HEAD" `isSuffixOf`) ls + ls <- S.split nl <$> pipeReadStrict [Param "show-ref", Param "--head"] repo + return $ any (" HEAD" `S.isSuffixOf`) ls + where + nl = fromIntegral (ord '\n') {- Get the sha of a fully qualified git ref, if it exists. -} sha :: Branch -> Repo -> IO (Maybe Sha) @@ -100,8 +105,9 @@ sha branch repo = process <$> showref repo , Param "--hash" -- get the hash , Param $ fromRef branch ] - process [] = Nothing - process s = Just $ Ref $ firstLine s + process s + | S.null s = Nothing + | otherwise = Just $ Ref $ decodeBS' $ firstLine' s headSha :: Repo -> IO (Maybe Sha) headSha = sha headRef @@ -116,7 +122,7 @@ matchingWithHEAD refs repo = matching' ("--head" : map fromRef refs) repo {- List of (shas, branches) matching a given ref spec. -} matching' :: [String] -> Repo -> IO [(Sha, Branch)] -matching' ps repo = map gen . lines <$> +matching' ps repo = map gen . lines . decodeBS' <$> pipeReadStrict (Param "show-ref" : map Param ps) repo where gen l = let (r, b) = separate (== ' ') l @@ -148,7 +154,7 @@ delete oldvalue ref = run - The ref may be something like a branch name, and it could contain - ":subdir" if a subtree is wanted. -} tree :: Ref -> Repo -> IO (Maybe Sha) -tree (Ref ref) = extractSha <$$> pipeReadStrict +tree (Ref ref) = extractSha . decodeBS <$$> pipeReadStrict [ Param "rev-parse", Param "--verify", Param "--quiet", Param ref' ] where ref' = if ":" `isInfixOf` ref diff --git a/Git/RefLog.hs b/Git/RefLog.hs index 2c5a65d74a..7ba8713af7 100644 --- a/Git/RefLog.hs +++ b/Git/RefLog.hs @@ -21,7 +21,7 @@ getMulti :: [Branch] -> Repo -> IO [Sha] getMulti bs = get' (map (Param . fromRef) bs) get' :: [CommandParam] -> Repo -> IO [Sha] -get' ps = mapMaybe extractSha . lines <$$> pipeReadStrict ps' +get' ps = mapMaybe extractSha . lines . decodeBS <$$> pipeReadStrict ps' where ps' = catMaybes [ Just $ Param "log" diff --git a/Git/Remote.hs b/Git/Remote.hs index fa336013e7..5ff88a84fd 100644 --- a/Git/Remote.hs +++ b/Git/Remote.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} module Git.Remote where @@ -15,18 +16,21 @@ import Git.Types import Data.Char import qualified Data.Map as M +import qualified Data.ByteString as S +import qualified Data.ByteString.Char8 as S8 import Network.URI #ifdef mingw32_HOST_OS import Git.FilePath #endif {- Is a git config key one that specifies the location of a remote? -} -isRemoteKey :: String -> Bool -isRemoteKey k = "remote." `isPrefixOf` k && ".url" `isSuffixOf` k +isRemoteKey :: ConfigKey -> Bool +isRemoteKey (ConfigKey k) = "remote." `S.isPrefixOf` k && ".url" `S.isSuffixOf` k {- Get a remote's name from the config key that specifies its location. -} -remoteKeyToRemoteName :: String -> RemoteName -remoteKeyToRemoteName k = intercalate "." $ dropFromEnd 1 $ drop 1 $ splitc '.' k +remoteKeyToRemoteName :: ConfigKey -> RemoteName +remoteKeyToRemoteName (ConfigKey k) = decodeBS' $ + S.intercalate "." $ dropFromEnd 1 $ drop 1 $ S8.split '.' k {- Construct a legal git remote name out of an arbitrary input string. - @@ -76,16 +80,16 @@ parseRemoteLocation s repo = ret $ calcloc s -- insteadof config can rewrite remote location calcloc l | null insteadofs = l - | otherwise = replacement ++ drop (length bestvalue) l + | otherwise = replacement ++ drop (S.length bestvalue) l where - replacement = drop (length prefix) $ - take (length bestkey - length suffix) bestkey - (bestkey, bestvalue) = maximumBy longestvalue insteadofs + replacement = decodeBS' $ S.drop (S.length prefix) $ + S.take (S.length bestkey - S.length suffix) bestkey + (ConfigKey bestkey, ConfigValue bestvalue) = maximumBy longestvalue insteadofs longestvalue (_, a) (_, b) = compare b a - insteadofs = filterconfig $ \(k, v) -> - prefix `isPrefixOf` k && - suffix `isSuffixOf` k && - v `isPrefixOf` l + insteadofs = filterconfig $ \(ConfigKey k, ConfigValue v) -> + prefix `S.isPrefixOf` k && + suffix `S.isSuffixOf` k && + v `S.isPrefixOf` encodeBS l filterconfig f = filter f $ concatMap splitconfigs $ M.toList $ fullconfig repo splitconfigs (k, vs) = map (\v -> (k, v)) vs diff --git a/Git/Repair.hs b/Git/Repair.hs index e6267a5f54..6031f4dd73 100644 --- a/Git/Repair.hs +++ b/Git/Repair.hs @@ -284,7 +284,7 @@ findUncorruptedCommit missing goodcommits branch r = do , Param "--format=%H" , Param (fromRef branch) ] r - let branchshas = catMaybes $ map extractSha ls + let branchshas = catMaybes $ map (extractSha . decodeBL) ls reflogshas <- RefLog.get branch r -- XXX Could try a bit harder here, and look -- for uncorrupted old commits in branches in the @@ -313,7 +313,7 @@ verifyCommit missing goodcommits commit r , Param "--format=%H %T" , Param (fromRef commit) ] r - let committrees = map parse ls + let committrees = map (parse . decodeBL) ls if any isNothing committrees || null committrees then do void cleanup @@ -342,7 +342,7 @@ verifyTree missing treesha r | S.member treesha missing = return False | otherwise = do (ls, cleanup) <- pipeNullSplit (LsTree.lsTreeParams LsTree.LsTreeRecursive treesha []) r - let objshas = map (LsTree.sha . LsTree.parseLsTree) ls + let objshas = mapMaybe (LsTree.sha <$$> eitherToMaybe . LsTree.parseLsTree) ls if any (`S.member` missing) objshas then do void cleanup @@ -366,7 +366,7 @@ checkIndex r = do - itself is not corrupt. -} checkIndexFast :: Repo -> IO Bool checkIndexFast r = do - (indexcontents, cleanup) <- LsFiles.stagedDetails [repoPath r] r + (indexcontents, cleanup) <- LsFiles.stagedDetails [toRawFilePath (repoPath r)] r length indexcontents `seq` cleanup missingIndex :: Repo -> IO Bool @@ -375,7 +375,7 @@ missingIndex r = not <$> doesFileExist (localGitDir r "index") {- Finds missing and ok files staged in the index. -} partitionIndex :: Repo -> IO ([LsFiles.StagedDetails], [LsFiles.StagedDetails], IO Bool) partitionIndex r = do - (indexcontents, cleanup) <- LsFiles.stagedDetails [repoPath r] r + (indexcontents, cleanup) <- LsFiles.stagedDetails [toRawFilePath (repoPath r)] r l <- forM indexcontents $ \i -> case i of (_file, Just sha, Just _mode) -> (,) <$> isMissing sha r <*> pure i _ -> pure (False, i) @@ -394,12 +394,12 @@ rewriteIndex r UpdateIndex.streamUpdateIndex r =<< (catMaybes <$> mapM reinject good) void cleanup - return $ map fst3 bad + return $ map (fromRawFilePath . fst3) bad where reinject (file, Just sha, Just mode) = case toTreeItemType mode of Nothing -> return Nothing Just treeitemtype -> Just <$> - UpdateIndex.stageFile sha treeitemtype file r + UpdateIndex.stageFile sha treeitemtype (fromRawFilePath file) r reinject _ = return Nothing newtype GoodCommits = GoodCommits (S.Set Sha) diff --git a/Git/Status.hs b/Git/Status.hs index 5a1077baf8..c15a11bd63 100644 --- a/Git/Status.hs +++ b/Git/Status.hs @@ -69,7 +69,7 @@ parseStatusZ = go [] getStatus :: [CommandParam] -> [FilePath] -> Repo -> IO ([StagedUnstaged Status], IO Bool) getStatus ps fs r = do (ls, cleanup) <- pipeNullSplit ps' r - return (parseStatusZ ls, cleanup) + return (parseStatusZ (map decodeBL ls), cleanup) where ps' = concat [ [Param "status"] diff --git a/Git/Tree.hs b/Git/Tree.hs index 3a8851099a..8a69c53a2a 100644 --- a/Git/Tree.hs +++ b/Git/Tree.hs @@ -115,7 +115,7 @@ mkTreeOutput :: FileMode -> ObjectType -> Sha -> TopFilePath -> String mkTreeOutput fm ot s f = concat [ showOct fm "" , " " - , show ot + , decodeBS (fmtObjectType ot) , " " , fromRef s , "\t" @@ -134,7 +134,7 @@ treeItemToTreeContent (TreeItem f m s) = case toTreeItemType m of treeItemToLsTreeItem :: TreeItem -> LsTree.TreeItem treeItemToLsTreeItem (TreeItem f mode sha) = LsTree.TreeItem { LsTree.mode = mode - , LsTree.typeobj = show BlobObject + , LsTree.typeobj = fmtObjectType BlobObject , LsTree.sha = sha , LsTree.file = f } @@ -239,7 +239,7 @@ adjustTree adjusttreeitem addtreeitems resolveaddconflict removefiles r repo = Just CommitObject -> do let ti = TreeCommit (LsTree.file i) (LsTree.mode i) (LsTree.sha i) go h wasmodified (ti:c) depth intree is - _ -> error ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"") + _ -> error ("unexpected object type \"" ++ decodeBS (LsTree.typeobj i) ++ "\"") | otherwise = return (c, wasmodified, i:is) adjustlist h depth ishere underhere l = do @@ -328,7 +328,7 @@ graftTree' subtree graftloc basetree repo hdl = go basetree graftdirs -- For a graftloc of "foo/bar/baz", this generates -- ["foo", "foo/bar", "foo/bar/baz"] - graftdirs = map (asTopFilePath . toInternalGitPath) $ + graftdirs = map (asTopFilePath . decodeBS . toInternalGitPath . encodeBS) $ mkpaths [] $ splitDirectories $ gitPath graftloc mkpaths _ [] = [] mkpaths base (d:rest) = (joinPath base d) : mkpaths (base ++ [d]) rest @@ -355,7 +355,7 @@ extractTree l = case go [] inTopTree l of Just CommitObject -> let c = TreeCommit (LsTree.file i) (LsTree.mode i) (LsTree.sha i) in go (c:t) intree is - _ -> parseerr ("unexpected object type \"" ++ LsTree.typeobj i ++ "\"") + _ -> parseerr ("unexpected object type \"" ++ decodeBS (LsTree.typeobj i) ++ "\"") | otherwise = Right (t, i:is) parseerr = Left diff --git a/Git/Types.hs b/Git/Types.hs index 4a4dff0c53..f15e334732 100644 --- a/Git/Types.hs +++ b/Git/Types.hs @@ -1,16 +1,23 @@ {- git data types - - - Copyright 2010-2018 Joey Hess + - Copyright 2010-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} + module Git.Types where import Network.URI +import Data.String +import Data.Default import qualified Data.Map as M +import qualified Data.ByteString as S import System.Posix.Types import Utility.SafeCommand +import Utility.FileSystemEncoding {- Support repositories on local disk, and repositories accessed via an URL. - @@ -31,9 +38,9 @@ data RepoLocation data Repo = Repo { location :: RepoLocation - , config :: M.Map String String + , config :: M.Map ConfigKey ConfigValue -- a given git config key can actually have multiple values - , fullconfig :: M.Map String [String] + , fullconfig :: M.Map ConfigKey [ConfigValue] -- remoteName holds the name used for this repo in some other -- repo's list of remotes, when this repo is such a remote , remoteName :: Maybe RemoteName @@ -44,6 +51,33 @@ data Repo = Repo , gitGlobalOpts :: [CommandParam] } deriving (Show, Eq, Ord) +newtype ConfigKey = ConfigKey S.ByteString + deriving (Ord, Eq) + +newtype ConfigValue = ConfigValue S.ByteString + deriving (Ord, Eq, Semigroup, Monoid) + +instance Default ConfigValue where + def = ConfigValue mempty + +fromConfigKey :: ConfigKey -> String +fromConfigKey (ConfigKey s) = decodeBS' s + +instance Show ConfigKey where + show = fromConfigKey + +fromConfigValue :: ConfigValue -> String +fromConfigValue (ConfigValue s) = decodeBS' s + +instance Show ConfigValue where + show = fromConfigValue + +instance IsString ConfigKey where + fromString = ConfigKey . encodeBS' + +instance IsString ConfigValue where + fromString = ConfigValue . encodeBS' + type RemoteName = String {- A git ref. Can be a sha1, or a branch or tag name. -} @@ -64,32 +98,31 @@ newtype RefDate = RefDate String {- Types of objects that can be stored in git. -} data ObjectType = BlobObject | CommitObject | TreeObject - deriving (Eq) -instance Show ObjectType where - show BlobObject = "blob" - show CommitObject = "commit" - show TreeObject = "tree" - -readObjectType :: String -> Maybe ObjectType +readObjectType :: S.ByteString -> Maybe ObjectType readObjectType "blob" = Just BlobObject readObjectType "commit" = Just CommitObject readObjectType "tree" = Just TreeObject readObjectType _ = Nothing +fmtObjectType :: ObjectType -> S.ByteString +fmtObjectType BlobObject = "blob" +fmtObjectType CommitObject = "commit" +fmtObjectType TreeObject = "tree" + {- Types of items in a tree. -} data TreeItemType = TreeFile | TreeExecutable | TreeSymlink | TreeSubmodule - deriving (Eq) + deriving (Eq, Show) {- Git uses magic numbers to denote the type of a tree item. -} -readTreeItemType :: String -> Maybe TreeItemType +readTreeItemType :: S.ByteString -> Maybe TreeItemType readTreeItemType "100644" = Just TreeFile readTreeItemType "100755" = Just TreeExecutable readTreeItemType "120000" = Just TreeSymlink readTreeItemType "160000" = Just TreeSubmodule readTreeItemType _ = Nothing -fmtTreeItemType :: TreeItemType -> String +fmtTreeItemType :: TreeItemType -> S.ByteString fmtTreeItemType TreeFile = "100644" fmtTreeItemType TreeExecutable = "100755" fmtTreeItemType TreeSymlink = "120000" diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs index ea9dd3500e..fc3c30e2ac 100644 --- a/Git/UnionMerge.hs +++ b/Git/UnionMerge.hs @@ -10,6 +10,7 @@ module Git.UnionMerge ( mergeIndex ) where +import qualified Data.ByteString.Lazy as L import qualified Data.ByteString.Lazy.Char8 as L8 import qualified Data.Set as S @@ -69,7 +70,7 @@ diffOpts = ["--raw", "-z", "-r", "--no-renames", "-l0"] doMerge :: HashObjectHandle -> CatFileHandle -> [String] -> Repo -> Streamer doMerge hashhandle ch differ repo streamer = do (diff, cleanup) <- pipeNullSplit (map Param differ) repo - go diff + go (map decodeBL' diff) void $ cleanup where go [] = noop @@ -80,7 +81,7 @@ doMerge hashhandle ch differ repo streamer = do {- Given an info line from a git raw diff, and the filename, generates - a line suitable for update-index that union merges the two sides of the - diff. -} -mergeFile :: String -> FilePath -> HashObjectHandle -> CatFileHandle -> IO (Maybe String) +mergeFile :: String -> FilePath -> HashObjectHandle -> CatFileHandle -> IO (Maybe L.ByteString) mergeFile info file hashhandle h = case filter (/= nullSha) [Ref asha, Ref bsha] of [] -> return Nothing (sha:[]) -> use sha diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs index 435c4f28e2..76094a3432 100644 --- a/Git/UpdateIndex.hs +++ b/Git/UpdateIndex.hs @@ -1,11 +1,11 @@ {- git-update-index library - - - Copyright 2011-2018 Joey Hess + - Copyright 2011-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} -{-# LANGUAGE BangPatterns, CPP #-} +{-# LANGUAGE BangPatterns, OverloadedStrings, CPP #-} module Git.UpdateIndex ( Streamer, @@ -32,12 +32,14 @@ import Git.FilePath import Git.Sha import qualified Git.DiffTreeItem as Diff +import qualified Data.ByteString.Lazy as L + {- Streamers are passed a callback and should feed it lines in the form - read by update-index, and generated by ls-tree. -} -type Streamer = (String -> IO ()) -> IO () +type Streamer = (L.ByteString -> IO ()) -> IO () {- A streamer with a precalculated value. -} -pureStreamer :: String -> Streamer +pureStreamer :: L.ByteString -> Streamer pureStreamer !s = \streamer -> streamer s {- Streams content into update-index from a list of Streamers. -} @@ -49,8 +51,8 @@ data UpdateIndexHandle = UpdateIndexHandle ProcessHandle Handle streamUpdateIndex' :: UpdateIndexHandle -> Streamer -> IO () streamUpdateIndex' (UpdateIndexHandle _ h) a = a $ \s -> do - hPutStr h s - hPutStr h "\0" + L.hPutStr h s + L.hPutStr h "\0" startUpdateIndex :: Repo -> IO UpdateIndexHandle startUpdateIndex repo = do @@ -84,14 +86,13 @@ lsSubTree (Ref x) p repo streamer = do {- Generates a line suitable to be fed into update-index, to add - a given file with a given sha. -} -updateIndexLine :: Sha -> TreeItemType -> TopFilePath -> String -updateIndexLine sha treeitemtype file = concat - [ fmtTreeItemType treeitemtype - , " blob " - , fromRef sha - , "\t" - , indexPath file - ] +updateIndexLine :: Sha -> TreeItemType -> TopFilePath -> L.ByteString +updateIndexLine sha treeitemtype file = L.fromStrict $ + fmtTreeItemType treeitemtype + <> " blob " + <> encodeBS (fromRef sha) + <> "\t" + <> indexPath file stageFile :: Sha -> TreeItemType -> FilePath -> Repo -> IO Streamer stageFile sha treeitemtype file repo = do @@ -105,7 +106,11 @@ unstageFile file repo = do return $ unstageFile' p unstageFile' :: TopFilePath -> Streamer -unstageFile' p = pureStreamer $ "0 " ++ fromRef nullSha ++ "\t" ++ indexPath p +unstageFile' p = pureStreamer $ L.fromStrict $ + "0 " + <> encodeBS' (fromRef nullSha) + <> "\t" + <> indexPath p {- A streamer that adds a symlink to the index. -} stageSymlink :: FilePath -> Sha -> Repo -> IO Streamer @@ -123,7 +128,7 @@ stageDiffTreeItem d = case toTreeItemType (Diff.dstmode d) of Just t -> pureStreamer $ updateIndexLine (Diff.dstsha d) t (Diff.file d) indexPath :: TopFilePath -> InternalGitPath -indexPath = toInternalGitPath . getTopFilePath +indexPath = toInternalGitPath . toRawFilePath . getTopFilePath {- Refreshes the index, by checking file stat information. -} refreshIndex :: Repo -> ((FilePath -> IO ()) -> IO ()) -> IO Bool diff --git a/Key.hs b/Key.hs index d666c0c45e..85739dc0f1 100644 --- a/Key.hs +++ b/Key.hs @@ -8,10 +8,12 @@ {-# OPTIONS_GHC -fno-warn-orphans #-} module Key ( - Key(..), + Key, + KeyData(..), AssociatedFile(..), - stubKey, - buildKey, + fromKey, + mkKey, + alterKey, keyParser, serializeKey, serializeKey', @@ -28,13 +30,7 @@ module Key ( import qualified Data.Text as T import qualified Data.ByteString as S -import qualified Data.ByteString.Char8 as S8 -import qualified Data.ByteString.Lazy as L -import Data.ByteString.Builder -import Data.ByteString.Builder.Extra import qualified Data.Attoparsec.ByteString as A -import qualified Data.Attoparsec.ByteString.Char8 as A8 -import Foreign.C.Types import Common import Types.Key @@ -43,98 +39,29 @@ import Utility.Bloom import Utility.Aeson import qualified Utility.SimpleProtocol as Proto -stubKey :: Key -stubKey = Key - { keyName = mempty - , keyVariety = OtherKey mempty - , keySize = Nothing - , keyMtime = Nothing - , keyChunkSize = Nothing - , keyChunkNum = Nothing - } - -- Gets the parent of a chunk key. nonChunkKey :: Key -> Key -nonChunkKey k = k - { keyChunkSize = Nothing - , keyChunkNum = Nothing - } +nonChunkKey k + | fromKey keyChunkSize k == Nothing && fromKey keyChunkNum k == Nothing = k + | otherwise = alterKey k $ \d -> d + { keyChunkSize = Nothing + , keyChunkNum = Nothing + } -- Where a chunk key is offset within its parent. chunkKeyOffset :: Key -> Maybe Integer chunkKeyOffset k = (*) - <$> keyChunkSize k - <*> (pred <$> keyChunkNum k) + <$> fromKey keyChunkSize k + <*> (pred <$> fromKey keyChunkNum k) isChunkKey :: Key -> Bool -isChunkKey k = isJust (keyChunkSize k) && isJust (keyChunkNum k) - --- Checks if a string looks like at least the start of a key. -isKeyPrefix :: String -> Bool -isKeyPrefix s = [fieldSep, fieldSep] `isInfixOf` s - -fieldSep :: Char -fieldSep = '-' - -{- Builds a ByteString from a Key. - - - - The name field is always shown last, separated by doubled fieldSeps, - - and is the only field allowed to contain the fieldSep. - -} -buildKey :: Key -> Builder -buildKey k = byteString (formatKeyVariety (keyVariety k)) - <> 's' ?: (integerDec <$> keySize k) - <> 'm' ?: (integerDec . (\(CTime t) -> fromIntegral t) <$> keyMtime k) - <> 'S' ?: (integerDec <$> keyChunkSize k) - <> 'C' ?: (integerDec <$> keyChunkNum k) - <> sepbefore (sepbefore (byteString (keyName k))) - where - sepbefore s = char7 fieldSep <> s - c ?: (Just b) = sepbefore (char7 c <> b) - _ ?: Nothing = mempty +isChunkKey k = isJust (fromKey keyChunkSize k) && isJust (fromKey keyChunkNum k) serializeKey :: Key -> String -serializeKey = decodeBL' . serializeKey' +serializeKey = decodeBS' . serializeKey' -serializeKey' :: Key -> L.ByteString -serializeKey' = toLazyByteStringWith (safeStrategy 128 smallChunkSize) L.empty . buildKey - -{- This is a strict parser for security reasons; a key - - can contain only 4 fields, which all consist only of numbers. - - Any key containing other fields, or non-numeric data will fail - - to parse. - - - - If a key contained non-numeric fields, they could be used to - - embed data used in a SHA1 collision attack, which would be a - - problem since the keys are committed to git. - -} -keyParser :: A.Parser Key -keyParser = do - -- key variety cannot be empty - v <- (parseKeyVariety <$> A8.takeWhile1 (/= fieldSep)) - s <- parsesize - m <- parsemtime - cs <- parsechunksize - cn <- parsechunknum - _ <- A8.char fieldSep - _ <- A8.char fieldSep - n <- A.takeByteString - if validKeyName v n - then return $ Key - { keyName = n - , keyVariety = v - , keySize = s - , keyMtime = m - , keyChunkSize = cs - , keyChunkNum = cn - } - else fail "invalid keyName" - where - parseopt p = (Just <$> (A8.char fieldSep *> p)) <|> pure Nothing - parsesize = parseopt $ A8.char 's' *> A8.decimal - parsemtime = parseopt $ CTime <$> (A8.char 'm' *> A8.decimal) - parsechunksize = parseopt $ A8.char 'S' *> A8.decimal - parsechunknum = parseopt $ A8.char 'C' *> A8.decimal +serializeKey' :: Key -> S.ByteString +serializeKey' = keySerialization deserializeKey :: String -> Maybe Key deserializeKey = deserializeKey' . encodeBS' @@ -142,35 +69,7 @@ deserializeKey = deserializeKey' . encodeBS' deserializeKey' :: S.ByteString -> Maybe Key deserializeKey' = eitherToMaybe . A.parseOnly keyParser -{- This splits any extension out of the keyName, returning the - - keyName minus extension, and the extension (including leading dot). - -} -splitKeyNameExtension :: Key -> (S.ByteString, S.ByteString) -splitKeyNameExtension = splitKeyNameExtension' . keyName - -splitKeyNameExtension' :: S.ByteString -> (S.ByteString, S.ByteString) -splitKeyNameExtension' keyname = S8.span (/= '.') keyname - -{- Limits the length of the extension in the keyName to mitigate against - - SHA1 collision attacks. - - - - In such an attack, the extension of the key could be made to contain - - the collision generation data, with the result that a signed git commit - - including such keys would not be secure. - - - - The maximum extension length ever generated for such a key was 8 - - characters, but they may be unicode which could use up to 4 bytes each, - - so 32 bytes. 64 bytes is used here to give a little future wiggle-room. - - The SHA1 common-prefix attack needs 128 bytes of data. - -} -validKeyName :: KeyVariety -> S.ByteString -> Bool -validKeyName kv name - | hasExt kv = - let ext = snd $ splitKeyNameExtension' name - in S.length ext <= 64 - | otherwise = True - -instance Arbitrary Key where +instance Arbitrary KeyData where arbitrary = Key <$> (encodeBS <$> (listOf1 $ elements $ ['A'..'Z'] ++ ['a'..'z'] ++ ['0'..'9'] ++ "-_\r\n \t")) <*> (parseKeyVariety . encodeBS <$> (listOf1 $ elements ['A'..'Z'])) -- BACKEND @@ -179,6 +78,17 @@ instance Arbitrary Key where <*> ((abs <$>) <$> arbitrary) -- chunksize cannot be negative <*> ((succ . abs <$>) <$> arbitrary) -- chunknum cannot be 0 or negative +-- AssociatedFile cannot be empty, and cannot contain a NUL +-- (but can be Nothing) +instance Arbitrary AssociatedFile where + arbitrary = (AssociatedFile . fmap mk <$> arbitrary) + `suchThat` (/= AssociatedFile (Just S.empty)) + where + mk = toRawFilePath . filter (/= '\NUL') + +instance Arbitrary Key where + arbitrary = mkKey . const <$> arbitrary + instance Hashable Key where hashIO32 = hashIO32 . serializeKey' hashIO64 = hashIO64 . serializeKey' @@ -196,3 +106,4 @@ instance Proto.Serializable Key where prop_isomorphic_key_encode :: Key -> Bool prop_isomorphic_key_encode k = Just k == (deserializeKey . serializeKey) k + diff --git a/Limit.hs b/Limit.hs index 5ac0fe636a..7511e39abc 100644 --- a/Limit.hs +++ b/Limit.hs @@ -97,7 +97,7 @@ matchGlobFile glob = go go (MatchingFile fi) = pure $ matchGlob cglob (matchFile fi) go (MatchingInfo p) = matchGlob cglob <$> getInfo (providedFilePath p) go (MatchingKey _ (AssociatedFile Nothing)) = pure False - go (MatchingKey _ (AssociatedFile (Just af))) = pure $ matchGlob cglob af + go (MatchingKey _ (AssociatedFile (Just af))) = pure $ matchGlob cglob (fromRawFilePath af) addMimeType :: String -> Annex () addMimeType = addMagicLimit "mimetype" getMagicMimeType providedMimeType @@ -110,13 +110,13 @@ addMagicLimit limitname querymagic selectprovidedinfo glob = do magic <- liftIO initMagicMime addLimit $ matchMagic limitname querymagic' selectprovidedinfo magic glob where - querymagic' magic f = liftIO (isPointerFile f) >>= \case + querymagic' magic f = liftIO (isPointerFile (toRawFilePath f)) >>= \case -- Avoid getting magic of a pointer file, which would -- wrongly be detected as text. Just _ -> return Nothing -- When the file is an annex symlink, get magic of the -- object file. - Nothing -> isAnnexLink f >>= \case + Nothing -> isAnnexLink (toRawFilePath f) >>= \case Just k -> withObjectLoc k $ querymagic magic Nothing -> querymagic magic f @@ -143,7 +143,7 @@ matchLockStatus :: Bool -> MatchInfo -> Annex Bool matchLockStatus _ (MatchingKey _ _) = pure False matchLockStatus _ (MatchingInfo _) = pure False matchLockStatus wantlocked (MatchingFile fi) = liftIO $ do - islocked <- isPointerFile (currFile fi) >>= \case + islocked <- isPointerFile (toRawFilePath (currFile fi)) >>= \case Just _key -> return False Nothing -> isSymbolicLink <$> getSymbolicLinkStatus (currFile fi) @@ -192,7 +192,7 @@ limitInDir dir = const go where go (MatchingFile fi) = checkf $ matchFile fi go (MatchingKey _ (AssociatedFile Nothing)) = return False - go (MatchingKey _ (AssociatedFile (Just af))) = checkf af + go (MatchingKey _ (AssociatedFile (Just af))) = checkf (fromRawFilePath af) go (MatchingInfo p) = checkf =<< getInfo (providedFilePath p) checkf = return . elem dir . splitPath . takeDirectory @@ -294,7 +294,7 @@ addInBackend = addLimit . limitInBackend limitInBackend :: MkLimit Annex limitInBackend name = Right $ const $ checkKey check where - check key = pure $ keyVariety key == variety + check key = pure $ fromKey keyVariety key == variety variety = parseKeyVariety (encodeBS name) {- Adds a limit to skip files not using a secure hash. -} @@ -302,7 +302,7 @@ addSecureHash :: Annex () addSecureHash = addLimit $ Right limitSecureHash limitSecureHash :: MatchFiles Annex -limitSecureHash _ = checkKey $ pure . cryptographicallySecure . keyVariety +limitSecureHash _ = checkKey $ pure . cryptographicallySecure . fromKey keyVariety {- Adds a limit to skip files that are too large or too small -} addLargerThan :: String -> Annex () @@ -327,7 +327,7 @@ limitSize lb vs s = case readSize dataUnits s of go sz _ (MatchingInfo p) = getInfo (providedFileSize p) >>= \sz' -> return (Just sz' `vs` Just sz) - checkkey sz key = return $ keySize key `vs` Just sz + checkkey sz key = return $ fromKey keySize key `vs` Just sz addMetaData :: String -> Annex () addMetaData = addLimit . limitMetaData @@ -368,7 +368,7 @@ addAccessedWithin duration = do secs = fromIntegral (durationSeconds duration) lookupFileKey :: FileInfo -> Annex (Maybe Key) -lookupFileKey = lookupFile . currFile +lookupFileKey = lookupFile . toRawFilePath . currFile checkKey :: (Key -> Annex Bool) -> MatchInfo -> Annex Bool checkKey a (MatchingFile fi) = lookupFileKey fi >>= maybe (return False) a diff --git a/Limit/Wanted.hs b/Limit/Wanted.hs index adbcafbfba..668614ce28 100644 --- a/Limit/Wanted.hs +++ b/Limit/Wanted.hs @@ -21,6 +21,6 @@ addWantDrop = addLimit $ Right $ const $ checkWant $ wantDrop False Nothing Nothing checkWant :: (AssociatedFile -> Annex Bool) -> MatchInfo -> Annex Bool -checkWant a (MatchingFile fi) = a (AssociatedFile (Just $ matchFile fi)) +checkWant a (MatchingFile fi) = a (AssociatedFile (Just $ toRawFilePath $ matchFile fi)) checkWant a (MatchingKey _ af) = a af checkWant _ (MatchingInfo {}) = return False diff --git a/Logs.hs b/Logs.hs index e7b15be3c6..d612aa8d56 100644 --- a/Logs.hs +++ b/Logs.hs @@ -5,11 +5,15 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Logs where import Annex.Common import Annex.DirHashes +import qualified Data.ByteString as S + {- There are several varieties of log file formats. -} data LogVariety = OldUUIDBasedLog @@ -22,7 +26,7 @@ data LogVariety {- Converts a path from the git-annex branch into one of the varieties - of logs used by git-annex, if it's a known path. -} -getLogVariety :: FilePath -> Maybe LogVariety +getLogVariety :: RawFilePath -> Maybe LogVariety getLogVariety f | f `elem` topLevelOldUUIDBasedLogs = Just OldUUIDBasedLog | f `elem` topLevelNewUUIDBasedLogs = Just NewUUIDBasedLog @@ -34,7 +38,7 @@ getLogVariety f | otherwise = PresenceLog <$> firstJust (presenceLogs f) {- All the old-format uuid-based logs stored in the top of the git-annex branch. -} -topLevelOldUUIDBasedLogs :: [FilePath] +topLevelOldUUIDBasedLogs :: [RawFilePath] topLevelOldUUIDBasedLogs = [ uuidLog , remoteLog @@ -49,161 +53,172 @@ topLevelOldUUIDBasedLogs = ] {- All the new-format uuid-based logs stored in the top of the git-annex branch. -} -topLevelNewUUIDBasedLogs :: [FilePath] +topLevelNewUUIDBasedLogs :: [RawFilePath] topLevelNewUUIDBasedLogs = [ exportLog ] {- All the ways to get a key from a presence log file -} -presenceLogs :: FilePath -> [Maybe Key] +presenceLogs :: RawFilePath -> [Maybe Key] presenceLogs f = [ urlLogFileKey f , locationLogFileKey f ] {- Top-level logs that are neither UUID based nor presence logs. -} -otherLogs :: [FilePath] +otherLogs :: [RawFilePath] otherLogs = [ numcopiesLog , groupPreferredContentLog ] -uuidLog :: FilePath +uuidLog :: RawFilePath uuidLog = "uuid.log" -numcopiesLog :: FilePath +numcopiesLog :: RawFilePath numcopiesLog = "numcopies.log" -configLog :: FilePath +configLog :: RawFilePath configLog = "config.log" -remoteLog :: FilePath +remoteLog :: RawFilePath remoteLog = "remote.log" -trustLog :: FilePath +trustLog :: RawFilePath trustLog = "trust.log" -groupLog :: FilePath +groupLog :: RawFilePath groupLog = "group.log" -preferredContentLog :: FilePath +preferredContentLog :: RawFilePath preferredContentLog = "preferred-content.log" -requiredContentLog :: FilePath +requiredContentLog :: RawFilePath requiredContentLog = "required-content.log" -groupPreferredContentLog :: FilePath +groupPreferredContentLog :: RawFilePath groupPreferredContentLog = "group-preferred-content.log" -scheduleLog :: FilePath +scheduleLog :: RawFilePath scheduleLog = "schedule.log" -activityLog :: FilePath +activityLog :: RawFilePath activityLog = "activity.log" -differenceLog :: FilePath +differenceLog :: RawFilePath differenceLog = "difference.log" -multicastLog :: FilePath +multicastLog :: RawFilePath multicastLog = "multicast.log" -exportLog :: FilePath +exportLog :: RawFilePath exportLog = "export.log" {- The pathname of the location log file for a given key. -} -locationLogFile :: GitConfig -> Key -> String -locationLogFile config key = branchHashDir config key keyFile key ++ ".log" +locationLogFile :: GitConfig -> Key -> RawFilePath +locationLogFile config key = toRawFilePath $ + branchHashDir config key keyFile key ++ ".log" {- The filename of the url log for a given key. -} -urlLogFile :: GitConfig -> Key -> FilePath -urlLogFile config key = branchHashDir config key keyFile key ++ urlLogExt +urlLogFile :: GitConfig -> Key -> RawFilePath +urlLogFile config key = toRawFilePath $ + branchHashDir config key keyFile key ++ decodeBS' urlLogExt {- Old versions stored the urls elsewhere. -} -oldurlLogs :: GitConfig -> Key -> [FilePath] -oldurlLogs config key = +oldurlLogs :: GitConfig -> Key -> [RawFilePath] +oldurlLogs config key = map toRawFilePath [ "remote/web" hdir serializeKey key ++ ".log" , "remote/web" hdir keyFile key ++ ".log" ] where hdir = branchHashDir config key -urlLogExt :: String +urlLogExt :: S.ByteString urlLogExt = ".log.web" {- Does not work on oldurllogs. -} -isUrlLog :: FilePath -> Bool -isUrlLog file = urlLogExt `isSuffixOf` file +isUrlLog :: RawFilePath -> Bool +isUrlLog file = urlLogExt `S.isSuffixOf` file {- The filename of the remote state log for a given key. -} -remoteStateLogFile :: GitConfig -> Key -> FilePath -remoteStateLogFile config key = branchHashDir config key - keyFile key ++ remoteStateLogExt +remoteStateLogFile :: GitConfig -> Key -> RawFilePath +remoteStateLogFile config key = + toRawFilePath (branchHashDir config key keyFile key) + <> remoteStateLogExt -remoteStateLogExt :: String +remoteStateLogExt :: S.ByteString remoteStateLogExt = ".log.rmt" -isRemoteStateLog :: FilePath -> Bool -isRemoteStateLog path = remoteStateLogExt `isSuffixOf` path +isRemoteStateLog :: RawFilePath -> Bool +isRemoteStateLog path = remoteStateLogExt `S.isSuffixOf` path {- The filename of the chunk log for a given key. -} -chunkLogFile :: GitConfig -> Key -> FilePath -chunkLogFile config key = branchHashDir config key keyFile key ++ chunkLogExt +chunkLogFile :: GitConfig -> Key -> RawFilePath +chunkLogFile config key = + toRawFilePath (branchHashDir config key keyFile key) + <> chunkLogExt -chunkLogExt :: String +chunkLogExt :: S.ByteString chunkLogExt = ".log.cnk" -isChunkLog :: FilePath -> Bool -isChunkLog path = chunkLogExt `isSuffixOf` path +isChunkLog :: RawFilePath -> Bool +isChunkLog path = chunkLogExt `S.isSuffixOf` path {- The filename of the metadata log for a given key. -} -metaDataLogFile :: GitConfig -> Key -> FilePath -metaDataLogFile config key = branchHashDir config key keyFile key ++ metaDataLogExt +metaDataLogFile :: GitConfig -> Key -> RawFilePath +metaDataLogFile config key = + toRawFilePath (branchHashDir config key keyFile key) + <> metaDataLogExt -metaDataLogExt :: String +metaDataLogExt :: S.ByteString metaDataLogExt = ".log.met" -isMetaDataLog :: FilePath -> Bool -isMetaDataLog path = metaDataLogExt `isSuffixOf` path +isMetaDataLog :: RawFilePath -> Bool +isMetaDataLog path = metaDataLogExt `S.isSuffixOf` path {- The filename of the remote metadata log for a given key. -} -remoteMetaDataLogFile :: GitConfig -> Key -> FilePath -remoteMetaDataLogFile config key = branchHashDir config key keyFile key ++ remoteMetaDataLogExt +remoteMetaDataLogFile :: GitConfig -> Key -> RawFilePath +remoteMetaDataLogFile config key = + toRawFilePath (branchHashDir config key keyFile key) + <> remoteMetaDataLogExt -remoteMetaDataLogExt :: String +remoteMetaDataLogExt :: S.ByteString remoteMetaDataLogExt = ".log.rmet" -isRemoteMetaDataLog :: FilePath -> Bool -isRemoteMetaDataLog path = remoteMetaDataLogExt `isSuffixOf` path +isRemoteMetaDataLog :: RawFilePath -> Bool +isRemoteMetaDataLog path = remoteMetaDataLogExt `S.isSuffixOf` path {- The filename of the remote content identifier log for a given key. -} -remoteContentIdentifierLogFile :: GitConfig -> Key -> FilePath -remoteContentIdentifierLogFile config key = branchHashDir config key keyFile key ++ remoteContentIdentifierExt +remoteContentIdentifierLogFile :: GitConfig -> Key -> RawFilePath +remoteContentIdentifierLogFile config key = + toRawFilePath (branchHashDir config key keyFile key) + <> remoteContentIdentifierExt -remoteContentIdentifierExt :: String +remoteContentIdentifierExt :: S.ByteString remoteContentIdentifierExt = ".log.cid" -isRemoteContentIdentifierLog :: FilePath -> Bool -isRemoteContentIdentifierLog path = remoteContentIdentifierExt `isSuffixOf` path +isRemoteContentIdentifierLog :: RawFilePath -> Bool +isRemoteContentIdentifierLog path = remoteContentIdentifierExt `S.isSuffixOf` path {- From an extension and a log filename, get the key that it's a log for. -} -extLogFileKey :: String -> FilePath -> Maybe Key +extLogFileKey :: S.ByteString -> RawFilePath -> Maybe Key extLogFileKey expectedext path - | ext == expectedext = fileKey base + | encodeBS' ext == expectedext = fileKey base | otherwise = Nothing where - file = takeFileName path + file = takeFileName (fromRawFilePath path) (base, ext) = splitAt (length file - extlen) file - extlen = length expectedext + extlen = S.length expectedext {- Converts a url log file into a key. - (Does not work on oldurlLogs.) -} -urlLogFileKey :: FilePath -> Maybe Key +urlLogFileKey :: RawFilePath -> Maybe Key urlLogFileKey = extLogFileKey urlLogExt {- Converts a pathname into a key if it's a location log. -} -locationLogFileKey :: FilePath -> Maybe Key +locationLogFileKey :: RawFilePath -> Maybe Key locationLogFileKey path -- Want only xx/yy/foo.log, not .log files in other places. - | length (splitDirectories path) /= 3 = Nothing + | length (splitDirectories (fromRawFilePath path)) /= 3 = Nothing | otherwise = extLogFileKey ".log" path diff --git a/Logs/Config.hs b/Logs/Config.hs index ca6387e4e0..1271c9826c 100644 --- a/Logs/Config.hs +++ b/Logs/Config.hs @@ -6,8 +6,8 @@ -} module Logs.Config ( - ConfigName, - ConfigValue, + ConfigKey(..), + ConfigValue(..), setGlobalConfig, unsetGlobalConfig, getGlobalConfig, @@ -18,48 +18,51 @@ import Annex.Common import Logs import Logs.MapLog import qualified Annex.Branch +import Git.Types (ConfigKey(..), ConfigValue(..)) import qualified Data.Map as M +import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L import qualified Data.Attoparsec.ByteString.Lazy as A import Data.ByteString.Builder -type ConfigName = String -type ConfigValue = String - -setGlobalConfig :: ConfigName -> ConfigValue -> Annex () +setGlobalConfig :: ConfigKey -> ConfigValue -> Annex () setGlobalConfig name new = do curr <- getGlobalConfig name when (curr /= Just new) $ setGlobalConfig' name new -setGlobalConfig' :: ConfigName -> ConfigValue -> Annex () +setGlobalConfig' :: ConfigKey -> ConfigValue -> Annex () setGlobalConfig' name new = do c <- liftIO currentVectorClock Annex.Branch.change configLog $ buildGlobalConfig . changeMapLog c name new . parseGlobalConfig -unsetGlobalConfig :: ConfigName -> Annex () +unsetGlobalConfig :: ConfigKey -> Annex () unsetGlobalConfig name = do curr <- getGlobalConfig name when (curr /= Nothing) $ - setGlobalConfig' name "" -- set to empty string to unset + -- set to empty string to unset + setGlobalConfig' name (ConfigValue mempty) -- Reads the global config log every time. -getGlobalConfig :: ConfigName -> Annex (Maybe ConfigValue) +getGlobalConfig :: ConfigKey -> Annex (Maybe ConfigValue) getGlobalConfig name = M.lookup name <$> loadGlobalConfig -buildGlobalConfig :: MapLog ConfigName ConfigValue -> Builder -buildGlobalConfig = buildMapLog fieldbuilder valuebuilder +buildGlobalConfig :: MapLog ConfigKey ConfigValue -> Builder +buildGlobalConfig = buildMapLog configkeybuilder valuebuilder where - fieldbuilder = byteString . encodeBS - valuebuilder = byteString . encodeBS + configkeybuilder (ConfigKey k) = byteString k + valuebuilder (ConfigValue v) = byteString v -parseGlobalConfig :: L.ByteString -> MapLog ConfigName ConfigValue -parseGlobalConfig = parseMapLog string string +parseGlobalConfig :: L.ByteString -> MapLog ConfigKey ConfigValue +parseGlobalConfig = parseMapLog configkeyparser valueparser where - string = decodeBS <$> A.takeByteString + configkeyparser = ConfigKey <$> A.takeByteString + valueparser = ConfigValue <$> A.takeByteString -loadGlobalConfig :: Annex (M.Map ConfigName ConfigValue) -loadGlobalConfig = M.filter (not . null) . simpleMap . parseGlobalConfig +loadGlobalConfig :: Annex (M.Map ConfigKey ConfigValue) +loadGlobalConfig = M.filter (\(ConfigValue v) -> not (S.null v)) + . simpleMap + . parseGlobalConfig <$> Annex.Branch.get configLog diff --git a/Logs/Export.hs b/Logs/Export.hs index 6ab1c231c7..fd2ebfe504 100644 --- a/Logs/Export.hs +++ b/Logs/Export.hs @@ -37,6 +37,8 @@ import qualified Data.ByteString.Lazy as L import qualified Data.Attoparsec.ByteString.Lazy as A import qualified Data.Attoparsec.ByteString.Char8 as A8 import Data.ByteString.Builder +import Data.Either +import Data.Char -- This constuctor is not itself exported to other modules, to enforce -- consistent use of exportedTreeishes. @@ -176,8 +178,9 @@ logExportExcluded u a = do getExportExcluded :: UUID -> Annex [Git.Tree.TreeItem] getExportExcluded u = do logf <- fromRepo $ gitAnnexExportExcludeLog u - liftIO $ catchDefaultIO [] $ - (map parser . lines) - <$> readFile logf + liftIO $ catchDefaultIO [] $ parser <$> L.readFile logf where - parser = Git.Tree.lsTreeItemToTreeItem . Git.LsTree.parseLsTree + parser = map Git.Tree.lsTreeItemToTreeItem + . rights + . map Git.LsTree.parseLsTree + . L.split (fromIntegral $ ord '\n') diff --git a/Logs/Location.hs b/Logs/Location.hs index d70f364849..66532ae413 100644 --- a/Logs/Location.hs +++ b/Logs/Location.hs @@ -71,7 +71,7 @@ loggedLocationsHistorical = getLoggedLocations . historicalLogInfo loggedLocationsRef :: Ref -> Annex [UUID] loggedLocationsRef ref = map (toUUID . fromLogInfo) . getLog <$> catObject ref -getLoggedLocations :: (FilePath -> Annex [LogInfo]) -> Key -> Annex [UUID] +getLoggedLocations :: (RawFilePath -> Annex [LogInfo]) -> Key -> Annex [UUID] getLoggedLocations getter key = do config <- Annex.getGitConfig map (toUUID . fromLogInfo) <$> getter (locationLogFile config key) diff --git a/Logs/MetaData.hs b/Logs/MetaData.hs index c139e7aa3e..ea1462c61f 100644 --- a/Logs/MetaData.hs +++ b/Logs/MetaData.hs @@ -57,7 +57,7 @@ import qualified Data.Map as M getCurrentMetaData :: Key -> Annex MetaData getCurrentMetaData = getCurrentMetaData' metaDataLogFile -getCurrentMetaData' :: (GitConfig -> Key -> FilePath) -> Key -> Annex MetaData +getCurrentMetaData' :: (GitConfig -> Key -> RawFilePath) -> Key -> Annex MetaData getCurrentMetaData' getlogfile k = do config <- Annex.getGitConfig ls <- S.toAscList <$> readLog (getlogfile config k) @@ -95,7 +95,7 @@ getCurrentRemoteMetaData (RemoteStateHandle u) k = extractRemoteMetaData u <$> addMetaData :: Key -> MetaData -> Annex () addMetaData = addMetaData' metaDataLogFile -addMetaData' :: (GitConfig -> Key -> FilePath) -> Key -> MetaData -> Annex () +addMetaData' :: (GitConfig -> Key -> RawFilePath) -> Key -> MetaData -> Annex () addMetaData' getlogfile k metadata = addMetaDataClocked' getlogfile k metadata =<< liftIO currentVectorClock @@ -106,7 +106,7 @@ addMetaData' getlogfile k metadata = addMetaDataClocked :: Key -> MetaData -> VectorClock -> Annex () addMetaDataClocked = addMetaDataClocked' metaDataLogFile -addMetaDataClocked' :: (GitConfig -> Key -> FilePath) -> Key -> MetaData -> VectorClock -> Annex () +addMetaDataClocked' :: (GitConfig -> Key -> RawFilePath) -> Key -> MetaData -> VectorClock -> Annex () addMetaDataClocked' getlogfile k d@(MetaData m) c | d == emptyMetaData = noop | otherwise = do @@ -151,5 +151,5 @@ copyMetaData oldkey newkey const $ buildLog l return True -readLog :: FilePath -> Annex (Log MetaData) +readLog :: RawFilePath -> Annex (Log MetaData) readLog = parseLog <$$> Annex.Branch.get diff --git a/Logs/PreferredContent/Raw.hs b/Logs/PreferredContent/Raw.hs index fb9393ce6e..fb95b8c264 100644 --- a/Logs/PreferredContent/Raw.hs +++ b/Logs/PreferredContent/Raw.hs @@ -28,7 +28,7 @@ preferredContentSet = setLog preferredContentLog requiredContentSet :: UUID -> PreferredContentExpression -> Annex () requiredContentSet = setLog requiredContentLog -setLog :: FilePath -> UUID -> PreferredContentExpression -> Annex () +setLog :: RawFilePath -> UUID -> PreferredContentExpression -> Annex () setLog logfile uuid@(UUID _) val = do c <- liftIO currentVectorClock Annex.Branch.change logfile $ diff --git a/Logs/Presence.hs b/Logs/Presence.hs index 5987460857..486af7ee13 100644 --- a/Logs/Presence.hs +++ b/Logs/Presence.hs @@ -30,7 +30,7 @@ import Git.Types (RefDate) {- Adds a LogLine to the log, removing any LogLines that are obsoleted by - adding it. -} -addLog :: FilePath -> LogLine -> Annex () +addLog :: RawFilePath -> LogLine -> Annex () addLog file line = Annex.Branch.change file $ \b -> buildLog $ compactLog (line : parseLog b) @@ -38,14 +38,14 @@ addLog file line = Annex.Branch.change file $ \b -> - older timestamp, that LogLine is preserved, rather than updating the log - with a newer timestamp. -} -maybeAddLog :: FilePath -> LogLine -> Annex () +maybeAddLog :: RawFilePath -> LogLine -> Annex () maybeAddLog file line = Annex.Branch.maybeChange file $ \s -> do m <- insertNewStatus line $ logMap $ parseLog s return $ buildLog $ mapLog m {- Reads a log file. - Note that the LogLines returned may be in any order. -} -readLog :: FilePath -> Annex [LogLine] +readLog :: RawFilePath -> Annex [LogLine] readLog = parseLog <$$> Annex.Branch.get {- Generates a new LogLine with the current time. -} @@ -55,10 +55,10 @@ logNow s i = do return $ LogLine c s i {- Reads a log and returns only the info that is still in effect. -} -currentLogInfo :: FilePath -> Annex [LogInfo] +currentLogInfo :: RawFilePath -> Annex [LogInfo] currentLogInfo file = map info <$> currentLog file -currentLog :: FilePath -> Annex [LogLine] +currentLog :: RawFilePath -> Annex [LogLine] currentLog file = filterPresent <$> readLog file {- Reads a historical version of a log and returns the info that was in @@ -66,6 +66,6 @@ currentLog file = filterPresent <$> readLog file - - The date is formatted as shown in gitrevisions man page. -} -historicalLogInfo :: RefDate -> FilePath -> Annex [LogInfo] +historicalLogInfo :: RefDate -> RawFilePath -> Annex [LogInfo] historicalLogInfo refdate file = map info . filterPresent . parseLog <$> Annex.Branch.getHistorical refdate file diff --git a/Logs/SingleValue.hs b/Logs/SingleValue.hs index 6a4e283a14..8edbd50786 100644 --- a/Logs/SingleValue.hs +++ b/Logs/SingleValue.hs @@ -25,13 +25,13 @@ import Annex.VectorClock import qualified Data.Set as S -readLog :: (Ord v, SingleValueSerializable v) => FilePath -> Annex (Log v) +readLog :: (Ord v, SingleValueSerializable v) => RawFilePath -> Annex (Log v) readLog = parseLog <$$> Annex.Branch.get -getLog :: (Ord v, SingleValueSerializable v) => FilePath -> Annex (Maybe v) +getLog :: (Ord v, SingleValueSerializable v) => RawFilePath -> Annex (Maybe v) getLog = newestValue <$$> readLog -setLog :: (SingleValueSerializable v) => FilePath -> v -> Annex () +setLog :: (SingleValueSerializable v) => RawFilePath -> v -> Annex () setLog f v = do c <- liftIO currentVectorClock let ent = LogEntry c v diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 58e035c1a0..2dabe5cf34 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -1,6 +1,6 @@ {- git-annex transfer information files and lock files - - - Copyright 2012 Joey Hess + - Copyright 2012-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -31,7 +31,7 @@ describeTransfer :: Transfer -> TransferInfo -> String describeTransfer t info = unwords [ show $ transferDirection t , show $ transferUUID t - , actionItemDesc $ ActionItemAssociatedFile + , decodeBS' $ actionItemDesc $ ActionItemAssociatedFile (associatedFile info) (transferKey t) , show $ bytesComplete info @@ -41,12 +41,14 @@ describeTransfer t info = unwords equivilantTransfer :: Transfer -> Transfer -> Bool equivilantTransfer t1 t2 | transferDirection t1 == Download && transferDirection t2 == Download && - transferKey t1 == transferKey t2 = True + transferKeyData t1 == transferKeyData t2 = True | otherwise = t1 == t2 percentComplete :: Transfer -> TransferInfo -> Maybe Percentage -percentComplete (Transfer { transferKey = key }) info = - percentage <$> keySize key <*> Just (fromMaybe 0 $ bytesComplete info) +percentComplete t info = + percentage + <$> keySize (transferKeyData t) + <*> Just (fromMaybe 0 $ bytesComplete info) {- Generates a callback that can be called as transfer progresses to update - the transfer info file. Also returns the file it'll be updating, @@ -72,7 +74,7 @@ mkProgressUpdater t info = do {- The minimum change in bytesComplete that is worth - updating a transfer info file for is 1% of the total - keySize, rounded down. -} - mindelta = case keySize (transferKey t) of + mindelta = case keySize (transferKeyData t) of Just sz -> sz `div` 100 Nothing -> 100 * 1024 -- arbitrarily, 100 kb @@ -155,7 +157,7 @@ sizeOfDownloadsInProgress wanted = sum . map remaining <$> getTransfers' [Download] wanted where remaining (t, info) = - case (keySize (transferKey t), bytesComplete info) of + case (fromKey keySize (transferKey t), bytesComplete info) of (Just sz, Just done) -> sz - done (Just sz, Nothing) -> sz (Nothing, _) -> 0 @@ -191,14 +193,14 @@ recordFailedTransfer t info = do {- The transfer information file to use for a given Transfer. -} transferFile :: Transfer -> Git.Repo -> FilePath -transferFile (Transfer direction u key) r = transferDir direction r +transferFile (Transfer direction u kd) r = transferDir direction r filter (/= '/') (fromUUID u) - keyFile key + keyFile (mkKey (const kd)) {- The transfer information file to use to record a failed Transfer -} failedTransferFile :: Transfer -> Git.Repo -> FilePath -failedTransferFile (Transfer direction u key) r = failedTransferDir u direction r - keyFile key +failedTransferFile (Transfer direction u kd) r = failedTransferDir u direction r + keyFile (mkKey (const kd)) {- The transfer lock file corresponding to a given transfer info file. -} transferLockFile :: FilePath -> FilePath @@ -213,7 +215,7 @@ parseTransferFile file [direction, u, key] -> Transfer <$> parseDirection direction <*> pure (toUUID u) - <*> fileKey key + <*> fmap (fromKey id) (fileKey key) _ -> Nothing where bits = splitDirectories file @@ -243,7 +245,7 @@ writeTransferInfo info = unlines #endif -- comes last; arbitrary content , let AssociatedFile afile = associatedFile info - in fromMaybe "" afile + in maybe "" fromRawFilePath afile ] readTransferInfoFile :: Maybe PID -> FilePath -> IO (Maybe TransferInfo) @@ -261,7 +263,7 @@ readTransferInfo mpid s = TransferInfo <*> pure Nothing <*> pure Nothing <*> bytes - <*> pure (AssociatedFile (if null filename then Nothing else Just filename)) + <*> pure (AssociatedFile (if null filename then Nothing else Just (toRawFilePath filename))) <*> pure False where #ifdef mingw32_HOST_OS diff --git a/Logs/Transitions.hs b/Logs/Transitions.hs index 232a47aada..26a7eeb3eb 100644 --- a/Logs/Transitions.hs +++ b/Logs/Transitions.hs @@ -12,6 +12,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Logs.Transitions where import Annex.Common @@ -26,7 +28,7 @@ import qualified Data.ByteString.Lazy as L import qualified Data.Attoparsec.ByteString.Lazy as A import qualified Data.Attoparsec.ByteString.Char8 as A8 -transitionsLog :: FilePath +transitionsLog :: RawFilePath transitionsLog = "transitions.log" data Transition @@ -94,6 +96,6 @@ knownTransitionList = nub . rights . map transition . S.elems {- Typically ran with Annex.Branch.change, but we can't import Annex.Branch - here since it depends on this module. -} -recordTransitions :: (FilePath -> (L.ByteString -> Builder) -> Annex ()) -> Transitions -> Annex () +recordTransitions :: (RawFilePath -> (L.ByteString -> Builder) -> Annex ()) -> Transitions -> Annex () recordTransitions changer t = changer transitionsLog $ buildTransitions . S.union t . parseTransitionsStrictly "local" diff --git a/Logs/Web.hs b/Logs/Web.hs index b057a6580e..a59ea99205 100644 --- a/Logs/Web.hs +++ b/Logs/Web.hs @@ -93,7 +93,7 @@ knownUrls = do Annex.Branch.update Annex.Branch.commit =<< Annex.Branch.commitMessage Annex.Branch.withIndex $ do - top <- fromRepo Git.repoPath + top <- toRawFilePath <$> fromRepo Git.repoPath (l, cleanup) <- inRepo $ Git.LsFiles.stagedDetails [top] r <- mapM getkeyurls l void $ liftIO cleanup diff --git a/Messages.hs b/Messages.hs index a99aff6271..77ebdb9714 100644 --- a/Messages.hs +++ b/Messages.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Messages ( showStart, showStart', @@ -53,6 +55,7 @@ import System.Log.Formatter import System.Log.Handler (setFormatter) import System.Log.Handler.Simple import Control.Concurrent +import qualified Data.ByteString as S import Common import Types @@ -66,21 +69,21 @@ import Messages.Concurrent import qualified Messages.JSON as JSON import qualified Annex -showStart :: String -> FilePath -> Annex () +showStart :: String -> RawFilePath -> Annex () showStart command file = outputMessage json $ - command ++ " " ++ file ++ " " + encodeBS' command <> " " <> file <> " " where json = JSON.start command (Just file) Nothing showStart' :: String -> Maybe String -> Annex () -showStart' command mdesc = outputMessage json $ +showStart' command mdesc = outputMessage json $ encodeBS' $ command ++ (maybe "" (" " ++) mdesc) ++ " " where json = JSON.start command Nothing Nothing showStartKey :: String -> Key -> ActionItem -> Annex () showStartKey command key i = outputMessage json $ - command ++ " " ++ actionItemDesc i ++ " " + encodeBS' command <> " " <> actionItemDesc i <> " " where json = JSON.start command (actionItemWorkTreeFile i) (Just key) @@ -112,7 +115,7 @@ showEndMessage (StartNoMessage _) = const noop showEndMessage (CustomOutput _) = const noop showNote :: String -> Annex () -showNote s = outputMessage (JSON.note s) $ "(" ++ s ++ ") " +showNote s = outputMessage (JSON.note s) $ encodeBS' $ "(" ++ s ++ ") " showAction :: String -> Annex () showAction s = showNote $ s ++ "..." @@ -127,7 +130,7 @@ showSideAction m = Annex.getState Annex.output >>= go Annex.changeState $ \s -> s { Annex.output = st' } | sideActionBlock st == InBlock = return () | otherwise = p - p = outputMessage JSON.none $ "(" ++ m ++ "...)\n" + p = outputMessage JSON.none $ encodeBS' $ "(" ++ m ++ "...)\n" showStoringStateAction :: Annex () showStoringStateAction = showSideAction "recording state in git" @@ -171,7 +174,7 @@ showOutput = unlessM commandProgressDisabled $ outputMessage JSON.none "\n" showLongNote :: String -> Annex () -showLongNote s = outputMessage (JSON.note s) (formatLongNote s) +showLongNote s = outputMessage (JSON.note s) (encodeBS' (formatLongNote s)) formatLongNote :: String -> String formatLongNote s = '\n' : indent s ++ "\n" @@ -179,7 +182,8 @@ formatLongNote s = '\n' : indent s ++ "\n" -- Used by external special remote, displayed same as showLongNote -- to console, but json object containing the info is emitted immediately. showInfo :: String -> Annex () -showInfo s = outputMessage' outputJSON (JSON.info s) (formatLongNote s) +showInfo s = outputMessage' outputJSON (JSON.info s) $ + encodeBS' (formatLongNote s) showEndOk :: Annex () showEndOk = showEndResult True @@ -188,9 +192,9 @@ showEndFail :: Annex () showEndFail = showEndResult False showEndResult :: Bool -> Annex () -showEndResult ok = outputMessage (JSON.end ok) $ endResult ok ++ "\n" +showEndResult ok = outputMessage (JSON.end ok) $ endResult ok <> "\n" -endResult :: Bool -> String +endResult :: Bool -> S.ByteString endResult True = "ok" endResult False = "failed" @@ -238,11 +242,11 @@ showCustom command a = do r <- a outputMessage (JSON.end r) "" -showHeader :: String -> Annex () -showHeader h = outputMessage JSON.none $ (h ++ ": ") +showHeader :: S.ByteString -> Annex () +showHeader h = outputMessage JSON.none (h <> ": ") -showRaw :: String -> Annex () -showRaw s = outputMessage JSON.none (s ++ "\n") +showRaw :: S.ByteString -> Annex () +showRaw s = outputMessage JSON.none (s <> "\n") setupConsole :: IO () setupConsole = do diff --git a/Messages/Internal.hs b/Messages/Internal.hs index edfb38d5d7..79829ac151 100644 --- a/Messages/Internal.hs +++ b/Messages/Internal.hs @@ -14,17 +14,19 @@ import Messages.Concurrent import qualified Messages.JSON as JSON import Messages.JSON (JSONBuilder) +import qualified Data.ByteString as S + withMessageState :: (MessageState -> Annex a) -> Annex a withMessageState a = Annex.getState Annex.output >>= a -outputMessage :: JSONBuilder -> String -> Annex () +outputMessage :: JSONBuilder -> S.ByteString -> Annex () outputMessage = outputMessage' bufferJSON -outputMessage' :: (JSONBuilder -> MessageState -> Annex Bool) -> JSONBuilder -> String -> Annex () +outputMessage' :: (JSONBuilder -> MessageState -> Annex Bool) -> JSONBuilder -> S.ByteString -> Annex () outputMessage' jsonoutputter jsonbuilder msg = withMessageState $ \s -> case outputType s of NormalOutput - | concurrentOutputEnabled s -> concurrentMessage s False msg q - | otherwise -> liftIO $ flushed $ putStr msg + | concurrentOutputEnabled s -> concurrentMessage s False (decodeBS msg) q + | otherwise -> liftIO $ flushed $ S.putStr msg JSONOutput _ -> void $ jsonoutputter jsonbuilder s QuietOutput -> q diff --git a/Messages/JSON.hs b/Messages/JSON.hs index 976baf6e1d..7561c61261 100644 --- a/Messages/JSON.hs +++ b/Messages/JSON.hs @@ -43,6 +43,7 @@ import Key import Utility.Metered import Utility.Percentage import Utility.Aeson +import Utility.FileSystemEncoding -- A global lock to avoid concurrent threads emitting json at the same time. {-# NOINLINE emitLock #-} @@ -63,13 +64,13 @@ type JSONBuilder = Maybe (Object, Bool) -> Maybe (Object, Bool) none :: JSONBuilder none = id -start :: String -> Maybe FilePath -> Maybe Key -> JSONBuilder +start :: String -> Maybe RawFilePath -> Maybe Key -> JSONBuilder start command file key _ = Just (o, False) where Object o = toJSON' $ JSONActionItem { itemCommand = Just command , itemKey = key - , itemFile = file + , itemFile = fromRawFilePath <$> file , itemAdded = Nothing } diff --git a/Messages/Progress.hs b/Messages/Progress.hs index 48615918ae..113c3f5286 100644 --- a/Messages/Progress.hs +++ b/Messages/Progress.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE OverloadedStrings #-} module Messages.Progress where @@ -36,7 +37,7 @@ instance MeterSize FileSize where getMeterSize = pure . Just instance MeterSize Key where - getMeterSize = pure . keySize + getMeterSize = pure . fromKey keySize instance MeterSize InodeCache where getMeterSize = pure . Just . inodeCacheFileSize @@ -51,7 +52,7 @@ instance MeterSize KeySource where data KeySizer = KeySizer Key (Annex (Maybe FilePath)) instance MeterSize KeySizer where - getMeterSize (KeySizer k getsrcfile) = case keySize k of + getMeterSize (KeySizer k getsrcfile) = case fromKey keySize k of Just sz -> return (Just sz) Nothing -> do srcfile <- getsrcfile diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs index ac105f2d21..e9895d3de4 100644 --- a/P2P/Protocol.hs +++ b/P2P/Protocol.hs @@ -22,6 +22,7 @@ import Utility.AuthToken import Utility.Applicative import Utility.PartialPrelude import Utility.Metered +import Utility.FileSystemEncoding import Git.FilePath import Annex.ChangedRefs (ChangedRefs) @@ -166,17 +167,17 @@ instance Proto.Serializable Service where instance Proto.Serializable AssociatedFile where serialize (AssociatedFile Nothing) = "" serialize (AssociatedFile (Just af)) = - toInternalGitPath $ concatMap esc af + decodeBS' $ toInternalGitPath $ encodeBS' $ concatMap esc $ fromRawFilePath af where esc '%' = "%%" esc c | isSpace c = "%" | otherwise = [c] - deserialize s = case fromInternalGitPath $ deesc [] s of + deserialize s = case fromRawFilePath $ fromInternalGitPath $ toRawFilePath $ deesc [] s of [] -> Just (AssociatedFile Nothing) f - | isRelative f -> Just (AssociatedFile (Just f)) + | isRelative f -> Just $ AssociatedFile $ Just $ toRawFilePath f | otherwise -> Nothing where deesc b [] = reverse b diff --git a/Remote.hs b/Remote.hs index e54f711acf..fb096736ee 100644 --- a/Remote.hs +++ b/Remote.hs @@ -74,7 +74,7 @@ import Logs.Web import Remote.List import Config import Config.DynamicConfig -import Git.Types (RemoteName) +import Git.Types (RemoteName, ConfigKey(..), fromConfigValue) import Utility.Aeson {- Map from UUIDs of Remotes to a calculated value. -} @@ -147,10 +147,12 @@ byName' n = go . filter matching <$> remoteList {- Finds the remote or remote group matching the name. -} byNameOrGroup :: RemoteName -> Annex [Remote] -byNameOrGroup n = go =<< getConfigMaybe (ConfigKey ("remotes." ++ n)) +byNameOrGroup n = go =<< getConfigMaybe (ConfigKey ("remotes." <> encodeBS' n)) where - go (Just l) = catMaybes <$> mapM (byName . Just) (splitc ' ' l) - go Nothing = maybeToList <$> byName (Just n) + go (Just l) = catMaybes + <$> mapM (byName . Just) (splitc ' ' (fromConfigValue l)) + go Nothing = maybeToList + <$> byName (Just n) {- Only matches remote name, not UUID -} byNameOnly :: RemoteName -> Annex (Maybe Remote) diff --git a/Remote/Adb.hs b/Remote/Adb.hs index 50e708826a..03e3819cff 100644 --- a/Remote/Adb.hs +++ b/Remote/Adb.hs @@ -269,7 +269,7 @@ listImportableContentsM serial adir = liftIO $ let (stat, fn) = separate (== '\t') l sz = fromMaybe 0 (readish (takeWhile (/= ' ') stat)) cid = ContentIdentifier (encodeBS' stat) - loc = mkImportLocation $ + loc = mkImportLocation $ toRawFilePath $ Posix.makeRelative (fromAndroidPath adir) fn in Just (loc, (cid, sz)) mk _ = Nothing @@ -331,7 +331,7 @@ checkPresentExportWithContentIdentifierM serial adir _k loc knowncids = androidExportLocation :: AndroidPath -> ExportLocation -> AndroidPath androidExportLocation adir loc = AndroidPath $ - fromAndroidPath adir ++ "/" ++ fromExportLocation loc + fromAndroidPath adir ++ "/" ++ fromRawFilePath (fromExportLocation loc) -- | List all connected Android devices. enumerateAdbConnected :: IO [AndroidSerial] diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs index b18e0334a2..09fa5ed744 100644 --- a/Remote/BitTorrent.hs +++ b/Remote/BitTorrent.hs @@ -258,7 +258,7 @@ downloadTorrentContent k u dest filenum p = do , return False ) where - download torrent tmpdir = ariaProgress (keySize k) p + download torrent tmpdir = ariaProgress (fromKey keySize k) p [ Param $ "--select-file=" ++ show filenum , File torrent , Param "-d" diff --git a/Remote/Bup.hs b/Remote/Bup.hs index ba06939c8e..8fa00cbc41 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -1,6 +1,6 @@ {- Using bup as a remote. - - - Copyright 2011-2014 Joey Hess + - Copyright 2011-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -8,6 +8,7 @@ module Remote.Bup (remote) where import qualified Data.Map as M +import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L import Data.ByteString.Lazy.UTF8 (fromString) @@ -15,6 +16,7 @@ import Annex.Common import qualified Annex import Types.Remote import Types.Creds +import Git.Types (ConfigValue(..), fromConfigKey) import qualified Git import qualified Git.Command import qualified Git.Config @@ -207,12 +209,12 @@ storeBupUUID u buprepo = do then do showAction "storing uuid" unlessM (onBupRemote r boolSystem "git" - [Param "config", Param "annex.uuid", Param v]) $ + [Param "config", Param (fromConfigKey configkeyUUID), Param v]) $ giveup "ssh failed" else liftIO $ do r' <- Git.Config.read r - let olduuid = Git.Config.get "annex.uuid" "" r' - when (olduuid == "") $ + let ConfigValue olduuid = Git.Config.get configkeyUUID mempty r' + when (S.null olduuid) $ Git.Command.run [ Param "config" , Param "annex.uuid" @@ -248,7 +250,7 @@ getBupUUID r u | otherwise = liftIO $ do ret <- tryIO $ Git.Config.read r case ret of - Right r' -> return (toUUID $ Git.Config.get "annex.uuid" "" r', r') + Right r' -> return (toUUID $ Git.Config.get configkeyUUID mempty r', r') Left _ -> return (NoUUID, r) {- Converts a bup remote path spec into a Git.Repo. There are some diff --git a/Remote/Directory.hs b/Remote/Directory.hs index e5b397b3e9..0387474f9a 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -295,18 +295,18 @@ renameExportM d _k oldloc newloc = liftIO $ Just <$> go dest = exportPath d newloc exportPath :: FilePath -> ExportLocation -> FilePath -exportPath d loc = d fromExportLocation loc +exportPath d loc = d fromRawFilePath (fromExportLocation loc) {- Removes the ExportLocation's parent directory and its parents, so long as - they're empty, up to but not including the topdir. -} removeExportLocation :: FilePath -> ExportLocation -> IO () removeExportLocation topdir loc = - go (Just $ takeDirectory $ fromExportLocation loc) (Right ()) + go (Just $ takeDirectory $ fromRawFilePath $ fromExportLocation loc) (Right ()) where go _ (Left _e) = return () go Nothing _ = return () go (Just loc') _ = go (upFrom loc') - =<< tryIO (removeDirectory $ exportPath topdir (mkExportLocation loc')) + =<< tryIO (removeDirectory $ exportPath topdir (mkExportLocation (toRawFilePath loc'))) listImportableContentsM :: FilePath -> Annex (Maybe (ImportableContents (ContentIdentifier, ByteSize))) listImportableContentsM dir = catchMaybeIO $ liftIO $ do @@ -319,7 +319,7 @@ listImportableContentsM dir = catchMaybeIO $ liftIO $ do mkContentIdentifier f st >>= \case Nothing -> return Nothing Just cid -> do - relf <- relPathDirToFile dir f + relf <- toRawFilePath <$> relPathDirToFile dir f sz <- getFileSize' f st return $ Just (mkImportLocation relf, (cid, sz)) diff --git a/Remote/External.hs b/Remote/External.hs index cbf3e57b7a..c172bc71cd 100644 --- a/Remote/External.hs +++ b/Remote/External.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Remote.External (remote) where import Remote.External.Types @@ -716,7 +718,7 @@ checkKeyUrl :: Git.Repo -> CheckPresent checkKeyUrl r k = do showChecking r us <- getWebUrls k - anyM (\u -> withUrlOptions $ checkBoth u (keySize k)) us + anyM (\u -> withUrlOptions $ checkBoth u (fromKey keySize k)) us getWebUrls :: Key -> Annex [URLString] getWebUrls key = filter supported <$> getUrls key diff --git a/Remote/External/Types.hs b/Remote/External/Types.hs index 9c1e207aa1..b9785cb140 100644 --- a/Remote/External/Types.hs +++ b/Remote/External/Types.hs @@ -101,10 +101,10 @@ newtype SafeKey = SafeKey Key mkSafeKey :: Key -> Either String SafeKey mkSafeKey k - | any isSpace (decodeBS $ keyName k) = Left $ concat + | any isSpace (decodeBS $ fromKey keyName k) = Left $ concat [ "Sorry, this file cannot be stored on an external special remote because its key's name contains a space. " , "To avoid this problem, you can run: git-annex migrate --backend=" - , decodeBS (formatKeyVariety (keyVariety k)) + , decodeBS (formatKeyVariety (fromKey keyVariety k)) , " and pass it the name of the file" ] | otherwise = Right (SafeKey k) @@ -384,12 +384,12 @@ instance Proto.Serializable URI where deserialize = parseURI instance Proto.Serializable ExportLocation where - serialize = fromExportLocation - deserialize = Just . mkExportLocation + serialize = fromRawFilePath . fromExportLocation + deserialize = Just . mkExportLocation . toRawFilePath instance Proto.Serializable ExportDirectory where - serialize = fromExportDirectory - deserialize = Just . mkExportDirectory + serialize = fromRawFilePath . fromExportDirectory + deserialize = Just . mkExportDirectory . toRawFilePath instance Proto.Serializable ExtensionList where serialize (ExtensionList l) = unwords l diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs index ff948ba0d6..4682637eaf 100644 --- a/Remote/GCrypt.hs +++ b/Remote/GCrypt.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Remote.GCrypt ( remote, chainGen, @@ -16,6 +18,7 @@ module Remote.GCrypt ( ) where import qualified Data.Map as M +import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L import Control.Exception import Data.Default @@ -27,6 +30,7 @@ import Types.GitConfig import Types.Crypto import Types.Creds import Types.Transfer +import Git.Types (ConfigKey(..), fromConfigKey, fromConfigValue) import qualified Git import qualified Git.Command import qualified Git.Config @@ -96,7 +100,7 @@ gen baser u c gc rs = do (Just remotename, Just c') -> do setGcryptEncryption c' remotename storeUUIDIn (remoteConfig baser "uuid") u' - setConfig (ConfigKey $ Git.GCrypt.remoteConfigKey "gcrypt-id" remotename) gcryptid + setConfig (Git.GCrypt.remoteConfigKey "gcrypt-id" remotename) gcryptid gen' r u' c' gc rs _ -> do warning $ "not using unknown gcrypt repository pointed to by remote " ++ Git.repoDescribe r @@ -159,11 +163,12 @@ rsyncTransportToObjects r gc = do rsyncTransport :: Git.Repo -> RemoteGitConfig -> Annex (Annex [CommandParam], String, AccessMethod) rsyncTransport r gc - | "ssh://" `isPrefixOf` loc = sshtransport $ break (== '/') $ drop (length "ssh://") loc + | sshprefix `isPrefixOf` loc = sshtransport $ break (== '/') $ drop (length sshprefix) loc | "//:" `isInfixOf` loc = othertransport | ":" `isInfixOf` loc = sshtransport $ separate (== ':') loc | otherwise = othertransport where + sshprefix = "ssh://" :: String loc = Git.repoLocation r sshtransport (host, path) = do let rsyncpath = if "/~/" `isPrefixOf` path @@ -252,7 +257,7 @@ setupRepo gcryptid r | otherwise = localsetup r where localsetup r' = do - let setconfig k v = liftIO $ Git.Command.run [Param "config", Param k, Param v] r' + let setconfig k v = liftIO $ Git.Command.run [Param "config", Param (fromConfigKey k), Param v] r' setconfig coreGCryptId gcryptid setconfig denyNonFastForwards (Git.Config.boolConfig False) return AccessDirect @@ -272,8 +277,8 @@ setupRepo gcryptid r , Param tmpconfig ] liftIO $ do - void $ Git.Config.changeFile tmpconfig coreGCryptId gcryptid - void $ Git.Config.changeFile tmpconfig denyNonFastForwards (Git.Config.boolConfig False) + void $ Git.Config.changeFile tmpconfig coreGCryptId (encodeBS' gcryptid) + void $ Git.Config.changeFile tmpconfig denyNonFastForwards (Git.Config.boolConfig' False) ok <- liftIO $ rsync $ opts ++ [ Param "--recursive" , Param $ tmp ++ "/" @@ -289,7 +294,7 @@ setupRepo gcryptid r (\f p -> liftIO (boolSystem f p), return False) "gcryptsetup" [ Param gcryptid ] [] - denyNonFastForwards = "receive.denyNonFastForwards" + denyNonFastForwards = ConfigKey "receive.denyNonFastForwards" accessShell :: Remote -> Bool accessShell = accessShellConfig . gitconfig @@ -326,7 +331,7 @@ setGcryptEncryption c remotename = do Nothing -> noop Just (KeyIds { keyIds = ks}) -> do setConfig participants (unwords ks) - let signingkey = ConfigKey $ Git.GCrypt.remoteSigningKey remotename + let signingkey = Git.GCrypt.remoteSigningKey remotename cmd <- gpgCmd <$> Annex.getGitConfig skeys <- M.keys <$> liftIO (secretKeys cmd) case filter (`elem` ks) skeys of @@ -335,7 +340,7 @@ setGcryptEncryption c remotename = do setConfig (remoteconfig Git.GCrypt.remotePublishParticipantConfigKey) (Git.Config.boolConfig True) where - remoteconfig n = ConfigKey $ n remotename + remoteconfig n = n remotename store :: Remote -> Remote.Rsync.RsyncOpts -> Storer store r rsyncopts k s p = do @@ -435,7 +440,7 @@ getGCryptUUID fast r = do (genUUIDInNameSpace gCryptNameSpace <$>) . fst <$> getGCryptId fast r dummycfg -coreGCryptId :: String +coreGCryptId :: ConfigKey coreGCryptId = "core.gcrypt-id" {- gcrypt repos set up by git-annex as special remotes have a @@ -457,9 +462,9 @@ getGCryptId fast r gc | otherwise = return (Nothing, r) where extract Nothing = (Nothing, r) - extract (Just r') = (Git.Config.getMaybe coreGCryptId r', r') + extract (Just r') = (fromConfigValue <$> Git.Config.getMaybe coreGCryptId r', r') -getConfigViaRsync :: Git.Repo -> RemoteGitConfig -> Annex (Either SomeException (Git.Repo, String)) +getConfigViaRsync :: Git.Repo -> RemoteGitConfig -> Annex (Either SomeException (Git.Repo, S.ByteString)) getConfigViaRsync r gc = do (rsynctransport, rsyncurl, _) <- rsyncTransport r gc opts <- rsynctransport diff --git a/Remote/Git.hs b/Remote/Git.hs index 6e1b31f748..7dc85aa629 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -6,6 +6,7 @@ -} {-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} module Remote.Git ( remote, @@ -68,6 +69,7 @@ import Utility.FileMode import Control.Concurrent import Control.Concurrent.MSampleVar import qualified Data.Map as M +import qualified Data.ByteString as S import Network.URI remote :: RemoteType @@ -86,14 +88,14 @@ list autoinit = do rs <- mapM (tweakurl c) =<< Annex.getGitRemotes mapM (configRead autoinit) rs where - annexurl n = "remote." ++ n ++ ".annexurl" + annexurl n = Git.ConfigKey ("remote." <> encodeBS' n <> ".annexurl") tweakurl c r = do let n = fromJust $ Git.remoteName r case M.lookup (annexurl n) c of Nothing -> return r Just url -> inRepo $ \g -> Git.Construct.remoteNamed n $ - Git.Construct.fromRemoteLocation url g + Git.Construct.fromRemoteLocation (Git.fromConfigValue url) g {- Git remotes are normally set up using standard git command, not - git-annex initremote and enableremote. @@ -254,7 +256,7 @@ tryGitConfigRead autoinit r v <- liftIO $ Git.Config.fromPipe r cmd params case v of Right (r', val) -> do - unless (isUUIDConfigured r' || null val) $ do + unless (isUUIDConfigured r' || S.null val) $ do warning $ "Failed to get annex.uuid configuration of repository " ++ Git.repoDescribe r warning $ "Instead, got: " ++ show val warning $ "This is unexpected; please check the network transport!" @@ -367,7 +369,7 @@ inAnnex' repo rmt (State connpool duc _ _) key checkhttp = do showChecking repo gc <- Annex.getGitConfig - ifM (Url.withUrlOptions $ \uo -> anyM (\u -> Url.checkBoth u (keySize key) uo) (keyUrls gc repo rmt key)) + ifM (Url.withUrlOptions $ \uo -> anyM (\u -> Url.checkBoth u (fromKey keySize key) uo) (keyUrls gc repo rmt key)) ( return True , giveup "not found" ) @@ -511,7 +513,7 @@ copyFromRemote'' repo forcersync r st@(State connpool _ _ _) key file dest meter Nothing -> return (False, UnVerified) Just (object, checksuccess) -> do copier <- mkCopier hardlink st params - runTransfer (Transfer Download u key) + runTransfer (Transfer Download u (fromKey id key)) file stdRetry $ \p -> metered (Just (combineMeterUpdate p meterupdate)) key $ \_ p' -> copier object dest p' checksuccess @@ -549,7 +551,7 @@ copyFromRemote'' repo forcersync r st@(State connpool _ _ _) key file dest meter u <- getUUID let AssociatedFile afile = file let fields = (Fields.remoteUUID, fromUUID u) - : maybe [] (\f -> [(Fields.associatedFile, f)]) afile + : maybe [] (\f -> [(Fields.associatedFile, fromRawFilePath f)]) afile Just (cmd, params) <- Ssh.git_annex_shell ConsumeStdin repo "transferinfo" [Param $ serializeKey key] fields @@ -647,7 +649,7 @@ copyToRemote' repo r st@(State connpool duc _ _) key file meterupdate -- run copy from perspective of remote onLocalFast repo r $ ifM (Annex.Content.inAnnex key) ( return True - , runTransfer (Transfer Download u key) file stdRetry $ \p -> do + , runTransfer (Transfer Download u (fromKey id key)) file stdRetry $ \p -> do copier <- mkCopier hardlink st params let verify = Annex.Content.RemoteVerify r let rsp = RetrievalAllKeysSecure diff --git a/Remote/GitLFS.hs b/Remote/GitLFS.hs index 01f76a5a8b..fb4f2fce8c 100644 --- a/Remote/GitLFS.hs +++ b/Remote/GitLFS.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Remote.GitLFS (remote, gen, configKnownUrl) where import Annex.Common @@ -153,7 +155,7 @@ mySetup _ mu _ c gc = do -- (so it's also usable by git as a non-special remote), -- and set remote.name.annex-git-lfs = true gitConfigSpecialRemote u c' [("git-lfs", "true")] - setConfig (ConfigKey ("remote." ++ getRemoteName c ++ ".url")) url + setConfig (Git.ConfigKey ("remote." <> encodeBS' (getRemoteName c) <> ".url")) url return (c', u) where url = fromMaybe (giveup "Specify url=") (M.lookup "url" c) @@ -185,9 +187,9 @@ configKnownUrl r set "config-uuid" (fromUUID cu) r' Nothing -> return r' set k v r' = do - let ck@(ConfigKey k') = remoteConfig r' k - setConfig ck v - return $ Git.Config.store' k' v r' + let k' = remoteConfig r' k + setConfig k' v + return $ Git.Config.store' k' (Git.ConfigValue (encodeBS' v)) r' data LFSHandle = LFSHandle { downloadEndpoint :: Maybe LFS.Endpoint @@ -344,10 +346,10 @@ sendTransferRequest req endpoint = do LFS.ParseFailed err -> Left err extractKeySha256 :: Key -> Maybe LFS.SHA256 -extractKeySha256 k = case keyVariety k of +extractKeySha256 k = case fromKey keyVariety k of SHA2Key (HashSize 256) (HasExt hasext) | hasext -> eitherToMaybe $ E.decodeUtf8' (keyHash k) - | otherwise -> eitherToMaybe $ E.decodeUtf8' (keyName k) + | otherwise -> eitherToMaybe $ E.decodeUtf8' (fromKey keyName k) _ -> Nothing -- The size of an encrypted key is the size of the input data, but we need @@ -355,7 +357,7 @@ extractKeySha256 k = case keyVariety k of extractKeySize :: Key -> Maybe Integer extractKeySize k | isEncKey k = Nothing - | otherwise = keySize k + | otherwise = fromKey keySize k mkUploadRequest :: RemoteStateHandle -> Key -> FilePath -> Annex (LFS.TransferRequest, LFS.SHA256, Integer) mkUploadRequest rs k content = case (extractKeySha256 k, extractKeySize k) of diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs index 6823179d12..00d623f50f 100644 --- a/Remote/Glacier.hs +++ b/Remote/Glacier.hs @@ -117,7 +117,7 @@ prepareStore r = checkPrepare nonEmpty (byteStorer $ store r) nonEmpty :: Key -> Annex Bool nonEmpty k - | keySize k == Just 0 = do + | fromKey keySize k == Just 0 = do warning "Cannot store empty files in Glacier." return False | otherwise = return True diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs index a011d9b5a2..473760edb3 100644 --- a/Remote/Helper/Chunked.hs +++ b/Remote/Helper/Chunked.hs @@ -68,8 +68,10 @@ newtype ChunkKeyStream = ChunkKeyStream [Key] chunkKeyStream :: Key -> ChunkSize -> ChunkKeyStream chunkKeyStream basek chunksize = ChunkKeyStream $ map mk [1..] where - mk chunknum = sizedk { keyChunkNum = Just chunknum } - sizedk = basek { keyChunkSize = Just (toInteger chunksize) } + mk chunknum = alterKey sizedk $ \d -> d + { keyChunkNum = Just chunknum } + sizedk = alterKey basek $ \d -> d + { keyChunkSize = Just (toInteger chunksize) } nextChunkKeyStream :: ChunkKeyStream -> (Key, ChunkKeyStream) nextChunkKeyStream (ChunkKeyStream (k:l)) = (k, ChunkKeyStream l) @@ -80,7 +82,7 @@ takeChunkKeyStream n (ChunkKeyStream l) = genericTake n l -- Number of chunks already consumed from the stream. numChunks :: ChunkKeyStream -> Integer -numChunks = pred . fromJust . keyChunkNum . fst . nextChunkKeyStream +numChunks = pred . fromJust . fromKey keyChunkNum . fst . nextChunkKeyStream {- Splits up the key's content into chunks, passing each chunk to - the storer action, along with a corresponding chunk key and a @@ -173,7 +175,7 @@ seekResume -> Annex (ChunkKeyStream, BytesProcessed) seekResume h encryptor chunkkeys checker = do sz <- liftIO (hFileSize h) - if sz <= fromMaybe 0 (keyChunkSize $ fst $ nextChunkKeyStream chunkkeys) + if sz <= fromMaybe 0 (fromKey keyChunkSize $ fst $ nextChunkKeyStream chunkkeys) then return (chunkkeys, zeroBytesProcessed) else check 0 chunkkeys sz where @@ -193,7 +195,7 @@ seekResume h encryptor chunkkeys checker = do return (cks, toBytesProcessed pos) where (k, cks') = nextChunkKeyStream cks - pos' = pos + fromMaybe 0 (keyChunkSize k) + pos' = pos + fromMaybe 0 (fromKey keyChunkSize k) {- Removes all chunks of a key from a remote, by calling a remover - action on each. @@ -208,7 +210,7 @@ removeChunks remover u chunkconfig encryptor k = do ls <- chunkKeys u chunkconfig k ok <- allM (remover . encryptor) (concat ls) when ok $ do - let chunksizes = catMaybes $ map (keyChunkSize <=< headMaybe) ls + let chunksizes = catMaybes $ map (fromKey keyChunkSize <=< headMaybe) ls forM_ chunksizes $ chunksRemoved u k . FixedSizeChunks . fromIntegral return ok @@ -272,7 +274,7 @@ retrieveChunks retriever u chunkconfig encryptor basek dest basep sink bracketIO (maybe opennew openresume offset) hClose $ \h -> do void $ tosink (Just h) p content let sz = toBytesProcessed $ - fromMaybe 0 $ keyChunkSize k + fromMaybe 0 $ fromKey keyChunkSize k getrest p h sz sz ks `catchNonAsync` unable case v of @@ -333,7 +335,7 @@ setupResume :: [[Key]] -> Integer -> [[Key]] setupResume ls currsize = map dropunneeded ls where dropunneeded [] = [] - dropunneeded l@(k:_) = case keyChunkSize k of + dropunneeded l@(k:_) = case fromKey keyChunkSize k of Just chunksize | chunksize > 0 -> genericDrop (currsize `div` chunksize) l _ -> l diff --git a/Remote/Helper/ExportImport.hs b/Remote/Helper/ExportImport.hs index 399163ba23..21d9814c65 100644 --- a/Remote/Helper/ExportImport.hs +++ b/Remote/Helper/ExportImport.hs @@ -324,7 +324,7 @@ adjustExportImport r rs = case M.lookup "exporttree" (config r) of liftIO $ Export.getExportTree db k retrieveKeyFileFromExport dbv k _af dest p = unVerified $ - if maybe False (isJust . verifyKeyContent) (maybeLookupBackendVariety (keyVariety k)) + if maybe False (isJust . verifyKeyContent) (maybeLookupBackendVariety (fromKey keyVariety k)) then do locs <- getexportlocs dbv k case locs of @@ -336,5 +336,5 @@ adjustExportImport r rs = case M.lookup "exporttree" (config r) of return False (l:_) -> retrieveExport (exportActions r) k l dest p else do - warning $ "exported content cannot be verified due to using the " ++ decodeBS (formatKeyVariety (keyVariety k)) ++ " backend" + warning $ "exported content cannot be verified due to using the " ++ decodeBS (formatKeyVariety (fromKey keyVariety k)) ++ " backend" return False diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs index 510793a8db..40934c6f08 100644 --- a/Remote/Helper/Special.hs +++ b/Remote/Helper/Special.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Remote.Helper.Special ( findSpecialRemotes, gitConfigSpecialRemote, @@ -51,7 +53,9 @@ import Annex.Content import Messages.Progress import qualified Git import qualified Git.Construct +import Git.Types +import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L import qualified Data.Map as M @@ -65,14 +69,17 @@ findSpecialRemotes s = do liftIO $ mapM construct $ remotepairs m where remotepairs = M.toList . M.filterWithKey match - construct (k,_) = Git.Construct.remoteNamedFromKey k (pure Git.Construct.fromUnknown) - match k _ = "remote." `isPrefixOf` k && (".annex-"++s) `isSuffixOf` k + construct (k,_) = Git.Construct.remoteNamedFromKey k + (pure Git.Construct.fromUnknown) + match (ConfigKey k) _ = + "remote." `S.isPrefixOf` k + && (".annex-" <> encodeBS' s) `S.isSuffixOf` k {- Sets up configuration for a special remote in .git/config. -} gitConfigSpecialRemote :: UUID -> RemoteConfig -> [(String, String)] -> Annex () gitConfigSpecialRemote u c cfgs = do forM_ cfgs $ \(k, v) -> - setConfig (remoteConfig c k) v + setConfig (remoteConfig c (encodeBS' k)) v storeUUIDIn (remoteConfig c "uuid") u -- RetrievalVerifiableKeysSecure unless overridden by git config. diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs index cc17220f28..ae4a680d9a 100644 --- a/Remote/Helper/Ssh.hs +++ b/Remote/Helper/Ssh.hs @@ -137,7 +137,7 @@ rsyncParamsRemote unlocked r direction key file (AssociatedFile afile) = do -- Send direct field for unlocked content, for backwards -- compatability. : (Fields.direct, if unlocked then "1" else "") - : maybe [] (\f -> [(Fields.associatedFile, f)]) afile + : maybe [] (\f -> [(Fields.associatedFile, fromRawFilePath f)]) afile repo <- getRepo r Just (shellcmd, shellparams) <- git_annex_shell ConsumeStdin repo (if direction == Download then "sendkey" else "recvkey") diff --git a/Remote/Hook.hs b/Remote/Hook.hs index 83c5e8ebc0..f0a67d808e 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -11,6 +11,7 @@ import Annex.Common import Types.Remote import Types.Creds import qualified Git +import Git.Types (fromConfigKey, fromConfigValue) import Config import Config.Cost import Annex.UUID @@ -107,19 +108,19 @@ hookEnv action k f = Just <$> mergeenv (fileenv f ++ keyenv) lookupHook :: HookName -> Action -> Annex (Maybe String) lookupHook hookname action = do - command <- getConfig (annexConfig hook) "" + command <- fromConfigValue <$> getConfig hook mempty if null command then do - fallback <- getConfig (annexConfig hookfallback) "" + fallback <- fromConfigValue <$> getConfig hookfallback mempty if null fallback then do - warning $ "missing configuration for " ++ hook ++ " or " ++ hookfallback + warning $ "missing configuration for " ++ fromConfigKey hook ++ " or " ++ fromConfigKey hookfallback return Nothing else return $ Just fallback else return $ Just command where - hook = hookname ++ "-" ++ action ++ "-hook" - hookfallback = hookname ++ "-hook" + hook = annexConfig $ encodeBS' $ hookname ++ "-" ++ action ++ "-hook" + hookfallback = annexConfig $ encodeBS' $ hookname ++ "-hook" runHook :: HookName -> Action -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool runHook hook action k f a = maybe (return False) run =<< lookupHook hook action diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index af26fbc757..566f95bab6 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -268,22 +268,22 @@ storeExportM :: RsyncOpts -> FilePath -> Key -> ExportLocation -> MeterUpdate -> storeExportM o src _k loc meterupdate = storeGeneric o meterupdate basedest populatedest where - basedest = fromExportLocation loc + basedest = fromRawFilePath (fromExportLocation loc) populatedest = liftIO . createLinkOrCopy src retrieveExportM :: RsyncOpts -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool retrieveExportM o _k loc dest p = rsyncRetrieve o [rsyncurl] dest (Just p) where - rsyncurl = mkRsyncUrl o (fromExportLocation loc) + rsyncurl = mkRsyncUrl o (fromRawFilePath (fromExportLocation loc)) checkPresentExportM :: RsyncOpts -> Key -> ExportLocation -> Annex Bool checkPresentExportM o _k loc = checkPresentGeneric o [rsyncurl] where - rsyncurl = mkRsyncUrl o (fromExportLocation loc) + rsyncurl = mkRsyncUrl o (fromRawFilePath (fromExportLocation loc)) removeExportM :: RsyncOpts -> Key -> ExportLocation -> Annex Bool removeExportM o _k loc = - removeGeneric o (includes (fromExportLocation loc)) + removeGeneric o $ includes $ fromRawFilePath $ fromExportLocation loc where includes f = f : case upFrom f of Nothing -> [] @@ -292,7 +292,7 @@ removeExportM o _k loc = removeExportDirectoryM :: RsyncOpts -> ExportDirectory -> Annex Bool removeExportDirectoryM o ed = removeGeneric o (allbelow d : includes d) where - d = fromExportDirectory ed + d = fromRawFilePath $ fromExportDirectory ed allbelow f = f "***" includes f = f : case upFrom f of Nothing -> [] diff --git a/Remote/S3.hs b/Remote/S3.hs index 2787e3f554..55d0b85fde 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -347,7 +347,7 @@ checkKey hv r rs c info k = withS3Handle hv $ \case Right us -> do showChecking r let check u = withUrlOptions $ - Url.checkBoth u (keySize k) + Url.checkBoth u (fromKey keySize k) anyM check us checkKeyHelper :: S3Info -> S3Handle -> (Either S3.Object S3VersionID) -> Annex Bool @@ -417,7 +417,7 @@ checkPresentExportS3 hv r info k loc = withS3Handle hv $ \case Just h -> checkKeyHelper info h (Left (T.pack $ bucketExportLocation info loc)) Nothing -> case getPublicUrlMaker info of Just geturl -> withUrlOptions $ - Url.checkBoth (geturl $ bucketExportLocation info loc) (keySize k) + Url.checkBoth (geturl $ bucketExportLocation info loc) (fromKey keySize k) Nothing -> do warning $ needS3Creds (uuid r) giveup "No S3 credentials configured" @@ -881,7 +881,8 @@ getBucketObject c = munge . serializeKey _ -> getFilePrefix c ++ s getBucketExportLocation :: RemoteConfig -> ExportLocation -> BucketObject -getBucketExportLocation c loc = getFilePrefix c ++ fromExportLocation loc +getBucketExportLocation c loc = + getFilePrefix c ++ fromRawFilePath (fromExportLocation loc) getBucketImportLocation :: RemoteConfig -> BucketObject -> Maybe ImportLocation getBucketImportLocation c obj @@ -889,7 +890,8 @@ getBucketImportLocation c obj | obj == uuidfile = Nothing -- Only import files that are under the fileprefix, when -- one is configured. - | prefix `isPrefixOf` obj = Just $ mkImportLocation $ drop prefixlen obj + | prefix `isPrefixOf` obj = Just $ mkImportLocation $ + toRawFilePath $ drop prefixlen obj | otherwise = Nothing where prefix = getFilePrefix c diff --git a/Remote/Web.hs b/Remote/Web.hs index 645495d696..810c2f027e 100644 --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -117,7 +117,7 @@ checkKey' key us = firsthit us (Right False) $ \u -> do case downloader of YoutubeDownloader -> youtubeDlCheck u' _ -> catchMsgIO $ - Url.withUrlOptions $ Url.checkBoth u' (keySize key) + Url.withUrlOptions $ Url.checkBoth u' (fromKey keySize key) where firsthit [] miss _ = return miss firsthit (u:rest) _ a = do diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs index 9204495317..08c3d528cc 100644 --- a/Remote/WebDAV.hs +++ b/Remote/WebDAV.hs @@ -229,7 +229,7 @@ removeExportDav r _k loc = case exportLocation loc of removeExportDirectoryDav :: Remote -> ExportDirectory -> Annex Bool removeExportDirectoryDav r dir = withDAVHandle r $ \mh -> runExport mh $ \_dav -> do - let d = fromExportDirectory dir + let d = fromRawFilePath $ fromExportDirectory dir debugDav $ "delContent " ++ d safely (inLocation d delContentM) >>= maybe (return False) (const $ return True) diff --git a/Remote/WebDAV/DavLocation.hs b/Remote/WebDAV/DavLocation.hs index 2f78923be5..4464ed2d36 100644 --- a/Remote/WebDAV/DavLocation.hs +++ b/Remote/WebDAV/DavLocation.hs @@ -17,6 +17,7 @@ import Utility.Url (URLString) #ifdef mingw32_HOST_OS import Utility.Split #endif +import Utility.FileSystemEncoding import System.FilePath.Posix -- for manipulating url paths import Network.Protocol.HTTP.DAV (inDAVLocation, DAVT) @@ -50,10 +51,12 @@ keyLocation k = keyDir k ++ keyFile k - those. -} exportLocation :: ExportLocation -> Either String DavLocation exportLocation l = - let p = fromExportLocation l - in if any (`elem` p) ['#', '?'] + let p = fromRawFilePath $ fromExportLocation l + in if any (`elem` p) illegalinurl then Left ("Cannot store file containing '#' or '?' on webdav: " ++ p) else Right p + where + illegalinurl = ['#', '?'] :: [Char] {- Where we store temporary data for a key as it's being uploaded. -} keyTmpLocation :: Key -> DavLocation diff --git a/Test.hs b/Test.hs index 115ea3bb66..c207dae4b6 100644 --- a/Test.hs +++ b/Test.hs @@ -624,7 +624,7 @@ test_lock_force = intmpclonerepo $ do git_annex "get" [annexedfile] @? "get of file failed" git_annex "unlock" [annexedfile] @? "unlock failed" annexeval $ do - Just k <- Annex.WorkTree.lookupFile annexedfile + Just k <- Annex.WorkTree.lookupFile (toRawFilePath annexedfile) Database.Keys.removeInodeCaches k Database.Keys.closeDb liftIO . nukeFile =<< Annex.fromRepo Annex.Locations.gitAnnexKeysDbIndexCache @@ -1146,7 +1146,7 @@ test_mixed_conflict_resolution = do @? (what ++ " too many variant files in: " ++ show v) indir d $ do git_annex "get" (conflictor:v) @? ("get failed in " ++ what) - git_annex_expectoutput "find" [conflictor] [Git.FilePath.toInternalGitPath subfile] + git_annex_expectoutput "find" [conflictor] [fromRawFilePath (Git.FilePath.toInternalGitPath (toRawFilePath subfile))] git_annex_expectoutput "find" v v {- Check merge conflict resolution when both repos start with an annexed @@ -1343,7 +1343,7 @@ test_conflict_resolution_symlink_bit = unlessM (hasUnlockedFiles <$> getTestMode where conflictor = "conflictor" check_is_link f what = do - git_annex_expectoutput "find" ["--include=*", f] [Git.FilePath.toInternalGitPath f] + git_annex_expectoutput "find" ["--include=*", f] [fromRawFilePath (Git.FilePath.toInternalGitPath (toRawFilePath f))] l <- annexeval $ Annex.inRepo $ Git.LsTree.lsTreeFiles Git.Ref.headRef [f] all (\i -> Git.Types.toTreeItemType (Git.LsTree.mode i) == Just Git.Types.TreeSymlink) l @? (what ++ " " ++ f ++ " lost symlink bit after merge: " ++ show l) @@ -1598,7 +1598,7 @@ test_crypto = do (c,k) <- annexeval $ do uuid <- Remote.nameToUUID "foo" rs <- Logs.Remote.readRemoteLog - Just k <- Annex.WorkTree.lookupFile annexedfile + Just k <- Annex.WorkTree.lookupFile (toRawFilePath annexedfile) return (fromJust $ M.lookup uuid rs, k) let key = if scheme `elem` ["hybrid","pubkey"] then Just $ Utility.Gpg.KeyIds [Utility.Gpg.testKeyId] diff --git a/Test/Framework.hs b/Test/Framework.hs index 93e9e3ad5b..8f3a773bd3 100644 --- a/Test/Framework.hs +++ b/Test/Framework.hs @@ -23,6 +23,7 @@ import qualified Types.RepoVersion import qualified Backend import qualified Git.CurrentRepo import qualified Git.Construct +import qualified Git.Types import qualified Types.KeySource import qualified Types.Backend import qualified Types @@ -88,8 +89,9 @@ inmainrepo a = do with_ssh_origin :: (Assertion -> Assertion) -> (Assertion -> Assertion) with_ssh_origin cloner a = cloner $ do - origindir <- absPath - =<< annexeval (Config.getConfig (Config.ConfigKey config) "/dev/null") + let k = Git.Types.ConfigKey (encodeBS' config) + let v = Git.Types.ConfigValue (toRawFilePath "/dev/null") + origindir <- absPath . Git.Types.fromConfigValue =<< annexeval (Config.getConfig k v) let originurl = "localhost:" ++ origindir boolSystem "git" [Param "config", Param config, Param originurl] @? "git config failed" a @@ -254,7 +256,7 @@ finalCleanup = whenM (doesDirectoryExist tmpdir) $ do checklink :: FilePath -> Assertion checklink f = ifM (annexeval Config.crippledFileSystem) - ( (isJust <$> annexeval (Annex.Link.getAnnexLinkTarget f)) + ( (isJust <$> annexeval (Annex.Link.getAnnexLinkTarget (toRawFilePath f))) @? f ++ " is not a (crippled) symlink" , do s <- getSymbolicLinkStatus f @@ -312,7 +314,7 @@ checkdangling f = ifM (annexeval Config.crippledFileSystem) checklocationlog :: FilePath -> Bool -> Assertion checklocationlog f expected = do thisuuid <- annexeval Annex.UUID.getUUID - r <- annexeval $ Annex.WorkTree.lookupFile f + r <- annexeval $ Annex.WorkTree.lookupFile (toRawFilePath f) case r of Just k -> do uuids <- annexeval $ Remote.keyLocations k @@ -323,11 +325,11 @@ checklocationlog f expected = do checkbackend :: FilePath -> Types.Backend -> Assertion checkbackend file expected = do b <- annexeval $ maybe (return Nothing) (Backend.getBackend file) - =<< Annex.WorkTree.lookupFile file + =<< Annex.WorkTree.lookupFile (toRawFilePath file) assertEqual ("backend for " ++ file) (Just expected) b checkispointerfile :: FilePath -> Assertion -checkispointerfile f = unlessM (isJust <$> Annex.Link.isPointerFile f) $ +checkispointerfile f = unlessM (isJust <$> Annex.Link.isPointerFile (toRawFilePath f)) $ assertFailure $ f ++ " is not a pointer file" inlocationlog :: FilePath -> Assertion diff --git a/Types/ActionItem.hs b/Types/ActionItem.hs index 1396c93002..fcb8c64345 100644 --- a/Types/ActionItem.hs +++ b/Types/ActionItem.hs @@ -12,15 +12,17 @@ module Types.ActionItem where import Key import Types.Transfer import Git.FilePath +import Utility.FileSystemEncoding import Data.Maybe +import qualified Data.ByteString as S data ActionItem = ActionItemAssociatedFile AssociatedFile Key | ActionItemKey Key | ActionItemBranchFilePath BranchFilePath Key | ActionItemFailedTransfer Transfer TransferInfo - | ActionItemWorkTreeFile FilePath + | ActionItemWorkTreeFile RawFilePath | ActionItemOther (Maybe String) -- Use to avoid more than one thread concurrently processing the -- same Key. @@ -39,10 +41,10 @@ instance MkActionItem (AssociatedFile, Key) where instance MkActionItem (Key, AssociatedFile) where mkActionItem = uncurry $ flip ActionItemAssociatedFile -instance MkActionItem (Key, FilePath) where +instance MkActionItem (Key, RawFilePath) where mkActionItem (key, file) = ActionItemAssociatedFile (AssociatedFile (Just file)) key -instance MkActionItem (FilePath, Key) where +instance MkActionItem (RawFilePath, Key) where mkActionItem (file, key) = mkActionItem (key, file) instance MkActionItem Key where @@ -54,16 +56,16 @@ instance MkActionItem (BranchFilePath, Key) where instance MkActionItem (Transfer, TransferInfo) where mkActionItem = uncurry ActionItemFailedTransfer -actionItemDesc :: ActionItem -> String +actionItemDesc :: ActionItem -> S.ByteString actionItemDesc (ActionItemAssociatedFile (AssociatedFile (Just f)) _) = f actionItemDesc (ActionItemAssociatedFile (AssociatedFile Nothing) k) = - serializeKey k -actionItemDesc (ActionItemKey k) = serializeKey k + serializeKey' k +actionItemDesc (ActionItemKey k) = serializeKey' k actionItemDesc (ActionItemBranchFilePath bfp _) = descBranchFilePath bfp actionItemDesc (ActionItemFailedTransfer t i) = actionItemDesc $ ActionItemAssociatedFile (associatedFile i) (transferKey t) actionItemDesc (ActionItemWorkTreeFile f) = f -actionItemDesc (ActionItemOther s) = fromMaybe "" s +actionItemDesc (ActionItemOther s) = encodeBS' (fromMaybe "" s) actionItemDesc (OnlyActionOn _ ai) = actionItemDesc ai actionItemKey :: ActionItem -> Maybe Key @@ -75,7 +77,7 @@ actionItemKey (ActionItemWorkTreeFile _) = Nothing actionItemKey (ActionItemOther _) = Nothing actionItemKey (OnlyActionOn _ ai) = actionItemKey ai -actionItemWorkTreeFile :: ActionItem -> Maybe FilePath +actionItemWorkTreeFile :: ActionItem -> Maybe RawFilePath actionItemWorkTreeFile (ActionItemAssociatedFile (AssociatedFile af) _) = af actionItemWorkTreeFile (ActionItemWorkTreeFile f) = Just f actionItemWorkTreeFile (OnlyActionOn _ ai) = actionItemWorkTreeFile ai diff --git a/Types/Difference.hs b/Types/Difference.hs index 774336c18e..a974e332a5 100644 --- a/Types/Difference.hs +++ b/Types/Difference.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Types.Difference ( Difference(..), Differences(..), @@ -20,9 +22,11 @@ module Types.Difference ( import Utility.PartialPrelude import qualified Git import qualified Git.Config +import Git.Types import Data.Maybe import Data.Monoid +import qualified Data.ByteString as B import qualified Data.Set as S import qualified Data.Semigroup as Sem import Prelude @@ -92,11 +96,11 @@ getDifferences :: Git.Repo -> Differences getDifferences r = mkDifferences $ S.fromList $ mapMaybe getmaybe [minBound .. maxBound] where - getmaybe d = case Git.Config.isTrue =<< Git.Config.getMaybe (differenceConfigKey d) r of + getmaybe d = case Git.Config.isTrue' =<< Git.Config.getMaybe (differenceConfigKey d) r of Just True -> Just d _ -> Nothing -differenceConfigKey :: Difference -> String +differenceConfigKey :: Difference -> ConfigKey differenceConfigKey ObjectHashLower = tunable "objecthashlower" differenceConfigKey OneLevelObjectHash = tunable "objecthash1" differenceConfigKey OneLevelBranchHash = tunable "branchhash1" @@ -104,8 +108,8 @@ differenceConfigKey OneLevelBranchHash = tunable "branchhash1" differenceConfigVal :: Difference -> String differenceConfigVal _ = Git.Config.boolConfig True -tunable :: String -> String -tunable k = "annex.tune." ++ k +tunable :: B.ByteString -> ConfigKey +tunable k = ConfigKey ("annex.tune." <> k) hasDifference :: Difference -> Differences -> Bool hasDifference _ UnknownDifferences = False diff --git a/Types/Distribution.hs b/Types/Distribution.hs index b41a90a1c0..c0ad2c02fd 100644 --- a/Types/Distribution.hs +++ b/Types/Distribution.hs @@ -21,7 +21,9 @@ type GitAnnexVersion = String data GitAnnexDistribution = GitAnnexDistribution { distributionUrl :: String - , distributionKey :: Key + , distributionKey :: KeyData + -- ^ This used to be a Key, but now KeyData serializes + -- to Key { ... }, so back-compat for Read and Show is preserved. , distributionVersion :: GitAnnexVersion , distributionReleasedate :: UTCTime , distributionUrgentUpgrade :: Maybe GitAnnexVersion @@ -46,7 +48,7 @@ parseInfoFile s = case lines s of formatGitAnnexDistribution :: GitAnnexDistribution -> String formatGitAnnexDistribution d = unlines [ distributionUrl d - , serializeKey (distributionKey d) + , serializeKey $ mkKey $ const $ distributionKey d , distributionVersion d , show (distributionReleasedate d) , maybe "" show (distributionUrgentUpgrade d) @@ -56,7 +58,7 @@ parseGitAnnexDistribution :: String -> Maybe GitAnnexDistribution parseGitAnnexDistribution s = case lines s of (u:k:v:d:uu:_) -> GitAnnexDistribution <$> pure u - <*> deserializeKey k + <*> fmap (fromKey id) (deserializeKey k) <*> pure v <*> readish d <*> pure (readish uu) diff --git a/Types/Export.hs b/Types/Export.hs index 437d74e286..b90b5cbe5e 100644 --- a/Types/Export.hs +++ b/Types/Export.hs @@ -17,38 +17,39 @@ module Types.Export ( import Git.FilePath import Utility.Split +import Utility.FileSystemEncoding import qualified System.FilePath.Posix as Posix -- A location on a remote that a key can be exported to. --- The FilePath will be relative to the top of the remote, +-- The RawFilePath will be relative to the top of the remote, -- and uses unix-style path separators. -newtype ExportLocation = ExportLocation FilePath +newtype ExportLocation = ExportLocation RawFilePath deriving (Show, Eq) -mkExportLocation :: FilePath -> ExportLocation +mkExportLocation :: RawFilePath -> ExportLocation mkExportLocation = ExportLocation . toInternalGitPath -fromExportLocation :: ExportLocation -> FilePath +fromExportLocation :: ExportLocation -> RawFilePath fromExportLocation (ExportLocation f) = f -newtype ExportDirectory = ExportDirectory FilePath +newtype ExportDirectory = ExportDirectory RawFilePath deriving (Show, Eq) -mkExportDirectory :: FilePath -> ExportDirectory +mkExportDirectory :: RawFilePath -> ExportDirectory mkExportDirectory = ExportDirectory . toInternalGitPath -fromExportDirectory :: ExportDirectory -> FilePath +fromExportDirectory :: ExportDirectory -> RawFilePath fromExportDirectory (ExportDirectory f) = f -- | All subdirectories down to the ExportLocation, with the deepest ones -- last. Does not include the top of the export. exportDirectories :: ExportLocation -> [ExportDirectory] exportDirectories (ExportLocation f) = - map (ExportDirectory . Posix.joinPath . reverse) (subs [] dirs) + map (ExportDirectory . encodeBS . Posix.joinPath . reverse) (subs [] dirs) where subs _ [] = [] subs ps (d:ds) = (d:ps) : subs (d:ps) ds dirs = map Posix.dropTrailingPathSeparator $ - dropFromEnd 1 $ Posix.splitPath f + dropFromEnd 1 $ Posix.splitPath $ decodeBS f diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index 50aa6f2379..df2cd6bb1f 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Types.GitConfig ( Configurable(..), GitConfig(..), @@ -199,16 +201,17 @@ extractGitConfig r = GitConfig } where getbool k d = fromMaybe d $ getmaybebool k - getmaybebool k = Git.Config.isTrue =<< getmaybe k + getmaybebool k = Git.Config.isTrue' =<< getmaybe' k getmayberead k = readish =<< getmaybe k - getmaybe k = Git.Config.getMaybe k r - getlist k = Git.Config.getList k r + getmaybe = fmap fromConfigValue . getmaybe' + getmaybe' k = Git.Config.getMaybe k r + getlist k = map fromConfigValue $ Git.Config.getList k r getwords k = fromMaybe [] $ words <$> getmaybe k configurable d Nothing = DefaultConfig d configurable _ (Just v) = HasConfig v - annex k = "annex." ++ k + annex k = ConfigKey $ "annex." <> k onemegabyte = 1000000 @@ -340,14 +343,16 @@ extractRemoteGitConfig r remotename = do } where getbool k d = fromMaybe d $ getmaybebool k - getmaybebool k = Git.Config.isTrue =<< getmaybe k + getmaybebool k = Git.Config.isTrue' =<< getmaybe' k getmayberead k = readish =<< getmaybe k - getmaybe k = mplus (Git.Config.getMaybe (key k) r) + getmaybe = fmap fromConfigValue . getmaybe' + getmaybe' k = mplus (Git.Config.getMaybe (key k) r) (Git.Config.getMaybe (remotekey k) r) getoptions k = fromMaybe [] $ words <$> getmaybe k - key k = "annex." ++ k - remotekey k = "remote." ++ remotename ++ ".annex-" ++ k + key k = ConfigKey $ "annex." <> k + remotekey k = ConfigKey $ + "remote." <> encodeBS' remotename <> ".annex-" <> k notempty :: Maybe String -> Maybe String notempty Nothing = Nothing diff --git a/Types/Import.hs b/Types/Import.hs index c6d94edb61..a297af76e6 100644 --- a/Types/Import.hs +++ b/Types/Import.hs @@ -19,10 +19,10 @@ import Utility.FileSystemEncoding - location on the remote. -} type ImportLocation = ExportLocation -mkImportLocation :: FilePath -> ImportLocation +mkImportLocation :: RawFilePath -> ImportLocation mkImportLocation = mkExportLocation -fromImportLocation :: ImportLocation -> FilePath +fromImportLocation :: ImportLocation -> RawFilePath fromImportLocation = fromExportLocation {- An identifier for content stored on a remote that has been imported into diff --git a/Types/Key.hs b/Types/Key.hs index 0d751bd736..9992fdcabb 100644 --- a/Types/Key.hs +++ b/Types/Key.hs @@ -7,19 +7,48 @@ {-# LANGUAGE OverloadedStrings, DeriveGeneric #-} -module Types.Key where +module Types.Key ( + KeyData(..), + Key, + fromKey, + mkKey, + alterKey, + isKeyPrefix, + splitKeyNameExtension, + keyParser, + keySerialization, + AssociatedFile(..), + KeyVariety(..), + HasExt(..), + HashSize(..), + hasExt, + sameExceptExt, + cryptographicallySecure, + isVerifiable, + formatKeyVariety, + parseKeyVariety, +) where import qualified Data.ByteString as S import qualified Data.ByteString.Char8 as S8 +import qualified Data.ByteString.Lazy as L +import Data.ByteString.Builder +import Data.ByteString.Builder.Extra +import qualified Data.Attoparsec.ByteString as A +import qualified Data.Attoparsec.ByteString.Char8 as A8 +import Utility.FileSystemEncoding +import Data.List import System.Posix.Types +import Foreign.C.Types import Data.Monoid +import Control.Applicative import GHC.Generics import Control.DeepSeq import Prelude {- A Key has a unique name, which is derived from a particular backend, - and may contain other optional metadata. -} -data Key = Key +data KeyData = Key { keyName :: S.ByteString , keyVariety :: KeyVariety , keySize :: Maybe Integer @@ -28,10 +57,151 @@ data Key = Key , keyChunkNum :: Maybe Integer } deriving (Eq, Ord, Read, Show, Generic) +instance NFData KeyData + +{- Caching the seralization of a key is an optimization. + - + - This constructor is not exported, and all smart constructors maintain + - the serialization. + -} +data Key = MkKey + { keyData :: KeyData + , keySerialization :: S.ByteString + } deriving (Show, Generic) + +instance Eq Key where + -- comparing the serialization would be unncessary work + a == b = keyData a == keyData b + +instance Ord Key where + compare a b = compare (keyData a) (keyData b) + instance NFData Key +{- Access a field of data from the KeyData. -} +{-# INLINE fromKey #-} +fromKey :: (KeyData -> a) -> Key -> a +fromKey f = f . keyData + +{- Smart constructor for a Key. The provided KeyData has all values empty. -} +mkKey :: (KeyData -> KeyData) -> Key +mkKey f = + let d = f stub + in MkKey d (mkKeySerialization d) + where + stub = Key + { keyName = mempty + , keyVariety = OtherKey mempty + , keySize = Nothing + , keyMtime = Nothing + , keyChunkSize = Nothing + , keyChunkNum = Nothing + } + +{- Alter a Key's data. -} +alterKey :: Key -> (KeyData -> KeyData) -> Key +alterKey k f = + let d = f (keyData k) + in MkKey d (mkKeySerialization d) + +-- Checks if a string looks like at least the start of a key. +isKeyPrefix :: String -> Bool +isKeyPrefix s = [fieldSep, fieldSep] `isInfixOf` s + +fieldSep :: Char +fieldSep = '-' + +mkKeySerialization :: KeyData -> S.ByteString +mkKeySerialization = L.toStrict + . toLazyByteStringWith (safeStrategy 128 smallChunkSize) L.empty + . buildKeyData + +{- Builds a ByteString from a KeyData. + - + - The name field is always shown last, separated by doubled fieldSeps, + - and is the only field allowed to contain the fieldSep. + -} +buildKeyData :: KeyData -> Builder +buildKeyData k = byteString (formatKeyVariety (keyVariety k)) + <> 's' ?: (integerDec <$> keySize k) + <> 'm' ?: (integerDec . (\(CTime t) -> fromIntegral t) <$> keyMtime k) + <> 'S' ?: (integerDec <$> keyChunkSize k) + <> 'C' ?: (integerDec <$> keyChunkNum k) + <> sepbefore (sepbefore (byteString (keyName k))) + where + sepbefore s = char7 fieldSep <> s + c ?: (Just b) = sepbefore (char7 c <> b) + _ ?: Nothing = mempty + +{- This is a strict parser for security reasons; a key + - can contain only 4 fields, which all consist only of numbers. + - Any key containing other fields, or non-numeric data will fail + - to parse. + - + - If a key contained non-numeric fields, they could be used to + - embed data used in a SHA1 collision attack, which would be a + - problem since the keys are committed to git. + -} +keyParser :: A.Parser Key +keyParser = do + -- key variety cannot be empty + v <- (parseKeyVariety <$> A8.takeWhile1 (/= fieldSep)) + s <- parsesize + m <- parsemtime + cs <- parsechunksize + cn <- parsechunknum + _ <- A8.char fieldSep + _ <- A8.char fieldSep + n <- A.takeByteString + if validKeyName v n + then + let d = Key + { keyName = n + , keyVariety = v + , keySize = s + , keyMtime = m + , keyChunkSize = cs + , keyChunkNum = cn + } + in pure $ MkKey d (mkKeySerialization d) + else fail "invalid keyName" + where + parseopt p = (Just <$> (A8.char fieldSep *> p)) <|> pure Nothing + parsesize = parseopt $ A8.char 's' *> A8.decimal + parsemtime = parseopt $ CTime <$> (A8.char 'm' *> A8.decimal) + parsechunksize = parseopt $ A8.char 'S' *> A8.decimal + parsechunknum = parseopt $ A8.char 'C' *> A8.decimal + +{- Limits the length of the extension in the keyName to mitigate against + - SHA1 collision attacks. + - + - In such an attack, the extension of the key could be made to contain + - the collision generation data, with the result that a signed git commit + - including such keys would not be secure. + - + - The maximum extension length ever generated for such a key was 8 + - characters, but they may be unicode which could use up to 4 bytes each, + - so 32 bytes. 64 bytes is used here to give a little future wiggle-room. + - The SHA1 common-prefix attack needs 128 bytes of data. + -} +validKeyName :: KeyVariety -> S.ByteString -> Bool +validKeyName kv name + | hasExt kv = + let ext = snd $ splitKeyNameExtension' name + in S.length ext <= 64 + | otherwise = True + +{- This splits any extension out of the keyName, returning the + - keyName minus extension, and the extension (including leading dot). + -} +splitKeyNameExtension :: Key -> (S.ByteString, S.ByteString) +splitKeyNameExtension = splitKeyNameExtension' . keyName . keyData + +splitKeyNameExtension' :: S.ByteString -> (S.ByteString, S.ByteString) +splitKeyNameExtension' keyname = S8.span (/= '.') keyname + {- A filename may be associated with a Key. -} -newtype AssociatedFile = AssociatedFile (Maybe FilePath) +newtype AssociatedFile = AssociatedFile (Maybe RawFilePath) deriving (Show, Eq, Ord) {- There are several different varieties of keys. -} diff --git a/Types/Transfer.hs b/Types/Transfer.hs index 2a89c6c39f..fed03cb0a3 100644 --- a/Types/Transfer.hs +++ b/Types/Transfer.hs @@ -11,9 +11,11 @@ module Types.Transfer where import Types import Types.Remote (Verification(..)) +import Types.Key import Utility.PID import Utility.QuickCheck import Utility.Url +import Utility.FileSystemEncoding import Data.Time.Clock.POSIX import Control.Concurrent @@ -24,9 +26,12 @@ import Prelude data Transfer = Transfer { transferDirection :: Direction , transferUUID :: UUID - , transferKey :: Key + , transferKeyData :: KeyData } - deriving (Eq, Ord, Read, Show) + deriving (Eq, Ord, Show, Read) + +transferKey :: Transfer -> Key +transferKey = mkKey . const . transferKeyData {- Information about a Transfer, stored in the transfer information file. - @@ -67,8 +72,7 @@ instance Arbitrary TransferInfo where <*> pure Nothing -- cannot generate a ThreadID <*> pure Nothing -- remote not needed <*> arbitrary - -- associated file cannot be empty (but can be Nothing) - <*> (AssociatedFile <$> arbitrary `suchThat` (/= Just "")) + <*> arbitrary <*> arbitrary class Observable a where @@ -97,7 +101,7 @@ class Transferrable t where descTransfrerrable :: t -> Maybe String instance Transferrable AssociatedFile where - descTransfrerrable (AssociatedFile af) = af + descTransfrerrable (AssociatedFile af) = fromRawFilePath <$> af instance Transferrable URLString where descTransfrerrable = Just diff --git a/Types/UUID.hs b/Types/UUID.hs index 726875b3a8..92f5ed9e17 100644 --- a/Types/UUID.hs +++ b/Types/UUID.hs @@ -17,6 +17,7 @@ import Data.String import Data.ByteString.Builder import qualified Data.Semigroup as Sem +import Git.Types (ConfigValue(..)) import Utility.FileSystemEncoding import Utility.QuickCheck import qualified Utility.SimpleProtocol as Proto @@ -52,6 +53,12 @@ instance FromUUID String where instance ToUUID String where toUUID s = toUUID (encodeBS' s) +instance FromUUID ConfigValue where + fromUUID s = (ConfigValue (fromUUID s)) + +instance ToUUID ConfigValue where + toUUID (ConfigValue v) = toUUID v + -- There is no matching FromUUID U.UUID because a git-annex UUID may -- be NoUUID or perhaps contain something not allowed in a canonical UUID. instance ToUUID U.UUID where diff --git a/Upgrade.hs b/Upgrade.hs index 7f54e8b334..019d38f9f7 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -87,4 +87,3 @@ upgrade automatic destversion = do up (RepoVersion 6) = Upgrade.V6.upgrade automatic up (RepoVersion 7) = Upgrade.V7.upgrade automatic up _ = return True - diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 93d9108db9..fd46108dd5 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -84,8 +84,8 @@ updateSymlinks :: Annex () updateSymlinks = do showAction "updating symlinks" top <- fromRepo Git.repoPath - (files, cleanup) <- inRepo $ LsFiles.inRepo [top] - forM_ files fixlink + (files, cleanup) <- inRepo $ LsFiles.inRepo [toRawFilePath top] + forM_ files (fixlink . fromRawFilePath) void $ liftIO cleanup where fixlink f = do @@ -134,7 +134,7 @@ oldlog2key l where len = length l - 4 k = readKey1 (take len l) - sane = (not . S.null $ keyName k) && (not . S.null $ formatKeyVariety $ keyVariety k) + sane = (not . S.null $ fromKey keyName k) && (not . S.null $ formatKeyVariety $ fromKey keyVariety k) -- WORM backend keys: "WORM:mtime:size:filename" -- all the rest: "backend:key" @@ -145,7 +145,7 @@ oldlog2key l readKey1 :: String -> Key readKey1 v | mixup = fromJust $ deserializeKey $ intercalate ":" $ Prelude.tail bits - | otherwise = stubKey + | otherwise = mkKey $ \d -> d { keyName = encodeBS n , keyVariety = parseKeyVariety (encodeBS b) , keySize = s @@ -165,12 +165,16 @@ readKey1 v mixup = wormy && isUpper (Prelude.head $ bits !! 1) showKey1 :: Key -> String -showKey1 Key { keyName = n , keyVariety = v, keySize = s, keyMtime = t } = - intercalate ":" $ filter (not . null) [b, showifhere t, showifhere s, decodeBS n] +showKey1 k = intercalate ":" $ filter (not . null) + [b, showifhere t, showifhere s, decodeBS n] where showifhere Nothing = "" showifhere (Just x) = show x b = decodeBS $ formatKeyVariety v + n = fromKey keyName k + v = fromKey keyVariety k + s = fromKey keySize k + t = fromKey keyMtime k keyFile1 :: Key -> FilePath keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ showKey1 key @@ -194,7 +198,7 @@ lookupFile1 file = do Right l -> makekey l where getsymlink = takeFileName <$> readSymbolicLink file - makekey l = case maybeLookupBackendVariety (keyVariety k) of + makekey l = case maybeLookupBackendVariety (fromKey keyVariety k) of Nothing -> do unless (null kname || null bname || not (isLinkToAnnex (toRawFilePath l))) $ @@ -203,8 +207,8 @@ lookupFile1 file = do Just backend -> return $ Just (k, backend) where k = fileKey1 l - bname = decodeBS (formatKeyVariety (keyVariety k)) - kname = decodeBS (keyName k) + bname = decodeBS (formatKeyVariety (fromKey keyVariety k)) + kname = decodeBS (fromKey keyName k) skip = "skipping " ++ file ++ " (unknown backend " ++ bname ++ ")" diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs index ffac8e49aa..9b29783e9d 100644 --- a/Upgrade/V2.hs +++ b/Upgrade/V2.hs @@ -50,7 +50,7 @@ upgrade = do e <- liftIO $ doesDirectoryExist old when e $ do config <- Annex.getGitConfig - mapM_ (\(k, f) -> inject f $ locationLogFile config k) =<< locationLogs + mapM_ (\(k, f) -> inject f $ fromRawFilePath $ locationLogFile config k) =<< locationLogs mapM_ (\f -> inject f f) =<< logFiles old saveState False @@ -76,13 +76,13 @@ locationLogs = do where tryDirContents d = catchDefaultIO [] $ dirContents d islogfile f = maybe Nothing (\k -> Just (k, f)) $ - locationLogFileKey f + locationLogFileKey (toRawFilePath f) inject :: FilePath -> FilePath -> Annex () inject source dest = do old <- fromRepo olddir new <- liftIO (readFile $ old source) - Annex.Branch.change dest $ \prev -> + Annex.Branch.change (toRawFilePath dest) $ \prev -> encodeBL $ unlines $ nub $ lines (decodeBL prev) ++ lines new logFiles :: FilePath -> Annex [FilePath] diff --git a/Upgrade/V5.hs b/Upgrade/V5.hs index 5d331c8787..ba897399f2 100644 --- a/Upgrade/V5.hs +++ b/Upgrade/V5.hs @@ -5,6 +5,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Upgrade.V5 where import Annex.Common @@ -106,7 +108,7 @@ convertDirect = do upgradeDirectWorkTree :: Annex () upgradeDirectWorkTree = do top <- fromRepo Git.repoPath - (l, clean) <- inRepo $ Git.LsFiles.stagedDetails [top] + (l, clean) <- inRepo $ Git.LsFiles.stagedDetails [toRawFilePath top] forM_ l go void $ liftIO clean where @@ -119,11 +121,11 @@ upgradeDirectWorkTree = do Just k -> do stagePointerFile f Nothing =<< hashPointerFile k ifM (isJust <$> getAnnexLinkTarget f) - ( writepointer f k - , fromdirect f k + ( writepointer (fromRawFilePath f) k + , fromdirect (fromRawFilePath f) k ) Database.Keys.addAssociatedFile k - =<< inRepo (toTopFilePath f) + =<< inRepo (toTopFilePath (fromRawFilePath f)) go _ = noop fromdirect f k = ifM (Direct.goodContent k f) diff --git a/Upgrade/V5/Direct.hs b/Upgrade/V5/Direct.hs index 8b67bb3926..3f67959976 100644 --- a/Upgrade/V5/Direct.hs +++ b/Upgrade/V5/Direct.hs @@ -7,6 +7,8 @@ - Licensed under the GNU AGPL version 3 or higher. -} +{-# LANGUAGE OverloadedStrings #-} + module Upgrade.V5.Direct ( switchHEADBack, setIndirect, @@ -44,12 +46,12 @@ setIndirect = do -- unset it when enabling direct mode, caching in -- core.indirect-worktree moveconfig indirectworktree coreworktree - setConfig (ConfigKey Git.Config.coreBare) val + setConfig Git.Config.coreBare val moveconfig src dest = getConfigMaybe src >>= \case Nothing -> noop Just wt -> do unsetConfig src - setConfig dest wt + setConfig dest (fromConfigValue wt) reloadConfig {- Converts a directBranch back to the original branch. diff --git a/Upgrade/V7.hs b/Upgrade/V7.hs index 6925cf0379..7b53794ddf 100644 --- a/Upgrade/V7.hs +++ b/Upgrade/V7.hs @@ -95,11 +95,11 @@ removeOldDb getdb = do populateKeysDb :: Annex () populateKeysDb = do top <- fromRepo Git.repoPath - (l, cleanup) <- inRepo $ LsFiles.inodeCaches [top] + (l, cleanup) <- inRepo $ LsFiles.inodeCaches [toRawFilePath top] forM_ l $ \case (_f, Nothing) -> giveup "Unable to parse git ls-files --debug output while upgrading git-annex sqlite databases." (f, Just ic) -> unlessM (liftIO $ isSymbolicLink <$> getSymbolicLinkStatus f) $ do - catKeyFile f >>= \case + catKeyFile (toRawFilePath f) >>= \case Nothing -> noop Just k -> do topf <- inRepo $ toTopFilePath f diff --git a/Utility/Applicative.hs b/Utility/Applicative.hs index 62d84bcf2b..fcd693264d 100644 --- a/Utility/Applicative.hs +++ b/Utility/Applicative.hs @@ -5,6 +5,8 @@ - License: BSD-2-clause -} +{-# OPTIONS_GHC -fno-warn-tabs #-} + module Utility.Applicative ( (<$$>), ) where diff --git a/Utility/Attoparsec.hs b/Utility/Attoparsec.hs new file mode 100644 index 0000000000..bd20e8e6d9 --- /dev/null +++ b/Utility/Attoparsec.hs @@ -0,0 +1,21 @@ +{- attoparsec utility functions + - + - Copyright 2019 Joey Hess + - Copyright 2007-2015 Bryan O'Sullivan + - + - License: BSD-3-clause + -} + +module Utility.Attoparsec where + +import qualified Data.Attoparsec.ByteString as A +import qualified Data.ByteString as B + +-- | Parse and decode an unsigned octal number. +-- +-- This parser does not accept a leading @\"0o\"@ string. +octal :: Integral a => A.Parser a +octal = B.foldl' step 0 `fmap` A.takeWhile1 isOctDigit + where + isOctDigit w = w >= 48 && w <= 55 + step a w = a * 8 + fromIntegral (w - 48) diff --git a/Utility/FileSize.hs b/Utility/FileSize.hs index c86528369a..8544ad4179 100644 --- a/Utility/FileSize.hs +++ b/Utility/FileSize.hs @@ -4,6 +4,7 @@ -} {-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.FileSize ( FileSize, @@ -32,7 +33,10 @@ getFileSize f = fmap (fromIntegral . fileSize) (getFileStatus f) getFileSize f = bracket (openFile f ReadMode) hClose hFileSize #endif -{- Gets the size of the file, when its FileStatus is already known. -} +{- Gets the size of the file, when its FileStatus is already known. + - + - On windows, uses getFileSize. Otherwise, the FileStatus contains the + - size, so this does not do any work. -} getFileSize' :: FilePath -> FileStatus -> IO FileSize #ifndef mingw32_HOST_OS getFileSize' _ s = return $ fromIntegral $ fileSize s diff --git a/Utility/Hash.hs b/Utility/Hash.hs index d198918e67..397ada5d35 100644 --- a/Utility/Hash.hs +++ b/Utility/Hash.hs @@ -24,6 +24,7 @@ module Utility.Hash ( blake2b_512, blake2bp_512, md5, + md5s, prop_hashes_stable, Mac(..), calcMac, @@ -106,6 +107,9 @@ blake2bp_512 = hashlazy md5 :: L.ByteString -> Digest MD5 md5 = hashlazy +md5s :: S.ByteString -> Digest MD5 +md5s = hash + {- Check that all the hashes continue to hash the same. -} prop_hashes_stable :: Bool prop_hashes_stable = all (\(hasher, result) -> hasher foo == result) diff --git a/Utility/InodeCache.hs b/Utility/InodeCache.hs index 7b46022dea..08f983bf01 100644 --- a/Utility/InodeCache.hs +++ b/Utility/InodeCache.hs @@ -23,6 +23,7 @@ module Utility.InodeCache ( readInodeCache, showInodeCache, genInodeCache, + genInodeCache', toInodeCache, InodeCacheKey, @@ -46,6 +47,7 @@ module Utility.InodeCache ( import Common import Utility.TimeStamp import Utility.QuickCheck +import qualified Utility.RawFilePath as R import System.PosixCompat.Types import Data.Time.Clock.POSIX @@ -187,6 +189,10 @@ genInodeCache :: FilePath -> TSDelta -> IO (Maybe InodeCache) genInodeCache f delta = catchDefaultIO Nothing $ toInodeCache delta f =<< getFileStatus f +genInodeCache' :: RawFilePath -> TSDelta -> IO (Maybe InodeCache) +genInodeCache' f delta = catchDefaultIO Nothing $ + toInodeCache delta (fromRawFilePath f) =<< R.getFileStatus f + toInodeCache :: TSDelta -> FilePath -> FileStatus -> IO (Maybe InodeCache) toInodeCache (TSDelta getdelta) f s | isRegularFile s = do diff --git a/Utility/Misc.hs b/Utility/Misc.hs index de77c949a0..2f1766ec23 100644 --- a/Utility/Misc.hs +++ b/Utility/Misc.hs @@ -12,6 +12,7 @@ module Utility.Misc ( readFileStrict, separate, firstLine, + firstLine', segment, segmentDelim, massReplace, @@ -28,6 +29,7 @@ import Data.Char import Data.List import System.Exit import Control.Applicative +import qualified Data.ByteString as S import Prelude {- A version of hgetContents that is not lazy. Ensures file is @@ -56,6 +58,11 @@ separate c l = unbreak $ break c l firstLine :: String -> String firstLine = takeWhile (/= '\n') +firstLine' :: S.ByteString -> S.ByteString +firstLine' = S.takeWhile (/= nl) + where + nl = fromIntegral (ord '\n') + {- Splits a list into segments that are delimited by items matching - a predicate. (The delimiters are not included in the segments.) - Segments may be empty. -} diff --git a/Utility/Path.hs b/Utility/Path.hs index 26d66066ad..3f34156e88 100644 --- a/Utility/Path.hs +++ b/Utility/Path.hs @@ -43,6 +43,7 @@ import Utility.Monad import Utility.UserInfo import Utility.Directory import Utility.Split +import Utility.FileSystemEncoding {- Simplifies a path, removing any "." component, collapsing "dir/..", - and removing the trailing path separator. @@ -200,20 +201,21 @@ prop_relPathDirToFile_regressionTest = same_dir_shortcurcuits_at_difference - we stop preserving ordering at that point. Presumably a user passing - that many paths in doesn't care too much about order of the later ones. -} -segmentPaths :: [FilePath] -> [FilePath] -> [[FilePath]] +segmentPaths :: [RawFilePath] -> [RawFilePath] -> [[RawFilePath]] segmentPaths [] new = [new] segmentPaths [_] new = [new] -- optimisation segmentPaths (l:ls) new = found : segmentPaths ls rest where (found, rest) = if length ls < 100 - then partition (l `dirContains`) new - else break (\p -> not (l `dirContains` p)) new + then partition inl new + else break (not . inl) new + inl f = fromRawFilePath l `dirContains` fromRawFilePath f {- This assumes that it's cheaper to call segmentPaths on the result, - than it would be to run the action separately with each path. In - the case of git file list commands, that assumption tends to hold. -} -runSegmentPaths :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [[FilePath]] +runSegmentPaths :: ([RawFilePath] -> IO [RawFilePath]) -> [RawFilePath] -> IO [[RawFilePath]] runSegmentPaths a paths = segmentPaths paths <$> a paths {- Converts paths in the home directory to use ~/ -} diff --git a/Utility/RawFilePath.hs b/Utility/RawFilePath.hs index 485346a0e6..ff7057783f 100644 --- a/Utility/RawFilePath.hs +++ b/Utility/RawFilePath.hs @@ -1,4 +1,11 @@ {- Portability shim around System.Posix.Files.ByteString + - + - On unix, this makes syscalls using RawFilesPaths as efficiently as + - possible. + - + - On Windows, filenames are in unicode, so RawFilePaths have to be + - decoded. So this library will work, but less efficiently than using + - FilePath would. - - Copyright 2019 Joey Hess - @@ -10,19 +17,20 @@ module Utility.RawFilePath ( RawFilePath, readSymbolicLink, + getFileStatus, ) where #ifndef mingw32_HOST_OS +import Utility.FileSystemEncoding (RawFilePath) import System.Posix.Files.ByteString -import System.Posix.ByteString.FilePath #else import qualified Data.ByteString as B -import System.IO.Error - -type RawFilePath = B.ByteString +import qualified System.PosixCompat as P +import Utility.FileSystemEncoding readSymbolicLink :: RawFilePath -> IO RawFilePath -readSymbolicLink _ = ioError $ mkIOError illegalOperationErrorType x Nothing Nothing - where - x = "Utility.RawFilePath.readSymbolicLink: not supported" +readSymbolicLink f = toRawFilePath <$> P.readSymbolicLink (fromRawFilePath f) + +getFileStatus :: RawFilePath -> IO FileStatus +getFileStatus = P.getFileStatus . fromRawFilePath #endif diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs index f820e69f19..19d5f2026e 100644 --- a/Utility/SafeCommand.hs +++ b/Utility/SafeCommand.hs @@ -7,7 +7,23 @@ {-# OPTIONS_GHC -fno-warn-tabs #-} -module Utility.SafeCommand where +module Utility.SafeCommand ( + CommandParam(..), + toCommand, + boolSystem, + boolSystem', + boolSystemEnv, + safeSystem, + safeSystem', + safeSystemEnv, + shellWrap, + shellEscape, + shellUnEscape, + segmentXargsOrdered, + segmentXargsUnordered, + prop_isomorphic_shellEscape, + prop_isomorphic_shellEscape_multiword, +) where import System.Exit import Utility.Process diff --git a/Utility/Scheduled/QuickCheck.hs b/Utility/Scheduled/QuickCheck.hs index a2051cd2aa..96ce046c30 100644 --- a/Utility/Scheduled/QuickCheck.hs +++ b/Utility/Scheduled/QuickCheck.hs @@ -7,7 +7,7 @@ {-# OPTIONS_GHC -fno-warn-orphans #-} -module Utility.Scheduled.QuickCheck where +module Utility.Scheduled.QuickCheck (prop_schedule_roundtrips) where import Utility.Scheduled import Utility.QuickCheck diff --git a/Utility/Shell.hs b/Utility/Shell.hs index 2f7cd3cb10..5053cdcba6 100644 --- a/Utility/Shell.hs +++ b/Utility/Shell.hs @@ -7,7 +7,11 @@ {-# LANGUAGE CPP #-} -module Utility.Shell where +module Utility.Shell ( + shellPath, + shebang, + findShellCommand, +) where import Utility.SafeCommand #ifdef mingw32_HOST_OS diff --git a/Utility/Split.hs b/Utility/Split.hs index ffea5d3f07..028218e006 100644 --- a/Utility/Split.hs +++ b/Utility/Split.hs @@ -7,7 +7,12 @@ {-# OPTIONS_GHC -fno-warn-tabs #-} -module Utility.Split where +module Utility.Split ( + split, + splitc, + replace, + dropFromEnd, +) where import Data.List (intercalate) import Data.List.Split (splitOn) @@ -29,6 +34,6 @@ splitc c s = case break (== c) s of replace :: Eq a => [a] -> [a] -> [a] -> [a] replace old new = intercalate new . split old --- | Only traverses the list once while dropping the last n characters. +-- | Only traverses the list once while dropping the last n items. dropFromEnd :: Int -> [a] -> [a] dropFromEnd n l = zipWith const l (drop n l) diff --git a/Utility/SshConfig.hs b/Utility/SshConfig.hs index 1f8581a280..111d73ec44 100644 --- a/Utility/SshConfig.hs +++ b/Utility/SshConfig.hs @@ -5,7 +5,24 @@ - License: BSD-2-clause -} -module Utility.SshConfig where +module Utility.SshConfig ( + SshConfig(..), + Comment(..), + SshSetting(..), + Indent, + Host, + Key, + Value, + parseSshConfig, + genSshConfig, + findHostConfigKey, + addToHostConfig, + modifyUserSshConfig, + changeUserSshConfig, + writeSshConfig, + setSshConfigMode, + sshDir, +) where import Common import Utility.UserInfo diff --git a/Utility/TList.hs b/Utility/TList.hs index 033c8ca02c..787ad4ba01 100644 --- a/Utility/TList.hs +++ b/Utility/TList.hs @@ -11,7 +11,17 @@ {-# LANGUAGE BangPatterns #-} -module Utility.TList where +module Utility.TList ( + TList, + newTList, + getTList, + setTList, + takeTList, + readTList, + consTList, + snocTList, + appendTList, +) where import Common diff --git a/Utility/Tense.hs b/Utility/Tense.hs index ef2454bdc7..a0e5fee368 100644 --- a/Utility/Tense.hs +++ b/Utility/Tense.hs @@ -7,7 +7,13 @@ {-# LANGUAGE OverloadedStrings #-} -module Utility.Tense where +module Utility.Tense ( + Tense(..), + TenseChunk(..), + TenseText, + renderTense, + tenseWords, +) where import qualified Data.Text as T import Data.Text (Text) @@ -52,6 +58,3 @@ tenseWords = TenseText . go [] go c ((Tensed w1 w2):ws) = go (Tensed (addspace w1) (addspace w2) : c) ws addspace w = T.append w " " - -unTensed :: Text -> TenseText -unTensed t = TenseText [UnTensed t] diff --git a/Utility/ThreadLock.hs b/Utility/ThreadLock.hs index e212fc11f7..d43e4f64ae 100644 --- a/Utility/ThreadLock.hs +++ b/Utility/ThreadLock.hs @@ -5,7 +5,11 @@ - License: BSD-2-clause -} -module Utility.ThreadLock where +module Utility.ThreadLock ( + Lock, + newLock, + withLock, +) where import Control.Concurrent.MVar diff --git a/Utility/ThreadScheduler.hs b/Utility/ThreadScheduler.hs index 5b46c92e3d..ef69ead81f 100644 --- a/Utility/ThreadScheduler.hs +++ b/Utility/ThreadScheduler.hs @@ -8,7 +8,14 @@ {-# LANGUAGE CPP #-} -module Utility.ThreadScheduler where +module Utility.ThreadScheduler ( + Seconds(..), + Microseconds, + runEvery, + threadDelaySeconds, + waitForTermination, + oneSecond, +) where import Control.Monad import Control.Concurrent diff --git a/Utility/TimeStamp.hs b/Utility/TimeStamp.hs index c9ecb89aba..b740d7bead 100644 --- a/Utility/TimeStamp.hs +++ b/Utility/TimeStamp.hs @@ -5,7 +5,11 @@ - License: BSD-2-clause -} -module Utility.TimeStamp where +module Utility.TimeStamp ( + parserPOSIXTime, + parsePOSIXTime, + formatPOSIXTime, +) where import Utility.Data diff --git a/Utility/Tmp.hs b/Utility/Tmp.hs index 84118dc47d..6ee592b865 100644 --- a/Utility/Tmp.hs +++ b/Utility/Tmp.hs @@ -8,7 +8,13 @@ {-# LANGUAGE CPP #-} {-# OPTIONS_GHC -fno-warn-tabs #-} -module Utility.Tmp where +module Utility.Tmp ( + Template, + viaTmp, + withTmpFile, + withTmpFileIn, + relatedTemplate, +) where import System.IO import System.FilePath diff --git a/Utility/Tmp/Dir.hs b/Utility/Tmp/Dir.hs index 64c57d60f0..c68ef86571 100644 --- a/Utility/Tmp/Dir.hs +++ b/Utility/Tmp/Dir.hs @@ -8,7 +8,10 @@ {-# LANGUAGE CPP #-} {-# OPTIONS_GHC -fno-warn-tabs #-} -module Utility.Tmp.Dir where +module Utility.Tmp.Dir ( + withTmpDir, + withTmpDirIn, +) where import Control.Monad.IfElse import System.FilePath diff --git a/Utility/Tor.hs b/Utility/Tor.hs index 094c91aacd..168e6448dc 100644 --- a/Utility/Tor.hs +++ b/Utility/Tor.hs @@ -7,7 +7,17 @@ {-# LANGUAGE CPP #-} -module Utility.Tor where +module Utility.Tor ( + OnionPort, + OnionAddress(..), + OnionSocket, + UniqueIdent, + AppName, + connectHiddenService, + addHiddenService, + getHiddenServiceSocketFile, + torIsInstalled, +) where import Common import Utility.ThreadScheduler diff --git a/Utility/Tuple.hs b/Utility/Tuple.hs index 25c6e8f36f..9638bcc0d9 100644 --- a/Utility/Tuple.hs +++ b/Utility/Tuple.hs @@ -5,7 +5,11 @@ - License: BSD-2-clause -} -module Utility.Tuple where +module Utility.Tuple ( + fst3, + snd3, + thd3, +) where fst3 :: (a,b,c) -> a fst3 (a,_,_) = a diff --git a/Utility/Verifiable.hs b/Utility/Verifiable.hs index 278c320aee..e24cd446d9 100644 --- a/Utility/Verifiable.hs +++ b/Utility/Verifiable.hs @@ -5,7 +5,14 @@ - License: BSD-2-clause -} -module Utility.Verifiable where +module Utility.Verifiable ( + Secret, + HMACDigest, + Verifiable(..), + mkVerifiable, + verify, + prop_verifiable_sane, +) where import Data.ByteString.UTF8 (fromString) import qualified Data.ByteString as S diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs index 5856bd756c..f51cdc1a12 100644 --- a/Utility/WebApp.hs +++ b/Utility/WebApp.hs @@ -7,7 +7,14 @@ {-# LANGUAGE OverloadedStrings, CPP, RankNTypes #-} -module Utility.WebApp where +module Utility.WebApp ( + browserProc, + runWebApp, + webAppSessionBackend, + checkAuthToken, + insertAuthToken, + writeHtmlShim, +) where import Common import Utility.Tmp @@ -19,11 +26,9 @@ import qualified Network.Wai as Wai import Network.Wai.Handler.Warp import Network.Wai.Handler.WarpTLS import Network.HTTP.Types -import qualified Data.CaseInsensitive as CI import Network.Socket import "crypto-api" Crypto.Random import qualified Web.ClientSession as CS -import qualified Data.ByteString as B import qualified Data.Text as T import qualified Data.Text.Encoding as TE import Blaze.ByteString.Builder.Char.Utf8 (fromText) @@ -119,9 +124,6 @@ getSocket h = do listen sock maxListenQueue return sock -lookupRequestField :: CI.CI B.ByteString -> Wai.Request -> B.ByteString -lookupRequestField k req = fromMaybe "" . lookup k $ Wai.requestHeaders req - {- Rather than storing a session key on disk, use a random key - that will only be valid for this run of the webapp. -} webAppSessionBackend :: Yesod.Yesod y => y -> IO (Maybe Yesod.SessionBackend) @@ -188,7 +190,6 @@ insertAuthToken extractAuthToken predicate webapp root pathbits params = writeHtmlShim :: String -> String -> FilePath -> IO () writeHtmlShim title url file = viaTmp writeFileProtected file $ genHtmlShim title url -{- TODO: generate this static file using Yesod. -} genHtmlShim :: String -> String -> String genHtmlShim title url = unlines [ "" diff --git a/debian/control b/debian/control index f1808631f7..c26f747ac0 100644 --- a/debian/control +++ b/debian/control @@ -81,7 +81,7 @@ Build-Depends: lsof [linux-any], ikiwiki, libimage-magick-perl, - git (>= 1:2.1), + git (>= 1:2.22), rsync, curl, openssh-client, @@ -98,7 +98,7 @@ Package: git-annex Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, - git (>= 1:1.8.1), + git (>= 1:2.22), netbase, rsync, curl, diff --git a/debian/patches/standalone-build b/debian/patches/standalone-build index bd2942db02..8e8e5f19c9 100644 --- a/debian/patches/standalone-build +++ b/debian/patches/standalone-build @@ -7,7 +7,7 @@ Last-Update: 2015-04-20 --- a/debian/control +++ b/debian/control @@ -85,6 +85,7 @@ Build-Depends: - git (>= 1:1.8.1), + git (>= 1:2.22), rsync, curl, + locales, @@ -23,7 +23,7 @@ Last-Update: 2015-04-20 Architecture: any Section: utils -Depends: ${misc:Depends}, ${shlibs:Depends}, -- git (>= 1:1.8.1), +- git (>= 1:2.22), +Conflicts: git-annex +Provides: git-annex +Depends: ${misc:Depends}, diff --git a/doc/backends.mdwn b/doc/backends.mdwn index ee6742902c..ee305b1684 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -1,4 +1,4 @@ -When a file is annexed, a key is generated from its content and/or filesystem +When a file is annexed, a [[key|internals/key_format]] is generated from its content and/or filesystem metadata. The file checked into git symlinks to the key. This key can later be used to retrieve the file's content (its value). diff --git a/doc/bugs/WSL1__58___git-annex-add_fails_in_DrvFs_filesystem.mdwn b/doc/bugs/WSL1__58___git-annex-add_fails_in_DrvFs_filesystem.mdwn new file mode 100644 index 0000000000..cc0bc621a0 --- /dev/null +++ b/doc/bugs/WSL1__58___git-annex-add_fails_in_DrvFs_filesystem.mdwn @@ -0,0 +1,123 @@ +### Please describe the problem. + +git-annex fails to add file to the repository because of permission problem (probably faulty permission handling in WSL). Interestingly, it is possible to add a file anyway, by executing `git annex add` twice. Unfortunately, files added this way are writeable, when they shouldn't. + +It's probably not in the scope of git-annex developing, but I think it's good to keep trace on the problem. + +### What steps will reproduce the problem? + +``` +cd /mnt/c +git init test +cd test +git annex init test +init test +touch file +git annex add file +``` + +### What version of git-annex are you using? On what operating system? + +Windows 10 Pro version 1909 build 18363.476 - WSL (Arch) + +``` +git-annex version: 7.20191114-ga95efcbc55 +build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite +dependency versions: aws-0.21.1 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.3 feed-1.2.0.1 ghc-8.6.5 http-client-0.6.4 persistent-sqlite-2.10.5 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs hook external +operating system: linux x86_64 +supported repository versions: 7 +upgrade supported from repository versions: 0 1 2 3 4 5 6 +local repository version: 7 +``` + +### Please provide any additional information below. + +[[!format sh """ +$ git annex init test +init test + Detected a filesystem without fifo support. + + Disabling ssh connection caching. +(scanning for unlocked files...) +ok +(recording state in git...) +$ touch file +$ git annex add file --debug +[2019-11-28 11:52:53.048398] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2019-11-28 11:52:53.0570463] process done ExitSuccess +[2019-11-28 11:52:53.0573639] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2019-11-28 11:52:53.0656397] process done ExitFailure 1 +[2019-11-28 11:52:53.0660529] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--others","--exclude-standard","-z","--","file"] +[2019-11-28 11:52:53.0742999] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2019-11-28 11:52:53.0822627] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2019-11-28 11:52:53.0853736] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +add file [2019-11-28 11:52:53.0949002] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2019-11-28 11:52:53.1027361] process done ExitSuccess +[2019-11-28 11:52:53.1030132] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2019-11-28 11:52:53.1122577] process done ExitFailure 1 + +[2019-11-28 11:52:53.1232169] call: cp ["--reflink=auto","--preserve=timestamps",".git/annex/objects/pX/ZJ/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","file"] +[2019-11-28 11:52:53.1606206] process done ExitSuccess + +git-annex: .git/annex/othertmp/file.0/file: rename: permission denied (Permission denied) +failed +[2019-11-28 11:52:53.1617248] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--modified","-z","--","file"] +[2019-11-28 11:52:53.1693198] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--name-only","--diff-filter=T","-z","--cached","--","file"] +[2019-11-28 11:52:53.1825925] process done ExitSuccess +[2019-11-28 11:52:53.1835521] process done ExitSuccess +[2019-11-28 11:52:53.1844047] process done ExitSuccess +git-annex: add: 1 failed +"""]] + +Second attempt: + +[[!format sh """ +$ git annex add file --debug +[2019-11-28 11:57:56.4029726] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2019-11-28 11:57:56.4114361] process done ExitSuccess +[2019-11-28 11:57:56.411681] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2019-11-28 11:57:56.4201317] process done ExitFailure 1 +[2019-11-28 11:57:56.420548] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--others","--exclude-standard","-z","--","file"] +[2019-11-28 11:57:56.4316368] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","annex.largefiles","--"] +[2019-11-28 11:57:56.4416827] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2019-11-28 11:57:56.4452357] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +add file [2019-11-28 11:57:56.4545013] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2019-11-28 11:57:56.4626846] process done ExitSuccess +[2019-11-28 11:57:56.4629866] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2019-11-28 11:57:56.4735385] process done ExitFailure 1 + +[2019-11-28 11:57:56.4848163] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2019-11-28 11:57:56.488706] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +ok +[2019-11-28 11:57:56.4964438] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--modified","-z","--","file"] +[2019-11-28 11:57:56.5043041] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--name-only","--diff-filter=T","-z","--cached","--","file"] +(recording state in git...) +[2019-11-28 11:57:56.5152453] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"] +[2019-11-28 11:57:56.5426207] process done ExitSuccess +[2019-11-28 11:57:56.5438586] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] +[2019-11-28 11:57:56.5478542] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2019-11-28 11:57:56.5713] process done ExitSuccess +[2019-11-28 11:57:56.5716027] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2019-11-28 11:57:56.5803067] process done ExitSuccess +[2019-11-28 11:57:56.5807703] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2019-11-28 11:57:56.6111405] process done ExitSuccess +[2019-11-28 11:57:56.6115303] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","ffa5a12eba0b2ea9bc5b529278597615f70c901c","--no-gpg-sign","-p","refs/heads/git-annex"] +[2019-11-28 11:57:56.6269742] process done ExitSuccess +[2019-11-28 11:57:56.6272697] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","0ece4a3a069693ea12cb61168cfb701040c8a7a7"] +[2019-11-28 11:57:56.6465065] process done ExitSuccess +[2019-11-28 11:57:56.6506175] process done ExitSuccess +[2019-11-28 11:57:56.651426] process done ExitSuccess +[2019-11-28 11:57:56.6520969] process done ExitSuccess +[2019-11-28 11:57:56.6527282] process done ExitSuccess +[2019-11-28 11:57:56.6536136] process done ExitSuccess +[2019-11-28 11:57:56.6554327] process done ExitSuccess +$ echo "this should fail" > file +$ cat file +this should fail +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes! Thank you very much Joey for your hard work and digging into WSL bugs :) diff --git a/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_3_2a9f0396df93d9500306b3d7039803a2._comment b/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_3_2a9f0396df93d9500306b3d7039803a2._comment new file mode 100644 index 0000000000..fb1206a7d1 --- /dev/null +++ b/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_3_2a9f0396df93d9500306b3d7039803a2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="ply" + avatar="http://cdn.libravatar.org/avatar/1270501a59ed4a4042366b00295fe236" + subject="comment 3" + date="2019-11-28T11:18:50Z" + content=""" +Thanks Joey for investigating this! It looks like I need to wait for WSL 2 to become available in windows public release. In the meantime I've submitted a bug on [faulty behaviour of `git annex add` on DrvFs](https://git-annex.branchable.com/bugs/WSL1__58___git-annex-add_fails_in_DrvFs_filesystem/). I don't think you can fix it, as it is apparantly a WSL problem, but I think it's good to keep track of it and warn potential users +"""]] diff --git a/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file.mdwn b/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file.mdwn new file mode 100644 index 0000000000..d93c3623a5 --- /dev/null +++ b/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file.mdwn @@ -0,0 +1,146 @@ +It is not a ground shaking issue, but probably would be best to handle it more gracefully. + +Initially mentioned while doing install using datalad. Account/permission is required to access this particular repo, ask Canadians for access if you don't have it yet Joey. credentials I guess got asked for and cached by git upon initial invocation, so upon subsequent calls didn't ask for any: + +[[!format sh """ +$> datalad install https://git.bic.mni.mcgill.ca/bic/Coffey-mri-bids +[INFO ] Cloning https://git.bic.mni.mcgill.ca/bic/Coffey-mri-bids [1 other candidates] into '/tmp/Coffey-mri-bids' +[INFO ] fatal: bad config line 1 in file /home/yoh/.tmp/git-annex96493-5.tmp +[INFO ] Remote origin not usable by git-annex; setting annex-ignore +install(ok): /tmp/Coffey-mri-bids (dataset) +"""]] + +which boiled down to that message being spited out during `git annex init` which samples the remote, but fails to download the config and gets instead a redirected html page: + +[[!format sh """ +$> git clone https://git.bic.mni.mcgill.ca/bic/Coffey-mri-bids +Cloning into 'Coffey-mri-bids'... +warning: redirecting to https://git.bic.mni.mcgill.ca/bic/Coffey-mri-bids.git/ +remote: Enumerating objects: 398, done. +remote: Counting objects: 100% (398/398), done. +remote: Compressing objects: 100% (282/282), done. +remote: Total 398 (delta 53), reused 393 (delta 48) +Receiving objects: 100% (398/398), 34.97 KiB | 795.00 KiB/s, done. +Resolving deltas: 100% (53/53), done. + + +$> git -C Coffey-mri-bids annex init --debug +... +[2019-11-27 19:27:01.341315979] Request { + host = "git.bic.mni.mcgill.ca" + port = 443 + secure = True + requestHeaders = [("Accept-Encoding","identity"),("User-Agent","git-annex/7.20190819+git2-g908476a9b-1~ndall+1")] + path = "/bic/Coffey-mri-bids/config" + queryString = "" + method = "GET" + proxy = Nothing + rawBody = False + redirectCount = 10 + responseTimeout = ResponseTimeoutDefault + requestVersion = HTTP/1.1 +} + +[2019-11-27 19:27:01.90016181] read: git ["config","--null","--list","--file","/home/yoh/.tmp/git-annex228094-5.tmp"] +fatal: bad config line 1 in file /home/yoh/.tmp/git-annex228094-5.tmp +[2019-11-27 19:27:01.913302324] process done ExitFailure 128 + + Remote origin not usable by git-annex; setting annex-ignore + +$> wget -S https://git.bic.mni.mcgill.ca/bic/Coffey-mri-bids/config +--2019-11-27 19:29:25-- https://git.bic.mni.mcgill.ca/bic/Coffey-mri-bids/config +Resolving git.bic.mni.mcgill.ca (git.bic.mni.mcgill.ca)... 132.216.133.92 +Connecting to git.bic.mni.mcgill.ca (git.bic.mni.mcgill.ca)|132.216.133.92|:443... connected. +HTTP request sent, awaiting response... + HTTP/1.1 302 Found + Server: nginx + Date: Thu, 28 Nov 2019 00:29:26 GMT + Content-Type: text/html; charset=utf-8 + Content-Length: 109 + Connection: keep-alive + Cache-Control: no-cache + Location: https://git.bic.mni.mcgill.ca/users/sign_in + Set-Cookie: _gitlab_session=8a4f8d5569636004aaebfb73588a2d53; path=/; secure; HttpOnly + X-Request-Id: xTcSyu4H36 + X-Runtime: 0.071681 + Strict-Transport-Security: max-age=31536000 + Referrer-Policy: strict-origin-when-cross-origin +Location: https://git.bic.mni.mcgill.ca/users/sign_in [following] +--2019-11-27 19:29:26-- https://git.bic.mni.mcgill.ca/users/sign_in +Reusing existing connection to git.bic.mni.mcgill.ca:443. +HTTP request sent, awaiting response... + HTTP/1.1 200 OK + Server: nginx + Date: Thu, 28 Nov 2019 00:29:26 GMT + Content-Type: text/html; charset=utf-8 + Transfer-Encoding: chunked + Connection: keep-alive + Vary: Accept-Encoding + Cache-Control: max-age=0, private, must-revalidate + Etag: W/"305857ff0ba591a1e4ee7fec83b5687c" + Referrer-Policy: strict-origin-when-cross-origin + Set-Cookie: _gitlab_session=8a4f8d5569636004aaebfb73588a2d53; path=/; expires=Thu, 28 Nov 2019 02:29:26 -0000; secure; HttpOnly + X-Content-Type-Options: nosniff + X-Download-Options: noopen + X-Frame-Options: DENY + X-Permitted-Cross-Domain-Policies: none + X-Request-Id: MHFi7Yjxe82 + X-Runtime: 0.063359 + X-Ua-Compatible: IE=edge + X-Xss-Protection: 1; mode=block + Strict-Transport-Security: max-age=31536000 + Referrer-Policy: strict-origin-when-cross-origin +Length: unspecified [text/html] +Saving to: ‘config’ + +config [ <=> ] 13.19K --.-KB/s in 0s + +2019-11-27 19:29:26 (89.1 MB/s) - ‘config’ saved [13505] + +$> cat config + + + + + + + + +... +"""]] + +I guess the problem is multi-faceted: + +1. in case of authenticated http remote, `git` caches credentials, but then `git annex` tries to download file directly (instead of somehow via git), it could not "sense" that remote to be a valid annex and/or get files from it. + +You can try with this simple one -- user "demo", password "demo": + +[[!format sh """ +$> git clone http://www.onerussian.com/tmp/secret-repo/.git +Cloning into 'secret-repo'... +Username for 'http://www.onerussian.com': demo +Password for 'http://demo@www.onerussian.com': + +$> git -C secret-repo annex init +init (merging origin/git-annex into git-annex...) +(recording state in git...) + + Remote origin not usable by git-annex; setting annex-ignore +ok +(recording state in git...) + +"""]] + +although remote is a proper annex, indeed `git annex` cannot use it since does not authenticate as git does. +So even though the error message is not incorrect, I would say the situation is suboptimal + +2. if remote server instead of just returning 404 or 403 error code (as eg github seems to do in similar cases of non-authenticated access) instead redirects to some login page, annex feeds that page as a config to git, ignores the error message and just marks that remote as ignored for annex, while leaking that obscure "fatal" error message from git. + +IMHO, ideally 1. should be addressed properly (authentication), and for 2. annex should spit out some more sensible message ("git failed to parse a config file fetched from the remote X. Please inspect it at this /path/config"), so keep that file around for debugging. As it is now I had to dig quite deep to figure out WTF is going on. + +git annex 7.20190819+git2-g908476a9b-1~ndall+1 and the same with bleeding edge 7.20191114+git43-ge29663773-1~ndall+1 (probably that commit is the one with my patch for stricter git versioning, so use the count of 42 ;)) + +[[!meta author=yoh]] +[[!tag projects/dandi]] + + diff --git a/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file/comment_1_08af564539efb1b0d85905c0aa862c43._comment b/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file/comment_1_08af564539efb1b0d85905c0aa862c43._comment new file mode 100644 index 0000000000..53716fecb2 --- /dev/null +++ b/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file/comment_1_08af564539efb1b0d85905c0aa862c43._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="related: shouldn't git annex try external remotes to download config?" + date="2019-11-28T01:22:53Z" + content=""" +I haven't tested, but I can see the situation where a specific repository URL could be handled by external special remote (such as datalad, downloaders of which do handle obscure setups such as this one without 403/404 but rather forwarding to login page) which would provide authenticated access to the URL. Would annex even try that config URL via external special remotes? +"""]] diff --git a/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file/comment_2_9a63b2918f5621efbfe8cdb33b23ff21._comment b/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file/comment_2_9a63b2918f5621efbfe8cdb33b23ff21._comment new file mode 100644 index 0000000000..bf9004d767 --- /dev/null +++ b/doc/bugs/leaks_git_config_error_message_upon_inability_to_read_downloaded___34__config__34___file/comment_2_9a63b2918f5621efbfe8cdb33b23ff21._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 2" + date="2019-11-29T18:09:45Z" + content=""" +one of the use-cases (will be) https://gin.g-node.org/ -- an archive of (primarily) electrophys data. The platform is based on gogs, but uses git-annex underneath. It \"will be\" because currently access to git-annex is provided only via ssh, but as of today it is already possible to `git clone` (tried on public, didn't try private) datasets via https, and developers are looking into exposing git-annex also via http. To access private datasets authentication will need to be handled +"""]] diff --git a/doc/devblog/day_608__easier_git-lfs_setup/comment_1_b72bea21d4445d12cade9f54ecc3767d._comment b/doc/devblog/day_608__easier_git-lfs_setup/comment_1_b72bea21d4445d12cade9f54ecc3767d._comment new file mode 100644 index 0000000000..8a45019d3b --- /dev/null +++ b/doc/devblog/day_608__easier_git-lfs_setup/comment_1_b72bea21d4445d12cade9f54ecc3767d._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="jkrenzer" + avatar="http://cdn.libravatar.org/avatar/49eda64e3bd130a5dfaf48f09218cfb1" + subject="Great work! Thank you!" + date="2019-12-05T12:10:47Z" + content=""" +Hi Joe, + +I have been using git-annex since several years now and am really happy with it. + +But the addition of git-lfs support was, in my humble opinion, an extraordinary leap forward! After gitlab ceased support in favour of lfs and with the advent of lfs in github, annex-integration with widespread used collaborative git-platforms was not very straight-forward anymore. And so it became harder to justify or promote it's use in my institute or other organizations I work for. This has change now again for the better! + +So thank you very much for all the hard work and regarding the daunting speed with which new developments and features (version 8 already!) arrive here: Please take good care and do not overstrain yourself. + +Kind regards, + +Jörn +"""]] diff --git a/doc/devblog/day_609__optimisation.mdwn b/doc/devblog/day_609__optimisation.mdwn new file mode 100644 index 0000000000..d89fd66179 --- /dev/null +++ b/doc/devblog/day_609__optimisation.mdwn @@ -0,0 +1,11 @@ +Today, sped up many git-annex commands by around 5%. Often git-annex +traverses the work tree and deserializes keys to its Key data type, only to +turn around and do something with a Key that needs it to be serialized +again. So caching the original serialization of a key avoids that work. I +had started on this in January but had to throw my first attempt away. + +The big bytestring conversion in January only yielded a 5-15% speedup, +so an extra 5% is a nice bonus for so relativly little work today. +It also feels like this optimisation approach is nearly paid out though; +only converting all filepath operations to bytestrings seems likely to +yield a similar widespread improvement. diff --git a/doc/devblog/day_610-611__ByteString_optimisation_early_days.mdwn b/doc/devblog/day_610-611__ByteString_optimisation_early_days.mdwn new file mode 100644 index 0000000000..c64240173d --- /dev/null +++ b/doc/devblog/day_610-611__ByteString_optimisation_early_days.mdwn @@ -0,0 +1,12 @@ +Two entire days spent making a branch where git-annex uses ByteString +instead of String, especially for filepaths. I commented out all the +commands except for find, but it still took thousands of lines of patches +to get it to compile. + +The result: git-annex find is between 28% and 66% faster when using +ByteString. The files just fly by! + +It's going to be a long, long road to finish this, but it's good to have a +start, and know it will be worth it. +[[todo/optimize_by_converting_String_to_ByteString]] is the tracking page +for this going forward. diff --git a/doc/devblog/day_610-611__ByteString_optimisation_early_days/comment_1_1601268fd4dba4df9cc3dc84932914a6._comment b/doc/devblog/day_610-611__ByteString_optimisation_early_days/comment_1_1601268fd4dba4df9cc3dc84932914a6._comment new file mode 100644 index 0000000000..45e0c377ed --- /dev/null +++ b/doc/devblog/day_610-611__ByteString_optimisation_early_days/comment_1_1601268fd4dba4df9cc3dc84932914a6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="anarcat" + avatar="http://cdn.libravatar.org/avatar/4ad594c1e13211c1ad9edb81ce5110b7" + subject="amazing!" + date="2019-11-26T21:07:32Z" + content=""" +66% performance improvements is an amazing number! i take it this will be especially good for repositories with a large number of files? if so this could make my life MUCH better! :) + +i wonder if this connects with the [problems gorzen identified in python 3 about POSIX paths](https://changelog.complete.org/archives/10063-the-fundamental-problem-in-python-3)... does Haskell have similar problems with non-unicode filenames? + +in any case, I thank you for this awesome work... +"""]] diff --git a/doc/devblog/day_610-611__ByteString_optimisation_early_days/comment_2_21872150f9b2ff9cc08f94d52dbdf3a6._comment b/doc/devblog/day_610-611__ByteString_optimisation_early_days/comment_2_21872150f9b2ff9cc08f94d52dbdf3a6._comment new file mode 100644 index 0000000000..7249b8dc09 --- /dev/null +++ b/doc/devblog/day_610-611__ByteString_optimisation_early_days/comment_2_21872150f9b2ff9cc08f94d52dbdf3a6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="parallelization" + date="2019-11-27T17:30:12Z" + content=""" +This is great. + +One other potential for speedup is fixing [[issues with parallel operations|forum/people's_experience_with_parallel_git-annex_operations]]. My current fix is to use `-J1`, giving up a potential 96X speedup. There may also be additional [[todo/parallel_possibilities]]. +"""]] diff --git a/doc/devblog/day_612__building_again.mdwn b/doc/devblog/day_612__building_again.mdwn new file mode 100644 index 0000000000..a6e4e518af --- /dev/null +++ b/doc/devblog/day_612__building_again.mdwn @@ -0,0 +1,11 @@ +I've gotten the `bs` branch to build everything again. Was not trivial, +the diff is over 7000 lines. + +Had hoped this was a mechanical enough conversion it would not introduce +many bugs, but the test suite quickly found a lot of problems. So that +branch is not ready for merging yet. + +I'm considering making a library that's like +[filepath](http://hackage.haskell.org/package/filepath) but for +RawFilePath. That would probably speed git-annex up by another 5% or so, +in places where it currently has to convert back to FilePath. diff --git a/doc/git-annex-benchmark.mdwn b/doc/git-annex-benchmark.mdwn index c81839f5de..73bb01eb82 100644 --- a/doc/git-annex-benchmark.mdwn +++ b/doc/git-annex-benchmark.mdwn @@ -45,13 +45,14 @@ instead of a command. N is the number of items to benchmark. # OUTPUT The output of the commands being benchmarked goes to standard output and -standard error as usual. It's often a good idea to sink it to /dev/null to -avoid the display of the output skewing the benchmark results. Of course ---quiet can also be used to avoid most git-annex output, as long as you -don't want to benchmark the generation of that output. +standard error as usual. It's often a good idea to use --quiet to avoid +unncessary output, unless the generation of that output is part of what +you want to benchmark. The benchmark report is output to standard output by default, although -criterion options can be used to redirect it to a file. +criterion options can be used to redirect it to a file. For example: + + git annex benchmark -o bench -- find >/dev/null # SEE ALSO diff --git a/doc/git-annex-unused/comment_4_5b208cb45335f0d383a456865068e659._comment b/doc/git-annex-unused/comment_4_5b208cb45335f0d383a456865068e659._comment new file mode 100644 index 0000000000..a10db0d1a5 --- /dev/null +++ b/doc/git-annex-unused/comment_4_5b208cb45335f0d383a456865068e659._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="atrent" + avatar="http://cdn.libravatar.org/avatar/6069dfebff03997460874771defa0fa4" + subject="can't find unused objects" + date="2019-12-02T07:26:41Z" + content=""" +I recently migrated an annex to SHA256 (without \"E\") and I'm now trying to clean the repo from unused data. +I have a strange situation: there are 62G of unused objects: + + $ du -ks .git/annex/objects/ + 64334024 .git/annex/objects/ + +but 'git annex unused' gives me only: + + $ git annex unused + unused ... + Some annexed data is no longer used by any files: + NUMBER KEY + 1 SHA256E-s27--32efec98dc9e05442fc2385bb85d855a8c7824c68abd4ab5bf55a4dfe412b334.pdf + (To see where data was previously used, try: git log --stat --no-textconv -S'KEY') + To remove unwanted data: git-annex dropunused NUMBER + ok + +I've checked (through a small shell script) that none of the object is in fact referenced by any symlink... + +May I delete them? Shall I do some other checking/fscking/repairing? + +Thank you +"""]] diff --git a/doc/git-annex-unused/comment_5_65ab9f687f2817199eda7455d9b82677._comment b/doc/git-annex-unused/comment_5_65ab9f687f2817199eda7455d9b82677._comment new file mode 100644 index 0000000000..f688589402 --- /dev/null +++ b/doc/git-annex-unused/comment_5_65ab9f687f2817199eda7455d9b82677._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="atrent" + avatar="http://cdn.libravatar.org/avatar/6069dfebff03997460874771defa0fa4" + subject="P.S. they are all SHA256E" + date="2019-12-02T07:29:06Z" + content=""" +the \"lost\" objects are all SHA256E (*with* the \"E\") +"""]] diff --git a/doc/git-annex-unused/comment_6_962097729f8b4a657bb7b9863fdffb68._comment b/doc/git-annex-unused/comment_6_962097729f8b4a657bb7b9863fdffb68._comment new file mode 100644 index 0000000000..564bbab667 --- /dev/null +++ b/doc/git-annex-unused/comment_6_962097729f8b4a657bb7b9863fdffb68._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="atrent" + avatar="http://cdn.libravatar.org/avatar/6069dfebff03997460874771defa0fa4" + subject="P.P.S. i dropped all local copies" + date="2019-12-02T08:03:01Z" + content=""" +I forgot to tell that after migrating I synced to all remotes and dropped everything in 'here', that's why I was expecting no more objects locally. +"""]] diff --git a/doc/git-annex-unused/comment_7_e21b036767651c5cfdd34bbd24a31fb5._comment b/doc/git-annex-unused/comment_7_e21b036767651c5cfdd34bbd24a31fb5._comment new file mode 100644 index 0000000000..bcc17e1d5b --- /dev/null +++ b/doc/git-annex-unused/comment_7_e21b036767651c5cfdd34bbd24a31fb5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="comment 7" + date="2019-12-02T16:02:47Z" + content=""" +Did you commit after git-migrate? Does the worktree have any uncommitted changes? +"""]] diff --git a/doc/git-annex-unused/comment_8_10ee2770ffe1cde367f3d310b2670539._comment b/doc/git-annex-unused/comment_8_10ee2770ffe1cde367f3d310b2670539._comment new file mode 100644 index 0000000000..b7377c955c --- /dev/null +++ b/doc/git-annex-unused/comment_8_10ee2770ffe1cde367f3d310b2670539._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="dropping contents of old keys after migration" + date="2019-12-02T16:48:47Z" + content=""" +\"May I delete them\" -- `git-annex-drop --force` may be safer, as it also updates [[location_tracking]]. You might also want to [[git-annex-dead]] the dropped keys to prevent [[git-annex-fsck]] from complaining about lost contents. + +Re: why [[git-annex-unused]] isn't finding the unused contents, try running it with `--used-refspec=+HEAD`, and make sure `annex.used-refspec` git config is not set. Note that this will mark as unused any annexed contents not referenced from the latest tree of the HEAD branch, e.g. annexed files that were removed in some older commit. +"""]] diff --git a/doc/git-annex-unused/comment_9_cfcc0bab810d58b7db66c2fa4e92f769._comment b/doc/git-annex-unused/comment_9_cfcc0bab810d58b7db66c2fa4e92f769._comment new file mode 100644 index 0000000000..41477d216b --- /dev/null +++ b/doc/git-annex-unused/comment_9_cfcc0bab810d58b7db66c2fa4e92f769._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="comment 9" + date="2019-12-02T16:58:25Z" + content=""" +\"I synced to all remotes and dropped everything in 'here'\" -- [[git-annex-unused]] \"Checks the *annex*\" for the unused contents (unless `--from=repository` is used), so if you dropped everything in `here`, there's nothing to find. But it seems from `du` results that contents wasn't actually dropped? [[git-annex-whereis]] tells where git-annex thinks contents is. +"""]] diff --git a/doc/install/openSUSE/comment_1_a5aea1ef644d0402d3caf593fef2456f._comment b/doc/install/openSUSE/comment_1_a5aea1ef644d0402d3caf593fef2456f._comment new file mode 100644 index 0000000000..ee0bf31099 --- /dev/null +++ b/doc/install/openSUSE/comment_1_a5aea1ef644d0402d3caf593fef2456f._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="nangal.vivek@08b8bc308cb03037792b7930fd839b9deec118df" + nickname="nangal.vivek" + avatar="http://cdn.libravatar.org/avatar/f8e7f170b837feb1008df116c7f9d0de" + subject="not able to find git-annex on openSUSE using zypper" + date="2019-12-01T17:04:53Z" + content=""" +Getting the following error on runnning `zypper in git-annex` + + Loading repository data... + Reading installed packages... + Package 'git-annex' not found. + +I am running openSUSE for WSL with the following info + + NAME=\"openSUSE Leap\" + VERSION=\"15.1 \" + ID=\"opensuse-leap\" + ID_LIKE=\"suse opensuse\" + VERSION_ID=\"15.1\" + PRETTY_NAME=\"openSUSE Leap 15.1\" + ANSI_COLOR=\"0;32\" + CPE_NAME=\"cpe:/o:opensuse:leap:15.1\" + BUG_REPORT_URL=\"https://bugs.opensuse.org\" + HOME_URL=\"https://www.opensuse.org/\" +"""]] diff --git a/doc/internals/comment_10_c4298babd96b2596bd4f6ad828212c92._comment b/doc/internals/comment_10_c4298babd96b2596bd4f6ad828212c92._comment new file mode 100644 index 0000000000..794735a02f --- /dev/null +++ b/doc/internals/comment_10_c4298babd96b2596bd4f6ad828212c92._comment @@ -0,0 +1,31 @@ +[[!comment format=mdwn + username="atrent" + avatar="http://cdn.libravatar.org/avatar/6069dfebff03997460874771defa0fa4" + subject="duplicate objects?" + date="2019-11-30T14:04:17Z" + content=""" +Do I understand correctly that in .git/annex/objects dir there should be no duplicates? +Here follows a run of 'rdfind' done in the objects dir: + + $ rdfind . + Now scanning \".\", found 12874 files. + Now have 12874 files in total. + Removed 0 files due to nonunique device and inode. + Total size is 75579281486 bytes or 70 GiB + Removed 8376 files due to unique sizes from list.4498 files left. + Now eliminating candidates based on first bytes:removed 68 files from list.4430 files left. + Now eliminating candidates based on last bytes:removed 66 files from list.4364 files left. + Now eliminating candidates based on sha1 checksum:removed 0 files from list.4364 files left. + It seems like you have 4364 files that are not unique + Totally, 10 GiB can be reduced. + Now making results file results.txt + +And here is an example pair of dupes (excerpt from the abovementioned 'results.txt'): + + DUPTYPE_FIRST_OCCURRENCE 2073 3 86558 26 21057567 1 ./53/zv/SHA256E-s86558--e79a0891bb94fc9212ce2f28178fe84591c5fb24c07b5239d367099118e12ede.jpg/SHA256E-s86558--e79a0891bb94fc9212ce2f28178fe84591c5fb24c07b5239d367099118e12ede.jpg + DUPTYPE_WITHIN_SAME_TREE -2073 3 86558 26 1080608 1 ./7w/w2/SHA256E-s86558--e79a0891bb94fc9212ce2f28178fe84591c5fb24c07b5239d367099118e12ede.56.jpeg/SHA256E-s86558--e79a0891bb94fc9212ce2f28178fe84591c5fb24c07b5239d367099118e12ede.56.jpeg + +Any clues? + +Thank you +"""]] diff --git a/doc/internals/comment_11_9758bb3a17f63b4dcf51742ea482dbe9._comment b/doc/internals/comment_11_9758bb3a17f63b4dcf51742ea482dbe9._comment new file mode 100644 index 0000000000..3eb9d34423 --- /dev/null +++ b/doc/internals/comment_11_9758bb3a17f63b4dcf51742ea482dbe9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="same contents with different keys" + date="2019-11-30T16:51:58Z" + content=""" +@atrent -- some [[backends]] (like SHA256E) base the key not just on object contents, but also on part of its filename (the extension). So the same content can exist with two different keys. In your example, the same contents exists in one file ending with .jpg and in another ending with .56.jpeg . (This is done to give the annexed contents the same extension as the original file had before annexing, to avoid confusing some programs). There are also backends like WORM and URL, not based on checksums, that could lead to different keys with same contents. There could also be same contents added under different backends (see also [[`git-annex-migrate`|git-annex-migrate]]). Finally, there is the theoretical possibility of hash collisions. +"""]] diff --git a/doc/internals/comment_12_f0325cefa5cd53a5a897046606137cef._comment b/doc/internals/comment_12_f0325cefa5cd53a5a897046606137cef._comment new file mode 100644 index 0000000000..fe35b66fa2 --- /dev/null +++ b/doc/internals/comment_12_f0325cefa5cd53a5a897046606137cef._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="atrent" + avatar="http://cdn.libravatar.org/avatar/6069dfebff03997460874771defa0fa4" + subject="no collisions" + date="2019-11-30T20:37:00Z" + content=""" +I can confirm that these are not collisions: these identical files are the same photos with different names, shame on Dropbox syncing from my smartphone. I was actually hoping to dedupe through git-annex ;-) + +Some more questions/suggestions/conversation-starters: + +* I suppose I can dedup them with rdfind (i.e., hardlinking identical files), do you foresee any side effects? + +* may I change the hash function of git-annex to something not depending on filenames? (I suppose so, I'll have a look at the docs) + +* if I can change the hash function can I regenerate the whole annex without re-creating it? (again I'll have a look at docs) + +Thanks +"""]] diff --git a/doc/internals/comment_13_e45b6fa035a30703618448a0f764f935._comment b/doc/internals/comment_13_e45b6fa035a30703618448a0f764f935._comment new file mode 100644 index 0000000000..69452682e1 --- /dev/null +++ b/doc/internals/comment_13_e45b6fa035a30703618448a0f764f935._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="comment 13" + date="2019-11-30T21:11:53Z" + content=""" +[[git-annex-migrate]] to a backend not ending in E (e.g. SHA256 not SHA256E), then [[git-annex-unused]] to drop the old keys. + +"""]] diff --git a/doc/internals/comment_14_3f62751c2dd041f4ead1c6580ea5eec1._comment b/doc/internals/comment_14_3f62751c2dd041f4ead1c6580ea5eec1._comment new file mode 100644 index 0000000000..a950c8f739 --- /dev/null +++ b/doc/internals/comment_14_3f62751c2dd041f4ead1c6580ea5eec1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="hardlinking identical files in annex may break invariants" + date="2019-11-30T21:36:38Z" + content=""" +P.S. Re: hardlinking identical files -- git-annex [[keeps track of inodes|todo/inode_based_clean_filter_for_less_surprising_git_add]] where contents is stored, so deleting a file might make that info stale. Also, dropping one key will drop another key's contents without updating [[location_tracking]] info. And dropping then getting files would lead to two separate copies again. So I wouldn't recommend that. + +See also [[tips/local_caching_of_annexed_files]]. +"""]] diff --git a/doc/internals/comment_15_c3d12d14e4d044f39829c5d92f523655._comment b/doc/internals/comment_15_c3d12d14e4d044f39829c5d92f523655._comment new file mode 100644 index 0000000000..aa630ff5b9 --- /dev/null +++ b/doc/internals/comment_15_c3d12d14e4d044f39829c5d92f523655._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="atrent" + avatar="http://cdn.libravatar.org/avatar/6069dfebff03997460874771defa0fa4" + subject="migrating..." + date="2019-11-30T22:30:06Z" + content=""" +I'm git-annex-migrating (to SHA256) now, thank you for all suggestions! +"""]] diff --git a/doc/profiling/comment_6_ca4ac016a0fb0132fc5c746dfb6fefb3._comment b/doc/profiling/comment_6_ca4ac016a0fb0132fc5c746dfb6fefb3._comment new file mode 100644 index 0000000000..7f8d3a2571 --- /dev/null +++ b/doc/profiling/comment_6_ca4ac016a0fb0132fc5c746dfb6fefb3._comment @@ -0,0 +1,70 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 6""" + date="2019-11-22T22:18:02Z" + content=""" +After caching serialized Keys. + + Fri Nov 22 19:06 2019 Time and Allocation Profiling Report (Final) + + git-annex +RTS -p -RTS find + + total time = 3.00 secs (2997 ticks @ 1000 us, 1 processor) + total alloc = 1,890,060,432 bytes (excludes profiling overheads) + + COST CENTRE MODULE SRC %time %alloc + + inAnnex'.\ Annex.Content Annex/Content.hs:(103,61)-(118,31) 28.2 40.6 + keyFile' Annex.Locations Annex/Locations.hs:(564,1)-(574,30) 3.8 5.3 + splitc Utility.Split Utility/Split.hs:(24,1)-(26,25) 3.7 5.1 + _encodeFilePath Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:(111,1)-(114,49) 3.1 2.7 + encodeW8 Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:(189,1)-(191,70) 3.0 3.6 + w82s Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:217:1-15 2.8 5.1 + >>=.\.succ' Data.Attoparsec.Internal.Types Data/Attoparsec/Internal/Types.hs:146:13-76 2.8 0.2 + >>=.\ Data.Attoparsec.Internal.Types Data/Attoparsec/Internal/Types.hs:(146,9)-(147,44) 2.5 0.7 + getAnnexLinkTarget'.probesymlink Annex.Link Annex/Link.hs:78:9-62 2.4 2.5 + fileKey' Annex.Locations Annex/Locations.hs:(583,1)-(593,41) 2.1 1.5 + parseLinkTarget Annex.Link Annex/Link.hs:(254,1)-(262,25) 2.0 3.8 + keyFile'.esc Annex.Locations Annex/Locations.hs:(570,9)-(574,30) 1.9 4.7 + s2w8 Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:214:1-15 1.7 3.5 + keyPath Annex.Locations Annex/Locations.hs:(603,1)-(605,23) 1.6 3.5 + getState Annex Annex.hs:(251,1)-(254,27) 1.6 0.3 + parseKeyVariety Types.Key Types/Key.hs:(322,1)-(370,42) 1.4 0.0 + withMutablePtrHint Basement.Block.Base Basement/Block/Base.hs:(468,1)-(482,50) 1.3 0.4 + hashUpdates.\.\.\ Crypto.Hash Crypto/Hash.hs:85:48-99 1.2 0.0 + parseLinkTargetOrPointer Annex.Link Annex/Link.hs:(239,1)-(243,25) 1.1 0.1 + primitive Basement.Monad Basement/Monad.hs:72:5-18 1.1 0.0 + withPtr.makeTrampoline Basement.Block.Base Basement/Block/Base.hs:(401,5)-(404,31) 1.0 0.6 + assertLocal Git Git.hs:(123,1)-(129,28) 0.8 1.6 + decodeBS' Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:151:1-31 0.4 2.4 + +Runtime improved by 5% or so, and getAnnexLinkTarget moved up, otherwise +not a lot of change. keyFile is looking like an optimization target, +although its percent of the runtime actually reduced. +However that's specific to this repo which has a lot of URL keys that +contain '/' and so need to be escaped. + + Fri Nov 22 19:09 2019 Time and Allocation Profiling Report (Final) + + git-annex +RTS -p -RTS find --not --in web + + total time = 8.42 secs (8421 ticks @ 1000 us, 1 processor) + total alloc = 1,887,547,744 bytes (excludes profiling overheads) + + COST CENTRE MODULE SRC %time %alloc + + catObjectDetails.\ Git.CatFile Git/CatFile.hs:(83,88)-(91,97) 8.1 4.1 + catchMaybeIO Utility.Exception Utility/Exception.hs:53:1-63 7.6 2.2 + parseResp Git.CatFile Git/CatFile.hs:(145,1)-(156,28) 5.2 5.6 + >>=.\ Data.Attoparsec.Internal.Types Data/Attoparsec/Internal/Types.hs:(146,9)-(147,44) 5.0 2.0 + >>=.\.succ' Data.Attoparsec.Internal.Types Data/Attoparsec/Internal/Types.hs:146:13-76 4.7 1.0 + MAIN MAIN 4.2 0.4 + getState Annex Annex.hs:(251,1)-(254,27) 2.0 1.1 + getAnnexLinkTarget'.probesymlink Annex.Link Annex/Link.hs:78:9-62 1.9 2.6 + splitc Utility.Split Utility/Split.hs:(24,1)-(26,25) 1.9 5.2 + _encodeFilePath Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:(111,1)-(114,49) 1.8 2.8 + query.send Git.CatFile Git/CatFile.hs:141:9-32 1.7 0.5 + keyFile' Annex.Locations Annex/Locations.hs:(564,1)-(574,30) 1.6 5.4 + +Ditto. +"""]] diff --git a/doc/thanks/list b/doc/thanks/list index 00670e948d..3a5de35e3e 100644 --- a/doc/thanks/list +++ b/doc/thanks/list @@ -70,3 +70,4 @@ Ryan Rix, Svenne Krap, Jelmer Vernooij, Rian McGuire, +Cesar, diff --git a/doc/todo/OPT__58_____34__bundle__34___get_+_check___40__of_checksum__41___in_a_single_operation.mdwn b/doc/todo/OPT__58_____34__bundle__34___get_+_check___40__of_checksum__41___in_a_single_operation.mdwn new file mode 100644 index 0000000000..a17aa8f02d --- /dev/null +++ b/doc/todo/OPT__58_____34__bundle__34___get_+_check___40__of_checksum__41___in_a_single_operation.mdwn @@ -0,0 +1,14 @@ +In neurophysiology we encounter HUGE files (HDF5 .nwb files). +Sizes reach hundreds of GBs per file (thus exceeding any possible file system memory cache size). While operating in the cloud or on a fast connection it is possible to fetch the files with speeds up to 100 MBps. +Upon successful download such files are then loaded back by git-annex for the checksum validation, and often at slower speeds (eg <60MBps on EC2 SSD drive). +So, ironically, it does not just double, but rather nearly triples overall time to obtain a file. + +I think ideally, + +- (at minimum) for built-in special remotes (such as web), it would be great if git-annex was check-summing incrementally as data comes in; +- made it possible to for external special remotes to provide desired checksum on obtained content. First git-annex should of cause inform them on type (backend) of the checksum it is interested in, and may be have some information reported by external remotes on what checksums they support. + +If needed example, here is http://datasets.datalad.org/allen-brain-observatory/visual-coding-2p/.git with >50GB files such as ophys_movies/ophys_experiment_576261945.h5 . + +[[!meta author=yoh]] +[[!tag projects/dandi]] diff --git a/doc/todo/OPT__58_____34__bundle__34___get_+_check___40__of_checksum__41___in_a_single_operation/comment_1_29e601ea3ea4f22301c6cf6eed403ba4._comment b/doc/todo/OPT__58_____34__bundle__34___get_+_check___40__of_checksum__41___in_a_single_operation/comment_1_29e601ea3ea4f22301c6cf6eed403ba4._comment new file mode 100644 index 0000000000..7b2bf7b358 --- /dev/null +++ b/doc/todo/OPT__58_____34__bundle__34___get_+_check___40__of_checksum__41___in_a_single_operation/comment_1_29e601ea3ea4f22301c6cf6eed403ba4._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="use named pipes?" + date="2019-11-25T16:45:26Z" + content=""" +For external remotes can pass to the `TRANSFER` request, as the `FILE` parameter, a named pipe, and use `tee` to create a separate stream for checksumming. + +An external remote could also do its own checksum checking and then set remote..annex-verify=false. +Could also make a “wrapper” external remote that delegates all requests to a given external remote but does checksum-checking in parallel with downloading (by creating a named pipe and passing that to the wrapped remote). +"""]] diff --git a/doc/todo/git-annex-cat.mdwn b/doc/todo/git-annex-cat.mdwn new file mode 100644 index 0000000000..ed49cca89c --- /dev/null +++ b/doc/todo/git-annex-cat.mdwn @@ -0,0 +1,5 @@ +It would be useful to have a [[`git-annex-cat`|forum/Is_there_a___34__git_annex_cat-file__34___type_command__63__/]] command that outputs the contents of an annexed file without storing it in the annex. This [[can be faster|OPT: "bundle" get + check (of checksum) in a single operation]] than `git-annex-get` followed by `cat`, even if file is already present. It avoids some failure modes of `git-annex-get` (like running out of local space, or contending for locks). It supports a common use case of just needing a file for some operation, without needing to remember to drop it later. It could be used to implement a web server or FUSE filesystem that serves git-annex repo files on demand. + +If file is not present, or `remote.here.cost` is higher than `remote.someremote.cost` where file is present, `someremote` would get a `TRANSFER` request where the `FILE` argument is a named pipe, and a `cat` of that named pipe would be started. + +If file is not annexed, for uniformity `git-annex-cat file` would just call `cat file`. diff --git a/doc/todo/git-lfs_special_remote_simpler_setup/comment_1_4e9f8b60dd1b705d4755200dada8801c._comment b/doc/todo/git-lfs_special_remote_simpler_setup/comment_1_4e9f8b60dd1b705d4755200dada8801c._comment new file mode 100644 index 0000000000..f9af1d11c4 --- /dev/null +++ b/doc/todo/git-lfs_special_remote_simpler_setup/comment_1_4e9f8b60dd1b705d4755200dada8801c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="reference original bug report" + date="2019-11-29T17:58:28Z" + content=""" +original bug report was https://git-annex.branchable.com/bugs/git-lfs_remote_URL_is_not_recorded__63__/ for an attempt to share some NWB data on github's LFS +"""]] diff --git a/doc/todo/git_status_smudges_unncessarily_after_unlock.mdwn b/doc/todo/git_status_smudges_unncessarily_after_unlock.mdwn new file mode 100644 index 0000000000..b1b59f0869 --- /dev/null +++ b/doc/todo/git_status_smudges_unncessarily_after_unlock.mdwn @@ -0,0 +1,11 @@ +After unlocking a file, `git status` runs the smudge filter. That is +unnecessary, and when many files were unlocked, it can take a long time +because [[git_smudge_clean_interface_suboptiomal]] means it runs git-annex +once per file. + +It should be possible to avoid that, as was done with git drop in [[!commit +1113caa53efedbe7ab1d98b74010160f20473e8d]]. I tried making Command.Unlock +use restagePointerFile, but that did not help; git update-index does then +smudge it during the `git annex unlock`, which is no faster (but at least +doing it then would avoid the surprise of a slow `git status` or `git +commit -a`). Afterwards, `git status` then smudged it again, unsure why! diff --git a/doc/todo/only_pass_unlocked_files_through_the_clean__47__smudge_filter/comment_2_8ddf77de6313df0157de8d24c2dc7951._comment b/doc/todo/only_pass_unlocked_files_through_the_clean__47__smudge_filter/comment_2_8ddf77de6313df0157de8d24c2dc7951._comment new file mode 100644 index 0000000000..dd8ce6fc47 --- /dev/null +++ b/doc/todo/only_pass_unlocked_files_through_the_clean__47__smudge_filter/comment_2_8ddf77de6313df0157de8d24c2dc7951._comment @@ -0,0 +1,46 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="moving unlocked file onto locked file isn't possible" + date="2019-11-24T16:36:24Z" + content=""" +`git mv` won't move an unlocked file onto a locked file (trace below). + +\"The right solution is to improve the smudge/clean filter interface\" -- of course, but realistically, do you think git devs can be persuaded to do [[this|todo/git_smudge_clean_interface_suboptiomal]] sometime soon? Even if yes, it still seems better to avoid adding a step to common git workflows, than to make the step fast. + + +[[!format sh \"\"\" +(master_env_v164_py36) 11:14 [t1] $ ls +bar foo +(master_env_v164_py36) 11:14 [t1] $ git init +Initialized empty Git repository in /tmp/t1/.git/ +(master_env_v164_py36) 11:14 [t1] $ git annex init +init (scanning for unlocked files...) +ok +(recording state in git...) +(master_env_v164_py36) 11:14 [t1] $ git annex add foo +add foo ok +(recording state in git...) +(master_env_v164_py36) 11:14 [t1] $ git annex add bar +add bar ok +(recording state in git...) +(master_env_v164_py36) 11:14 [t1] $ ls -alt +total 0 +drwxrwxr-x 8 ilya ilya 141 Nov 24 11:14 .git +drwxrwxr-x 3 ilya ilya 40 Nov 24 11:14 . +lrwxrwxrwx 1 ilya ilya 108 Nov 24 11:14 bar -> .git/annex/objects/jx/MV/MD5E-s4--c157a79031e1c40f85931829bc5fc552/MD5E-s4--c157a79031\ +e1c40f85931829bc5fc552 +lrwxrwxrwx 1 ilya ilya 108 Nov 24 11:14 foo -> .git/annex/objects/00/zZ/MD5E-s4--d3b07384d113edec49eaa6238ad5ff00/MD5E-s4--d3b07384d1\ +13edec49eaa6238ad5ff00 +drwxrwxrwt 12 root root 282 Nov 24 11:14 .. +(master_env_v164_py36) 11:14 [t1] $ git annex unlock bar +unlock bar ok +(recording state in git...) +(master_env_v164_py36) 11:16 [t1] $ git mv bar foo +fatal: destination exists, source=bar, destination=foo +(master_env_v164_py36) 11:17 [t1] $ + + + +\"\"\"]] +"""]] diff --git a/doc/todo/only_pass_unlocked_files_through_the_clean__47__smudge_filter/comment_3_51ea1a6c7c6c46322975cf051c191887._comment b/doc/todo/only_pass_unlocked_files_through_the_clean__47__smudge_filter/comment_3_51ea1a6c7c6c46322975cf051c191887._comment new file mode 100644 index 0000000000..9f59e5140c --- /dev/null +++ b/doc/todo/only_pass_unlocked_files_through_the_clean__47__smudge_filter/comment_3_51ea1a6c7c6c46322975cf051c191887._comment @@ -0,0 +1,126 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="even git mv -f seems to work correctly" + date="2019-11-24T17:25:32Z" + content=""" +Also, `git mv` seems to reuse the already-smudged object contents of the source file for the target file, so even with `git mv -f` only the checksum gets checked into git: + +[[!format sh \"\"\" ++ cat ./test-git-mv +#!/bin/bash + +set -eu -o pipefail -x + +cat $0 + +TEST_DIR=/tmp/test_dir +mkdir -p $TEST_DIR +chmod -R u+w $TEST_DIR +rm -rf $TEST_DIR +mkdir -p $TEST_DIR +pushd $TEST_DIR + +git init +git annex init + +git --version +git annex version + +rm .git/info/attributes +echo foo > foo +echo bar > bar +git annex add foo bar +git check-attr -a foo +git check-attr -a bar +echo 'bar filter=annex' > .gitattributes +git add .gitattributes +git check-attr -a foo +git check-attr -a bar + +git annex unlock bar +git mv bar foo || true +git mv -f bar foo +git commit -m add +git log -p + + ++ TEST_DIR=/tmp/test_dir ++ mkdir -p /tmp/test_dir ++ chmod -R u+w /tmp/test_dir ++ rm -rf /tmp/test_dir ++ mkdir -p /tmp/test_dir ++ pushd /tmp/test_dir +/tmp/test_dir /tmp ++ git init +Initialized empty Git repository in /tmp/test_dir/.git/ ++ git annex init +init (scanning for unlocked files...) +ok +(recording state in git...) ++ git --version +git version 2.20.1 ++ git annex version +git-annex version: 7.20191024-g6dc2272 +build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite +dependency versions: aws-0.21.1 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.3 feed-1.0.1.0 ghc-8.6.5 http-client-0.5.14 persistent-sqlite-2.9.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs hook external +operating system: linux x86_64 +supported repository versions: 7 +upgrade supported from repository versions: 0 1 2 3 4 5 6 +local repository version: 7 ++ rm .git/info/attributes ++ echo foo ++ echo bar ++ git annex add foo bar +add foo ok +add bar ok +(recording state in git...) ++ git check-attr -a foo ++ git check-attr -a bar ++ echo 'bar filter=annex' ++ git add .gitattributes ++ git check-attr -a foo ++ git check-attr -a bar +bar: filter: annex ++ git annex unlock bar +unlock bar ok +(recording state in git...) ++ git mv bar foo +fatal: destination exists, source=bar, destination=foo ++ true ++ git mv -f bar foo ++ git commit -m add +[master (root-commit) 8610c0d] add + 2 files changed, 2 insertions(+) + create mode 100644 .gitattributes + create mode 100644 foo ++ git log -p +commit 8610c0d8f327140608e71dc229f167731552d284 +Author: Ilya Shlyakhter +Date: Sun Nov 24 12:24:28 2019 -0500 + + add + +diff --git a/.gitattributes b/.gitattributes +new file mode 100644 +index 0000000..649f07e +--- /dev/null ++++ b/.gitattributes +@@ -0,0 +1 @@ ++bar filter=annex +diff --git a/foo b/foo +new file mode 100644 +index 0000000..266ae50 +--- /dev/null ++++ b/foo +@@ -0,0 +1 @@ ++/annex/objects/MD5E-s4--c157a79031e1c40f85931829bc5fc552 + +\"\"\"]] + + + + +"""]] diff --git a/doc/todo/optimize_by_converting_String_to_ByteString.mdwn b/doc/todo/optimize_by_converting_String_to_ByteString.mdwn new file mode 100644 index 0000000000..7ac7efe382 --- /dev/null +++ b/doc/todo/optimize_by_converting_String_to_ByteString.mdwn @@ -0,0 +1,37 @@ +git-annex uses FilePath (String) extensively. That's a slow data type. +Converting to ByteString, and RawFilePath, should speed it up +significantly, according to [[/profiling]]. + +I've made a test branch, `bs`, to see what kind of performance improvement +to expect. + +Benchmarking `git-annex find`, speedups range from 28-66%. The files fly by +much more snappily. Other commands likely also speed up, but do more work +than find so the improvement is not as large. + +The `bs` branch is in a mergeable state now, but still needs work: + +* Eliminate all the fromRawFilePath, toRawFilePath, encodeBS, + decodeBS conversions. Or at least most of them. There are likely + quite a few places where a value is converted back and forth several times. + + As a first step, profile and look for the hot spots. Known hot spots: + + * keyFile uses fromRawFilePath and that adds around 3% overhead in `git-annex find`. + Converting it to a RawFilePath needs a version of `` for RawFilePaths. + * getJournalFileStale uses fromRawFilePath, and adds 3-5% overhead in + `git-annex whereis`. Converting it to RawFilePath needs a version + of `` for RawFilePaths. It also needs a ByteString.readFile + for RawFilePath. + +* System.FilePath is not available for RawFilePath, and many of the + conversions are to get a FilePath in order to use that library. + + It should be entirely straightforward to make a version of System.FilePath + that can operate on RawFilePath, except possibly there could be some + complications due to Windows. + +* Use versions of IO actions like getFileStatus that take a RawFilePath, + avoiding a conversion. Note that these are only available on unix, not + windows, so a compatability shim will be needed. + (I can't seem to find any library that provides one.) diff --git a/doc/todo/optimize_by_converting_String_to_ByteString/comment_1_403601fa8ad6946eca8f598bdc31f2d7._comment b/doc/todo/optimize_by_converting_String_to_ByteString/comment_1_403601fa8ad6946eca8f598bdc31f2d7._comment new file mode 100644 index 0000000000..0d24a70d0c --- /dev/null +++ b/doc/todo/optimize_by_converting_String_to_ByteString/comment_1_403601fa8ad6946eca8f598bdc31f2d7._comment @@ -0,0 +1,44 @@ +[[!comment format=mdwn + username="joey" + subject="""profiling""" + date="2019-11-26T20:05:28Z" + content=""" +Profiling the early version of the `bs` branch. + + Tue Nov 26 16:05 2019 Time and Allocation Profiling Report (Final) + + git-annex +RTS -p -RTS find + + total time = 2.75 secs (2749 ticks @ 1000 us, 1 processor) + total alloc = 1,642,607,120 bytes (excludes profiling overheads) + + COST CENTRE MODULE SRC %time %alloc + + inAnnex'.\ Annex.Content Annex/Content.hs:(103,61)-(118,31) 31.2 46.8 + keyFile' Annex.Locations Annex/Locations.hs:(567,1)-(577,30) 5.3 6.2 + encodeW8 Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:(189,1)-(191,70) 3.3 4.2 + >>=.\ Data.Attoparsec.Internal.Types Data/Attoparsec/Internal/Types.hs:(146,9)-(147,44) 2.9 0.8 + >>=.\.succ' Data.Attoparsec.Internal.Types Data/Attoparsec/Internal/Types.hs:146:13-76 2.6 0.3 + keyFile'.esc Annex.Locations Annex/Locations.hs:(573,9)-(577,30) 2.5 5.5 + parseLinkTarget Annex.Link Annex/Link.hs:(254,1)-(262,25) 2.4 4.4 + getAnnexLinkTarget'.probesymlink Annex.Link Annex/Link.hs:78:9-46 2.4 2.8 + w82s Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:217:1-15 2.3 6.0 + keyPath Annex.Locations Annex/Locations.hs:(606,1)-(608,23) 1.9 4.0 + parseKeyVariety Types.Key Types/Key.hs:(323,1)-(371,42) 1.8 0.0 + getState Annex Annex.hs:(251,1)-(254,27) 1.7 0.4 + fileKey'.go Annex.Locations Annex/Locations.hs:588:9-55 1.4 0.8 + fileKey' Annex.Locations Annex/Locations.hs:(586,1)-(596,41) 1.4 1.7 + hashUpdates.\.\.\ Crypto.Hash Crypto/Hash.hs:85:48-99 1.3 0.0 + parseLinkTargetOrPointer Annex.Link Annex/Link.hs:(239,1)-(243,25) 1.2 0.1 + withPtr Basement.Block.Base Basement/Block/Base.hs:(395,1)-(404,31) 1.2 0.6 + primitive Basement.Monad Basement/Monad.hs:72:5-18 1.0 0.1 + decodeBS' Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:151:1-31 1.0 2.8 + mkKeySerialization Types.Key Types/Key.hs:(115,1)-(117,22) 0.7 1.1 + w82c Utility.FileSystemEncoding Utility/FileSystemEncoding.hs:211:1-28 0.6 1.1 + +Comparing with [[/profiling]] results, the alloc is down significantly. +And the main IO actions are getting a larger share of the runtime. + +There is still significantly conversion going on, encodeW8 and w82s and +decodeBS' and w82c. Likely another 5% or so speedup if that's eliminated. +"""]] diff --git a/doc/todo/optimize_by_converting_String_to_ByteString/comment_2_9c51e1986aeb16b3138b6824be9f5a58._comment b/doc/todo/optimize_by_converting_String_to_ByteString/comment_2_9c51e1986aeb16b3138b6824be9f5a58._comment new file mode 100644 index 0000000000..d3f545d556 --- /dev/null +++ b/doc/todo/optimize_by_converting_String_to_ByteString/comment_2_9c51e1986aeb16b3138b6824be9f5a58._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="representing paths" + date="2019-11-27T15:08:40Z" + content=""" +Thanks for working on this Joey. + +I don't know Haskell or git-annex architecture, so my thoughts might make no sense, but I'll post just in case. + +\"There are likely quite a few places where a value is converted back and forth several times\" -- as a quick/temp fix, could memoization speed this up? Or memoizing the results of some system calls? + +The many filenames flying around often share long prefixes. Could that be used to speed things up? E.g. if they could be represented as pointers into some compact storage, maybe cache performance would improve. + +\"git annex find... files fly by much more snappily\" -- does this mean `git-annex-find` is testing each file individually, as opposed to constructing a SQL query to an indexed db? Maybe, simpler `git-annex-find` queries that are fully mappable to SQL queries could be special-cased? + +Sorry for naive comments, I'll eventually read up on Haskell and make more sense... + +"""]] diff --git a/doc/todo/parallel_possibilities/comment_1_bad182de605b7b47d66dcfe583acd4f1._comment b/doc/todo/parallel_possibilities/comment_1_bad182de605b7b47d66dcfe583acd4f1._comment new file mode 100644 index 0000000000..36a15aef89 --- /dev/null +++ b/doc/todo/parallel_possibilities/comment_1_bad182de605b7b47d66dcfe583acd4f1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="parallelization" + date="2019-11-27T17:23:14Z" + content=""" +When operating on many files, maybe run N parallel commands where i'th command ignores paths for which `(hash(filename) module N) != i`. Or, if git index has size I, i'th command ignores paths that are not legixographically between `index[(I/N)*i]` and `index[(I/N)*(i+1)]` (for index state at command start). Extending [[git-annex-matching-options]] with `--block=i` would let this be done using `xargs`. +"""]] diff --git a/git-annex.cabal b/git-annex.cabal index 01527c1c66..fc21f36f4f 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1021,6 +1021,7 @@ Executable git-annex Utility.Aeson Utility.Android Utility.Applicative + Utility.Attoparsec Utility.AuthToken Utility.Base64 Utility.Batch diff --git a/templates/dashboard/transfers.hamlet b/templates/dashboard/transfers.hamlet index ee8ddb8ab9..c1b3e15eef 100644 --- a/templates/dashboard/transfers.hamlet +++ b/templates/dashboard/transfers.hamlet @@ -15,7 +15,7 @@ #{maybe "unknown" Remote.name $ transferRemote info} - $with size <- maybe "unknown" (roughSize dataUnits True) $ keySize $ transferKey transfer + $with size <- maybe "unknown" (roughSize dataUnits True) $ keySize $ transferKeyData transfer $if isJust $ startedTime info $if isrunning info #{percent} of #{size}