diff --git a/Annex/Branch.hs b/Annex/Branch.hs index 9663311d51..c90958ab05 100644 --- a/Annex/Branch.hs +++ b/Annex/Branch.hs @@ -61,6 +61,7 @@ import qualified Annex.Queue import Annex.Branch.Transitions import qualified Annex import Annex.Hook +import Utility.FileSystemEncoding {- Name of the branch that is used to store git-annex's information. -} name :: Git.Ref @@ -436,7 +437,6 @@ stageJournal jl = withIndex $ do g <- gitRepo let dir = gitAnnexJournalDir g (jlogf, jlogh) <- openjlog - liftIO $ fileEncoding jlogh h <- hashObjectHandle withJournalHandle $ \jh -> Git.UpdateIndex.streamUpdateIndex g diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs index b1d8fba287..25952dfecd 100644 --- a/Annex/CatFile.hs +++ b/Annex/CatFile.hs @@ -33,6 +33,7 @@ import Git.FilePath import Git.Index import qualified Git.Ref import Annex.Link +import Utility.FileSystemEncoding catFile :: Git.Branch -> FilePath -> Annex L.ByteString catFile branch file = do diff --git a/Annex/Content/Direct.hs b/Annex/Content/Direct.hs index 2007360e31..734a0c1b94 100644 --- a/Annex/Content/Direct.hs +++ b/Annex/Content/Direct.hs @@ -52,8 +52,7 @@ associatedFiles key = do associatedFilesRelative :: Key -> Annex [FilePath] associatedFilesRelative key = do mapping <- calcRepo $ gitAnnexMapping key - liftIO $ catchDefaultIO [] $ withFile mapping ReadMode $ \h -> do - fileEncoding h + liftIO $ catchDefaultIO [] $ withFile mapping ReadMode $ \h -> -- Read strictly to ensure the file is closed -- before changeAssociatedFiles tries to write to it. -- (Especially needed on Windows.) @@ -68,8 +67,7 @@ changeAssociatedFiles key transform = do let files' = transform files when (files /= files') $ modifyContent mapping $ - liftIO $ viaTmp writeFileAnyEncoding mapping $ - unlines files' + liftIO $ viaTmp writeFile mapping $ unlines files' top <- fromRepo Git.repoPath return $ map (top ) files' diff --git a/Annex/DirHashes.hs b/Annex/DirHashes.hs index 004536ca79..ed20cfb8a7 100644 --- a/Annex/DirHashes.hs +++ b/Annex/DirHashes.hs @@ -26,6 +26,7 @@ import Common import Types.Key import Types.GitConfig import Types.Difference +import Utility.FileSystemEncoding type Hasher = Key -> FilePath diff --git a/Annex/Journal.hs b/Annex/Journal.hs index e4faa48653..184bb0ab04 100644 --- a/Annex/Journal.hs +++ b/Annex/Journal.hs @@ -37,7 +37,6 @@ setJournalFile _jl file content = do let tmpfile = tmp takeFileName jfile liftIO $ do withFile tmpfile WriteMode $ \h -> do - fileEncoding h #ifdef mingw32_HOST_OS hSetNewlineMode h noNewlineTranslation #endif @@ -53,7 +52,7 @@ getJournalFile _jl = getJournalFileStale - changes. -} getJournalFileStale :: FilePath -> Annex (Maybe String) getJournalFileStale file = inRepo $ \g -> catchMaybeIO $ - readFileStrictAnyEncoding $ journalFile file g + readFileStrict $ journalFile file g {- List of files that have updated content in the journal. -} getJournalledFiles :: JournalLocked -> Annex [FilePath] diff --git a/Annex/Link.hs b/Annex/Link.hs index 90312a04a0..fcc300beec 100644 --- a/Annex/Link.hs +++ b/Annex/Link.hs @@ -24,6 +24,7 @@ import Git.Types import Git.FilePath import Annex.HashObject import Utility.FileMode +import Utility.FileSystemEncoding import qualified Data.ByteString.Lazy as L @@ -63,7 +64,6 @@ getAnnexLinkTarget' file coresymlinks = if coresymlinks Nothing -> fallback probefilecontent f = withFile f ReadMode $ \h -> do - fileEncoding h -- The first 8k is more than enough to read; link -- files are small. s <- take 8192 <$> hGetContents h diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs index 4f879436b5..512f0375c2 100644 --- a/Annex/Ssh.hs +++ b/Annex/Ssh.hs @@ -33,6 +33,7 @@ import qualified Git.Url import Config import Annex.Path import Utility.Env +import Utility.FileSystemEncoding import Types.CleanupActions import Git.Env #ifndef mingw32_HOST_OS diff --git a/Annex/VariantFile.hs b/Annex/VariantFile.hs index 9bf027b5ce..17658a9c61 100644 --- a/Annex/VariantFile.hs +++ b/Annex/VariantFile.hs @@ -8,6 +8,7 @@ module Annex.VariantFile where import Annex.Common +import Utility.FileSystemEncoding import Data.Hash.MD5 diff --git a/Assistant/TransferrerPool.hs b/Assistant/TransferrerPool.hs index 7c0cb44151..892e156e8b 100644 --- a/Assistant/TransferrerPool.hs +++ b/Assistant/TransferrerPool.hs @@ -74,8 +74,6 @@ mkTransferrer program batchmaker = do , std_in = CreatePipe , std_out = CreatePipe } - fileEncoding readh - fileEncoding writeh return $ Transferrer { transferrerRead = readh , transferrerWrite = writeh diff --git a/Assistant/WebApp/Control.hs b/Assistant/WebApp/Control.hs index 93b923fed9..8a0e82119d 100644 --- a/Assistant/WebApp/Control.hs +++ b/Assistant/WebApp/Control.hs @@ -74,5 +74,5 @@ getLogR :: Handler Html getLogR = page "Logs" Nothing $ do logfile <- liftAnnex $ fromRepo gitAnnexLogFile logs <- liftIO $ listLogs logfile - logcontent <- liftIO $ concat <$> mapM readFileStrictAnyEncoding logs + logcontent <- liftIO $ concat <$> mapM readFile logs $(widgetFile "control/log") diff --git a/Backend/Utilities.hs b/Backend/Utilities.hs index 04221650b4..d1fb94f2af 100644 --- a/Backend/Utilities.hs +++ b/Backend/Utilities.hs @@ -10,6 +10,7 @@ module Backend.Utilities where import Data.Hash.MD5 import Annex.Common +import Utility.FileSystemEncoding {- Generates a keyName from an input string. Takes care of sanitizing it. - If it's not too long, the full string is used as the keyName. diff --git a/Build/DistributionUpdate.hs b/Build/DistributionUpdate.hs index 814927e995..dd18a78837 100644 --- a/Build/DistributionUpdate.hs +++ b/Build/DistributionUpdate.hs @@ -14,6 +14,7 @@ import Build.Version (getChangelogVersion, Version) import Utility.UserInfo import Utility.Url import Utility.Tmp +import Utility.FileSystemEncoding import qualified Git.Construct import qualified Annex import Annex.Content @@ -50,6 +51,7 @@ autobuilds = main :: IO () main = do + useFileSystemEncoding version <- liftIO getChangelogVersion repodir <- getRepoDir changeWorkingDirectory repodir diff --git a/Build/EvilSplicer.hs b/Build/EvilSplicer.hs index ca690c2501..32d9a1c9fb 100644 --- a/Build/EvilSplicer.hs +++ b/Build/EvilSplicer.hs @@ -210,7 +210,6 @@ applySplices destdir imports splices@(first:_) = do when (oldcontent /= Just newcontent) $ do putStrLn $ "splicing " ++ f withFile dest WriteMode $ \h -> do - fileEncoding h hPutStr h newcontent hClose h where @@ -721,7 +720,9 @@ parsecAndReplace p s = case parse find "" s of find = many $ try (Right <$> p) <|> (Left <$> anyChar) main :: IO () -main = go =<< getArgs +main = do + useFileSystemEncoding + go =<< getArgs where go (destdir:log:header:[]) = run destdir log (Just header) go (destdir:log:[]) = run destdir log Nothing diff --git a/CHANGELOG b/CHANGELOG index 2d7ea22a70..7a0ca2eb20 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -24,6 +24,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium * enable-tor: No longer needs to be run as root. * enable-tor: When run as a regular user, test a connection back to the hidden service over tor. + * Always use filesystem encoding for all file and handle reads and + writes. * Fix build with directory-1.3. * Debian: Suggest tor and magic-wormhole. * Debian: Build webapp on armel. diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs index 6ef21372f5..82038314c0 100644 --- a/CmdLine/Batch.hs +++ b/CmdLine/Batch.hs @@ -57,9 +57,7 @@ batchInput parser a = go =<< batchLines parseerr s = giveup $ "Batch input parse failure: " ++ s batchLines :: Annex [String] -batchLines = liftIO $ do - fileEncoding stdin - lines <$> getContents +batchLines = liftIO $ lines <$> getContents -- Runs a CommandStart in batch mode. -- diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs index e49d2727c7..8cc1484406 100644 --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -27,6 +27,7 @@ import Types.UrlContents import Annex.FileMatcher import Logs.Location import Utility.Metered +import Utility.FileSystemEncoding import qualified Annex.Transfer as Transfer import Annex.Quvi import qualified Utility.Quvi as Quvi diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs index 832ec1246f..ea936e84a5 100644 --- a/Command/ImportFeed.hs +++ b/Command/ImportFeed.hs @@ -156,7 +156,7 @@ downloadFeed url liftIO $ withTmpFile "feed" $ \f h -> do hClose h ifM (Url.download url f uo) - ( parseFeedString <$> readFileStrictAnyEncoding f + ( parseFeedString <$> readFileStrict f , return Nothing ) diff --git a/Command/P2P.hs b/Command/P2P.hs index afa5f9dc68..4ba3e43d53 100644 --- a/Command/P2P.hs +++ b/Command/P2P.hs @@ -161,7 +161,6 @@ performPairing remotename addrs = do getcode ourcode = do putStr "Enter the other repository's pairing code: " hFlush stdout - fileEncoding stdin l <- getLine case Wormhole.toCode l of Just code @@ -236,7 +235,7 @@ wormholePairing remotename ouraddrs ui = do then return ReceiveFailed else do r <- liftIO $ tryIO $ - readFileStrictAnyEncoding recvf + readFileStrict recvf case r of Left _e -> return ReceiveFailed Right s -> maybe diff --git a/Command/TransferKeys.hs b/Command/TransferKeys.hs index 2ac7845890..d875f496d0 100644 --- a/Command/TransferKeys.hs +++ b/Command/TransferKeys.hs @@ -56,10 +56,7 @@ runRequests -> (TransferRequest -> Annex Bool) -> Annex () runRequests readh writeh a = do - liftIO $ do - hSetBuffering readh NoBuffering - fileEncoding readh - fileEncoding writeh + liftIO $ hSetBuffering readh NoBuffering go =<< readrequests where go (d:rn:k:f:rest) = do diff --git a/Command/Vicfg.hs b/Command/Vicfg.hs index 64daa598b7..d9e8b88232 100644 --- a/Command/Vicfg.hs +++ b/Command/Vicfg.hs @@ -41,7 +41,7 @@ start = do createAnnexDirectory $ parentDir f cfg <- getCfg descs <- uuidDescriptions - liftIO $ writeFileAnyEncoding f $ genCfg cfg descs + liftIO $ writeFile f $ genCfg cfg descs vicfg cfg f stop @@ -51,11 +51,11 @@ vicfg curcfg f = do -- Allow EDITOR to be processed by the shell, so it can contain options. unlessM (liftIO $ boolSystem "sh" [Param "-c", Param $ unwords [vi, shellEscape f]]) $ giveup $ vi ++ " exited nonzero; aborting" - r <- parseCfg (defCfg curcfg) <$> liftIO (readFileStrictAnyEncoding f) + r <- parseCfg (defCfg curcfg) <$> liftIO (readFileStrict f) liftIO $ nukeFile f case r of Left s -> do - liftIO $ writeFileAnyEncoding f s + liftIO $ writeFile f s vicfg curcfg f Right newcfg -> setCfg curcfg newcfg diff --git a/Common.hs b/Common.hs index 5213863b9f..2e28117b6c 100644 --- a/Common.hs +++ b/Common.hs @@ -29,7 +29,6 @@ import Utility.Directory as X import Utility.Monad as X import Utility.Data as X import Utility.Applicative as X -import Utility.FileSystemEncoding as X import Utility.PosixFiles as X hiding (fileSize) import Utility.FileSize as X import Utility.Network as X diff --git a/Config.hs b/Config.hs index be60852daa..84736cac35 100644 --- a/Config.hs +++ b/Config.hs @@ -112,7 +112,7 @@ configureSmudgeFilter = do createDirectoryIfMissing True (takeDirectory lf) writeFile lf (lfs ++ "\n" ++ stdattr) where - readattr = liftIO . catchDefaultIO "" . readFileStrictAnyEncoding + readattr = liftIO . catchDefaultIO "" . readFileStrict stdattr = unlines [ "* filter=annex" , ".* !filter" diff --git a/Database/Handle.hs b/Database/Handle.hs index 748feaa976..9071cd5380 100644 --- a/Database/Handle.hs +++ b/Database/Handle.hs @@ -69,7 +69,7 @@ openDb db tablename = do worker <- async (workerThread (T.pack db) tablename jobs) -- work around https://github.com/yesodweb/persistent/issues/474 - liftIO setConsoleEncoding + liftIO useFileSystemEncoding return $ DbHandle worker jobs diff --git a/Git/CatFile.hs b/Git/CatFile.hs index 061349f054..4935cdffa0 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -37,6 +37,7 @@ import Git.Command import Git.Types import Git.FilePath import qualified Utility.CoProcess as CoProcess +import Utility.FileSystemEncoding data CatFileHandle = CatFileHandle { catFileProcess :: CoProcess.CoProcessHandle diff --git a/Git/Command.hs b/Git/Command.hs index 2060563688..adea7622e0 100644 --- a/Git/Command.hs +++ b/Git/Command.hs @@ -53,7 +53,6 @@ runQuiet params repo = withQuietOutput createProcessSuccess $ pipeReadLazy :: [CommandParam] -> Repo -> IO (String, IO Bool) pipeReadLazy params repo = assertLocal repo $ do (_, Just h, _, pid) <- createProcess p { std_out = CreatePipe } - fileEncoding h c <- hGetContents h return (c, checkSuccessProcess pid) where @@ -66,7 +65,6 @@ pipeReadLazy params repo = assertLocal repo $ do pipeReadStrict :: [CommandParam] -> Repo -> IO String pipeReadStrict params repo = assertLocal repo $ withHandle StdoutHandle (createProcessChecked ignoreFailureProcess) p $ \h -> do - fileEncoding h output <- hGetContentsStrict h hClose h return output @@ -81,9 +79,7 @@ pipeWriteRead params writer repo = assertLocal repo $ writeReadProcessEnv "git" (toCommand $ gitCommandLine params repo) (gitEnv repo) writer (Just adjusthandle) where - adjusthandle h = do - fileEncoding h - hSetNewlineMode h noNewlineTranslation + adjusthandle h = hSetNewlineMode h noNewlineTranslation {- Runs a git command, feeding it input on a handle with an action. -} pipeWrite :: [CommandParam] -> Repo -> (Handle -> IO ()) -> IO () diff --git a/Git/Config.hs b/Git/Config.hs index 3d6239560f..65bd9b7ba3 100644 --- a/Git/Config.hs +++ b/Git/Config.hs @@ -79,10 +79,6 @@ global = do {- Reads git config from a handle and populates a repo with it. -} hRead :: Repo -> Handle -> IO Repo hRead repo h = do - -- We use the FileSystemEncoding when reading from git-config, - -- because it can contain arbitrary filepaths (and other strings) - -- in any encoding. - fileEncoding h val <- hGetContentsStrict h store val repo @@ -167,7 +163,6 @@ coreBare = "core.bare" fromPipe :: Repo -> String -> [CommandParam] -> IO (Either SomeException (Repo, String)) fromPipe r cmd params = try $ withHandle StdoutHandle createProcessSuccess p $ \h -> do - fileEncoding h val <- hGetContentsStrict h r' <- store val r return (r', val) diff --git a/Git/HashObject.hs b/Git/HashObject.hs index 4cd54ef54c..399e36d460 100644 --- a/Git/HashObject.hs +++ b/Git/HashObject.hs @@ -41,7 +41,6 @@ hashFile h file = CoProcess.query h send receive - interface does not allow batch hashing without using temp files. -} hashBlob :: HashObjectHandle -> String -> IO Sha hashBlob h s = withTmpFile "hash" $ \tmp tmph -> do - fileEncoding tmph #ifdef mingw32_HOST_OS hSetNewlineMode tmph noNewlineTranslation #endif diff --git a/Git/Queue.hs b/Git/Queue.hs index 0b0025b0a7..ee1f83ca9c 100644 --- a/Git/Queue.hs +++ b/Git/Queue.hs @@ -159,7 +159,6 @@ runAction repo action@(CommandAction {}) = do #ifndef mingw32_HOST_OS let p = (proc "xargs" $ "-0":"git":toCommand gitparams) { env = gitEnv repo } withHandle StdinHandle createProcessSuccess p $ \h -> do - fileEncoding h hPutStr h $ intercalate "\0" $ toCommand $ getFiles action hClose h #else diff --git a/Git/Repair.hs b/Git/Repair.hs index fcfc03600c..1baf51a644 100644 --- a/Git/Repair.hs +++ b/Git/Repair.hs @@ -614,4 +614,4 @@ successfulRepair = fst safeReadFile :: FilePath -> IO String safeReadFile f = do allowRead f - readFileStrictAnyEncoding f + readFileStrict f diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs index 9ae8295ae2..c6157a9ed0 100644 --- a/Git/UnionMerge.hs +++ b/Git/UnionMerge.hs @@ -22,6 +22,7 @@ import Git.UpdateIndex import Git.HashObject import Git.Types import Git.FilePath +import Utility.FileSystemEncoding {- Performs a union merge between two branches, staging it in the index. - Any previously staged changes in the index will be lost. @@ -94,8 +95,7 @@ mergeFile info file hashhandle h = case filter (/= nullSha) [Ref asha, Ref bsha] -- We don't know how the file is encoded, but need to -- split it into lines to union merge. Using the -- FileSystemEncoding for this is a hack, but ensures there - -- are no decoding errors. Note that this works because - -- hashObject sets fileEncoding on its write handle. + -- are no decoding errors. getcontents s = lines . encodeW8NUL . L.unpack <$> catObject h s {- Calculates a union merge between a list of refs, with contents. diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs index 55c5b3bb21..7fdc9450f4 100644 --- a/Git/UpdateIndex.hs +++ b/Git/UpdateIndex.hs @@ -55,7 +55,6 @@ startUpdateIndex :: Repo -> IO UpdateIndexHandle startUpdateIndex repo = do (Just h, _, _, p) <- createProcess (gitCreateProcess params repo) { std_in = CreatePipe } - fileEncoding h return $ UpdateIndexHandle p h where params = map Param ["update-index", "-z", "--index-info"] diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 65a4e3796d..28f7b0a263 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -220,8 +220,7 @@ parseTransferFile file bits = splitDirectories file writeTransferInfoFile :: TransferInfo -> FilePath -> IO () -writeTransferInfoFile info tfile = writeFileAnyEncoding tfile $ - writeTransferInfo info +writeTransferInfoFile info tfile = writeFile tfile $ writeTransferInfo info {- File format is a header line containing the startedTime and any - bytesComplete value. Followed by a newline and the associatedFile. @@ -243,7 +242,7 @@ writeTransferInfo info = unlines readTransferInfoFile :: Maybe PID -> FilePath -> IO (Maybe TransferInfo) readTransferInfoFile mpid tfile = catchDefaultIO Nothing $ - readTransferInfo mpid <$> readFileStrictAnyEncoding tfile + readTransferInfo mpid <$> readFileStrict tfile readTransferInfo :: Maybe PID -> String -> Maybe TransferInfo readTransferInfo mpid s = TransferInfo diff --git a/Logs/Unused.hs b/Logs/Unused.hs index 1035d1246c..2361fedbcf 100644 --- a/Logs/Unused.hs +++ b/Logs/Unused.hs @@ -66,7 +66,7 @@ updateUnusedLog prefix m = do writeUnusedLog :: FilePath -> UnusedLog -> Annex () writeUnusedLog prefix l = do logfile <- fromRepo $ gitAnnexUnusedLog prefix - liftIO $ viaTmp writeFileAnyEncoding logfile $ unlines $ map format $ M.toList l + liftIO $ viaTmp writeFile logfile $ unlines $ map format $ M.toList l where format (k, (i, Just t)) = show i ++ " " ++ key2file k ++ " " ++ show t format (k, (i, Nothing)) = show i ++ " " ++ key2file k @@ -76,7 +76,7 @@ readUnusedLog prefix = do f <- fromRepo $ gitAnnexUnusedLog prefix ifM (liftIO $ doesFileExist f) ( M.fromList . mapMaybe parse . lines - <$> liftIO (readFileStrictAnyEncoding f) + <$> liftIO (readFileStrict f) , return M.empty ) where diff --git a/Messages.hs b/Messages.hs index 0ab1f72bbb..0036e57596 100644 --- a/Messages.hs +++ b/Messages.hs @@ -183,7 +183,6 @@ setupConsole = do <$> streamHandler stderr DEBUG <*> pure preciseLogFormatter updateGlobalLogger rootLoggerName (setLevel NOTICE . setHandlers [s]) - setConsoleEncoding {- Force output to be line buffered. This is normally the case when - it's connected to a terminal, but may not be when redirected to - a file or a pipe. -} diff --git a/P2P/IO.hs b/P2P/IO.hs index ee1724d7b3..9ebb102f19 100644 --- a/P2P/IO.hs +++ b/P2P/IO.hs @@ -99,7 +99,6 @@ setupHandle s = do h <- socketToHandle s ReadWriteMode hSetBuffering h LineBuffering hSetBinaryMode h False - fileEncoding h return h -- Purposefully incomplete interpreter of Proto. diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs index 899c57e3eb..0ec78aa642 100644 --- a/Remote/BitTorrent.hs +++ b/Remote/BitTorrent.hs @@ -21,6 +21,7 @@ import Types.CleanupActions import Messages.Progress import Utility.Metered import Utility.Tmp +import Utility.FileSystemEncoding import Backend.URL import Annex.Perms import Annex.UUID diff --git a/Remote/External.hs b/Remote/External.hs index 0b0e1dc18b..7091a657c8 100644 --- a/Remote/External.hs +++ b/Remote/External.hs @@ -384,9 +384,6 @@ startExternal external = do p <- propgit g basep (Just hin, Just hout, Just herr, ph) <- createProcess p `catchIO` runerr - fileEncoding hin - fileEncoding hout - fileEncoding herr stderrelay <- async $ errrelayer herr checkearlytermination =<< getProcessExitCode ph cv <- newTVarIO $ externalDefaultConfig external diff --git a/Test.hs b/Test.hs index 3f67277211..0ab7bf1309 100644 --- a/Test.hs +++ b/Test.hs @@ -95,6 +95,7 @@ import qualified Utility.HumanTime import qualified Utility.ThreadScheduler import qualified Utility.Base64 import qualified Utility.Tmp +import qualified Utility.FileSystemEncoding import qualified Command.Uninit import qualified CmdLine.GitAnnex as GitAnnex #ifndef mingw32_HOST_OS @@ -1675,7 +1676,8 @@ test_add_subdirs = intmpclonerepo $ do - calculated correctly for files in subdirs. -} unlessM (unlockedFiles <$> getTestMode) $ do git_annex "sync" [] @? "sync failed" - l <- annexeval $ decodeBS <$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo") + l <- annexeval $ Utility.FileSystemEncoding.decodeBS + <$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo") "../.git/annex/" `isPrefixOf` l @? ("symlink from subdir to .git/annex is wrong: " ++ l) createDirectory "dir2" diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs index 94d5ac3bc4..2bae40fbae 100644 --- a/Utility/CoProcess.hs +++ b/Utility/CoProcess.hs @@ -47,10 +47,10 @@ start' s = do rawMode to return $ CoProcessState pid to from s where - rawMode h = do - fileEncoding h #ifdef mingw32_HOST_OS - hSetNewlineMode h noNewlineTranslation + rawMode h = hSetNewlineMode h noNewlineTranslation +#else + rawMode _ = return () #endif stop :: CoProcessHandle -> IO () diff --git a/Utility/ExternalSHA.hs b/Utility/ExternalSHA.hs index e581697aef..7b08820040 100644 --- a/Utility/ExternalSHA.hs +++ b/Utility/ExternalSHA.hs @@ -14,7 +14,6 @@ module Utility.ExternalSHA (externalSHA) where import Utility.SafeCommand import Utility.Process -import Utility.FileSystemEncoding import Utility.Misc import Utility.Exception @@ -30,7 +29,6 @@ externalSHA command shasize file = do Left _ -> Left (command ++ " failed") where readsha args = withHandle StdoutHandle createProcessSuccess p $ \h -> do - fileEncoding h output <- hGetContentsStrict h hClose h return output diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs index eab98337a8..be43ace95e 100644 --- a/Utility/FileSystemEncoding.hs +++ b/Utility/FileSystemEncoding.hs @@ -1,6 +1,6 @@ {- GHC File system encoding handling. - - - Copyright 2012-2014 Joey Hess + - Copyright 2012-2016 Joey Hess - - License: BSD-2-clause -} @@ -9,7 +9,7 @@ {-# OPTIONS_GHC -fno-warn-tabs #-} module Utility.FileSystemEncoding ( - fileEncoding, + useFileSystemEncoding, withFilePath, md5FilePath, decodeBS, @@ -19,7 +19,6 @@ module Utility.FileSystemEncoding ( encodeW8NUL, decodeW8NUL, truncateFilePath, - setConsoleEncoding, ) where import qualified GHC.Foreign as GHC @@ -39,19 +38,30 @@ import qualified Data.ByteString.Lazy.UTF8 as L8 import Utility.Exception -{- Sets a Handle to use the filesystem encoding. This causes data - - written or read from it to be encoded/decoded the same - - as ghc 7.4 does to filenames etc. This special encoding - - allows "arbitrary undecodable bytes to be round-tripped through it". +{- Makes all subsequent Handles that are opened, as well as stdio Handles, + - use the filesystem encoding, instead of the encoding of the current + - locale. + - + - The filesystem encoding allows "arbitrary undecodable bytes to be + - round-tripped through it". This avoids encoded failures when data is not + - encoded matching the current locale. + - + - Note that code can still use hSetEncoding to change the encoding of a + - Handle. This only affects the default encoding. -} -fileEncoding :: Handle -> IO () +useFileSystemEncoding :: IO () +useFileSystemEncoding = do #ifndef mingw32_HOST_OS -fileEncoding h = hSetEncoding h =<< Encoding.getFileSystemEncoding + e <- Encoding.getFileSystemEncoding #else -{- The file system encoding does not work well on Windows, - - and Windows only has utf FilePaths anyway. -} -fileEncoding h = hSetEncoding h Encoding.utf8 + {- The file system encoding does not work well on Windows, + - and Windows only has utf FilePaths anyway. -} + let e = Encoding.utf8 #endif + hSetEncoding stdin e + hSetEncoding stdout e + hSetEncoding stderr e + Encoding.setLocaleEncoding e {- Marshal a Haskell FilePath into a NUL terminated C string using temporary - storage. The FilePath is encoded using the filesystem encoding, @@ -165,10 +175,3 @@ truncateFilePath n = reverse . go [] n . L8.fromString else go (c:coll) (cnt - x') (L8.drop 1 bs) _ -> coll #endif - -{- This avoids ghc's output layer crashing on invalid encoded characters in - - filenames when printing them out. -} -setConsoleEncoding :: IO () -setConsoleEncoding = do - fileEncoding stdout - fileEncoding stderr diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs index 433b7c6799..27d34b5925 100644 --- a/Utility/Lsof.hs +++ b/Utility/Lsof.hs @@ -47,9 +47,8 @@ queryDir path = query ["+d", path] -} query :: [String] -> IO [(FilePath, LsofOpenMode, ProcessInfo)] query opts = - withHandle StdoutHandle (createProcessChecked checkSuccessProcess) p $ \h -> do - fileEncoding h - parse <$> hGetContentsStrict h + withHandle StdoutHandle (createProcessChecked checkSuccessProcess) p $ + parse <$$> hGetContentsStrict where p = proc "lsof" ("-F0can" : opts) diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs index 9a99cba33f..e217dcdca6 100644 --- a/Utility/MagicWormhole.hs +++ b/Utility/MagicWormhole.hs @@ -27,7 +27,6 @@ import Utility.Process import Utility.SafeCommand import Utility.Monad import Utility.Misc -import Utility.FileSystemEncoding import Utility.Env import Utility.Path @@ -105,8 +104,7 @@ sendFile f (CodeObserver observer) ps = do -- Work around stupid stdout buffering behavior of python. -- See https://github.com/warner/magic-wormhole/issues/108 environ <- addEntry "PYTHONUNBUFFERED" "1" <$> getEnvironment - runWormHoleProcess p { env = Just environ} $ \_hin hout -> do - fileEncoding hout + runWormHoleProcess p { env = Just environ} $ \_hin hout -> findcode =<< words <$> hGetContents hout where p = wormHoleProcess (Param "send" : ps ++ [File f]) diff --git a/Utility/Misc.hs b/Utility/Misc.hs index ebb42576b2..4498c0a03e 100644 --- a/Utility/Misc.hs +++ b/Utility/Misc.hs @@ -10,9 +10,6 @@ module Utility.Misc where -import Utility.FileSystemEncoding -import Utility.Monad - import System.IO import Control.Monad import Foreign @@ -35,20 +32,6 @@ hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s readFileStrict :: FilePath -> IO String readFileStrict = readFile >=> \s -> length s `seq` return s -{- Reads a file strictly, and using the FileSystemEncoding, so it will - - never crash on a badly encoded file. -} -readFileStrictAnyEncoding :: FilePath -> IO String -readFileStrictAnyEncoding f = withFile f ReadMode $ \h -> do - fileEncoding h - hClose h `after` hGetContentsStrict h - -{- Writes a file, using the FileSystemEncoding so it will never crash - - on a badly encoded content string. -} -writeFileAnyEncoding :: FilePath -> String -> IO () -writeFileAnyEncoding f content = withFile f WriteMode $ \h -> do - fileEncoding h - hPutStr h content - {- Like break, but the item matching the condition is not included - in the second result list. - diff --git a/Utility/Quvi.hs b/Utility/Quvi.hs index 417ab7041c..d33d79bb8e 100644 --- a/Utility/Quvi.hs +++ b/Utility/Quvi.hs @@ -153,11 +153,8 @@ httponly :: QuviParams httponly Quvi04 = [Param "-c", Param "http"] httponly _ = [] -- No way to do it with 0.9? -{- Both versions of quvi will output utf-8 encoded data even when - - the locale doesn't support it. -} readQuvi :: [String] -> IO String readQuvi ps = withHandle StdoutHandle createProcessSuccess p $ \h -> do - fileEncoding h r <- hGetContentsStrict h hClose h return r diff --git a/Utility/Shell.hs b/Utility/Shell.hs index 860ee11dda..7adb651289 100644 --- a/Utility/Shell.hs +++ b/Utility/Shell.hs @@ -48,9 +48,8 @@ findShellCommand f = do #ifndef mingw32_HOST_OS defcmd #else - l <- catchDefaultIO Nothing $ withFile f ReadMode $ \h -> do - fileEncoding h - headMaybe . lines <$> hGetContents h + l <- catchDefaultIO Nothing $ withFile f ReadMode $ + headMaybe . lines <$$> hGetContents h case l of Just ('#':'!':rest) -> case words rest of [] -> defcmd diff --git a/git-annex.hs b/git-annex.hs index d5fab7f477..e30d320b92 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -15,6 +15,7 @@ import qualified CmdLine.GitAnnex import qualified CmdLine.GitAnnexShell import qualified CmdLine.GitRemoteTorAnnex import qualified Test +import Utility.FileSystemEncoding #ifdef mingw32_HOST_OS import Utility.UserInfo @@ -23,6 +24,7 @@ import Utility.Env main :: IO () main = withSocketsDo $ do + useFileSystemEncoding ps <- getArgs #ifdef mingw32_HOST_OS winEnv diff --git a/git-union-merge.hs b/git-union-merge.hs index 3bf628c754..18c88b1a9e 100644 --- a/git-union-merge.hs +++ b/git-union-merge.hs @@ -14,6 +14,7 @@ import qualified Git.CurrentRepo import qualified Git.Branch import qualified Git.Index import qualified Git +import Utility.FileSystemEncoding header :: String header = "Usage: git-union-merge ref ref newref" @@ -39,6 +40,7 @@ parseArgs = do main :: IO () main = do + useFileSystemEncoding [aref, bref, newref] <- map Git.Ref <$> parseArgs g <- Git.Config.read =<< Git.CurrentRepo.get _ <- Git.Index.override (tmpIndex g) g