2012-03-09 23:08:10 +00:00
|
|
|
{- GHC File system encoding handling.
|
2012-03-09 22:52:03 +00:00
|
|
|
-
|
|
|
|
- Copyright 2012 Joey Hess <joey@kitenet.net>
|
|
|
|
-
|
|
|
|
- Licensed under the GNU GPL version 3 or higher.
|
|
|
|
-}
|
|
|
|
|
2012-09-13 23:14:00 +00:00
|
|
|
module Utility.FileSystemEncoding (
|
|
|
|
fileEncoding,
|
|
|
|
withFilePath,
|
|
|
|
md5FilePath,
|
|
|
|
decodeW8,
|
|
|
|
encodeW8
|
|
|
|
) where
|
2012-03-09 22:52:03 +00:00
|
|
|
|
fix key directory hash calculation code
Fix Key directory hash calculation code to behave as it did before version
3.20120227 when a key contains non-ascii.
The hash directories for a given Key are based on its md5sum.
Prior to ghc 7.4, Keys contained raw, undecoded bytes, so the md5sum was
taken of each byte in turn. With the ghc 7.4 filename encoding change,
keys contains decoded unicode characters (possibly with surrigates for
undecodable bytes). This changes the result of the md5sum, since the md5sum
used is pure haskell and supports unicode. And that won't do, as git-annex
will start looking in a different hash directory for the content of a key.
The surrigates are particularly bad, since that's essentially a ghc
implementation detail, so could change again at any time. Also, changing
the locale changes how the bytes are decoded, which can also change
the md5sum.
Symptoms would include things like:
* git annex fsck would complain that no copies existed of a file,
despite its symlink pointing to the content that was locally present
* git annex fix would change the symlink to use the wrong hash
directory.
Only WORM backend is likely to have been affected, since only it tends
to include much filename data (SHA1E could in theory also be affected).
I have not tried to support the hash directories used by git-annex versions
3.20120227 to 3.20120308, so things added with those versions with WORM
will require manual fixups. Sorry for the inconvenience!
2012-03-09 23:26:02 +00:00
|
|
|
import qualified GHC.Foreign as GHC
|
|
|
|
import qualified GHC.IO.Encoding as Encoding
|
2012-03-09 23:08:10 +00:00
|
|
|
import Foreign.C
|
fix key directory hash calculation code
Fix Key directory hash calculation code to behave as it did before version
3.20120227 when a key contains non-ascii.
The hash directories for a given Key are based on its md5sum.
Prior to ghc 7.4, Keys contained raw, undecoded bytes, so the md5sum was
taken of each byte in turn. With the ghc 7.4 filename encoding change,
keys contains decoded unicode characters (possibly with surrigates for
undecodable bytes). This changes the result of the md5sum, since the md5sum
used is pure haskell and supports unicode. And that won't do, as git-annex
will start looking in a different hash directory for the content of a key.
The surrigates are particularly bad, since that's essentially a ghc
implementation detail, so could change again at any time. Also, changing
the locale changes how the bytes are decoded, which can also change
the md5sum.
Symptoms would include things like:
* git annex fsck would complain that no copies existed of a file,
despite its symlink pointing to the content that was locally present
* git annex fix would change the symlink to use the wrong hash
directory.
Only WORM backend is likely to have been affected, since only it tends
to include much filename data (SHA1E could in theory also be affected).
I have not tried to support the hash directories used by git-annex versions
3.20120227 to 3.20120308, so things added with those versions with WORM
will require manual fixups. Sorry for the inconvenience!
2012-03-09 23:26:02 +00:00
|
|
|
import System.IO
|
|
|
|
import System.IO.Unsafe
|
2012-03-10 15:38:38 +00:00
|
|
|
import qualified Data.Hash.MD5 as MD5
|
2012-06-20 16:51:25 +00:00
|
|
|
import Data.Word
|
|
|
|
import Data.Bits.Utils
|
2012-03-09 23:08:10 +00:00
|
|
|
|
|
|
|
{- Sets a Handle to use the filesystem encoding. This causes data
|
|
|
|
- written or read from it to be encoded/decoded the same
|
|
|
|
- as ghc 7.4 does to filenames etc. This special encoding
|
|
|
|
- allows "arbitrary undecodable bytes to be round-tripped through it". -}
|
|
|
|
fileEncoding :: Handle -> IO ()
|
fix key directory hash calculation code
Fix Key directory hash calculation code to behave as it did before version
3.20120227 when a key contains non-ascii.
The hash directories for a given Key are based on its md5sum.
Prior to ghc 7.4, Keys contained raw, undecoded bytes, so the md5sum was
taken of each byte in turn. With the ghc 7.4 filename encoding change,
keys contains decoded unicode characters (possibly with surrigates for
undecodable bytes). This changes the result of the md5sum, since the md5sum
used is pure haskell and supports unicode. And that won't do, as git-annex
will start looking in a different hash directory for the content of a key.
The surrigates are particularly bad, since that's essentially a ghc
implementation detail, so could change again at any time. Also, changing
the locale changes how the bytes are decoded, which can also change
the md5sum.
Symptoms would include things like:
* git annex fsck would complain that no copies existed of a file,
despite its symlink pointing to the content that was locally present
* git annex fix would change the symlink to use the wrong hash
directory.
Only WORM backend is likely to have been affected, since only it tends
to include much filename data (SHA1E could in theory also be affected).
I have not tried to support the hash directories used by git-annex versions
3.20120227 to 3.20120308, so things added with those versions with WORM
will require manual fixups. Sorry for the inconvenience!
2012-03-09 23:26:02 +00:00
|
|
|
fileEncoding h = hSetEncoding h =<< Encoding.getFileSystemEncoding
|
2012-03-09 22:52:03 +00:00
|
|
|
|
|
|
|
{- Marshal a Haskell FilePath into a NUL terminated C string using temporary
|
|
|
|
- storage. The FilePath is encoded using the filesystem encoding,
|
|
|
|
- reversing the decoding that should have been done when the FilePath
|
|
|
|
- was obtained. -}
|
|
|
|
withFilePath :: FilePath -> (CString -> IO a) -> IO a
|
fix key directory hash calculation code
Fix Key directory hash calculation code to behave as it did before version
3.20120227 when a key contains non-ascii.
The hash directories for a given Key are based on its md5sum.
Prior to ghc 7.4, Keys contained raw, undecoded bytes, so the md5sum was
taken of each byte in turn. With the ghc 7.4 filename encoding change,
keys contains decoded unicode characters (possibly with surrigates for
undecodable bytes). This changes the result of the md5sum, since the md5sum
used is pure haskell and supports unicode. And that won't do, as git-annex
will start looking in a different hash directory for the content of a key.
The surrigates are particularly bad, since that's essentially a ghc
implementation detail, so could change again at any time. Also, changing
the locale changes how the bytes are decoded, which can also change
the md5sum.
Symptoms would include things like:
* git annex fsck would complain that no copies existed of a file,
despite its symlink pointing to the content that was locally present
* git annex fix would change the symlink to use the wrong hash
directory.
Only WORM backend is likely to have been affected, since only it tends
to include much filename data (SHA1E could in theory also be affected).
I have not tried to support the hash directories used by git-annex versions
3.20120227 to 3.20120308, so things added with those versions with WORM
will require manual fixups. Sorry for the inconvenience!
2012-03-09 23:26:02 +00:00
|
|
|
withFilePath fp f = Encoding.getFileSystemEncoding
|
|
|
|
>>= \enc -> GHC.withCString enc fp f
|
2012-03-09 23:08:10 +00:00
|
|
|
|
2012-09-13 23:14:00 +00:00
|
|
|
{- Encodes a FilePath into a String, applying the filesystem encoding.
|
|
|
|
-
|
|
|
|
- There are very few things it makes sense to do with such an encoded
|
|
|
|
- string. It's not a legal filename; it should not be displayed.
|
|
|
|
- So this function is not exported, but instead used by the few functions
|
|
|
|
- that can usefully consume it.
|
2012-03-09 23:08:10 +00:00
|
|
|
-
|
fix key directory hash calculation code
Fix Key directory hash calculation code to behave as it did before version
3.20120227 when a key contains non-ascii.
The hash directories for a given Key are based on its md5sum.
Prior to ghc 7.4, Keys contained raw, undecoded bytes, so the md5sum was
taken of each byte in turn. With the ghc 7.4 filename encoding change,
keys contains decoded unicode characters (possibly with surrigates for
undecodable bytes). This changes the result of the md5sum, since the md5sum
used is pure haskell and supports unicode. And that won't do, as git-annex
will start looking in a different hash directory for the content of a key.
The surrigates are particularly bad, since that's essentially a ghc
implementation detail, so could change again at any time. Also, changing
the locale changes how the bytes are decoded, which can also change
the md5sum.
Symptoms would include things like:
* git annex fsck would complain that no copies existed of a file,
despite its symlink pointing to the content that was locally present
* git annex fix would change the symlink to use the wrong hash
directory.
Only WORM backend is likely to have been affected, since only it tends
to include much filename data (SHA1E could in theory also be affected).
I have not tried to support the hash directories used by git-annex versions
3.20120227 to 3.20120308, so things added with those versions with WORM
will require manual fixups. Sorry for the inconvenience!
2012-03-09 23:26:02 +00:00
|
|
|
- This use of unsafePerformIO is belived to be safe; GHC's interface
|
|
|
|
- only allows doing this conversion with CStrings, and the CString buffer
|
|
|
|
- is allocated, used, and deallocated within the call, with no side
|
|
|
|
- effects.
|
|
|
|
-}
|
2012-09-13 23:14:00 +00:00
|
|
|
{-# NOINLINE _encodeFilePath #-}
|
|
|
|
_encodeFilePath :: FilePath -> String
|
|
|
|
_encodeFilePath fp = unsafePerformIO $ do
|
fix key directory hash calculation code
Fix Key directory hash calculation code to behave as it did before version
3.20120227 when a key contains non-ascii.
The hash directories for a given Key are based on its md5sum.
Prior to ghc 7.4, Keys contained raw, undecoded bytes, so the md5sum was
taken of each byte in turn. With the ghc 7.4 filename encoding change,
keys contains decoded unicode characters (possibly with surrigates for
undecodable bytes). This changes the result of the md5sum, since the md5sum
used is pure haskell and supports unicode. And that won't do, as git-annex
will start looking in a different hash directory for the content of a key.
The surrigates are particularly bad, since that's essentially a ghc
implementation detail, so could change again at any time. Also, changing
the locale changes how the bytes are decoded, which can also change
the md5sum.
Symptoms would include things like:
* git annex fsck would complain that no copies existed of a file,
despite its symlink pointing to the content that was locally present
* git annex fix would change the symlink to use the wrong hash
directory.
Only WORM backend is likely to have been affected, since only it tends
to include much filename data (SHA1E could in theory also be affected).
I have not tried to support the hash directories used by git-annex versions
3.20120227 to 3.20120308, so things added with those versions with WORM
will require manual fixups. Sorry for the inconvenience!
2012-03-09 23:26:02 +00:00
|
|
|
enc <- Encoding.getFileSystemEncoding
|
|
|
|
GHC.withCString enc fp $ GHC.peekCString Encoding.char8
|
2012-06-20 16:51:25 +00:00
|
|
|
|
2012-09-13 23:14:00 +00:00
|
|
|
{- Encodes a FilePath into a Md5.Str, applying the filesystem encoding. -}
|
|
|
|
md5FilePath :: FilePath -> MD5.Str
|
|
|
|
md5FilePath = MD5.Str . _encodeFilePath
|
|
|
|
|
2012-06-20 16:51:25 +00:00
|
|
|
{- Converts a [Word8] to a FilePath, encoding using the filesystem encoding.
|
|
|
|
-
|
|
|
|
- w82c produces a String, which may contain Chars that are invalid
|
|
|
|
- unicode. From there, this is really a simple matter of applying the
|
|
|
|
- file system encoding, only complicated by GHC's interface to doing so.
|
|
|
|
-}
|
|
|
|
{-# NOINLINE encodeW8 #-}
|
|
|
|
encodeW8 :: [Word8] -> FilePath
|
|
|
|
encodeW8 w8 = unsafePerformIO $ do
|
|
|
|
enc <- Encoding.getFileSystemEncoding
|
|
|
|
GHC.withCString Encoding.char8 (w82s w8) $ GHC.peekCString enc
|
2012-09-13 23:14:00 +00:00
|
|
|
|
|
|
|
{- Useful when you want the actual number of bytes that will be used to
|
|
|
|
- represent the FilePath on disk. -}
|
|
|
|
decodeW8 :: FilePath -> [Word8]
|
|
|
|
decodeW8 = s2w8 . _encodeFilePath
|