2013-08-29 22:51:22 +00:00
|
|
|
{- git-annex log file names
|
|
|
|
-
|
2015-01-21 16:50:09 +00:00
|
|
|
- Copyright 2013-2014 Joey Hess <id@joeyh.name>
|
2013-08-29 22:51:22 +00:00
|
|
|
-
|
|
|
|
- Licensed under the GNU GPL version 3 or higher.
|
|
|
|
-}
|
|
|
|
|
|
|
|
module Logs where
|
|
|
|
|
|
|
|
import Common.Annex
|
|
|
|
import Types.Key
|
|
|
|
|
add remote state logs
This allows a remote to store a piece of arbitrary state associated with a
key. This is needed to support Tahoe, where the file-cap is calculated from
the data stored in it, and used to retrieve a key later. Glacier also would
be much improved by using this.
GETSTATE and SETSTATE are added to the external special remote protocol.
Note that the state is left as-is even when a key is removed from a remote.
It's up to the remote to decide when it wants to clear the state.
The remote state log, $KEY.log.rmt, is a UUID-based log. However,
rather than using the old UUID-based log format, I created a new variant
of that format. The new varient is more space efficient (since it lacks the
"timestamp=" hack, and easier to parse (and the parser doesn't mess with
whitespace in the value), and avoids compatability cruft in the old one.
This seemed worth cleaning up for these new files, since there could be a
lot of them, while before UUID-based logs were only used for a few log
files at the top of the git-annex branch. The transition code has also
been updated to handle these new UUID-based logs.
This commit was sponsored by Daniel Hofer.
2014-01-03 20:35:57 +00:00
|
|
|
{- There are several varieties of log file formats. -}
|
2014-01-20 20:47:56 +00:00
|
|
|
data LogVariety
|
|
|
|
= UUIDBasedLog
|
|
|
|
| NewUUIDBasedLog
|
2014-07-24 20:23:36 +00:00
|
|
|
| ChunkLog Key
|
2014-01-20 20:47:56 +00:00
|
|
|
| PresenceLog Key
|
2014-02-13 01:12:22 +00:00
|
|
|
| OtherLog
|
2013-08-29 22:51:22 +00:00
|
|
|
deriving (Show)
|
|
|
|
|
|
|
|
{- Converts a path from the git-annex branch into one of the varieties
|
|
|
|
- of logs used by git-annex, if it's a known path. -}
|
|
|
|
getLogVariety :: FilePath -> Maybe LogVariety
|
|
|
|
getLogVariety f
|
add remote state logs
This allows a remote to store a piece of arbitrary state associated with a
key. This is needed to support Tahoe, where the file-cap is calculated from
the data stored in it, and used to retrieve a key later. Glacier also would
be much improved by using this.
GETSTATE and SETSTATE are added to the external special remote protocol.
Note that the state is left as-is even when a key is removed from a remote.
It's up to the remote to decide when it wants to clear the state.
The remote state log, $KEY.log.rmt, is a UUID-based log. However,
rather than using the old UUID-based log format, I created a new variant
of that format. The new varient is more space efficient (since it lacks the
"timestamp=" hack, and easier to parse (and the parser doesn't mess with
whitespace in the value), and avoids compatability cruft in the old one.
This seemed worth cleaning up for these new files, since there could be a
lot of them, while before UUID-based logs were only used for a few log
files at the top of the git-annex branch. The transition code has also
been updated to handle these new UUID-based logs.
This commit was sponsored by Daniel Hofer.
2014-01-03 20:35:57 +00:00
|
|
|
| f `elem` topLevelUUIDBasedLogs = Just UUIDBasedLog
|
|
|
|
| isRemoteStateLog f = Just NewUUIDBasedLog
|
2014-07-24 20:23:36 +00:00
|
|
|
| isChunkLog f = ChunkLog <$> chunkLogFileKey f
|
2014-03-15 17:44:31 +00:00
|
|
|
| isMetaDataLog f || f `elem` otherLogs = Just OtherLog
|
2013-08-29 22:51:22 +00:00
|
|
|
| otherwise = PresenceLog <$> firstJust (presenceLogs f)
|
|
|
|
|
add remote state logs
This allows a remote to store a piece of arbitrary state associated with a
key. This is needed to support Tahoe, where the file-cap is calculated from
the data stored in it, and used to retrieve a key later. Glacier also would
be much improved by using this.
GETSTATE and SETSTATE are added to the external special remote protocol.
Note that the state is left as-is even when a key is removed from a remote.
It's up to the remote to decide when it wants to clear the state.
The remote state log, $KEY.log.rmt, is a UUID-based log. However,
rather than using the old UUID-based log format, I created a new variant
of that format. The new varient is more space efficient (since it lacks the
"timestamp=" hack, and easier to parse (and the parser doesn't mess with
whitespace in the value), and avoids compatability cruft in the old one.
This seemed worth cleaning up for these new files, since there could be a
lot of them, while before UUID-based logs were only used for a few log
files at the top of the git-annex branch. The transition code has also
been updated to handle these new UUID-based logs.
This commit was sponsored by Daniel Hofer.
2014-01-03 20:35:57 +00:00
|
|
|
{- All the uuid-based logs stored in the top of the git-annex branch. -}
|
|
|
|
topLevelUUIDBasedLogs :: [FilePath]
|
|
|
|
topLevelUUIDBasedLogs =
|
2013-08-29 22:51:22 +00:00
|
|
|
[ uuidLog
|
|
|
|
, remoteLog
|
|
|
|
, trustLog
|
|
|
|
, groupLog
|
|
|
|
, preferredContentLog
|
2014-03-29 18:43:34 +00:00
|
|
|
, requiredContentLog
|
2013-10-07 20:06:34 +00:00
|
|
|
, scheduleLog
|
2013-08-29 22:51:22 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
{- All the ways to get a key from a presence log file -}
|
|
|
|
presenceLogs :: FilePath -> [Maybe Key]
|
|
|
|
presenceLogs f =
|
|
|
|
[ urlLogFileKey f
|
|
|
|
, locationLogFileKey f
|
|
|
|
]
|
|
|
|
|
2014-03-15 17:44:31 +00:00
|
|
|
{- Logs that are neither UUID based nor presence logs. -}
|
|
|
|
otherLogs :: [FilePath]
|
|
|
|
otherLogs =
|
|
|
|
[ numcopiesLog
|
|
|
|
, groupPreferredContentLog
|
|
|
|
]
|
|
|
|
|
2013-08-29 22:51:22 +00:00
|
|
|
uuidLog :: FilePath
|
|
|
|
uuidLog = "uuid.log"
|
|
|
|
|
2014-01-20 20:47:56 +00:00
|
|
|
numcopiesLog :: FilePath
|
|
|
|
numcopiesLog = "numcopies.log"
|
|
|
|
|
2013-08-29 22:51:22 +00:00
|
|
|
remoteLog :: FilePath
|
|
|
|
remoteLog = "remote.log"
|
|
|
|
|
|
|
|
trustLog :: FilePath
|
|
|
|
trustLog = "trust.log"
|
|
|
|
|
|
|
|
groupLog :: FilePath
|
|
|
|
groupLog = "group.log"
|
|
|
|
|
|
|
|
preferredContentLog :: FilePath
|
|
|
|
preferredContentLog = "preferred-content.log"
|
|
|
|
|
2014-03-29 18:43:34 +00:00
|
|
|
requiredContentLog :: FilePath
|
|
|
|
requiredContentLog = "required-content.log"
|
|
|
|
|
2014-03-15 17:44:31 +00:00
|
|
|
groupPreferredContentLog :: FilePath
|
|
|
|
groupPreferredContentLog = "group-preferred-content.log"
|
|
|
|
|
2013-10-07 20:06:34 +00:00
|
|
|
scheduleLog :: FilePath
|
|
|
|
scheduleLog = "schedule.log"
|
|
|
|
|
2013-08-29 22:51:22 +00:00
|
|
|
{- The pathname of the location log file for a given key. -}
|
|
|
|
locationLogFile :: Key -> String
|
|
|
|
locationLogFile key = hashDirLower key ++ keyFile key ++ ".log"
|
|
|
|
|
|
|
|
{- Converts a pathname into a key if it's a location log. -}
|
|
|
|
locationLogFileKey :: FilePath -> Maybe Key
|
|
|
|
locationLogFileKey path
|
|
|
|
| ["remote", "web"] `isPrefixOf` splitDirectories dir = Nothing
|
2014-10-09 19:09:26 +00:00
|
|
|
| ext == ".log" = fileKey base
|
|
|
|
| otherwise = Nothing
|
2013-08-29 22:51:22 +00:00
|
|
|
where
|
|
|
|
(dir, file) = splitFileName path
|
2014-10-09 19:09:26 +00:00
|
|
|
(base, ext) = splitAt (length file - 4) file
|
2013-08-29 22:51:22 +00:00
|
|
|
|
|
|
|
{- The filename of the url log for a given key. -}
|
|
|
|
urlLogFile :: Key -> FilePath
|
|
|
|
urlLogFile key = hashDirLower key </> keyFile key ++ urlLogExt
|
|
|
|
|
|
|
|
{- Old versions stored the urls elsewhere. -}
|
|
|
|
oldurlLogs :: Key -> [FilePath]
|
|
|
|
oldurlLogs key =
|
|
|
|
[ "remote/web" </> hashDirLower key </> key2file key ++ ".log"
|
|
|
|
, "remote/web" </> hashDirLower key </> keyFile key ++ ".log"
|
|
|
|
]
|
|
|
|
|
|
|
|
urlLogExt :: String
|
|
|
|
urlLogExt = ".log.web"
|
|
|
|
|
|
|
|
{- Converts a url log file into a key.
|
|
|
|
- (Does not work on oldurlLogs.) -}
|
|
|
|
urlLogFileKey :: FilePath -> Maybe Key
|
|
|
|
urlLogFileKey path
|
|
|
|
| ext == urlLogExt = fileKey base
|
|
|
|
| otherwise = Nothing
|
|
|
|
where
|
2014-10-09 18:53:13 +00:00
|
|
|
file = takeFileName path
|
2013-08-29 22:51:22 +00:00
|
|
|
(base, ext) = splitAt (length file - extlen) file
|
|
|
|
extlen = length urlLogExt
|
|
|
|
|
|
|
|
{- Does not work on oldurllogs. -}
|
|
|
|
isUrlLog :: FilePath -> Bool
|
|
|
|
isUrlLog file = urlLogExt `isSuffixOf` file
|
|
|
|
|
add remote state logs
This allows a remote to store a piece of arbitrary state associated with a
key. This is needed to support Tahoe, where the file-cap is calculated from
the data stored in it, and used to retrieve a key later. Glacier also would
be much improved by using this.
GETSTATE and SETSTATE are added to the external special remote protocol.
Note that the state is left as-is even when a key is removed from a remote.
It's up to the remote to decide when it wants to clear the state.
The remote state log, $KEY.log.rmt, is a UUID-based log. However,
rather than using the old UUID-based log format, I created a new variant
of that format. The new varient is more space efficient (since it lacks the
"timestamp=" hack, and easier to parse (and the parser doesn't mess with
whitespace in the value), and avoids compatability cruft in the old one.
This seemed worth cleaning up for these new files, since there could be a
lot of them, while before UUID-based logs were only used for a few log
files at the top of the git-annex branch. The transition code has also
been updated to handle these new UUID-based logs.
This commit was sponsored by Daniel Hofer.
2014-01-03 20:35:57 +00:00
|
|
|
{- The filename of the remote state log for a given key. -}
|
|
|
|
remoteStateLogFile :: Key -> FilePath
|
|
|
|
remoteStateLogFile key = hashDirLower key </> keyFile key ++ remoteStateLogExt
|
|
|
|
|
|
|
|
remoteStateLogExt :: String
|
|
|
|
remoteStateLogExt = ".log.rmt"
|
|
|
|
|
|
|
|
isRemoteStateLog :: FilePath -> Bool
|
|
|
|
isRemoteStateLog path = remoteStateLogExt `isSuffixOf` path
|
|
|
|
|
2014-07-24 20:23:36 +00:00
|
|
|
{- The filename of the chunk log for a given key. -}
|
|
|
|
chunkLogFile :: Key -> FilePath
|
|
|
|
chunkLogFile key = hashDirLower key </> keyFile key ++ chunkLogExt
|
|
|
|
|
|
|
|
chunkLogFileKey :: FilePath -> Maybe Key
|
|
|
|
chunkLogFileKey path
|
|
|
|
| ext == chunkLogExt = fileKey base
|
|
|
|
| otherwise = Nothing
|
|
|
|
where
|
2014-10-09 18:53:13 +00:00
|
|
|
file = takeFileName path
|
2014-07-24 20:23:36 +00:00
|
|
|
(base, ext) = splitAt (length file - extlen) file
|
|
|
|
extlen = length chunkLogExt
|
|
|
|
|
|
|
|
chunkLogExt :: String
|
|
|
|
chunkLogExt = ".log.cnk"
|
|
|
|
|
|
|
|
isChunkLog :: FilePath -> Bool
|
|
|
|
isChunkLog path = chunkLogExt `isSuffixOf` path
|
|
|
|
|
2014-02-13 01:12:22 +00:00
|
|
|
{- The filename of the metadata log for a given key. -}
|
|
|
|
metaDataLogFile :: Key -> FilePath
|
|
|
|
metaDataLogFile key = hashDirLower key </> keyFile key ++ metaDataLogExt
|
|
|
|
|
|
|
|
metaDataLogExt :: String
|
|
|
|
metaDataLogExt = ".log.met"
|
|
|
|
|
|
|
|
isMetaDataLog :: FilePath -> Bool
|
|
|
|
isMetaDataLog path = metaDataLogExt `isSuffixOf` path
|
|
|
|
|
2013-08-29 22:51:22 +00:00
|
|
|
prop_logs_sane :: Key -> Bool
|
2014-02-11 05:35:11 +00:00
|
|
|
prop_logs_sane dummykey = and
|
2013-08-29 22:51:22 +00:00
|
|
|
[ isNothing (getLogVariety "unknown")
|
2014-07-24 20:23:36 +00:00
|
|
|
, expect gotUUIDBasedLog (getLogVariety uuidLog)
|
|
|
|
, expect gotPresenceLog (getLogVariety $ locationLogFile dummykey)
|
|
|
|
, expect gotPresenceLog (getLogVariety $ urlLogFile dummykey)
|
|
|
|
, expect gotNewUUIDBasedLog (getLogVariety $ remoteStateLogFile dummykey)
|
|
|
|
, expect gotChunkLog (getLogVariety $ chunkLogFile dummykey)
|
|
|
|
, expect gotOtherLog (getLogVariety $ metaDataLogFile dummykey)
|
2014-10-09 19:35:19 +00:00
|
|
|
, expect gotOtherLog (getLogVariety numcopiesLog)
|
2013-08-29 22:51:22 +00:00
|
|
|
]
|
|
|
|
where
|
2014-10-09 18:53:13 +00:00
|
|
|
expect = maybe False
|
2014-07-24 20:23:36 +00:00
|
|
|
gotUUIDBasedLog UUIDBasedLog = True
|
|
|
|
gotUUIDBasedLog _ = False
|
2014-10-09 18:53:13 +00:00
|
|
|
gotNewUUIDBasedLog NewUUIDBasedLog = True
|
2014-07-24 20:23:36 +00:00
|
|
|
gotNewUUIDBasedLog _ = False
|
|
|
|
gotChunkLog (ChunkLog k) = k == dummykey
|
|
|
|
gotChunkLog _ = False
|
|
|
|
gotPresenceLog (PresenceLog k) = k == dummykey
|
|
|
|
gotPresenceLog _ = False
|
|
|
|
gotOtherLog OtherLog = True
|
|
|
|
gotOtherLog _ = False
|