Add trust and untrust subcommands, to allow configuring remotes that are trusted to retain files without explicit checking.

This commit is contained in:
Joey Hess 2010-12-28 17:17:02 -04:00
parent 6c58a58393
commit aa4f91b2d6
11 changed files with 109 additions and 16 deletions

View file

@ -16,6 +16,7 @@ module Backend.File (backend, checkKey) where
import Control.Monad.State import Control.Monad.State
import System.Directory import System.Directory
import Data.List (intersect)
import TypeInternals import TypeInternals
import LocationLog import LocationLog
@ -91,11 +92,16 @@ checkRemoveKey key numcopiesM = do
if force || numcopiesM == Just 0 if force || numcopiesM == Just 0
then return True then return True
else do else do
g <- Annex.gitRepo
locations <- liftIO $ keyLocations g key
trusted <- getTrusted
let trustedlocations = intersect locations trusted
remotes <- Remotes.keyPossibilities key remotes <- Remotes.keyPossibilities key
untrustedremotes <- reposWithoutUUID remotes trusted
numcopies <- getNumCopies numcopiesM numcopies <- getNumCopies numcopiesM
if numcopies > length remotes if numcopies > length untrustedremotes
then notEnoughCopies numcopies (length remotes) [] then notEnoughCopies numcopies (length untrustedremotes) []
else findcopies numcopies 0 remotes [] else findcopies numcopies (length trustedlocations) untrustedremotes []
where where
findcopies need have [] bad findcopies need have [] bad
| have >= need = return True | have >= need = return True

View file

@ -34,6 +34,8 @@ import qualified Command.Lock
import qualified Command.PreCommit import qualified Command.PreCommit
import qualified Command.Find import qualified Command.Find
import qualified Command.Uninit import qualified Command.Uninit
import qualified Command.Trust
import qualified Command.Untrust
subCmds :: [SubCommand] subCmds :: [SubCommand]
subCmds = subCmds =
@ -61,6 +63,10 @@ subCmds =
"de-initialize git-annex and clean out repository" "de-initialize git-annex and clean out repository"
, SubCommand "pre-commit" path Command.PreCommit.seek , SubCommand "pre-commit" path Command.PreCommit.seek
"run by git pre-commit hook" "run by git pre-commit hook"
, SubCommand "trust" remote Command.Trust.seek
"trust a repository"
, SubCommand "untrust" remote Command.Untrust.seek
"do not trust a repository"
, SubCommand "fromkey" key Command.FromKey.seek , SubCommand "fromkey" key Command.FromKey.seek
"adds a file using a specific key" "adds a file using a specific key"
, SubCommand "dropkey" key Command.DropKey.seek , SubCommand "dropkey" key Command.DropKey.seek
@ -84,6 +90,7 @@ subCmds =
key = "KEY ..." key = "KEY ..."
desc = "DESCRIPTION" desc = "DESCRIPTION"
number = "NUMBER ..." number = "NUMBER ..."
remote = "REMOTE ..."
nothing = "" nothing = ""
-- Each dashed command-line option results in generation of an action -- Each dashed command-line option results in generation of an action

View file

@ -12,6 +12,7 @@ module Remotes (
inAnnex, inAnnex,
same, same,
commandLineRemote, commandLineRemote,
byName,
copyFromRemote, copyFromRemote,
copyToRemote, copyToRemote,
runCmd runCmd
@ -156,6 +157,11 @@ commandLineRemote = do
fromName <- Annex.flagGet "fromrepository" fromName <- Annex.flagGet "fromrepository"
toName <- Annex.flagGet "torepository" toName <- Annex.flagGet "torepository"
let name = if null fromName then toName else fromName let name = if null fromName then toName else fromName
byName name
{- Looks up a remote by name. -}
byName :: String -> Annex Git.Repo
byName name = do
when (null name) $ error "no remote specified" when (null name) $ error "no remote specified"
g <- Annex.gitRepo g <- Annex.gitRepo
let match = filter (\r -> name == Git.repoRemoteName r) $ let match = filter (\r -> name == Git.repoRemoteName r) $

45
UUID.hs
View file

@ -14,9 +14,13 @@ module UUID (
prepUUID, prepUUID,
genUUID, genUUID,
reposByUUID, reposByUUID,
reposWithoutUUID,
prettyPrintUUIDs, prettyPrintUUIDs,
describeUUID, describeUUID,
uuidLog uuidLog,
trustLog,
getTrusted,
setTrusted
) where ) where
import Control.Monad.State import Control.Monad.State
@ -24,9 +28,7 @@ import Data.Maybe
import Data.List import Data.List
import System.Cmd.Utils import System.Cmd.Utils
import System.IO import System.IO
import System.Directory
import qualified Data.Map as M import qualified Data.Map as M
import System.Posix.Process
import qualified GitRepo as Git import qualified GitRepo as Git
import Types import Types
@ -85,6 +87,14 @@ reposByUUID repos uuids = filterM match repos
u <- getUUID r u <- getUUID r
return $ isJust $ elemIndex u uuids return $ isJust $ elemIndex u uuids
{- Filters a list of repos to ones that do not have the listed UUIDs. -}
reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
reposWithoutUUID repos uuids = filterM unmatch repos
where
unmatch r = do
u <- getUUID r
return $ not $ isJust $ elemIndex u uuids
{- Pretty-prints a list of UUIDs -} {- Pretty-prints a list of UUIDs -}
prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs :: [UUID] -> Annex String
prettyPrintUUIDs uuids = do prettyPrintUUIDs uuids = do
@ -103,11 +113,7 @@ describeUUID uuid desc = do
m <- uuidMap m <- uuidMap
let m' = M.insert uuid desc m let m' = M.insert uuid desc m
logfile <- uuidLog logfile <- uuidLog
pid <- liftIO $ getProcessID liftIO $ safeWriteFile logfile (serialize m')
let tmplogfile = logfile ++ ".tmp" ++ show pid
liftIO $ createDirectoryIfMissing True (parentDir logfile)
liftIO $ writeFile tmplogfile $ serialize m'
liftIO $ renameFile tmplogfile logfile
where where
serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
@ -125,7 +131,28 @@ uuidMap = do
ignoreerror _ = return "" ignoreerror _ = return ""
{- Filename of uuid.log. -} {- Filename of uuid.log. -}
uuidLog :: Annex String uuidLog :: Annex FilePath
uuidLog = do uuidLog = do
g <- Annex.gitRepo g <- Annex.gitRepo
return $ gitStateDir g ++ "uuid.log" return $ gitStateDir g ++ "uuid.log"
{- Filename of trust.log. -}
trustLog :: Annex FilePath
trustLog = do
g <- Annex.gitRepo
return $ gitStateDir g ++ "trust.log"
{- List of trusted UUIDs. -}
getTrusted :: Annex [UUID]
getTrusted = do
logfile <- trustLog
s <- liftIO $ catch (readFile logfile) ignoreerror
return $ map (\l -> head $ words l) $ lines s
where
ignoreerror _ = return ""
{- Changes the list of trusted UUIDs. -}
setTrusted :: [UUID] -> Annex ()
setTrusted u = do
logfile <- trustLog
liftIO $ safeWriteFile logfile $ unlines u

View file

@ -15,7 +15,8 @@ module Utility (
boolSystem, boolSystem,
shellEscape, shellEscape,
unsetFileMode, unsetFileMode,
readMaybe readMaybe,
safeWriteFile
) where ) where
import System.IO import System.IO
@ -139,3 +140,12 @@ readMaybe :: (Read a) => String -> Maybe a
readMaybe s = case reads s of readMaybe s = case reads s of
((x,_):_) -> Just x ((x,_):_) -> Just x
_ -> Nothing _ -> Nothing
{- Writes a file using a temp file that is renamed atomically into place. -}
safeWriteFile :: FilePath -> String -> IO ()
safeWriteFile file content = do
pid <- getProcessID
let tmpfile = file ++ ".tmp" ++ show pid
createDirectoryIfMissing True (parentDir file)
writeFile tmpfile content
renameFile tmpfile file

2
debian/changelog vendored
View file

@ -1,6 +1,8 @@
git-annex (0.15) UNRELEASED; urgency=low git-annex (0.15) UNRELEASED; urgency=low
* Support scp-style urls for remotes (host:path). * Support scp-style urls for remotes (host:path).
* Add trust and untrust subcommands, to allow configuring remotes
that are trusted to retain files without explicit checking.
-- Joey Hess <joeyh@debian.org> Tue, 28 Dec 2010 13:13:20 -0400 -- Joey Hess <joeyh@debian.org> Tue, 28 Dec 2010 13:13:20 -0400

View file

@ -11,7 +11,8 @@ setting in `.gitattributes` files.
`git annex drop` attempts to check with other git remotes, to check that N `git annex drop` attempts to check with other git remotes, to check that N
copies of the file exist. If enough repositories cannot be verified to have copies of the file exist. If enough repositories cannot be verified to have
it, it will retain the file content to avoid data loss. it, it will retain the file content to avoid data loss. Note that
[[trusted_remotes|trust]] are not explicitly checked.
For example, consider three repositories: Server, Laptop, and USB. Both Server For example, consider three repositories: Server, Laptop, and USB. Both Server
and USB have a copy of a file, and N=1. If on Laptop, you `git annex get and USB have a copy of a file, and N=1. If on Laptop, you `git annex get

View file

@ -171,6 +171,15 @@ Many git-annex subcommands will stage changes for later `git commit` by you.
This is meant to be called from git's pre-commit hook. `git annex init` This is meant to be called from git's pre-commit hook. `git annex init`
automatically creates a pre-commit hook using this. automatically creates a pre-commit hook using this.
* trust [repository ...]
Records that a repository is [[trusted]] to not unexpectedly lose content.
Use with care.
* untrust [repository ...]
Undoes a trust command.
* fromkey file * fromkey file
This can be used to maually set up a file to link to a specified key This can be used to maually set up a file to link to a specified key
@ -333,7 +342,9 @@ These files are used by git-annex, in your git repository:
available. Annexed files in your git repository symlink to that content. available. Annexed files in your git repository symlink to that content.
`.git-annex/uuid.log` is used to map between repository UUID and `.git-annex/uuid.log` is used to map between repository UUID and
decscriptions. You may edit it. decscriptions.
`.git-annex/trust.log` is used to list the UUIDs of trusted repositories.
`.git-annex/*.log` is where git-annex records its content tracking `.git-annex/*.log` is where git-annex records its content tracking
information. These files should be committed to git. information. These files should be committed to git.

View file

@ -26,3 +26,6 @@ descriptions to help you with finding them:
Try making some of these repositories available: Try making some of these repositories available:
c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1
e1938fee-d95b-11df-96cc-002170d25c55 e1938fee-d95b-11df-96cc-002170d25c55
In certian cases you may want to configure git-annex to [[trust]]
that location tracking information is always correct for a repository.

20
doc/trust.mdwn Normal file
View file

@ -0,0 +1,20 @@
Normally, git-annex does not fully trust its stored [[location_tracking]]
information. When removing content, it will directly check
that other repositories have [[copies]].
Generally that explicit checking is a good idea. Consider that the current
[[location_tracking]] information for a remote may not yet have propigated
out. Or, a remote may have suffered a catastrophic loss of data, or itself
been lost.
Sometimes though, you may have reasons to trust the location tracking
information for a remote repository. For example, it may be an offline
archival drive, from which you rarely or never remove content. Deciding
when it makes sense to trust the tracking info is up to you.
One way to handle this is just to use `--force` when a command cannot
access a remote you trust.
Another option is to configure which remotes you trust with the
`git annex trust` command, or by manually adding the UUIDs of trusted remotes
to `.git-annex/trust.log`.

View file

@ -134,7 +134,7 @@ you'll see something like this.
(Use --force to override this check, or adjust annex.numcopies.) (Use --force to override this check, or adjust annex.numcopies.)
failed failed
Here you might --force it to drop `important_file` if you trust your backup. Here you might --force it to drop `important_file` if you [[trust]] your backup.
But `other.iso` looks to have never been copied to anywhere else, so if But `other.iso` looks to have never been copied to anywhere else, so if
it's something you want to hold onto, you'd need to transfer it to it's something you want to hold onto, you'd need to transfer it to
some other repository before dropping it. some other repository before dropping it.