In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files.

This commit is contained in:
Joey Hess 2010-11-28 15:28:20 -04:00
parent 92e5d28ca8
commit 653ad35a9f
14 changed files with 87 additions and 75 deletions

View file

@ -104,8 +104,8 @@ retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool
retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest
{- Removes a key from a backend. -} {- Removes a key from a backend. -}
removeKey :: Backend -> Key -> Annex Bool removeKey :: Backend -> Key -> Maybe Int -> Annex Bool
removeKey backend key = (Internals.removeKey backend) key removeKey backend key numcopies = (Internals.removeKey backend) key numcopies
{- Checks if a key is present in its backend. -} {- Checks if a key is present in its backend. -}
hasKey :: Key -> Annex Bool hasKey :: Key -> Annex Bool
@ -114,8 +114,8 @@ hasKey key = do
(Internals.hasKey backend) key (Internals.hasKey backend) key
{- Checks a key's backend for problems. -} {- Checks a key's backend for problems. -}
fsckKey :: Backend -> Key -> Annex Bool fsckKey :: Backend -> Key -> Maybe Int -> Annex Bool
fsckKey backend key = (Internals.fsckKey backend) key fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies
{- Looks up the key and backend corresponding to an annexed file, {- Looks up the key and backend corresponding to an annexed file,
- by examining what the file symlinks to. -} - by examining what the file symlinks to. -}

View file

@ -86,14 +86,14 @@ copyKeyFile key file = do
{- Checks remotes to verify that enough copies of a key exist to allow {- Checks remotes to verify that enough copies of a key exist to allow
- for a key to be safely removed (with no data loss), and fails with an - for a key to be safely removed (with no data loss), and fails with an
- error if not. -} - error if not. -}
checkRemoveKey :: Key -> Annex Bool checkRemoveKey :: Key -> Maybe Int -> Annex Bool
checkRemoveKey key = do checkRemoveKey key numcopiesM = do
force <- Annex.flagIsSet "force" force <- Annex.flagIsSet "force"
if force if force || numcopiesM == Just 0
then return True then return True
else do else do
remotes <- Remotes.keyPossibilities key remotes <- Remotes.keyPossibilities key
numcopies <- getNumCopies numcopies <- getNumCopies numcopiesM
if numcopies > length remotes if numcopies > length remotes
then notEnoughCopies numcopies (length remotes) [] then notEnoughCopies numcopies (length remotes) []
else findcopies numcopies 0 remotes [] else findcopies numcopies 0 remotes []
@ -139,8 +139,9 @@ showTriedRemotes remotes =
showLongNote $ "I was unable to access these remotes: " ++ showLongNote $ "I was unable to access these remotes: " ++
Remotes.list remotes Remotes.list remotes
getNumCopies :: Annex Int getNumCopies :: Maybe Int -> Annex Int
getNumCopies = do getNumCopies (Just n) = return n
getNumCopies Nothing = do
g <- Annex.gitRepo g <- Annex.gitRepo
return $ read $ Git.configGet g config "1" return $ read $ Git.configGet g config "1"
where where
@ -153,15 +154,15 @@ getNumCopies = do
- The passed action is first run to allow backends deriving this one - The passed action is first run to allow backends deriving this one
- to do their own checks. - to do their own checks.
-} -}
checkKey :: (Key -> Annex Bool) -> Key -> Annex Bool checkKey :: (Key -> Annex Bool) -> Key -> Maybe Int -> Annex Bool
checkKey a key = do checkKey a key numcopies = do
a_ok <- a key a_ok <- a key
copies_ok <- checkKeyNumCopies key copies_ok <- checkKeyNumCopies key numcopies
return $ a_ok && copies_ok return $ a_ok && copies_ok
checkKeyNumCopies :: Key -> Annex Bool checkKeyNumCopies :: Key -> Maybe Int -> Annex Bool
checkKeyNumCopies key = do checkKeyNumCopies key numcopies = do
needed <- getNumCopies needed <- getNumCopies numcopies
remotes <- Remotes.keyPossibilities key remotes <- Remotes.keyPossibilities key
inannex <- inAnnex key inannex <- inAnnex key
let present = length remotes + if inannex then 1 else 0 let present = length remotes + if inannex then 1 else 0

View file

@ -22,11 +22,11 @@ backend = Backend {
retrieveKeyFile = downloadUrl, retrieveKeyFile = downloadUrl,
-- allow keys to be removed; presumably they can always be -- allow keys to be removed; presumably they can always be
-- downloaded again -- downloaded again
removeKey = dummyOk, removeKey = dummyRemove,
-- similarly, keys are always assumed to be out there on the web -- similarly, keys are always assumed to be out there on the web
hasKey = dummyOk, hasKey = dummyOk,
-- and nothing needed to fsck -- and nothing needed to fsck
fsckKey = dummyOk fsckKey = dummyFsck
} }
-- cannot generate url from filename -- cannot generate url from filename
@ -37,6 +37,12 @@ keyValue _ = return Nothing
dummyStore :: FilePath -> Key -> Annex Bool dummyStore :: FilePath -> Key -> Annex Bool
dummyStore _ _ = return False dummyStore _ _ = return False
dummyRemove :: Key -> Maybe a -> Annex Bool
dummyRemove _ _ = return False
dummyFsck :: Key -> Maybe a -> Annex Bool
dummyFsck _ _ = return True
dummyOk :: Key -> Annex Bool dummyOk :: Key -> Annex Bool
dummyOk _ = return True dummyOk _ = return True

View file

@ -44,6 +44,9 @@ type SubCmdStartString = String -> SubCmdStart
type BackendFile = (FilePath, Maybe Backend) type BackendFile = (FilePath, Maybe Backend)
type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek
type SubCmdStartBackendFile = BackendFile -> SubCmdStart type SubCmdStartBackendFile = BackendFile -> SubCmdStart
type AttrFile = (FilePath, String)
type SubCmdSeekAttrFiles = SubCmdStartAttrFile -> SubCmdSeek
type SubCmdStartAttrFile = AttrFile -> SubCmdStart
type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek
type SubCmdStartNothing = SubCmdStart type SubCmdStartNothing = SubCmdStart
@ -104,6 +107,13 @@ withFilesInGit a params = do
repo <- Annex.gitRepo repo <- Annex.gitRepo
files <- liftIO $ mapM (Git.inRepo repo) params files <- liftIO $ mapM (Git.inRepo repo) params
return $ map a $ filter notState $ foldl (++) [] files return $ map a $ filter notState $ foldl (++) [] files
withAttrFilesInGit :: String -> SubCmdSeekAttrFiles
withAttrFilesInGit attr a params = do
repo <- Annex.gitRepo
files <- liftIO $ mapM (Git.inRepo repo) params
pairs <- liftIO $ Git.checkAttr repo attr $
filter notState $ foldl (++) [] files
return $ map a pairs
withFilesMissing :: SubCmdSeekStrings withFilesMissing :: SubCmdSeekStrings
withFilesMissing a params = do withFilesMissing a params = do
files <- liftIO $ filterM missing params files <- liftIO $ filterM missing params
@ -152,21 +162,21 @@ backendPairs a files = do
{- Default to acting on all files matching the seek action if {- Default to acting on all files matching the seek action if
- none are specified. -} - none are specified. -}
withAll :: SubCmdSeekStrings -> SubCmdSeekStrings withAll :: (a -> SubCmdSeek) -> a -> SubCmdSeek
withAll w a [] = do withAll w a [] = do
g <- Annex.gitRepo g <- Annex.gitRepo
w a [Git.workTree g] w a [Git.workTree g]
withAll w a p = w a p withAll w a p = w a p
{- Provides a default parameter to act on if none is specified. -} {- Provides a default parameter to act on if none is specified. -}
withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings withDefault :: String-> (a -> SubCmdSeek) -> (a -> SubCmdSeek)
withDefault d w a [] = w a [d] withDefault d w a [] = w a [d]
withDefault _ w a p = w a p withDefault _ w a p = w a p
{- filter out files from the state directory -} {- filter out files from the state directory -}
notState :: FilePath -> Bool notState :: FilePath -> Bool
notState f = stateLoc /= take (length stateLoc) f notState f = stateLoc /= take (length stateLoc) f
{- filter out symlinks -} {- filter out symlinks -}
notSymlink :: FilePath -> IO Bool notSymlink :: FilePath -> IO Bool
notSymlink f = do notSymlink f = do

View file

@ -15,24 +15,27 @@ import LocationLog
import Types import Types
import Core import Core
import Messages import Messages
import Utility
seek :: [SubCmdSeek] seek :: [SubCmdSeek]
seek = [withFilesInGit start] seek = [withAttrFilesInGit "git-annex-numcopies" start]
{- Indicates a file's content is not wanted anymore, and should be removed {- Indicates a file's content is not wanted anymore, and should be removed
- if it's safe to do so. -} - if it's safe to do so. -}
start :: SubCmdStartString start :: SubCmdStartAttrFile
start file = isAnnexed file $ \(key, backend) -> do start (file, attr) = isAnnexed file $ \(key, backend) -> do
inbackend <- Backend.hasKey key inbackend <- Backend.hasKey key
if not inbackend if not inbackend
then return Nothing then return Nothing
else do else do
showStart "drop" file showStart "drop" file
return $ Just $ perform key backend return $ Just $ perform key backend numcopies
where
numcopies = readMaybe attr :: Maybe Int
perform :: Key -> Backend -> SubCmdPerform perform :: Key -> Backend -> Maybe Int -> SubCmdPerform
perform key backend = do perform key backend numcopies = do
success <- Backend.removeKey backend key success <- Backend.removeKey backend key numcopies
if success if success
then return $ Just $ cleanup key then return $ Just $ cleanup key
else return Nothing else return Nothing

View file

@ -30,9 +30,7 @@ start s = do
Just key -> do Just key -> do
showStart "dropunused" s showStart "dropunused" s
backend <- keyBackend key backend <- keyBackend key
-- force drop, even if this is the only copy return $ Just $ Command.Drop.perform key backend (Just 0)
Annex.flagChange "force" $ FlagBool True
return $ Just $ Command.Drop.perform key backend
readUnusedLog :: Annex (M.Map String Key) readUnusedLog :: Annex (M.Map String Key)
readUnusedLog = do readUnusedLog = do

View file

@ -11,19 +11,22 @@ import Command
import qualified Backend import qualified Backend
import Types import Types
import Messages import Messages
import Utility
seek :: [SubCmdSeek] seek :: [SubCmdSeek]
seek = [withAll withFilesInGit start] seek = [withAll (withAttrFilesInGit "git-annex-numcopies") start]
{- Checks a file's backend data for problems. -} {- Checks a file's backend data for problems. -}
start :: SubCmdStartString start :: SubCmdStartAttrFile
start file = isAnnexed file $ \(key, backend) -> do start (file, attr) = isAnnexed file $ \(key, backend) -> do
showStart "fsck" file showStart "fsck" file
return $ Just $ perform key backend return $ Just $ perform key backend numcopies
where
numcopies = readMaybe attr :: Maybe Int
perform :: Key -> Backend -> SubCmdPerform perform :: Key -> Backend -> Maybe Int -> SubCmdPerform
perform key backend = do perform key backend numcopies = do
success <- Backend.fsckKey backend key success <- Backend.fsckKey backend key numcopies
if success if success
then return $ Just $ return True then return $ Just $ return True
else return Nothing else return Nothing

View file

@ -1,29 +0,0 @@
{- git-annex command
-
- Copyright 2010 Joey Hess <joey@kitenet.net>
-
- Licensed under the GNU GPL version 3 or higher.
-}
module Command.FsckFile where
import Command
import qualified Backend
import Types
import Messages
seek :: [SubCmdSeek]
seek = [withFilesInGit start]
{- Checks a file's backend data for problems. -}
start :: SubCmdStartString
start file = isAnnexed file $ \(key, backend) -> do
showStart "fsck" file
return $ Just $ perform key backend
perform :: Key -> Backend -> SubCmdPerform
perform key backend = do
success <- Backend.fsckKey backend key
if success
then return $ Just $ return True
else return Nothing

View file

@ -32,8 +32,7 @@ start file = isAnnexed file $ \(key, backend) -> do
perform :: FilePath -> Key -> Backend -> SubCmdPerform perform :: FilePath -> Key -> Backend -> SubCmdPerform
perform file key backend = do perform file key backend = do
-- force backend to always remove -- force backend to always remove
Annex.flagChange "force" $ FlagBool True ok <- Backend.removeKey backend key (Just 0)
ok <- Backend.removeKey backend key
if ok if ok
then return $ Just $ cleanup file key then return $ Just $ cleanup file key
else return Nothing else return Nothing

View file

@ -72,12 +72,15 @@ data Backend = Backend {
storeFileKey :: FilePath -> Key -> Annex Bool, storeFileKey :: FilePath -> Key -> Annex Bool,
-- retrieves a key's contents to a file -- retrieves a key's contents to a file
retrieveKeyFile :: Key -> FilePath -> Annex Bool, retrieveKeyFile :: Key -> FilePath -> Annex Bool,
-- removes a key -- removes a key, optionally checking that enough copies are stored
removeKey :: Key -> Annex Bool, -- elsewhere
removeKey :: Key -> Maybe Int -> Annex Bool,
-- checks if a backend is storing the content of a key -- checks if a backend is storing the content of a key
hasKey :: Key -> Annex Bool, hasKey :: Key -> Annex Bool,
-- called during fsck to check a key -- called during fsck to check a key
fsckKey :: Key -> Annex Bool -- (second parameter may be the number of copies that there should
-- be of the key)
fsckKey :: Key -> Maybe Int -> Annex Bool
} }
instance Show Backend where instance Show Backend where

View file

@ -12,7 +12,8 @@ module Utility (
relPathDirToDir, relPathDirToDir,
boolSystem, boolSystem,
shellEscape, shellEscape,
unsetFileMode unsetFileMode,
readMaybe
) where ) where
import System.IO import System.IO
@ -125,3 +126,9 @@ unsetFileMode :: FilePath -> FileMode -> IO ()
unsetFileMode f m = do unsetFileMode f m = do
s <- getFileStatus f s <- getFileStatus f
setFileMode f $ fileMode s `intersectFileModes` complement m setFileMode f $ fileMode s `intersectFileModes` complement m
{- Attempts to read a value from a String. -}
readMaybe :: (Read a) => String -> Maybe a
readMaybe s = case reads s of
((x,_):_) -> Just x
_ -> Nothing

2
debian/changelog vendored
View file

@ -1,6 +1,8 @@
git-annex (0.10) UNRELEASED; urgency=low git-annex (0.10) UNRELEASED; urgency=low
* precommit: Optimise to avoid calling git-check-attr more than once. * precommit: Optimise to avoid calling git-check-attr more than once.
* In .gitattributes, the git-annex-numcopies attribute can be used
to control the number of copies to retain of different types of files.
-- Joey Hess <joeyh@debian.org> Sun, 28 Nov 2010 14:19:15 -0400 -- Joey Hess <joeyh@debian.org> Sun, 28 Nov 2010 14:19:15 -0400

View file

@ -3,8 +3,11 @@ your git repository's `.git` directory, not in some external data store.
It's important that data not get lost by an ill-considered `git annex drop` It's important that data not get lost by an ill-considered `git annex drop`
command. So, then using those backends, git-annex can be configured to try command. So, then using those backends, git-annex can be configured to try
to keep N copies of a file's content available across all repositories. By to keep N copies of a file's content available across all repositories.
default, N is 1; it is configured by annex.numcopies.
By default, N is 1; it is configured by annex.numcopies. This default
can be overridden on a per-file-type basis by the git-annex-numcopies
setting in the `.gitattributes` file.
`git annex drop` attempts to check with other git remotes, to check that N `git annex drop` attempts to check with other git remotes, to check that N
copies of the file exist. If enough repositories cannot be verified to have copies of the file exist. If enough repositories cannot be verified to have

View file

@ -275,6 +275,12 @@ but the SHA1 backend for ogg files:
* git-annex-backend=WORM * git-annex-backend=WORM
*.ogg git-annex-backend=SHA1 *.ogg git-annex-backend=SHA1
The numcopies setting can also be configured on a per-file-type basis via
the `git-annex-numcopies` attribute. For example, this makes two copies
be needed for ogg files:
*.ogg git-annex-numcopies=2
# FILES # FILES
These files are used, in your git repository: These files are used, in your git repository: