prevent dropping required content of other file using same content
When two files have the same content, and a required content expression matches one but not the other, dropping the latter file will fail as it would also remove the content of the required file. This will slow down drop (w/o --auto), dropunused, mirror, and move, by one keys db lookup per file. But I did include an optimisation to avoid a double db lookup in the drop --auto / sync --content case. I suspect that dropunused could also use PreferredContentChecked True, but haven't entirely thought it through and it's rarely used with enough files for the optimisation to matter. Sponsored-by: Dartmouth College's Datalad project
This commit is contained in:
Joey Hess
parent
7029ef1c3d
commit
cedc28a783
9 changed files with 105 additions and 60 deletions
|
|
@ -21,8 +21,6 @@ import Annex.Content
|
|||
import Annex.Wanted
|
||||
import Annex.Notification
|
||||
|
||||
import qualified Data.Set as S
|
||||
|
||||
cmd :: Command
|
||||
cmd = withGlobalOptions [jobsOption, jsonOptions, annexedMatchingOptions] $
|
||||
command "drop" SectionCommon
|
||||
|
|
@ -88,31 +86,32 @@ start' o from key afile ai si =
|
|||
checkDropAuto (autoMode o) from afile key $ \numcopies mincopies ->
|
||||
stopUnless wantdrop $
|
||||
case from of
|
||||
Nothing -> startLocal afile ai si numcopies mincopies key []
|
||||
Just remote -> startRemote afile ai si numcopies mincopies key remote
|
||||
Nothing -> startLocal pcc afile ai si numcopies mincopies key []
|
||||
Just remote -> startRemote pcc afile ai si numcopies mincopies key remote
|
||||
where
|
||||
wantdrop
|
||||
| autoMode o = wantDrop False (Remote.uuid <$> from) (Just key) afile Nothing
|
||||
| otherwise = return True
|
||||
pcc = PreferredContentChecked (autoMode o)
|
||||
|
||||
startKeys :: DropOptions -> Maybe Remote -> (SeekInput, Key, ActionItem) -> CommandStart
|
||||
startKeys o from (si, key, ai) = start' o from key (AssociatedFile Nothing) ai si
|
||||
|
||||
startLocal :: AssociatedFile -> ActionItem -> SeekInput -> NumCopies -> MinCopies -> Key -> [VerifiedCopy] -> CommandStart
|
||||
startLocal afile ai si numcopies mincopies key preverified =
|
||||
startLocal :: PreferredContentChecked -> AssociatedFile -> ActionItem -> SeekInput -> NumCopies -> MinCopies -> Key -> [VerifiedCopy] -> CommandStart
|
||||
startLocal pcc afile ai si numcopies mincopies key preverified =
|
||||
starting "drop" (OnlyActionOn key ai) si $
|
||||
performLocal key afile numcopies mincopies preverified
|
||||
performLocal pcc key afile numcopies mincopies preverified
|
||||
|
||||
startRemote :: AssociatedFile -> ActionItem -> SeekInput -> NumCopies -> MinCopies -> Key -> Remote -> CommandStart
|
||||
startRemote afile ai si numcopies mincopies key remote =
|
||||
startRemote :: PreferredContentChecked -> AssociatedFile -> ActionItem -> SeekInput -> NumCopies -> MinCopies -> Key -> Remote -> CommandStart
|
||||
startRemote pcc afile ai si numcopies mincopies key remote =
|
||||
starting ("drop " ++ Remote.name remote) (OnlyActionOn key ai) si $
|
||||
performRemote key afile numcopies mincopies remote
|
||||
performRemote pcc key afile numcopies mincopies remote
|
||||
|
||||
performLocal :: Key -> AssociatedFile -> NumCopies -> MinCopies -> [VerifiedCopy] -> CommandPerform
|
||||
performLocal key afile numcopies mincopies preverified = lockContentForRemoval key fallback $ \contentlock -> do
|
||||
performLocal :: PreferredContentChecked -> Key -> AssociatedFile -> NumCopies -> MinCopies -> [VerifiedCopy] -> CommandPerform
|
||||
performLocal pcc key afile numcopies mincopies preverified = lockContentForRemoval key fallback $ \contentlock -> do
|
||||
u <- getUUID
|
||||
(tocheck, verified) <- verifiableCopies key [u]
|
||||
doDrop u (Just contentlock) key afile numcopies mincopies [] (preverified ++ verified) tocheck
|
||||
doDrop pcc u (Just contentlock) key afile numcopies mincopies [] (preverified ++ verified) tocheck
|
||||
( \proof -> do
|
||||
fastDebug "Command.Drop" $ unwords
|
||||
[ "Dropping from here"
|
||||
|
|
@ -134,12 +133,12 @@ performLocal key afile numcopies mincopies preverified = lockContentForRemoval k
|
|||
-- to be done except for cleaning up.
|
||||
fallback = next $ cleanupLocal key
|
||||
|
||||
performRemote :: Key -> AssociatedFile -> NumCopies -> MinCopies -> Remote -> CommandPerform
|
||||
performRemote key afile numcopies mincopies remote = do
|
||||
performRemote :: PreferredContentChecked -> Key -> AssociatedFile -> NumCopies -> MinCopies -> Remote -> CommandPerform
|
||||
performRemote pcc key afile numcopies mincopies remote = do
|
||||
-- Filter the uuid it's being dropped from out of the lists of
|
||||
-- places assumed to have the key, and places to check.
|
||||
(tocheck, verified) <- verifiableCopies key [uuid]
|
||||
doDrop uuid Nothing key afile numcopies mincopies [uuid] verified tocheck
|
||||
doDrop pcc uuid Nothing key afile numcopies mincopies [uuid] verified tocheck
|
||||
( \proof -> do
|
||||
fastDebug "Command.Drop" $ unwords
|
||||
[ "Dropping from remote"
|
||||
|
|
@ -169,12 +168,11 @@ cleanupRemote key remote ok = do
|
|||
- verify that enough copies of a key exist to allow it to be
|
||||
- safely removed (with no data loss).
|
||||
-
|
||||
- Also checks if it's required content, and refuses to drop if so.
|
||||
-
|
||||
- --force overrides and always allows dropping.
|
||||
-}
|
||||
doDrop
|
||||
:: UUID
|
||||
:: PreferredContentChecked
|
||||
-> UUID
|
||||
-> Maybe ContentRemovalLock
|
||||
-> Key
|
||||
-> AssociatedFile
|
||||
|
|
@ -185,10 +183,10 @@ doDrop
|
|||
-> [UnVerifiedCopy]
|
||||
-> (Maybe SafeDropProof -> CommandPerform, CommandPerform)
|
||||
-> CommandPerform
|
||||
doDrop dropfrom contentlock key afile numcopies mincopies skip preverified check (dropaction, nodropaction) =
|
||||
doDrop pcc dropfrom contentlock key afile numcopies mincopies skip preverified check (dropaction, nodropaction) =
|
||||
ifM (Annex.getState Annex.force)
|
||||
( dropaction Nothing
|
||||
, ifM (checkRequiredContent dropfrom key afile)
|
||||
, ifM (checkRequiredContent pcc dropfrom key afile)
|
||||
( verifyEnoughCopiesToDrop nolocmsg key
|
||||
contentlock numcopies mincopies
|
||||
skip preverified check
|
||||
|
|
@ -203,18 +201,27 @@ doDrop dropfrom contentlock key afile numcopies mincopies skip preverified check
|
|||
showLongNote "(Use --force to override this check, or adjust numcopies.)"
|
||||
a
|
||||
|
||||
checkRequiredContent :: UUID -> Key -> AssociatedFile -> Annex Bool
|
||||
checkRequiredContent u k afile =
|
||||
ifM (isRequiredContent (Just u) S.empty (Just k) afile False)
|
||||
( requiredContent
|
||||
, return True
|
||||
)
|
||||
{- Checking preferred content also checks required content, so when
|
||||
- auto mode causes preferred content to be checked, it's redundant
|
||||
- for checkRequiredContent to separately check required content, and
|
||||
- providing this avoids that extra work. -}
|
||||
newtype PreferredContentChecked = PreferredContentChecked Bool
|
||||
|
||||
requiredContent :: Annex Bool
|
||||
requiredContent = do
|
||||
showLongNote "That file is required content, it cannot be dropped!"
|
||||
showLongNote "(Use --force to override this check, or adjust required content configuration.)"
|
||||
return False
|
||||
checkRequiredContent :: PreferredContentChecked -> UUID -> Key -> AssociatedFile -> Annex Bool
|
||||
checkRequiredContent (PreferredContentChecked True) _ _ _ = return True
|
||||
checkRequiredContent (PreferredContentChecked False) u k afile =
|
||||
checkDrop isRequiredContent False (Just u) (Just k) afile Nothing >>= \case
|
||||
Nothing -> return True
|
||||
Just afile' -> do
|
||||
if afile == afile'
|
||||
then showLongNote "That file is required content. It cannot be dropped!"
|
||||
else showLongNote $ "That file has the same content as another file"
|
||||
++ case afile' of
|
||||
AssociatedFile (Just f) -> " (" ++ fromRawFilePath f ++ "),"
|
||||
AssociatedFile Nothing -> ""
|
||||
++ " which is required content. It cannot be dropped!"
|
||||
showLongNote "(Use --force to override this check, or adjust required content configuration.)"
|
||||
return False
|
||||
|
||||
{- In auto mode, only runs the action if there are enough
|
||||
- copies on other semitrusted repositories. -}
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ perform :: Maybe Remote -> NumCopies -> MinCopies -> Key -> CommandPerform
|
|||
perform from numcopies mincopies key = case from of
|
||||
Just r -> do
|
||||
showAction $ "from " ++ Remote.name r
|
||||
Command.Drop.performRemote key (AssociatedFile Nothing) numcopies mincopies r
|
||||
Command.Drop.performRemote pcc key (AssociatedFile Nothing) numcopies mincopies r
|
||||
Nothing -> ifM (inAnnex key)
|
||||
( droplocal
|
||||
, ifM (objectFileExists key)
|
||||
|
|
@ -63,7 +63,8 @@ perform from numcopies mincopies key = case from of
|
|||
)
|
||||
)
|
||||
where
|
||||
droplocal = Command.Drop.performLocal key (AssociatedFile Nothing) numcopies mincopies []
|
||||
droplocal = Command.Drop.performLocal pcc key (AssociatedFile Nothing) numcopies mincopies []
|
||||
pcc = Command.Drop.PreferredContentChecked False
|
||||
|
||||
performOther :: (Key -> Git.Repo -> RawFilePath) -> Key -> CommandPerform
|
||||
performOther filespec key = do
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ startKey o afile (si, key, ai) = case fromToOptions o of
|
|||
( Command.Move.toStart Command.Move.RemoveNever afile key ai si =<< getParsed r
|
||||
, do
|
||||
(numcopies, mincopies) <- getnummincopies
|
||||
Command.Drop.startRemote afile ai si numcopies mincopies key =<< getParsed r
|
||||
Command.Drop.startRemote pcc afile ai si numcopies mincopies key =<< getParsed r
|
||||
)
|
||||
FromRemote r -> checkFailedTransferDirection ai Download $ do
|
||||
haskey <- flip Remote.hasKey key =<< getParsed r
|
||||
|
|
@ -82,10 +82,11 @@ startKey o afile (si, key, ai) = case fromToOptions o of
|
|||
Right False -> ifM (inAnnex key)
|
||||
( do
|
||||
(numcopies, mincopies) <- getnummincopies
|
||||
Command.Drop.startLocal afile ai si numcopies mincopies key []
|
||||
Command.Drop.startLocal pcc afile ai si numcopies mincopies key []
|
||||
, stop
|
||||
)
|
||||
where
|
||||
getnummincopies = case afile of
|
||||
AssociatedFile Nothing -> (,) <$> getNumCopies <*> getMinCopies
|
||||
AssociatedFile (Just af) -> getFileNumMinCopies af
|
||||
pcc = Command.Drop.PreferredContentChecked False
|
||||
|
|
|
|||
|
|
@ -293,7 +293,7 @@ toHereStart removewhen afile key ai si =
|
|||
- repository reduces the number of copies, and should fail if
|
||||
- that would violate numcopies settings.
|
||||
-
|
||||
- On the other hand, when the destiation repository does not already
|
||||
- On the other hand, when the destination repository does not already
|
||||
- have a copy of a file, it can be dropped without making numcopies
|
||||
- worse, so the move is allowed even if numcopies is not met.
|
||||
-
|
||||
|
|
@ -311,7 +311,7 @@ toHereStart removewhen afile key ai si =
|
|||
-}
|
||||
willDropMakeItWorse :: UUID -> UUID -> DestStartedWithCopy -> Key -> AssociatedFile -> Annex DropCheck
|
||||
willDropMakeItWorse srcuuid destuuid (DestStartedWithCopy deststartedwithcopy) key afile =
|
||||
ifM (Command.Drop.checkRequiredContent srcuuid key afile)
|
||||
ifM (Command.Drop.checkRequiredContent (Command.Drop.PreferredContentChecked False) srcuuid key afile)
|
||||
( if deststartedwithcopy
|
||||
then unlessforced DropCheckNumCopies
|
||||
else ifM checktrustlevel
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue