2011-10-30 03:48:46 +00:00
{- git - annex command seeking
-
- These functions find appropriate files or other things based on
- the values a user passes to a command , and prepare actions operating
- on them .
-
2021-01-04 19:25:28 +00:00
- Copyright 2010 - 2021 Joey Hess < id @ joeyh . name >
2011-10-30 03:48:46 +00:00
-
2019-03-13 19:48:14 +00:00
- Licensed under the GNU AGPL version 3 or higher .
2011-10-30 03:48:46 +00:00
- }
2014-01-26 20:25:55 +00:00
module CmdLine.Seek where
2011-10-30 03:48:46 +00:00
2016-01-20 20:36:33 +00:00
import Annex.Common
2011-10-30 03:48:46 +00:00
import Types.Command
2013-05-25 03:07:26 +00:00
import Types.FileMatcher
2011-10-30 03:48:46 +00:00
import qualified Annex
import qualified Git
import qualified Git.LsFiles as LsFiles
2014-04-17 22:41:24 +00:00
import qualified Git.LsTree as LsTree
2020-07-10 19:11:14 +00:00
import qualified Git.Types as Git
2020-07-13 21:04:02 +00:00
import qualified Git.Ref
2021-03-02 17:46:12 +00:00
import Git.Types ( toTreeItemType , TreeItemType ( .. ) )
2014-04-17 22:41:24 +00:00
import Git.FilePath
2011-10-30 03:48:46 +00:00
import qualified Limit
2015-07-08 21:59:06 +00:00
import CmdLine.GitAnnex.Options
2020-10-30 19:55:59 +00:00
import CmdLine.Action
sped up the --all option by 2x to 16x by using git cat-file --buffer
This assumes that no location log files will have a newline or carriage
return in their name. catObjectStream skips any such files due to
cat-file not supporting them.
Keys have been prevented from containing newlines since 2011,
commit 480495beb4a3422f006ee529df807a20cc944727. If some old repo
had a key with a newline in it, --all will just skip processing that key.
Other things, like .git/annex/unused files certianly assume no newlines in
keys too, and AFAICR, such keys never actually worked.
Carriage return is escaped by preSanitizeKeyName since 2013. WORM keys
generated before that point could perhaps contain a CR. (URL probably not,
http probably doesn't support an URL with a raw CR in it.) So, added
a warning in fsck about such keys. Although, fsck --all will naturally
skip them, so won't be able to warn about them. Not entirely
satisfactory, but I'll bet there are not really any such keys in
existence.
Thanks to Lukey for finding this optimisation.
2020-07-07 17:46:45 +00:00
import Logs
2013-07-03 19:26:59 +00:00
import Logs.Unused
2016-08-03 16:37:12 +00:00
import Types.Transfer
import Logs.Transfer
2021-03-02 17:46:12 +00:00
import Types.Link
2016-08-03 16:37:12 +00:00
import Remote.List
import qualified Remote
2013-08-22 17:57:07 +00:00
import Annex.CatFile
2020-07-13 18:09:08 +00:00
import Git.CatFile
2018-10-19 21:51:25 +00:00
import Annex.CurrentBranch
2015-06-02 18:20:38 +00:00
import Annex.Content
2020-07-10 17:54:52 +00:00
import Annex.Link
2018-09-12 19:20:34 +00:00
import Annex.InodeSentinal
2020-07-10 17:54:52 +00:00
import Annex.Concurrent
2020-09-18 17:33:35 +00:00
import Annex.CheckIgnore
2021-01-04 19:25:28 +00:00
import Annex.Action
sped up the --all option by 2x to 16x by using git cat-file --buffer
This assumes that no location log files will have a newline or carriage
return in their name. catObjectStream skips any such files due to
cat-file not supporting them.
Keys have been prevented from containing newlines since 2011,
commit 480495beb4a3422f006ee529df807a20cc944727. If some old repo
had a key with a newline in it, --all will just skip processing that key.
Other things, like .git/annex/unused files certianly assume no newlines in
keys too, and AFAICR, such keys never actually worked.
Carriage return is escaped by preSanitizeKeyName since 2013. WORM keys
generated before that point could perhaps contain a CR. (URL probably not,
http probably doesn't support an URL with a raw CR in it.) So, added
a warning in fsck about such keys. Although, fsck --all will naturally
skip them, so won't be able to warn about them. Not entirely
satisfactory, but I'll bet there are not really any such keys in
existence.
Thanks to Lukey for finding this optimisation.
2020-07-07 17:46:45 +00:00
import qualified Annex.Branch
2018-09-12 19:20:34 +00:00
import qualified Database.Keys
2019-12-06 19:37:12 +00:00
import qualified Utility.RawFilePath as R
2020-07-10 17:54:52 +00:00
import Utility.Tuple
2021-01-04 19:25:28 +00:00
import Utility.HumanTime
2020-07-10 17:54:52 +00:00
import Control.Concurrent.Async
import System.Posix.Types
2020-10-13 20:31:01 +00:00
import Data.IORef
2021-01-04 19:25:28 +00:00
import Data.Time.Clock.POSIX
2020-11-04 18:20:37 +00:00
import qualified System.FilePath.ByteString as P
2011-11-08 19:34:10 +00:00
2020-07-13 21:04:02 +00:00
data AnnexedFileSeeker = AnnexedFileSeeker
2020-09-14 20:49:33 +00:00
{ startAction :: SeekInput -> RawFilePath -> Key -> CommandStart
2020-07-13 21:04:02 +00:00
, checkContentPresent :: Maybe Bool
, usesLocationLog :: Bool
}
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
withFilesInGitAnnex :: WarnUnmatchWhen -> AnnexedFileSeeker -> WorkTreeItems -> CommandSeek
2020-07-10 18:17:35 +00:00
withFilesInGitAnnex ww a l = seekFilteredKeys a $
2020-07-10 17:54:52 +00:00
seekHelper fst3 ww LsFiles . inRepoDetails l
2011-10-30 03:48:46 +00:00
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
withFilesInGitAnnexNonRecursive :: WarnUnmatchWhen -> String -> AnnexedFileSeeker -> WorkTreeItems -> CommandSeek
withFilesInGitAnnexNonRecursive ww needforce a ( WorkTreeItems l ) = ifM ( Annex . getState Annex . force )
( withFilesInGitAnnex ww a ( WorkTreeItems l )
2017-10-16 18:10:03 +00:00
, if null l
2016-11-16 01:29:54 +00:00
then giveup needforce
2020-07-10 19:40:06 +00:00
else seekFilteredKeys a ( getfiles [] l )
2015-02-10 20:06:53 +00:00
)
where
2020-09-25 15:38:42 +00:00
getfiles c [] = return ( reverse c , pure True )
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
getfiles c ( p : ps ) = do
2020-05-28 19:55:17 +00:00
os <- seekOptions ww
2020-07-10 19:40:06 +00:00
( fs , cleanup ) <- inRepo $ LsFiles . inRepoDetails os [ toRawFilePath p ]
2020-09-25 15:38:42 +00:00
r <- case fs of
2015-02-10 20:06:53 +00:00
[ f ] -> do
void $ liftIO $ cleanup
2020-09-25 15:38:42 +00:00
fst <$> getfiles ( ( SeekInput [ p ] , f ) : c ) ps
2015-02-10 20:06:53 +00:00
[] -> do
void $ liftIO $ cleanup
2020-09-25 15:38:42 +00:00
fst <$> getfiles c ps
_ -> do
void $ liftIO $ cleanup
giveup needforce
return ( r , pure True )
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
withFilesInGitAnnexNonRecursive _ _ _ NoWorkTreeItems = noop
2015-02-10 20:06:53 +00:00
2020-09-18 17:33:35 +00:00
withFilesNotInGit :: CheckGitIgnore -> WarnUnmatchWhen -> ( ( SeekInput , RawFilePath ) -> CommandSeek ) -> WorkTreeItems -> CommandSeek
withFilesNotInGit ( CheckGitIgnore ci ) ww a l = do
2020-09-14 20:49:33 +00:00
force <- Annex . getState Annex . force
2020-09-18 17:33:35 +00:00
let include_ignored = force || not ci
2020-09-25 15:38:42 +00:00
seekFiltered ( const ( pure True ) ) a $
2020-09-18 17:33:35 +00:00
seekHelper id ww ( const $ LsFiles . notInRepo [] include_ignored ) l
2011-10-30 03:48:46 +00:00
2020-11-04 18:20:37 +00:00
withPathContents :: ( ( RawFilePath , RawFilePath ) -> CommandSeek ) -> CmdParams -> CommandSeek
2015-02-06 19:58:06 +00:00
withPathContents a params = do
matcher <- Limit . getMatcher
2021-01-04 19:25:28 +00:00
checktimelimit <- mkCheckTimeLimit
go matcher checktimelimit params []
2012-11-11 04:51:07 +00:00
where
2021-01-04 19:25:28 +00:00
go _ _ [] [] = return ()
go matcher checktimelimit ( p : ps ) [] =
go matcher checktimelimit ps =<< liftIO ( get p )
go matcher checktimelimit ps ( f : fs ) = checktimelimit noop $ do
whenM ( checkmatch matcher f ) $
a f
go matcher checktimelimit ps fs
2012-11-11 04:51:07 +00:00
get p = ifM ( isDirectory <$> getFileStatus p )
2020-11-04 18:20:37 +00:00
( map ( \ f ->
let f' = toRawFilePath f
in ( f' , P . makeRelative ( P . takeDirectory ( P . dropTrailingPathSeparator p' ) ) f' ) )
2013-12-18 19:05:29 +00:00
<$> dirContentsRecursiveSkipping ( " .git " ` isSuffixOf ` ) True p
2020-11-04 18:20:37 +00:00
, return [ ( p' , P . takeFileName p' ) ]
2012-11-11 04:51:07 +00:00
)
2020-11-04 18:20:37 +00:00
where
p' = toRawFilePath p
2015-02-06 19:58:06 +00:00
checkmatch matcher ( f , relf ) = matcher $ MatchingFile $ FileInfo
2021-03-01 20:34:40 +00:00
{ contentFile = f
2020-11-04 18:20:37 +00:00
, matchFile = relf
2020-12-14 21:42:02 +00:00
, matchKey = Nothing
2015-02-06 19:58:06 +00:00
}
2012-05-31 23:47:18 +00:00
2018-10-01 18:12:06 +00:00
withWords :: ( [ String ] -> CommandSeek ) -> CmdParams -> CommandSeek
2020-07-10 17:54:52 +00:00
withWords a params = a params
2011-10-30 03:48:46 +00:00
2018-10-01 18:12:06 +00:00
withStrings :: ( String -> CommandSeek ) -> CmdParams -> CommandSeek
2020-07-10 17:54:52 +00:00
withStrings a params = sequence_ $ map a params
2011-10-30 03:48:46 +00:00
2020-09-14 20:49:33 +00:00
withPairs :: ( ( SeekInput , ( String , String ) ) -> CommandSeek ) -> CmdParams -> CommandSeek
withPairs a params = sequence_ $
map ( \ p @ ( x , y ) -> a ( SeekInput [ x , y ] , p ) ) ( pairs [] params )
2012-11-11 04:51:07 +00:00
where
pairs c [] = reverse c
pairs c ( x : y : xs ) = pairs ( ( x , y ) : c ) xs
2016-11-16 01:29:54 +00:00
pairs _ _ = giveup " expected pairs "
2012-02-16 20:36:35 +00:00
2020-09-14 20:49:33 +00:00
withFilesToBeCommitted :: ( ( SeekInput , RawFilePath ) -> CommandSeek ) -> WorkTreeItems -> CommandSeek
2020-09-25 15:38:42 +00:00
withFilesToBeCommitted a l = seekFiltered ( const ( pure True ) ) a $
2020-07-10 17:54:52 +00:00
seekHelper id WarnUnmatchWorkTreeItems ( const LsFiles . stagedNotDeleted ) l
2011-10-30 03:48:46 +00:00
2019-08-30 17:54:57 +00:00
{- unlocked pointer files that are staged, and whose content has not been
2018-09-12 19:20:34 +00:00
- modified - }
2020-09-14 20:49:33 +00:00
withUnmodifiedUnlockedPointers :: WarnUnmatchWhen -> ( ( SeekInput , RawFilePath ) -> CommandSeek ) -> WorkTreeItems -> CommandSeek
2020-09-25 15:38:42 +00:00
withUnmodifiedUnlockedPointers ww a l =
seekFiltered ( isUnmodifiedUnlocked . snd ) a $
seekHelper id ww ( const LsFiles . typeChangedStaged ) l
2018-09-12 17:53:03 +00:00
2019-11-25 20:18:19 +00:00
isUnmodifiedUnlocked :: RawFilePath -> Annex Bool
2019-08-30 17:54:57 +00:00
isUnmodifiedUnlocked f = catKeyFile f >>= \ case
2018-09-12 19:20:34 +00:00
Nothing -> return False
2019-12-11 18:12:22 +00:00
Just k -> sameInodeCache f =<< Database . Keys . getInodeCaches k
2018-09-12 17:53:03 +00:00
2013-02-20 18:12:55 +00:00
{- Finds files that may be modified. -}
2020-09-14 20:49:33 +00:00
withFilesMaybeModified :: WarnUnmatchWhen -> ( ( SeekInput , RawFilePath ) -> CommandSeek ) -> WorkTreeItems -> CommandSeek
2020-09-25 15:38:42 +00:00
withFilesMaybeModified ww a params = seekFiltered ( const ( pure True ) ) a $
2020-07-10 17:54:52 +00:00
seekHelper id ww LsFiles . modified params
2013-02-20 18:12:55 +00:00
2020-09-14 20:49:33 +00:00
withKeys :: ( ( SeekInput , Key ) -> CommandSeek ) -> CmdParams -> CommandSeek
withKeys a ls = sequence_ $ map ( \ l -> a ( SeekInput [ l ] , parse l ) ) ls
2012-11-11 04:51:07 +00:00
where
2019-01-14 17:17:47 +00:00
parse p = fromMaybe ( giveup " bad key " ) $ deserializeKey p
2011-10-30 03:48:46 +00:00
2018-10-01 18:12:06 +00:00
withNothing :: CommandSeek -> CmdParams -> CommandSeek
withNothing a [] = a
2016-11-16 01:29:54 +00:00
withNothing _ _ = giveup " This command takes no parameters. "
2011-10-30 03:48:46 +00:00
2016-08-03 16:37:12 +00:00
{- Handles the - - all, - - branch, - - unused, - - failed, - - key, and
- --incomplete options, which specify particular keys to run an
- action on .
2013-07-03 19:26:59 +00:00
-
2015-06-02 18:20:38 +00:00
- In a bare repo , --all is the default.
2013-07-03 19:26:59 +00:00
-
2015-06-02 18:20:38 +00:00
- Otherwise falls back to a regular CommandSeek action on
2018-10-01 18:12:06 +00:00
- whatever params were passed .
- }
2016-07-20 19:22:55 +00:00
withKeyOptions
:: Maybe KeyOptions
-> Bool
2020-07-24 16:05:28 +00:00
-> AnnexedFileSeeker
2020-09-14 20:49:33 +00:00
-> ( ( SeekInput , Key , ActionItem ) -> CommandSeek )
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
-> ( WorkTreeItems -> CommandSeek )
-> WorkTreeItems
2016-07-20 19:22:55 +00:00
-> CommandSeek
2020-07-24 16:05:28 +00:00
withKeyOptions ko auto seeker keyaction = withKeyOptions' ko auto mkkeyaction
2015-06-16 20:50:03 +00:00
where
--branch, stage 1
Added --branch option to copy, drop, fsck, get, metadata, mirror, move, and
whereis commands. This option makes git-annex operate on files that are
included in a specified branch (or other treeish).
The names of the files from the branch that are being operated on are not
displayed yet; only the keys. Displaying the filenames will need changes
to every affected command.
Also, note that --branch can be specified repeatedly. This is not really
documented, but seemed worth supporting, especially since we may later want
the ability to operate on all branches matching a refspec. However, when
operating on two branches that contain the same key, that key will be
operated on twice.
2016-07-20 16:05:22 +00:00
mkkeyaction = do
matcher <- Limit . getMatcher
2021-03-02 17:46:12 +00:00
return $ \ lt v @ ( _si , k , ai ) -> checkseeker k $
support findred and --branch with file matching options
* findref: Support file matching options: --include, --exclude,
--want-get, --want-drop, --largerthan, --smallerthan, --accessedwithin
* Commands supporting --branch now apply file matching options --include,
--exclude, --want-get, --want-drop to filenames from the branch.
Previously, combining --branch with those would fail to match anything.
* add, import, findref: Support --time-limit.
This commit was sponsored by Jake Vosloo on Patreon.
2018-12-09 17:38:35 +00:00
let i = case ai of
2019-06-06 16:53:24 +00:00
ActionItemBranchFilePath ( BranchFilePath _ topf ) _ ->
2021-03-02 16:47:23 +00:00
ProvidedInfo
{ providedFilePath = Just $
getTopFilePath topf
, providedKey = Just k
, providedFileSize = Nothing
, providedMimeType = Nothing
, providedMimeEncoding = Nothing
2021-03-02 17:46:12 +00:00
, providedLinkType = lt
2021-03-02 16:47:23 +00:00
}
_ -> ProvidedInfo
{ providedFilePath = Nothing
, providedKey = Just k
, providedFileSize = Nothing
, providedMimeType = Nothing
, providedMimeEncoding = Nothing
2021-03-02 17:46:12 +00:00
, providedLinkType = lt
2021-03-02 16:47:23 +00:00
}
in whenM ( matcher ( MatchingInfo i ) ) $
2018-10-01 18:12:06 +00:00
keyaction v
2020-07-24 16:05:28 +00:00
checkseeker k a = case checkContentPresent seeker of
Nothing -> a
Just v -> do
present <- inAnnex k
when ( present == v ) a
2016-07-20 19:22:55 +00:00
withKeyOptions'
:: Maybe KeyOptions
-> Bool
2021-03-02 17:46:12 +00:00
-> Annex ( Maybe LinkType -> ( SeekInput , Key , ActionItem ) -> Annex () )
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
-> ( WorkTreeItems -> CommandSeek )
-> WorkTreeItems
2016-07-20 19:22:55 +00:00
-> CommandSeek
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
withKeyOptions' ko auto mkkeyaction fallbackaction worktreeitems = do
2013-07-03 19:26:59 +00:00
bare <- fromRepo Git . repoIsLocalBare
2014-01-26 18:59:47 +00:00
when ( auto && bare ) $
2016-11-16 01:29:54 +00:00
giveup " Cannot use --auto in a bare repository "
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
case ( noworktreeitems , ko ) of
2015-07-09 16:44:03 +00:00
( True , Nothing )
2021-03-01 20:10:42 +00:00
| bare -> nofilename $ noauto runallkeys
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
| otherwise -> fallbackaction worktreeitems
( False , Nothing ) -> fallbackaction worktreeitems
2021-03-01 20:10:42 +00:00
( True , Just WantAllKeys ) -> nofilename $ noauto runallkeys
( True , Just WantUnusedKeys ) -> nofilename $ noauto $ runkeyaction unusedKeys'
( True , Just WantFailedTransfers ) -> nofilename $ noauto runfailedtransfers
( True , Just ( WantSpecificKey k ) ) -> nofilename $ noauto $ runkeyaction ( return [ k ] )
( True , Just WantIncompleteKeys ) -> nofilename $ noauto $ runkeyaction incompletekeys
2021-03-02 17:46:12 +00:00
( True , Just ( WantBranchKeys bs ) ) -> noauto $ runbranchkeys bs
2016-11-16 01:29:54 +00:00
( False , Just _ ) -> giveup " Can only specify one of file names, --all, --branch, --unused, --failed, --key, or --incomplete "
2013-07-03 17:02:42 +00:00
where
--branch, stage 1
Added --branch option to copy, drop, fsck, get, metadata, mirror, move, and
whereis commands. This option makes git-annex operate on files that are
included in a specified branch (or other treeish).
The names of the files from the branch that are being operated on are not
displayed yet; only the keys. Displaying the filenames will need changes
to every affected command.
Also, note that --branch can be specified repeatedly. This is not really
documented, but seemed worth supporting, especially since we may later want
the ability to operate on all branches matching a refspec. However, when
operating on two branches that contain the same key, that key will be
operated on twice.
2016-07-20 16:05:22 +00:00
noauto a
2016-11-16 01:29:54 +00:00
| auto = giveup " Cannot use --auto with --all or --branch or --unused or --key or --incomplete "
--branch, stage 1
Added --branch option to copy, drop, fsck, get, metadata, mirror, move, and
whereis commands. This option makes git-annex operate on files that are
included in a specified branch (or other treeish).
The names of the files from the branch that are being operated on are not
displayed yet; only the keys. Displaying the filenames will need changes
to every affected command.
Also, note that --branch can be specified repeatedly. This is not really
documented, but seemed worth supporting, especially since we may later want
the ability to operate on all branches matching a refspec. However, when
operating on two branches that contain the same key, that key will be
operated on twice.
2016-07-20 16:05:22 +00:00
| otherwise = a
2021-03-01 20:10:42 +00:00
nofilename a = ifM ( Limit . introspect matchNeedsFileName )
( do
bare <- fromRepo Git . repoIsLocalBare
if bare
then giveup " Cannot use options that match on file names in a bare repository. "
else giveup " Cannot use --all or --unused or --key or --incomplete with options that match on file names. "
, a
)
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
noworktreeitems = case worktreeitems of
WorkTreeItems [] -> True
WorkTreeItems _ -> False
NoWorkTreeItems -> False
2015-06-02 18:20:38 +00:00
incompletekeys = staleKeysPrune gitAnnexTmpObjectDir True
sped up the --all option by 2x to 16x by using git cat-file --buffer
This assumes that no location log files will have a newline or carriage
return in their name. catObjectStream skips any such files due to
cat-file not supporting them.
Keys have been prevented from containing newlines since 2011,
commit 480495beb4a3422f006ee529df807a20cc944727. If some old repo
had a key with a newline in it, --all will just skip processing that key.
Other things, like .git/annex/unused files certianly assume no newlines in
keys too, and AFAICR, such keys never actually worked.
Carriage return is escaped by preSanitizeKeyName since 2013. WORM keys
generated before that point could perhaps contain a CR. (URL probably not,
http probably doesn't support an URL with a raw CR in it.) So, added
a warning in fsck about such keys. Although, fsck --all will naturally
skip them, so won't be able to warn about them. Not entirely
satisfactory, but I'll bet there are not really any such keys in
existence.
Thanks to Lukey for finding this optimisation.
2020-07-07 17:46:45 +00:00
-- List all location log files on the git-annex branch,
-- and use those to get keys. Pass through cat-file
-- to get the contents of the location logs, and pre-cache
-- those. This significantly speeds up typical operations
-- that need to look at the location log for each key.
runallkeys = do
2021-01-04 19:25:28 +00:00
checktimelimit <- mkCheckTimeLimit
sped up the --all option by 2x to 16x by using git cat-file --buffer
This assumes that no location log files will have a newline or carriage
return in their name. catObjectStream skips any such files due to
cat-file not supporting them.
Keys have been prevented from containing newlines since 2011,
commit 480495beb4a3422f006ee529df807a20cc944727. If some old repo
had a key with a newline in it, --all will just skip processing that key.
Other things, like .git/annex/unused files certianly assume no newlines in
keys too, and AFAICR, such keys never actually worked.
Carriage return is escaped by preSanitizeKeyName since 2013. WORM keys
generated before that point could perhaps contain a CR. (URL probably not,
http probably doesn't support an URL with a raw CR in it.) So, added
a warning in fsck about such keys. Although, fsck --all will naturally
skip them, so won't be able to warn about them. Not entirely
satisfactory, but I'll bet there are not really any such keys in
existence.
Thanks to Lukey for finding this optimisation.
2020-07-07 17:46:45 +00:00
keyaction <- mkkeyaction
config <- Annex . getGitConfig
2021-04-21 18:19:58 +00:00
let getk = locationLogFileKey config
2021-01-04 19:25:28 +00:00
let discard reader = reader >>= \ case
Nothing -> noop
Just _ -> discard reader
2021-04-21 19:40:32 +00:00
let go reader = reader >>= \ case
2021-04-21 18:19:58 +00:00
Just ( k , f , content ) -> checktimelimit ( discard reader ) $ do
2021-04-21 18:02:15 +00:00
maybe noop ( Annex . Branch . precache f ) content
2021-03-02 17:46:12 +00:00
keyaction Nothing ( SeekInput [] , k , mkActionItem k )
sped up the --all option by 2x to 16x by using git cat-file --buffer
This assumes that no location log files will have a newline or carriage
return in their name. catObjectStream skips any such files due to
cat-file not supporting them.
Keys have been prevented from containing newlines since 2011,
commit 480495beb4a3422f006ee529df807a20cc944727. If some old repo
had a key with a newline in it, --all will just skip processing that key.
Other things, like .git/annex/unused files certianly assume no newlines in
keys too, and AFAICR, such keys never actually worked.
Carriage return is escaped by preSanitizeKeyName since 2013. WORM keys
generated before that point could perhaps contain a CR. (URL probably not,
http probably doesn't support an URL with a raw CR in it.) So, added
a warning in fsck about such keys. Although, fsck --all will naturally
skip them, so won't be able to warn about them. Not entirely
satisfactory, but I'll bet there are not really any such keys in
existence.
Thanks to Lukey for finding this optimisation.
2020-07-07 17:46:45 +00:00
go reader
2021-04-21 18:19:58 +00:00
Nothing -> return ()
2021-12-27 18:30:51 +00:00
Annex . Branch . overBranchFileContents getk go >>= \ case
Just r -> return r
Nothing -> giveup " This repository is read-only, and there are unmerged git-annex branches, which prevents operating on all keys. (Set annex.merge-annex-branches to false to ignore the unmerged git-annex branches.) "
sped up the --all option by 2x to 16x by using git cat-file --buffer
This assumes that no location log files will have a newline or carriage
return in their name. catObjectStream skips any such files due to
cat-file not supporting them.
Keys have been prevented from containing newlines since 2011,
commit 480495beb4a3422f006ee529df807a20cc944727. If some old repo
had a key with a newline in it, --all will just skip processing that key.
Other things, like .git/annex/unused files certianly assume no newlines in
keys too, and AFAICR, such keys never actually worked.
Carriage return is escaped by preSanitizeKeyName since 2013. WORM keys
generated before that point could perhaps contain a CR. (URL probably not,
http probably doesn't support an URL with a raw CR in it.) So, added
a warning in fsck about such keys. Although, fsck --all will naturally
skip them, so won't be able to warn about them. Not entirely
satisfactory, but I'll bet there are not really any such keys in
existence.
Thanks to Lukey for finding this optimisation.
2020-07-07 17:46:45 +00:00
runkeyaction getks = do
--branch, stage 1
Added --branch option to copy, drop, fsck, get, metadata, mirror, move, and
whereis commands. This option makes git-annex operate on files that are
included in a specified branch (or other treeish).
The names of the files from the branch that are being operated on are not
displayed yet; only the keys. Displaying the filenames will need changes
to every affected command.
Also, note that --branch can be specified repeatedly. This is not really
documented, but seemed worth supporting, especially since we may later want
the ability to operate on all branches matching a refspec. However, when
operating on two branches that contain the same key, that key will be
operated on twice.
2016-07-20 16:05:22 +00:00
keyaction <- mkkeyaction
2016-07-20 19:22:55 +00:00
ks <- getks
2021-03-02 17:46:12 +00:00
forM_ ks $ \ k -> keyaction Nothing ( SeekInput [] , k , mkActionItem k )
sped up the --all option by 2x to 16x by using git cat-file --buffer
This assumes that no location log files will have a newline or carriage
return in their name. catObjectStream skips any such files due to
cat-file not supporting them.
Keys have been prevented from containing newlines since 2011,
commit 480495beb4a3422f006ee529df807a20cc944727. If some old repo
had a key with a newline in it, --all will just skip processing that key.
Other things, like .git/annex/unused files certianly assume no newlines in
keys too, and AFAICR, such keys never actually worked.
Carriage return is escaped by preSanitizeKeyName since 2013. WORM keys
generated before that point could perhaps contain a CR. (URL probably not,
http probably doesn't support an URL with a raw CR in it.) So, added
a warning in fsck about such keys. Although, fsck --all will naturally
skip them, so won't be able to warn about them. Not entirely
satisfactory, but I'll bet there are not really any such keys in
existence.
Thanks to Lukey for finding this optimisation.
2020-07-07 17:46:45 +00:00
--branch, stage 1
Added --branch option to copy, drop, fsck, get, metadata, mirror, move, and
whereis commands. This option makes git-annex operate on files that are
included in a specified branch (or other treeish).
The names of the files from the branch that are being operated on are not
displayed yet; only the keys. Displaying the filenames will need changes
to every affected command.
Also, note that --branch can be specified repeatedly. This is not really
documented, but seemed worth supporting, especially since we may later want
the ability to operate on all branches matching a refspec. However, when
operating on two branches that contain the same key, that key will be
operated on twice.
2016-07-20 16:05:22 +00:00
runbranchkeys bs = do
keyaction <- mkkeyaction
forM_ bs $ \ b -> do
2021-03-23 16:44:29 +00:00
( l , cleanup ) <- inRepo $ LsTree . lsTree LsTree . LsTreeRecursive ( LsTree . LsTreeLong False ) b
2019-06-06 16:53:24 +00:00
forM_ l $ \ i -> catKey ( LsTree . sha i ) >>= \ case
Just k ->
let bfp = mkActionItem ( BranchFilePath b ( LsTree . file i ) , k )
2021-03-02 17:46:12 +00:00
lt = case toTreeItemType ( LsTree . mode i ) of
Just TreeSymlink -> Just LockedLink
Just TreeFile -> Just UnlockedLink
Just TreeExecutable -> Just UnlockedLink
_ -> Nothing
in keyaction lt ( SeekInput [] , k , bfp )
Nothing -> noop
2016-07-20 17:44:33 +00:00
unlessM ( liftIO cleanup ) $
error ( " git ls-tree " ++ Git . fromRef b ++ " failed " )
sped up the --all option by 2x to 16x by using git cat-file --buffer
This assumes that no location log files will have a newline or carriage
return in their name. catObjectStream skips any such files due to
cat-file not supporting them.
Keys have been prevented from containing newlines since 2011,
commit 480495beb4a3422f006ee529df807a20cc944727. If some old repo
had a key with a newline in it, --all will just skip processing that key.
Other things, like .git/annex/unused files certianly assume no newlines in
keys too, and AFAICR, such keys never actually worked.
Carriage return is escaped by preSanitizeKeyName since 2013. WORM keys
generated before that point could perhaps contain a CR. (URL probably not,
http probably doesn't support an URL with a raw CR in it.) So, added
a warning in fsck about such keys. Although, fsck --all will naturally
skip them, so won't be able to warn about them. Not entirely
satisfactory, but I'll bet there are not really any such keys in
existence.
Thanks to Lukey for finding this optimisation.
2020-07-07 17:46:45 +00:00
2016-08-03 16:37:12 +00:00
runfailedtransfers = do
keyaction <- mkkeyaction
rs <- remoteList
ts <- concat <$> mapM ( getFailedTransfers . Remote . uuid ) rs
forM_ ts $ \ ( t , i ) ->
2021-03-02 17:46:12 +00:00
keyaction Nothing ( SeekInput [] , transferKey t , mkActionItem ( t , i ) )
2011-10-30 03:48:46 +00:00
2020-09-25 15:38:42 +00:00
seekFiltered :: ( ( SeekInput , RawFilePath ) -> Annex Bool ) -> ( ( SeekInput , RawFilePath ) -> CommandSeek ) -> Annex ( [ ( SeekInput , RawFilePath ) ] , IO Bool ) -> Annex ()
seekFiltered prefilter a listfs = do
2011-10-30 03:48:46 +00:00
matcher <- Limit . getMatcher
2021-01-04 19:25:28 +00:00
checktimelimit <- mkCheckTimeLimit
2020-09-25 15:38:42 +00:00
( fs , cleanup ) <- listfs
2021-01-04 19:25:28 +00:00
go matcher checktimelimit fs
2020-09-25 15:38:42 +00:00
liftIO $ void cleanup
2012-11-11 04:51:07 +00:00
where
2021-01-04 19:25:28 +00:00
go _ _ [] = return ()
go matcher checktimelimit ( v @ ( _si , f ) : rest ) = checktimelimit noop $ do
2020-09-25 15:38:42 +00:00
whenM ( prefilter v ) $
2021-03-01 20:34:40 +00:00
whenM ( matcher $ MatchingFile $ FileInfo f f Nothing ) $
2020-09-25 15:38:42 +00:00
a v
2021-01-04 19:25:28 +00:00
go matcher checktimelimit rest
2011-10-30 03:48:46 +00:00
2020-09-24 21:59:05 +00:00
data MatcherInfo = MatcherInfo
{ matcherAction :: MatchInfo -> Annex Bool
, matcherNeedsFileName :: Bool
, matcherNeedsKey :: Bool
, matcherNeedsLocationLog :: Bool
}
checkMatcherWhen :: MatcherInfo -> Bool -> MatchInfo -> Annex () -> Annex ()
checkMatcherWhen mi c i a
| c = whenM ( matcherAction mi i ) a
| otherwise = a
2020-07-13 21:04:02 +00:00
-- This is significantly faster than using lookupKey after seekFiltered,
-- because of the way data is streamed through git cat-file.
--
-- It can also precache location logs using the same efficient streaming.
2020-09-25 15:38:42 +00:00
seekFilteredKeys :: AnnexedFileSeeker -> Annex ( [ ( SeekInput , ( RawFilePath , Git . Sha , FileMode ) ) ] , IO Bool ) -> Annex ()
2020-07-13 21:04:02 +00:00
seekFilteredKeys seeker listfs = do
2020-07-10 17:54:52 +00:00
g <- Annex . gitRepo
2020-09-24 21:59:05 +00:00
mi <- MatcherInfo
<$> Limit . getMatcher
<*> Limit . introspect matchNeedsFileName
<*> Limit . introspect matchNeedsKey
<*> Limit . introspect matchNeedsLocationLog
2020-07-13 21:04:02 +00:00
config <- Annex . getGitConfig
2020-09-25 15:38:42 +00:00
( l , cleanup ) <- listfs
2021-01-04 19:25:28 +00:00
checktimelimit <- mkCheckTimeLimit
2020-07-13 18:09:08 +00:00
catObjectMetaDataStream g $ \ mdfeeder mdcloser mdreader ->
2020-07-13 21:04:02 +00:00
catObjectStream g $ \ ofeeder ocloser oreader -> do
2020-07-13 18:09:08 +00:00
processertid <- liftIO . async =<< forkState
2020-09-24 21:59:05 +00:00
( process mi ofeeder mdfeeder mdcloser False l )
2020-07-13 18:09:08 +00:00
mdprocessertid <- liftIO . async =<< forkState
2020-09-24 21:59:05 +00:00
( mdprocess mi mdreader ofeeder ocloser )
2021-12-27 18:08:50 +00:00
ifM ( precachell mi )
( catObjectStream g $ \ lfeeder lcloser lreader -> do
2020-07-13 21:04:02 +00:00
precachertid <- liftIO . async =<< forkState
2020-09-24 21:59:05 +00:00
( precacher mi config oreader lfeeder lcloser )
2021-01-04 19:25:28 +00:00
precachefinisher mi lreader checktimelimit
2020-07-13 21:04:02 +00:00
join ( liftIO ( wait precachertid ) )
2021-12-27 18:08:50 +00:00
, finisher mi oreader checktimelimit
)
2020-07-13 18:09:08 +00:00
join ( liftIO ( wait mdprocessertid ) )
join ( liftIO ( wait processertid ) )
2020-09-25 15:38:42 +00:00
liftIO $ void cleanup
2020-07-10 17:54:52 +00:00
where
2021-01-04 19:25:28 +00:00
finisher mi oreader checktimelimit = liftIO oreader >>= \ case
2021-04-21 19:40:32 +00:00
Just ( ( si , f ) , content ) -> checktimelimit ( liftIO discard ) $ do
2020-09-24 21:59:05 +00:00
keyaction f mi content $
commandAction . startAction seeker si f
2021-01-04 19:25:28 +00:00
finisher mi oreader checktimelimit
2020-07-13 21:04:02 +00:00
Nothing -> return ()
2021-01-04 19:25:28 +00:00
where
discard = oreader >>= \ case
Nothing -> return ()
Just _ -> discard
2020-07-13 21:04:02 +00:00
2021-01-04 19:25:28 +00:00
precachefinisher mi lreader checktimelimit = liftIO lreader >>= \ case
2021-04-21 19:40:32 +00:00
Just ( ( logf , ( si , f ) , k ) , logcontent ) -> checktimelimit ( liftIO discard ) $ do
2021-04-21 18:02:15 +00:00
maybe noop ( Annex . Branch . precache logf ) logcontent
2020-09-24 21:59:05 +00:00
checkMatcherWhen mi
( matcherNeedsLocationLog mi && not ( matcherNeedsFileName mi ) )
2021-03-01 20:34:40 +00:00
( MatchingFile $ FileInfo f f ( Just k ) )
2020-09-24 21:59:05 +00:00
( commandAction $ startAction seeker si f k )
2021-01-04 19:25:28 +00:00
precachefinisher mi lreader checktimelimit
2020-07-10 19:11:14 +00:00
Nothing -> return ()
2021-01-04 19:25:28 +00:00
where
discard = lreader >>= \ case
Nothing -> return ()
Just _ -> discard
2020-07-10 19:11:14 +00:00
2020-09-24 21:59:05 +00:00
precacher mi config oreader lfeeder lcloser = liftIO oreader >>= \ case
2020-09-14 20:49:33 +00:00
Just ( ( si , f ) , content ) -> do
2020-09-24 21:59:05 +00:00
keyaction f mi content $ \ k ->
let logf = locationLogFile config k
ref = Git . Ref . branchFileRef Annex . Branch . fullname logf
in liftIO $ lfeeder ( ( logf , ( si , f ) , k ) , ref )
precacher mi config oreader lfeeder lcloser
2020-07-13 21:04:02 +00:00
Nothing -> liftIO $ void lcloser
2020-09-24 21:59:05 +00:00
feedmatches mi ofeeder si f sha = checkMatcherWhen mi
-- When the matcher needs a key or location log
-- (and does not need a worktree filename), it will be
-- checked later, to avoid a slow lookup here.
( not ( ( matcherNeedsKey mi || matcherNeedsLocationLog mi )
&& not ( matcherNeedsFileName mi ) ) )
2021-03-01 20:34:40 +00:00
( MatchingFile $ FileInfo f f Nothing )
2020-09-24 21:59:05 +00:00
( liftIO $ ofeeder ( ( si , f ) , sha ) )
keyaction f mi content a =
case parseLinkTargetOrPointerLazy =<< content of
Just k -> checkMatcherWhen mi
( matcherNeedsKey mi && not ( matcherNeedsFileName mi || matcherNeedsLocationLog mi ) )
2021-03-01 20:34:40 +00:00
( MatchingFile $ FileInfo f f ( Just k ) )
2020-09-24 21:59:05 +00:00
( checkpresence k ( a k ) )
Nothing -> noop
checkpresence k cont = case checkContentPresent seeker of
Just v -> do
present <- inAnnex k
when ( present == v ) cont
Nothing -> cont
2020-07-10 19:11:14 +00:00
2020-09-24 21:59:05 +00:00
process mi ofeeder mdfeeder mdcloser seenpointer ( ( si , ( f , sha , mode ) ) : rest ) =
2020-07-13 18:09:08 +00:00
case Git . toTreeItemType mode of
Just Git . TreeSymlink -> do
fix reversion in skipping deleted files
And add a test case for that.
This certianly loses some of the 2x performance improvement in file
seeking that seekFilteredKeys led to, because now it has to stat the
worktree files again. Without benchmarking, I expect there will still be
a sizable improvement, and also the git-annex branch precaching that
seekFilteredKeys can do will still be a win of its approach.
Also worth noting that lookupKey, when the file DNE, check if it's in an
adjusted branch with hidden files, and if so, finds the key for the
file anyway. That was intended to make git-annex sync --content be able
to process those files, but a side effect was that, when a file was
deleted but the deletion not yet staged, git-annex commands used to
still list it. That was actually a bug. This commit fixes that bug too.
(git-annex sync --content on such a branch does not use seekFilteredKeys
so was not affected by the reversion or by this behavior change)
This commit was sponsored by Jake Vosloo on Patreon.
2020-07-20 00:33:10 +00:00
whenM ( exists f ) $
-- Once a pointer file has been seen,
-- symlinks have to be sent via the
2020-09-24 21:59:05 +00:00
-- metadata processor too. That is
-- slightly slower, but preserves the
-- requested file order.
fix reversion in skipping deleted files
And add a test case for that.
This certianly loses some of the 2x performance improvement in file
seeking that seekFilteredKeys led to, because now it has to stat the
worktree files again. Without benchmarking, I expect there will still be
a sizable improvement, and also the git-annex branch precaching that
seekFilteredKeys can do will still be a win of its approach.
Also worth noting that lookupKey, when the file DNE, check if it's in an
adjusted branch with hidden files, and if so, finds the key for the
file anyway. That was intended to make git-annex sync --content be able
to process those files, but a side effect was that, when a file was
deleted but the deletion not yet staged, git-annex commands used to
still list it. That was actually a bug. This commit fixes that bug too.
(git-annex sync --content on such a branch does not use seekFilteredKeys
so was not affected by the reversion or by this behavior change)
This commit was sponsored by Jake Vosloo on Patreon.
2020-07-20 00:33:10 +00:00
if seenpointer
2020-09-14 20:49:33 +00:00
then liftIO $ mdfeeder ( ( si , f ) , sha )
2020-09-24 21:59:05 +00:00
else feedmatches mi ofeeder si f sha
process mi ofeeder mdfeeder mdcloser seenpointer rest
2020-07-13 18:09:08 +00:00
Just Git . TreeSubmodule ->
2020-09-24 21:59:05 +00:00
process mi ofeeder mdfeeder mdcloser seenpointer rest
2020-07-10 19:11:14 +00:00
-- Might be a pointer file, might be other
-- file in git, possibly large. Avoid catting
-- large files by first looking up the size.
2020-07-13 18:09:08 +00:00
Just _ -> do
fix reversion in skipping deleted files
And add a test case for that.
This certianly loses some of the 2x performance improvement in file
seeking that seekFilteredKeys led to, because now it has to stat the
worktree files again. Without benchmarking, I expect there will still be
a sizable improvement, and also the git-annex branch precaching that
seekFilteredKeys can do will still be a win of its approach.
Also worth noting that lookupKey, when the file DNE, check if it's in an
adjusted branch with hidden files, and if so, finds the key for the
file anyway. That was intended to make git-annex sync --content be able
to process those files, but a side effect was that, when a file was
deleted but the deletion not yet staged, git-annex commands used to
still list it. That was actually a bug. This commit fixes that bug too.
(git-annex sync --content on such a branch does not use seekFilteredKeys
so was not affected by the reversion or by this behavior change)
This commit was sponsored by Jake Vosloo on Patreon.
2020-07-20 00:33:10 +00:00
whenM ( exists f ) $
2020-09-14 20:49:33 +00:00
liftIO $ mdfeeder ( ( si , f ) , sha )
2020-09-24 21:59:05 +00:00
process mi ofeeder mdfeeder mdcloser True rest
2020-07-13 18:09:08 +00:00
Nothing ->
2020-09-24 21:59:05 +00:00
process mi ofeeder mdfeeder mdcloser seenpointer rest
2020-07-13 18:09:08 +00:00
process _ _ _ mdcloser _ [] = liftIO $ void mdcloser
fix reversion in skipping deleted files
And add a test case for that.
This certianly loses some of the 2x performance improvement in file
seeking that seekFilteredKeys led to, because now it has to stat the
worktree files again. Without benchmarking, I expect there will still be
a sizable improvement, and also the git-annex branch precaching that
seekFilteredKeys can do will still be a win of its approach.
Also worth noting that lookupKey, when the file DNE, check if it's in an
adjusted branch with hidden files, and if so, finds the key for the
file anyway. That was intended to make git-annex sync --content be able
to process those files, but a side effect was that, when a file was
deleted but the deletion not yet staged, git-annex commands used to
still list it. That was actually a bug. This commit fixes that bug too.
(git-annex sync --content on such a branch does not use seekFilteredKeys
so was not affected by the reversion or by this behavior change)
This commit was sponsored by Jake Vosloo on Patreon.
2020-07-20 00:33:10 +00:00
-- Check if files exist, because a deleted file will still be
-- listed by ls-tree, but should not be processed.
exists p = isJust <$> liftIO ( catchMaybeIO $ R . getSymbolicLinkStatus p )
2020-07-13 18:09:08 +00:00
2020-09-24 21:59:05 +00:00
mdprocess mi mdreader ofeeder ocloser = liftIO mdreader >>= \ case
2020-09-14 20:49:33 +00:00
Just ( ( si , f ) , Just ( sha , size , _type ) )
2020-07-13 18:09:08 +00:00
| size < maxPointerSz -> do
2020-09-24 21:59:05 +00:00
feedmatches mi ofeeder si f sha
mdprocess mi mdreader ofeeder ocloser
Just _ -> mdprocess mi mdreader ofeeder ocloser
2020-07-13 21:04:02 +00:00
Nothing -> liftIO $ void ocloser
2020-07-10 19:11:14 +00:00
2021-12-27 18:08:50 +00:00
-- Precache location logs if it will speed things up.
--
-- When there are git-annex branches that are not able to be
-- merged, the precaching is disabled, since it only looks at the
-- git-annex branch and not at those.
precachell mi
| usesLocationLog seeker || matcherNeedsLocationLog mi =
null <$> Annex . Branch . getUnmergedRefs
| otherwise = pure False
2020-09-25 15:38:42 +00:00
seekHelper :: ( a -> RawFilePath ) -> WarnUnmatchWhen -> ( [ LsFiles . Options ] -> [ RawFilePath ] -> Git . Repo -> IO ( [ a ] , IO Bool ) ) -> WorkTreeItems -> Annex ( [ ( SeekInput , a ) ] , IO Bool )
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
seekHelper c ww a ( WorkTreeItems l ) = do
2020-05-28 19:55:17 +00:00
os <- seekOptions ww
2020-10-13 20:31:01 +00:00
v <- liftIO $ newIORef []
r <- inRepo $ \ g -> concat . concat <$> forM ( segmentXargsOrdered l )
( runSegmentPaths' mk c ( \ fs -> go v os fs g ) . map toRawFilePath )
return ( r , cleanupall v )
2020-09-14 20:49:33 +00:00
where
mk ( Just i ) f = ( SeekInput [ fromRawFilePath i ] , f )
-- This is not accurate, but it only happens when there are a
-- great many input WorkTreeItems.
mk Nothing f = ( SeekInput [ fromRawFilePath ( c f ) ] , f )
2020-09-25 15:38:42 +00:00
2020-10-13 20:31:01 +00:00
go v os fs g = do
2020-10-19 18:48:21 +00:00
( ls , cleanup ) <- a os fs g
2020-10-13 20:31:01 +00:00
liftIO $ modifyIORef' v ( cleanup : )
2020-10-19 18:48:21 +00:00
return ls
2020-10-13 20:31:01 +00:00
cleanupall v = do
cleanups <- readIORef v
and <$> sequence cleanups
2020-09-25 15:38:42 +00:00
seekHelper _ _ _ NoWorkTreeItems = return ( [] , pure True )
2017-10-16 18:10:03 +00:00
2020-05-28 19:55:17 +00:00
data WarnUnmatchWhen = WarnUnmatchLsFiles | WarnUnmatchWorkTreeItems
seekOptions :: WarnUnmatchWhen -> Annex [ LsFiles . Options ]
seekOptions WarnUnmatchLsFiles =
ifM ( annexSkipUnknown <$> Annex . getGitConfig )
( return []
, return [ LsFiles . ErrorUnmatch ]
)
seekOptions WarnUnmatchWorkTreeItems = return []
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
-- Items in the work tree, which may be files or directories.
data WorkTreeItems
= WorkTreeItems [ FilePath ]
-- ^ An empty list often means all files.
| NoWorkTreeItems
-- ^ Used when no work tree items should be operated on.
deriving ( Show )
2013-02-06 16:40:59 +00:00
2018-10-19 21:51:25 +00:00
-- When in an adjusted branch that hides some files, it may not exist
-- in the current work tree, but in the original branch. This allows
-- seeking for such files.
newtype AllowHidden = AllowHidden Bool
2020-05-28 19:55:17 +00:00
-- git ls-files without --error-unmatch seeks work tree items matching
-- some criteria, and silently skips over anything that does not exist.
2020-05-11 19:03:35 +00:00
-- Also, when two directories are symlinked, referring to a file
2020-05-28 19:55:17 +00:00
-- inside the symlinked directory will be silently skipped by
-- git ls-files without --error-unmatch.
--
-- Sometimes a command needs to use git-lsfiles that way, perhaps repeatedly.
-- But users expect an error message when one of the files they provided
-- as a command-line parameter doesn't exist, so this checks that each
-- exists when run with WarnUnmatchWorkTreeItems.
--
-- Note that, unlike --error-unmatch, using this does not warn
-- about command-line parameters that exist, but are not checked into git.
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
workTreeItems :: WarnUnmatchWhen -> CmdParams -> Annex WorkTreeItems
2018-10-19 21:51:25 +00:00
workTreeItems = workTreeItems' ( AllowHidden False )
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
workTreeItems' :: AllowHidden -> WarnUnmatchWhen -> CmdParams -> Annex WorkTreeItems
workTreeItems' ( AllowHidden allowhidden ) ww ps = case ww of
WarnUnmatchWorkTreeItems -> runcheck
WarnUnmatchLsFiles ->
ifM ( annexSkipUnknown <$> Annex . getGitConfig )
( runcheck
, return $ WorkTreeItems ps
)
2018-10-19 21:51:25 +00:00
where
2020-05-28 19:55:17 +00:00
runcheck = do
currbranch <- getCurrentBranch
2020-08-07 00:14:30 +00:00
stopattop <- prepviasymlink
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
ps' <- flip filterM ps $ \ p -> do
2020-10-30 19:55:59 +00:00
relf <- liftIO $ relPathCwdToFile $ toRawFilePath p
2020-05-28 19:55:17 +00:00
ifM ( not <$> ( exists p <||> hidden currbranch relf ) )
( prob ( p ++ " not found " )
2020-08-07 00:14:30 +00:00
, ifM ( viasymlink stopattop ( upFrom relf ) )
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
( prob ( p ++ " is beyond a symbolic link " )
, return True
)
2020-05-28 19:55:17 +00:00
)
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
if null ps' && not ( null ps )
then return NoWorkTreeItems
else return ( WorkTreeItems ps' )
2020-05-28 19:55:17 +00:00
2018-10-19 21:51:25 +00:00
exists p = isJust <$> liftIO ( catchMaybeIO $ getSymbolicLinkStatus p )
2020-05-11 19:03:35 +00:00
2020-08-07 00:14:30 +00:00
prepviasymlink = do
repotopst <- inRepo $
maybe
( pure Nothing )
( catchMaybeIO . R . getSymbolicLinkStatus )
. Git . repoWorkTree
return $ \ st -> case repotopst of
Nothing -> False
Just tst -> fileID st == fileID tst
&& deviceID st == deviceID tst
viasymlink _ Nothing = return False
viasymlink stopattop ( Just p ) = do
2020-10-30 19:55:59 +00:00
st <- liftIO $ R . getSymbolicLinkStatus p
2020-08-07 00:14:30 +00:00
if stopattop st
then return False
else if isSymbolicLink st
then return True
else viasymlink stopattop ( upFrom p )
2020-05-11 19:03:35 +00:00
hidden currbranch f
| allowhidden = isJust
2020-10-30 19:55:59 +00:00
<$> catObjectMetaDataHidden f currbranch
2018-10-19 21:51:25 +00:00
| otherwise = return False
2017-06-01 15:40:47 +00:00
2020-05-11 19:03:35 +00:00
prob msg = do
toplevelWarning False msg
Annex . incError
when workTreeItems finds a problem with a parameter, don't go on to process it
Part of workTreeItems is trying detect a case
where git porcelain refuses to process a file, and where
git ls-files silently outputs nothing. But, it's hard to perfectly
replicate git's behavior, and besides, git's behavior could change.
So it could be that we warn, but then git ls-files does not skip over
it, and so git-annex also processes it after warning about it.
So, if we think we have a problem with a parameter, display the warning,
and skip processing it at all.
Implementing this was complicated by needing to handle the case where
all command-line parameters get filtered out this way. Which is
different than the case where there are none, because we don't want to
operate on all files in this new case..
2020-08-06 17:47:45 +00:00
return False
2020-05-11 19:03:35 +00:00
2019-11-25 20:18:19 +00:00
notSymlink :: RawFilePath -> IO Bool
2019-12-06 19:37:12 +00:00
notSymlink f = liftIO $ not . isSymbolicLink <$> R . getSymbolicLinkStatus f
2021-01-04 19:25:28 +00:00
{- Returns an action that, when there's a time limit, can be used
2021-04-21 19:40:32 +00:00
- to check it before processing a file . The first action is run when over the
- time limit , otherwise the second action is run . - }
mkCheckTimeLimit :: Annex ( Annex () -> Annex () -> Annex () )
2021-01-04 19:25:28 +00:00
mkCheckTimeLimit = Annex . getState Annex . timelimit >>= \ case
Nothing -> return $ \ _ a -> a
Just ( duration , cutoff ) -> return $ \ cleanup a -> do
now <- liftIO getPOSIXTime
if now > cutoff
then do
warning $ " Time limit ( " ++ fromDuration duration ++ " ) reached! Shutting down... "
shutdown True
2021-04-21 19:40:32 +00:00
cleanup
2021-01-04 19:25:28 +00:00
liftIO $ exitWith $ ExitFailure 101
else a