findcomputed: New command, displays information about computed files.

This commit is contained in:
Joey Hess 2025-03-18 12:55:48 -04:00
parent 1b23823787
commit bcfd554a0f
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
10 changed files with 188 additions and 15 deletions

View file

@ -4,6 +4,7 @@ git-annex (10.20250116) UNRELEASED; urgency=medium
* addcomputed: New command, adds a file that is generated by a compute
special remote.
* recompute: New command, recomputes computed files.
* findcomputed: New command, displays information about computed files.
* Support help.autocorrect settings "prompt", "never", and "immediate".
* Allow setting remote.foo.annex-tracking-branch to a branch name
that contains "/", as long as it's not a remote tracking branch.

View file

@ -135,6 +135,7 @@ import qualified Command.MaxSize
import qualified Command.Sim
import qualified Command.AddComputed
import qualified Command.Recompute
import qualified Command.FindComputed
import qualified Command.Version
import qualified Command.RemoteDaemon
#ifdef WITH_ASSISTANT
@ -269,6 +270,7 @@ cmds testoptparser testrunner mkbenchmarkgenerator = map addGitAnnexCommonOption
, Command.Sim.cmd
, Command.AddComputed.cmd
, Command.Recompute.cmd
, Command.FindComputed.cmd
, Command.Version.cmd
, Command.RemoteDaemon.cmd
#ifdef WITH_ASSISTANT

93
Command/FindComputed.hs Normal file
View file

@ -0,0 +1,93 @@
{- git-annex command
-
- Copyright 2025 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
{-# LANGUAGE OverloadedStrings, TupleSections #-}
module Command.FindComputed where
import Command
import Git.FilePath
import qualified Utility.Format
import Utility.Terminal
import Command.Find (showFormatted, formatVars)
import Remote.Compute (isComputeRemote, getComputeState, ComputeState(..))
import qualified Remote
import qualified Types.Remote as Remote
cmd :: Command
cmd = withAnnexOptions [annexedMatchingOptions] $ noCommit $ noMessages $
withAnnexOptions [jsonOptions] $
command "findcomputed" SectionQuery "lists computed files"
paramPaths (seek <$$> optParser)
data FindComputedOptions = FindComputedOptions
{ findThese :: CmdParams
, formatOption :: Maybe Utility.Format.Format
, keyOptions :: Maybe KeyOptions
}
optParser :: CmdParamsDesc -> Parser FindComputedOptions
optParser desc = FindComputedOptions
<$> cmdParams desc
<*> optional parseFormatOption
<*> optional parseBranchKeysOption
parseFormatOption :: Parser Utility.Format.Format
parseFormatOption =
option (Utility.Format.gen <$> str)
( long "format" <> metavar paramFormat
<> help "control format of output"
)
seek :: FindComputedOptions -> CommandSeek
seek o = do
unless (isJust (keyOptions o)) $
checkNotBareRepo
isterminal <- liftIO $ checkIsTerminal stdout
computeremotes <- filter isComputeRemote <$> Remote.remoteList
let seeker = AnnexedFileSeeker
{ startAction = const (start o isterminal computeremotes)
, checkContentPresent = Nothing
, usesLocationLog = True
}
withKeyOptions (keyOptions o) False seeker
(commandAction . startKeys o isterminal computeremotes)
(withFilesInGitAnnex ww seeker)
=<< workTreeItems ww (findThese o)
where
ww = WarnUnmatchLsFiles "findcomputed"
start :: FindComputedOptions -> IsTerminal -> [Remote] -> SeekInput -> OsPath -> Key -> CommandStart
start o isterminal computeremotes _ file key = do
rs <- Remote.remotesWithUUID computeremotes
<$> Remote.keyLocations key
rcs <- catMaybes <$> forM rs get
if null rcs
then stop
else startingCustomOutput key $ do
forM_ rcs $ \(r, c) -> do
let computation = unwords (computeParams c)
let unformatted = fromOsPath file
<> " (" <> encodeBS (Remote.name r)
<> ") -- "
<> encodeBS computation
let formatvars =
[ ("remote", Remote.name r)
, ("computation", computation)
] ++ formatVars key (AssociatedFile (Just file))
showFormatted isterminal (formatOption o)
unformatted formatvars
next $ return True
where
get r = fmap (r, )
<$> getComputeState (Remote.remoteStateHandle r) key
startKeys :: FindComputedOptions -> IsTerminal -> [Remote] -> (SeekInput, Key, ActionItem) -> CommandStart
startKeys o isterminal computeremotes (si, key, ActionItemBranchFilePath (BranchFilePath _ topf) _) =
start o isterminal computeremotes si (getTopFilePath topf) key
startKeys _ _ _ _ = stop

View file

@ -55,8 +55,8 @@ remoteLocations' (IncludeIgnored ii) locations trusted rs = do
-- remotes that match uuids that have the key
allremotes <- if not ii
then filterM (not <$$> liftIO . getDynamicConfig . remoteAnnexIgnore . gitconfig) rs
else return rs
then filterM (not <$$> liftIO . getDynamicConfig . remoteAnnexIgnore . gitconfig) rs
else return rs
let validremotes = remotesWithUUID allremotes locations
return (sortBy (comparing cost) validremotes, validtrustedlocations)

View file

@ -99,6 +99,8 @@ the parameters provided to `git-annex addcomputed`.
[[git-annex-recompute]](1)
[[git-annex-findcomputed]](1)
[[git-annex-initremote]](1)
# AUTHOR

View file

@ -0,0 +1,75 @@
# NAME
git-annex findcomputed - lists computed files
# SYNOPSIS
git annex findcomputed `[path ...]`
# DESCRIPTION
Outputs a list of files in the specified path that can be computed by
enabled compute special remotes. With no path, lists files in the current
directory and its subdirectories.
Along with the name of each computed file, this displays the input that
was provided to [[git-annex-addcomputed]](1).
For example:
# git-annex findcomputed
foo.png (imageconvert) -- convert file.raw file.jpeg passes=10
bar.gz (compressor) -- compress bar --level=9
# OPTIONS
* matching options
The [[git-annex-matching-options]](1)
can be used to specify files to list.
* `--branch=ref`
List computed files in the specified branch or treeish.
* `--format=value`
Use custom output formatting.
This option works the same as in [[git-annex-find]](1), with these
additional variables available for use in it:
remote, computation
The default output format is the same as
`--format='${file} (${remote}) -- ${computation}\\n'`,
except when outputting to a terminal, control characters will be escaped.
* `--json`
Output the list of files in JSON format.
This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
# SEE ALSO
[[git-annex]](1)
[[git-annex-addcomputed]](1)
[[git-annex-recompute]](1)
[[git-annex-findcomputed]](1)
[[git-annex-find]](1)
# AUTHOR
Joey Hess <id@joeyh.name>
Warning: Automatically converted into a man page by mdwn2man. Edit with care.

View file

@ -9,7 +9,7 @@ git-annex recompute [path ...]`
# DESCRIPTION
This updates computed files that were added with
[[git-annex-addcomputed]](1).
[[git-annex-addcomputed]](1).
By default, this only recomputes files whose input files have changed.
The new contents of the input files are used to re-run the computation.
@ -21,7 +21,7 @@ updated with the new content. The updated file is staged in git.
* `--original`
Re-run the computation with the original input files.
Re-run the computation with the original input files content.
* `--remote=name`
@ -66,6 +66,8 @@ updated with the new content. The updated file is staged in git.
[[git-annex-addcomputed]](1)
[[git-annex-findcomputed]](1)
# AUTHOR
Joey Hess <id@joeyh.name>

View file

@ -198,6 +198,12 @@ content from the key-value store.
See [[git-annex-recompute]](1) for details.
* `findcomputed`
Lists computed files.
See [[git-annex-findcomputed]](1) for details.
* `multicast`
Multicast file distribution.

View file

@ -15,18 +15,9 @@ compute special remote. --[[Joey]]
* annex.diskreserve can also be violated if computing a file gets source
files that are larger than the disk reserve. This could be checked.
* would be nice to have a way to see what computations are used by a
compute remote for a file. Put it in `whereis` output? But it's not an
url. Maybe a separate command? That would also allow querying for eg,
what files are inputs for another file.
* Maybe add a file matching option, eg:
Or it could be exposed in the
Remote interface, and made into a file matching option:
git-annex find --inputof=foo
But that would require running expensive find over the whole tree,
and wouldn't work if the input file is no longer in the tree.
git-annex find --inputof=remote:file
* allow git-annex enableremote with program= explicitly specified,
without checking annex.security.allowed-compute-programs

View file

@ -684,6 +684,7 @@ Executable git-annex
Command.FilterBranch
Command.FilterProcess
Command.Find
Command.FindComputed
Command.FindKeys
Command.FindRef
Command.Fix