--json-exceptions

Added a --json-exceptions option, which makes some exceptions be output in json.

The distinction is that --json-error-messages is for messages relating
to a particular ActionItem, while --json-exceptions is for messages that
are not, eg ones for a file that does not exist.

It's unfortunate that we need two switches with such a fine distinction
between them, but I'm worried about maintaining backwards compatability
in the json output, to avoid breaking anything that parses it, and this was
the way to make sure I didn't.

toplevelWarning is generally used for the latter kind of message. And
the other calls to toplevelWarning could be converted to showException. The
only possible gotcha is that if toplevelWarning is ever called after
starting acting on a file, it will add to the --json-error-messages of the
json displayed for that file and converting to showException would be a
behavior change. That seems unlikely, but I didn't convery everything to
avoid needing to satisfy myself it was not a concern.

Sponsored-by: Dartmouth College's Datalad project
This commit is contained in:
Joey Hess 2023-04-25 17:02:25 -04:00
parent a474c9c63b
commit a325524454
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
32 changed files with 304 additions and 81 deletions

View file

@ -25,6 +25,8 @@ git-annex (10.20230408) UNRELEASED; urgency=medium
* assistant --autostop: Avoid crashing when ~/.config/git-annex/autostart
lists a directory that it cannot chdir to.
* Honor --force option when operating on a local git remote.
* Added a --json-exceptions option, which makes some exceptions be output
in json.
-- Joey Hess <id@joeyh.name> Sat, 08 Apr 2023 13:57:18 -0400

View file

@ -445,13 +445,20 @@ jsonOptions =
<> help "include error messages in JSON"
<> hidden
)
, annexFlag (setAnnexState $ Annex.setOutput (JSONOutput jsonexceptionsoptions))
( long "json-exceptions"
<> help "include exceptions in JSON"
<> hidden
)
]
where
stdjsonoptions = JSONOptions
{ jsonProgress = False
, jsonErrorMessages = False
, jsonExceptions = False
}
jsonerrormessagesoptions = stdjsonoptions { jsonErrorMessages = True }
jsonexceptionsoptions = stdjsonoptions { jsonExceptions = True }
jsonProgressOption :: [AnnexOption]
jsonProgressOption =
@ -465,6 +472,7 @@ jsonProgressOption =
jsonoptions = JSONOptions
{ jsonProgress = True
, jsonErrorMessages = False
, jsonExceptions = False
}
-- Note that a command that adds this option should wrap its seek

View file

@ -568,9 +568,9 @@ workTreeItems' (AllowHidden allowhidden) ww ps = case ww of
let p' = toRawFilePath p
relf <- liftIO $ relPathCwdToFile p'
ifM (not <$> (exists p' <||> hidden currbranch relf))
( prob (QuotedPath (toRawFilePath p) <> " not found")
( prob FileNotFound (QuotedPath (toRawFilePath p) <> " not found")
, ifM (viasymlink stopattop (upFrom relf))
( prob (QuotedPath (toRawFilePath p) <> " is beyond a symbolic link")
( prob FileBeyondSymbolicLink (QuotedPath (toRawFilePath p) <> " is beyond a symbolic link")
, return True
)
)
@ -605,8 +605,8 @@ workTreeItems' (AllowHidden allowhidden) ww ps = case ww of
<$> catObjectMetaDataHidden f currbranch
| otherwise = return False
prob msg = do
toplevelWarning False msg
prob eid msg = do
showException False eid msg
Annex.incError
return False

View file

@ -27,6 +27,8 @@ module Messages (
showEndFail,
showEndResult,
endResult,
ExceptionId(..),
showException,
toplevelWarning,
warning,
earlyWarning,
@ -197,6 +199,26 @@ endResult :: Bool -> S.ByteString
endResult True = "ok"
endResult False = "failed"
{- Unique ids for different exceptions. Do not change the constructors. -}
data ExceptionId
= FileNotFound
| FileBeyondSymbolicLink
deriving (Show)
{- Displays an message that is not associated with any file being
- processed. -}
showException :: Bool -> ExceptionId -> StringContainingQuotedPath -> Annex ()
showException makeway eid msg = do
when makeway $
outputMessage JSON.none id "\n"
outputException (show eid) (mentionedfile msg)
("git-annex: " <> msg <> "\n")
where
mentionedfile (QuotedPath p) = Just p
mentionedfile (a :+: b) = mentionedfile a <|> mentionedfile b
mentionedfile (UnquotedString _) = Nothing
mentionedfile (UnquotedByteString _) = Nothing
toplevelWarning :: Bool -> StringContainingQuotedPath -> Annex ()
toplevelWarning makeway s = warning' makeway id ("git-annex: " <> s)

View file

@ -90,11 +90,14 @@ outputError consolewhitespacef msg = withMessageState $ \s -> case (outputType s
qp <- coreQuotePath <$> Annex.getGitConfig
liftIO $ outputSerialized h $ OutputError $ decodeBS $
consolewhitespacef $ quote qp msg
_
| concurrentOutputEnabled s -> do
qp <- coreQuotePath <$> Annex.getGitConfig
concurrentMessage s True (decodeBS $ consolewhitespacef $ quote qp msg) go
| otherwise -> go
_ -> outputError' consolewhitespacef msg s
outputError' :: (S.ByteString -> S.ByteString) -> StringContainingQuotedPath -> MessageState -> Annex ()
outputError' consolewhitespacef msg s
| concurrentOutputEnabled s = do
qp <- coreQuotePath <$> Annex.getGitConfig
concurrentMessage s True (decodeBS $ consolewhitespacef $ quote qp msg) go
| otherwise = go
where
go = do
qp <- coreQuotePath <$> Annex.getGitConfig
@ -102,6 +105,17 @@ outputError consolewhitespacef msg = withMessageState $ \s -> case (outputType s
liftIO $ S.hPutStr stderr (consolewhitespacef $ quote qp msg)
liftIO $ hFlush stderr
outputException :: String -> Maybe RawFilePath -> StringContainingQuotedPath -> Annex ()
outputException eid mfile msg = withMessageState $ \s -> case outputType s of
(JSONOutput jsonoptions) | jsonExceptions jsonoptions ->
liftIO $ flushed $ JSON.emit $
JSON.exceptionObject eid (decodeBS (noquote msg)) mfile
(SerializedOutput h _) -> do
qp <- coreQuotePath <$> Annex.getGitConfig
liftIO $ outputSerialized h $ OutputException eid mfile $ decodeBS $
quote qp msg
_ -> outputError' id msg s
q :: Monad m => m ()
q = noop

View file

@ -1,6 +1,6 @@
{- git-annex command-line JSON output and input
-
- Copyright 2011-2021 Joey Hess <id@joeyh.name>
- Copyright 2011-2023 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
@ -27,6 +27,7 @@ module Messages.JSON (
ObjectMap(..),
JSONActionItem(..),
AddJSONActionItemFields(..),
exceptionObject,
) where
import Control.Applicative
@ -218,3 +219,14 @@ newtype AddJSONActionItemFields a = AddJSONActionItemFields a
instance ToJSON' a => ToJSON' (AddJSONActionItemFields a) where
toJSON' (AddJSONActionItemFields a) = object [ ("fields", toJSON' a) ]
exceptionObject :: String -> String -> Maybe RawFilePath -> Object
exceptionObject eid msg mfile = case o of
Object o' -> o'
_ -> error "internal"
where
o = object
[ "exception" .= toJSON' eid
, "message" .= toJSON' msg
, "file" .= toJSON' (maybe "" decodeBS mfile)
]

View file

@ -58,6 +58,9 @@ relaySerializedOutput getso sendsor meterreport runannex = go Nothing
Left (OutputError msg) -> do
runannex $ outputError id $ UnquotedString msg
loop st
Left (OutputException eid mfile msg) -> do
runannex $ outputException eid mfile $ UnquotedString msg
loop st
Left (JSONObject b) -> do
runannex $ withMessageState $ \s -> case outputType s of
JSONOutput _ -> liftIO $ flushed $ JSON.emit' b

View file

@ -9,6 +9,7 @@ module Types.Messages where
import qualified Utility.Aeson as Aeson
import Utility.Metered
import Utility.RawFilePath
import Control.Concurrent
import System.Console.Regions (ConsoleRegion)
@ -26,6 +27,7 @@ data OutputType
data JSONOptions = JSONOptions
{ jsonProgress :: Bool
, jsonErrorMessages :: Bool
, jsonExceptions :: Bool
}
deriving (Show)
@ -33,6 +35,7 @@ adjustOutputType :: OutputType -> OutputType -> OutputType
adjustOutputType (JSONOutput old) (JSONOutput new) = JSONOutput $ JSONOptions
{ jsonProgress = jsonProgress old || jsonProgress new
, jsonErrorMessages = jsonErrorMessages old || jsonErrorMessages new
, jsonExceptions = jsonExceptions old || jsonExceptions new
}
adjustOutputType _old new = new
@ -70,6 +73,7 @@ newMessageState = do
data SerializedOutput
= OutputMessage S.ByteString
| OutputError String
| OutputException String (Maybe RawFilePath) String
| BeginProgressMeter
| UpdateProgressMeterTotalSize TotalSize
| UpdateProgressMeter BytesProcessed

View file

@ -84,7 +84,8 @@ annexed content, and other symlinks.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-progress`
@ -92,8 +93,13 @@ annexed content, and other symlinks.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* `--batch`

View file

@ -125,7 +125,8 @@ be used to get better filenames.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to an url
being processed.
* `--json-progress`
@ -133,8 +134,13 @@ be used to get better filenames.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing an url.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular url.
* `--backend`

View file

@ -112,11 +112,11 @@ Paths of files or directories to operate on can be specified.
* `-z`
Makes batch input be delimited by nulls instead of the usual newlines.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-progress`
@ -124,8 +124,13 @@ Paths of files or directories to operate on can be specified.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -126,12 +126,18 @@ Paths of files or directories to drop can be specified.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -27,12 +27,19 @@ exist; using it can easily result in data loss.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a key
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a key.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular key.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -40,12 +40,18 @@ that can be determined purely by looking at the key.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a key
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a key.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular key.
* `--migrate-to-backend=backend`

View file

@ -102,7 +102,8 @@ so the overwritten modification is not lost.)
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-progress`
@ -110,8 +111,13 @@ so the overwritten modification is not lost.)
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -55,15 +55,19 @@ finds files in the current directory and its subdirectories.
* `--json`
Output the list of files in JSON format.
This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* `--batch`

View file

@ -50,15 +50,19 @@ Outputs a list of keys known to git-annex.
* `--json`
Output the list of keys in JSON format.
This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object corresponding to a key
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a key.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular key.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -103,12 +103,18 @@ better format.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* `--quiet`

View file

@ -126,7 +126,8 @@ be specified.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-progress`
@ -134,8 +135,13 @@ be specified.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -202,7 +202,8 @@ link, and that symbolic link will be followed.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-progress`
@ -210,8 +211,13 @@ link, and that symbolic link will be followed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -24,12 +24,18 @@ for the local repository and all annexed content.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a item
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a item.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular item.
* `--bytes`

View file

@ -26,12 +26,18 @@ can commit.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -119,6 +119,16 @@ the modified file.
{"command":"metadata","file":"foo","key":"...","author":["bar"],...,"note":"...","success":true}
* `--json-error-messages`
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* `--json-error-messages`
Messages that would normally be output to standard error are included in

View file

@ -71,7 +71,8 @@ contents. Use [[git-annex-sync]](1) for that.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-progress`
@ -79,8 +80,13 @@ contents. Use [[git-annex-sync]](1) for that.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -111,7 +111,8 @@ Paths of files or directories to operate on can be specified.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-progress`
@ -119,8 +120,13 @@ Paths of files or directories to operate on can be specified.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -51,12 +51,19 @@ special remote that claims it. (Usually the web special remote.)
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a key
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a key.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular key.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -24,12 +24,17 @@ modified (M), added but not committed (A), and type changed/unlocked (T).
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -53,12 +53,19 @@ repository. So, enable annex.thin with care.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -42,12 +42,19 @@ for it, because the content may still be present on the remote.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a key
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a key.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular key.
* Also the [[git-annex-common-options]](1) can be used.

View file

@ -71,12 +71,18 @@ received from remotes.
* `--json`
Enable JSON output. This is intended to be parsed by programs that use
git-annex. Each line of output is a JSON object.
git-annex. Each line of output is a JSON object corresponding to a file
being processed.
* `--json-error-messages`
Messages that would normally be output to standard error are included in
the JSON instead.
Adds an "error-messages" field to the JSON that contains messages that
would normally be output to the standard error when processing a file.
* `--json-exceptions`
Output additional JSON objects for some exceptions that are not
associated with a particular file.
* `--format=value`

View file

@ -31,7 +31,7 @@ bpoldrack makes a good point
So eg:
{"exception":"UNIQUEID", "file":"foo", "error-messages":["foo not found"]}
{"exception":"UNIQUEID", "file":"foo", "message":"foo not found"}
That seems about right to me, and it future proofs git-annex to be able to
report other exceptions in json output later on.

View file

@ -0,0 +1,24 @@
[[!comment format=mdwn
username="joey"
subject="""comment 5"""
date="2023-04-25T18:43:20Z"
content="""
I've implemented --json-exceptions:
joey@darkstar:~/tmp/xxx>git-annex add dne --json-exceptions
{"exception":"FileNotFound","file":"dne","message":"git-annex: dne not found\n"}
add: 1 failed
Note that when a command like `git-annex get` is run on a file that exists,
but is not checked into git, `git ls-files` still displays its own error message,
so --json-exceptions doesn't help with this:
joey@darkstar:~/tmp/xxx>touch foo
joey@darkstar:~/tmp/xxx>git-annex get foo --json-exceptions
error: pathspec 'foo' did not match any file(s) known to git
Did you forget to 'git add'?
get: 1 failed
Datalad will still have to parse the git output if it wants to know what file
it failed on in that case.
"""]]