Improve @mentions tokenization

This commit is contained in:
Evan Hahn 2021-08-17 13:55:25 -05:00 committed by GitHub
parent 6dcceb81ac
commit ed83061651
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 64 additions and 10 deletions

6
ts/Intl.d.ts vendored
View file

@ -10,6 +10,12 @@ declare namespace Intl {
index: number;
input: string;
segment: string;
// According to [the proposal][0], `isWordLike` is a boolean when `granularity` is
// "word" and undefined otherwise. There may be a more rigid way to enforce this
// with TypeScript, but an optional property is okay for now.
//
// [0]: https://github.com/tc39/proposal-intl-segmenter/blob/e5f982f51cef810111dfeab835d6a934a7cae045/README.md
isWordLike?: boolean;
};
interface Segments {