From d3590487479c5b57a61ce5b2ee03baa9c1acdec8 Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Thu, 21 Oct 2021 15:53:14 -0700 Subject: [PATCH] Use read stream for fetching debug logs --- ACKNOWLEDGMENTS.md | 16 +++++++++ package.json | 2 ++ ts/logging/main_process_logging.ts | 54 ++++++++++++++++++------------ yarn.lock | 12 +++++++ 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index e6c4920af..9495e53a1 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -3250,6 +3250,22 @@ Signal Desktop makes use of the following open source projects. See the License for the specific language governing permissions and limitations under the License. +## split2 + + Copyright (c) 2014-2018, Matteo Collina + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR + IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ## testcheck BSD License diff --git a/package.json b/package.json index 4ae4e9f8e..502fa24c3 100644 --- a/package.json +++ b/package.json @@ -159,6 +159,7 @@ "sanitize.css": "11.0.0", "semver": "5.4.1", "sharp": "0.28.1", + "split2": "4.0.0", "testcheck": "1.0.0-rc.2", "tmp": "0.0.33", "typeface-inter": "3.10.0", @@ -225,6 +226,7 @@ "@types/semver": "5.5.0", "@types/sharp": "0.28.0", "@types/sinon": "10.0.2", + "@types/split2": "3.2.1", "@types/storybook__addon-actions": "3.4.3", "@types/storybook__addon-knobs": "5.0.3", "@types/storybook__react": "4.0.2", diff --git a/ts/logging/main_process_logging.ts b/ts/logging/main_process_logging.ts index e5c1c359f..785e0ce5d 100644 --- a/ts/logging/main_process_logging.ts +++ b/ts/logging/main_process_logging.ts @@ -6,12 +6,13 @@ /* eslint-disable no-console */ import { join } from 'path'; -import { readdirSync, readFile, unlinkSync, writeFileSync } from 'fs'; +import split2 from 'split2'; +import { readdirSync, createReadStream, unlinkSync, writeFileSync } from 'fs'; import { BrowserWindow, app, ipcMain as ipc } from 'electron'; import pinoms from 'pino-multi-stream'; import pino from 'pino'; import * as mkdirp from 'mkdirp'; -import { compact, filter, flatten, map, pick, sortBy } from 'lodash'; +import { filter, flatten, map, pick, sortBy } from 'lodash'; import readFirstLine from 'firstline'; import { read as readLastLines } from 'read-last-lines'; import rimraf from 'rimraf'; @@ -41,6 +42,8 @@ declare global { } } +const MAX_LOG_LINES = 1000000; + let globalLogger: undefined | pino.Logger; let shouldRestart = false; @@ -283,28 +286,37 @@ export async function eliminateOldEntries( } // Exported for testing only. -export function fetchLog(logFile: string): Promise> { - return new Promise((resolve, reject) => { - readFile(logFile, { encoding: 'utf8' }, (err, text) => { - if (err) { - return reject(err); +export async function fetchLog(logFile: string): Promise> { + const results = new Array(); + + const rawStream = createReadStream(logFile); + const jsonStream = rawStream.pipe( + split2(line => { + try { + return JSON.parse(line); + } catch (e) { + return undefined; } + }) + ); - const lines = compact(text.split('\n')); - const data = compact( - lines.map(line => { - try { - const result = pick(JSON.parse(line), ['level', 'time', 'msg']); - return isLogEntry(result) ? result : null; - } catch (e) { - return null; - } - }) - ); + // Propagate fs errors down to the json stream so that for loop below handles + // them. + rawStream.on('error', error => jsonStream.emit('error', error)); - return resolve(data); - }); - }); + for await (const line of jsonStream) { + const result = line && pick(line, ['level', 'time', 'msg']); + if (!isLogEntry(result)) { + continue; + } + + results.push(result); + if (results.length > MAX_LOG_LINES) { + results.shift(); + } + } + + return results; } // Exported for testing only. diff --git a/yarn.lock b/yarn.lock index f2dbf145e..92787c397 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3004,6 +3004,13 @@ resolved "https://registry.yarnpkg.com/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9" integrity sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA== +"@types/split2@3.2.1": + version "3.2.1" + resolved "https://registry.yarnpkg.com/@types/split2/-/split2-3.2.1.tgz#97c42b560a1b41064d46cd728cf1267d5755ee34" + integrity sha512-7uz3yU+LooBq4yNOzlZD9PU9/1Eu0rTD1MjQ6apOVEoHsPrMUrFw7W8XrvWtesm2vK67SBK9AyJcOXtMpl9bgQ== + dependencies: + "@types/node" "*" + "@types/storybook__addon-actions@3.4.3": version "3.4.3" resolved "https://registry.yarnpkg.com/@types/storybook__addon-actions/-/storybook__addon-actions-3.4.3.tgz#bb0c3fd066e1f495d3f2b724487e2b99194b3f1a" @@ -17041,6 +17048,11 @@ split-string@^3.0.1, split-string@^3.0.2: dependencies: extend-shallow "^3.0.0" +split2@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.0.0.tgz#c76cb53ad55040ddff3a9c3b73fc88a4690f047c" + integrity sha512-gjmavJzvQCAZzaEHWoJBOwqIUAiEvUOlguQ6uO0+0LTS1tlLa2YetTLWCrm049ouvLOa1l6SOGm3XaiRiCg9SQ== + split@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/split/-/split-1.0.0.tgz#c4395ce683abcd254bc28fe1dabb6e5c27dcffae"