Use read stream for fetching debug logs

ACKNOWLEDGMENTS.md

@@ -3250,6 +3250,22 @@ Signal Desktop makes use of the following open source projects.
     See the License for the specific language governing permissions and
     limitations under the License.
 
+## split2
+
+    Copyright (c) 2014-2018, Matteo Collina <hello@matteocollina.com>
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+    MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+    IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
 ## testcheck
 
     BSD License

package.json

@@ -159,6 +159,7 @@
     "sanitize.css": "11.0.0",
     "semver": "5.4.1",
     "sharp": "0.28.1",
+    "split2": "4.0.0",
     "testcheck": "1.0.0-rc.2",
     "tmp": "0.0.33",
     "typeface-inter": "3.10.0",
@@ -225,6 +226,7 @@
     "@types/semver": "5.5.0",
     "@types/sharp": "0.28.0",
     "@types/sinon": "10.0.2",
+    "@types/split2": "3.2.1",
     "@types/storybook__addon-actions": "3.4.3",
     "@types/storybook__addon-knobs": "5.0.3",
     "@types/storybook__react": "4.0.2",

ts/logging/main_process_logging.ts

@@ -6,12 +6,13 @@
 /* eslint-disable no-console */
 
 import { join } from 'path';
-import { readdirSync, readFile, unlinkSync, writeFileSync } from 'fs';
+import split2 from 'split2';
+import { readdirSync, createReadStream, unlinkSync, writeFileSync } from 'fs';
 import { BrowserWindow, app, ipcMain as ipc } from 'electron';
 import pinoms from 'pino-multi-stream';
 import pino from 'pino';
 import * as mkdirp from 'mkdirp';
-import { compact, filter, flatten, map, pick, sortBy } from 'lodash';
+import { filter, flatten, map, pick, sortBy } from 'lodash';
 import readFirstLine from 'firstline';
 import { read as readLastLines } from 'read-last-lines';
 import rimraf from 'rimraf';
@@ -41,6 +42,8 @@ declare global {
   }
 }
 
+const MAX_LOG_LINES = 1000000;
+
 let globalLogger: undefined | pino.Logger;
 let shouldRestart = false;
 
@@ -283,28 +286,37 @@ export async function eliminateOldEntries(
 }
 
 // Exported for testing only.
-export function fetchLog(logFile: string): Promise<Array<LogEntryType>> {
+export async function fetchLog(logFile: string): Promise<Array<LogEntryType>> {
-  return new Promise((resolve, reject) => {
+  const results = new Array<LogEntryType>();
-    readFile(logFile, { encoding: 'utf8' }, (err, text) => {
+
-      if (err) {
+  const rawStream = createReadStream(logFile);
-        return reject(err);
+  const jsonStream = rawStream.pipe(
+    split2(line => {
+      try {
+        return JSON.parse(line);
+      } catch (e) {
+        return undefined;
       }
+    })
+  );
 
-      const lines = compact(text.split('\n'));
+  // Propagate fs errors down to the json stream so that for loop below handles
-      const data = compact(
+  // them.
-        lines.map(line => {
+  rawStream.on('error', error => jsonStream.emit('error', error));
-          try {
-            const result = pick(JSON.parse(line), ['level', 'time', 'msg']);
-            return isLogEntry(result) ? result : null;
-          } catch (e) {
-            return null;
-          }
-        })
-      );
 
-      return resolve(data);
+  for await (const line of jsonStream) {
-    });
+    const result = line && pick(line, ['level', 'time', 'msg']);
-  });
+    if (!isLogEntry(result)) {
+      continue;
+    }
+
+    results.push(result);
+    if (results.length > MAX_LOG_LINES) {
+      results.shift();
+    }
+  }
+
+  return results;
 }
 
 // Exported for testing only.

yarn.lock

@@ -3004,6 +3004,13 @@
   resolved "https://registry.yarnpkg.com/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9"
   integrity sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA==
 
+"@types/split2@3.2.1":
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/@types/split2/-/split2-3.2.1.tgz#97c42b560a1b41064d46cd728cf1267d5755ee34"
+  integrity sha512-7uz3yU+LooBq4yNOzlZD9PU9/1Eu0rTD1MjQ6apOVEoHsPrMUrFw7W8XrvWtesm2vK67SBK9AyJcOXtMpl9bgQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/storybook__addon-actions@3.4.3":
   version "3.4.3"
   resolved "https://registry.yarnpkg.com/@types/storybook__addon-actions/-/storybook__addon-actions-3.4.3.tgz#bb0c3fd066e1f495d3f2b724487e2b99194b3f1a"
@@ -17041,6 +17048,11 @@ split-string@^3.0.1, split-string@^3.0.2:
   dependencies:
     extend-shallow "^3.0.0"
 
+split2@4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/split2/-/split2-4.0.0.tgz#c76cb53ad55040ddff3a9c3b73fc88a4690f047c"
+  integrity sha512-gjmavJzvQCAZzaEHWoJBOwqIUAiEvUOlguQ6uO0+0LTS1tlLa2YetTLWCrm049ouvLOa1l6SOGm3XaiRiCg9SQ==
+
 split@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/split/-/split-1.0.0.tgz#c4395ce683abcd254bc28fe1dabb6e5c27dcffae"