Eliminate orphaned external message files on startup

Attachments, visual attachment thumbnails, video attachment screenshots
Quote thumbnails
Contact avatars

app/attachments.js

@@ -1,12 +1,22 @@
 const crypto = require('crypto');
 const path = require('path');
 
+const pify = require('pify');
+const glob = require('glob');
 const fse = require('fs-extra');
 const toArrayBuffer = require('to-arraybuffer');
-const { isArrayBuffer, isString } = require('lodash');
+const { map, isArrayBuffer, isString } = require('lodash');
 
 const PATH = 'attachments.noindex';
 
+exports.getAllAttachments = async userDataPath => {
+  const dir = exports.getPath(userDataPath);
+  const pattern = path.join(dir, '**', '*');
+
+  const files = await pify(glob)(pattern, { nodir: true });
+  return map(files, file => path.relative(dir, file));
+};
+
 //      getPath :: AbsolutePath -> AbsolutePath
 exports.getPath = userDataPath => {
   if (!isString(userDataPath)) {
@@ -120,6 +130,18 @@ exports.createDeleter = root => {
   };
 };
 
+exports.deleteAll = async ({ userDataPath, attachments }) => {
+  const deleteFromDisk = exports.createDeleter(exports.getPath(userDataPath));
+
+  for (let index = 0, max = attachments.length; index < max; index += 1) {
+    const file = attachments[index];
+    // eslint-disable-next-line no-await-in-loop
+    await deleteFromDisk(file);
+  }
+
+  console.log(`deleteAll: deleted ${attachments.length} files`);
+};
+
 //      createName :: Unit -> IO String
 exports.createName = () => {
   const buffer = crypto.randomBytes(32);

app/sql.js

@@ -4,7 +4,7 @@ const rimraf = require('rimraf');
 const sql = require('@journeyapps/sqlcipher');
 const pify = require('pify');
 const uuidv4 = require('uuid/v4');
-const { map, isString } = require('lodash');
+const { map, isString, fromPairs, forEach, last } = require('lodash');
 
 // To get long stack traces
 //   https://github.com/mapbox/node-sqlite3/wiki/API#sqlite3verbose
@@ -15,6 +15,7 @@ module.exports = {
   close,
   removeDB,
 
+  getMessageCount,
   saveMessage,
   saveMessages,
   removeMessage,
@@ -39,6 +40,8 @@ module.exports = {
   getMessagesNeedingUpgrade,
   getMessagesWithVisualMediaAttachments,
   getMessagesWithFileAttachments,
+
+  removeKnownAttachments,
 };
 
 function generateUUID() {
@@ -260,6 +263,16 @@ async function removeDB() {
   rimraf.sync(filePath);
 }
 
+async function getMessageCount() {
+  const row = await db.get('SELECT count(*) from messages;');
+
+  if (!row) {
+    throw new Error('getMessageCount: Unable to get count of messages');
+  }
+
+  return row['count(*)'];
+}
+
 async function saveMessage(data, { forceSave } = {}) {
   const {
     conversationId,
@@ -710,3 +723,92 @@ async function getMessagesWithFileAttachments(conversationId, { limit }) {
 
   return map(rows, row => jsonToObject(row.json));
 }
+
+function getExternalFilesForMessage(message) {
+  const { attachments, contact, quote } = message;
+  const files = [];
+
+  forEach(attachments, attachment => {
+    const { path: file, thumbnail, screenshot } = attachment;
+    if (file) {
+      files.push(file);
+    }
+
+    if (thumbnail && thumbnail.path) {
+      files.push(thumbnail.path);
+    }
+
+    if (screenshot && screenshot.path) {
+      files.push(screenshot.path);
+    }
+  });
+
+  if (quote && quote.attachments && quote.attachments.length) {
+    forEach(quote.attachments, attachment => {
+      const { thumbnail } = attachment;
+
+      if (thumbnail && thumbnail.path) {
+        files.push(thumbnail.path);
+      }
+    });
+  }
+
+  if (contact && contact.length) {
+    forEach(contact, item => {
+      const { avatar } = item;
+
+      if (avatar && avatar.avatar && avatar.avatar.path) {
+        files.push(avatar.avatar.path);
+      }
+    });
+  }
+
+  return files;
+}
+
+async function removeKnownAttachments(allAttachments) {
+  const lookup = fromPairs(map(allAttachments, file => [file, true]));
+  const chunkSize = 50;
+
+  const total = await getMessageCount();
+  console.log(
+    `removeKnownAttachments: About to iterate through ${total} messages`
+  );
+
+  let count = 0;
+  let complete = false;
+  let id = '';
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const rows = await db.all(
+      `SELECT json FROM messages
+       WHERE id > $id
+       ORDER BY id ASC
+       LIMIT $chunkSize;`,
+      {
+        $id: id,
+        $chunkSize: chunkSize,
+      }
+    );
+
+    const messages = map(rows, row => jsonToObject(row.json));
+    forEach(messages, message => {
+      const externalFiles = getExternalFilesForMessage(message);
+      forEach(externalFiles, file => {
+        delete lookup[file];
+      });
+    });
+
+    const lastMessage = last(messages);
+    if (lastMessage) {
+      ({ id } = lastMessage);
+    }
+    complete = messages.length < chunkSize;
+    count += messages.length;
+  }
+
+  console.log(`removeKnownAttachments: Done processing ${count} messages`);
+
+  return Object.keys(lookup);
+}

js/modules/data.js

@@ -34,6 +34,7 @@ module.exports = {
   close,
   removeDB,
 
+  getMessageCount,
   saveMessage,
   saveLegacyMessage,
   saveMessages,
@@ -201,6 +202,10 @@ async function removeDB() {
   await channels.removeDB();
 }
 
+async function getMessageCount() {
+  return channels.getMessageCount();
+}
+
 async function saveMessage(data, { forceSave, Message } = {}) {
   const id = await channels.saveMessage(_cleanData(data), { forceSave });
   Message.refreshExpirationTimer();

main.js

@@ -26,6 +26,7 @@ const packageJson = require('./package.json');
 
 const sql = require('./app/sql');
 const sqlChannels = require('./app/sql_channel');
+const attachments = require('./app/attachments');
 const attachmentChannel = require('./app/attachment_channel');
 const autoUpdate = require('./app/auto_update');
 const createTrayIcon = require('./app/tray_icon');
@@ -630,6 +631,13 @@ app.on('ready', async () => {
   await sql.initialize({ configDir: userDataPath, key });
   await sqlChannels.initialize({ userConfig });
 
+  const allAttachments = await attachments.getAllAttachments(userDataPath);
+  const orphanedAttachments = await sql.removeKnownAttachments(allAttachments);
+  await attachments.deleteAll({
+    userDataPath,
+    attachments: orphanedAttachments,
+  });
+
   ready = true;
 
   autoUpdate.initialize(getMainWindow, locale.messages);

package.json

@@ -63,6 +63,7 @@
     "firstline": "^1.2.1",
     "form-data": "^2.3.2",
     "fs-extra": "^5.0.0",
+    "glob": "^7.1.2",
     "google-libphonenumber": "^3.0.7",
     "got": "^8.2.0",
     "intl-tel-input": "^12.1.15",
@@ -119,7 +120,6 @@
     "eslint-plugin-mocha": "^4.12.1",
     "eslint-plugin-more": "^0.3.1",
     "extract-zip": "^1.6.6",
-    "glob": "^7.1.2",
     "grunt": "^1.0.1",
     "grunt-cli": "^1.2.0",
     "grunt-contrib-concat": "^1.0.1",