diff --git a/app/attachments.js b/app/attachments.js index b1691e0a99..f8b8627fe8 100644 --- a/app/attachments.js +++ b/app/attachments.js @@ -1,12 +1,22 @@ const crypto = require('crypto'); const path = require('path'); +const pify = require('pify'); +const glob = require('glob'); const fse = require('fs-extra'); const toArrayBuffer = require('to-arraybuffer'); -const { isArrayBuffer, isString } = require('lodash'); +const { map, isArrayBuffer, isString } = require('lodash'); const PATH = 'attachments.noindex'; +exports.getAllAttachments = async userDataPath => { + const dir = exports.getPath(userDataPath); + const pattern = path.join(dir, '**', '*'); + + const files = await pify(glob)(pattern, { nodir: true }); + return map(files, file => path.relative(dir, file)); +}; + // getPath :: AbsolutePath -> AbsolutePath exports.getPath = userDataPath => { if (!isString(userDataPath)) { @@ -120,6 +130,18 @@ exports.createDeleter = root => { }; }; +exports.deleteAll = async ({ userDataPath, attachments }) => { + const deleteFromDisk = exports.createDeleter(exports.getPath(userDataPath)); + + for (let index = 0, max = attachments.length; index < max; index += 1) { + const file = attachments[index]; + // eslint-disable-next-line no-await-in-loop + await deleteFromDisk(file); + } + + console.log(`deleteAll: deleted ${attachments.length} files`); +}; + // createName :: Unit -> IO String exports.createName = () => { const buffer = crypto.randomBytes(32); diff --git a/app/sql.js b/app/sql.js index 80b09627e2..c4bfc92884 100644 --- a/app/sql.js +++ b/app/sql.js @@ -4,7 +4,7 @@ const rimraf = require('rimraf'); const sql = require('@journeyapps/sqlcipher'); const pify = require('pify'); const uuidv4 = require('uuid/v4'); -const { map, isString } = require('lodash'); +const { map, isString, fromPairs, forEach, last } = require('lodash'); // To get long stack traces // https://github.com/mapbox/node-sqlite3/wiki/API#sqlite3verbose @@ -15,6 +15,7 @@ module.exports = { close, removeDB, + getMessageCount, saveMessage, saveMessages, removeMessage, @@ -39,6 +40,8 @@ module.exports = { getMessagesNeedingUpgrade, getMessagesWithVisualMediaAttachments, getMessagesWithFileAttachments, + + removeKnownAttachments, }; function generateUUID() { @@ -260,6 +263,16 @@ async function removeDB() { rimraf.sync(filePath); } +async function getMessageCount() { + const row = await db.get('SELECT count(*) from messages;'); + + if (!row) { + throw new Error('getMessageCount: Unable to get count of messages'); + } + + return row['count(*)']; +} + async function saveMessage(data, { forceSave } = {}) { const { conversationId, @@ -710,3 +723,92 @@ async function getMessagesWithFileAttachments(conversationId, { limit }) { return map(rows, row => jsonToObject(row.json)); } + +function getExternalFilesForMessage(message) { + const { attachments, contact, quote } = message; + const files = []; + + forEach(attachments, attachment => { + const { path: file, thumbnail, screenshot } = attachment; + if (file) { + files.push(file); + } + + if (thumbnail && thumbnail.path) { + files.push(thumbnail.path); + } + + if (screenshot && screenshot.path) { + files.push(screenshot.path); + } + }); + + if (quote && quote.attachments && quote.attachments.length) { + forEach(quote.attachments, attachment => { + const { thumbnail } = attachment; + + if (thumbnail && thumbnail.path) { + files.push(thumbnail.path); + } + }); + } + + if (contact && contact.length) { + forEach(contact, item => { + const { avatar } = item; + + if (avatar && avatar.avatar && avatar.avatar.path) { + files.push(avatar.avatar.path); + } + }); + } + + return files; +} + +async function removeKnownAttachments(allAttachments) { + const lookup = fromPairs(map(allAttachments, file => [file, true])); + const chunkSize = 50; + + const total = await getMessageCount(); + console.log( + `removeKnownAttachments: About to iterate through ${total} messages` + ); + + let count = 0; + let complete = false; + let id = ''; + + while (!complete) { + // eslint-disable-next-line no-await-in-loop + const rows = await db.all( + `SELECT json FROM messages + WHERE id > $id + ORDER BY id ASC + LIMIT $chunkSize;`, + { + $id: id, + $chunkSize: chunkSize, + } + ); + + const messages = map(rows, row => jsonToObject(row.json)); + forEach(messages, message => { + const externalFiles = getExternalFilesForMessage(message); + forEach(externalFiles, file => { + delete lookup[file]; + }); + }); + + const lastMessage = last(messages); + if (lastMessage) { + ({ id } = lastMessage); + } + complete = messages.length < chunkSize; + count += messages.length; + } + + console.log(`removeKnownAttachments: Done processing ${count} messages`); + + return Object.keys(lookup); +} diff --git a/js/modules/data.js b/js/modules/data.js index 2aae4d14a7..1ec904eb86 100644 --- a/js/modules/data.js +++ b/js/modules/data.js @@ -34,6 +34,7 @@ module.exports = { close, removeDB, + getMessageCount, saveMessage, saveLegacyMessage, saveMessages, @@ -201,6 +202,10 @@ async function removeDB() { await channels.removeDB(); } +async function getMessageCount() { + return channels.getMessageCount(); +} + async function saveMessage(data, { forceSave, Message } = {}) { const id = await channels.saveMessage(_cleanData(data), { forceSave }); Message.refreshExpirationTimer(); diff --git a/main.js b/main.js index 4a80ccd84c..6d7c11fb90 100644 --- a/main.js +++ b/main.js @@ -26,6 +26,7 @@ const packageJson = require('./package.json'); const sql = require('./app/sql'); const sqlChannels = require('./app/sql_channel'); +const attachments = require('./app/attachments'); const attachmentChannel = require('./app/attachment_channel'); const autoUpdate = require('./app/auto_update'); const createTrayIcon = require('./app/tray_icon'); @@ -630,6 +631,13 @@ app.on('ready', async () => { await sql.initialize({ configDir: userDataPath, key }); await sqlChannels.initialize({ userConfig }); + const allAttachments = await attachments.getAllAttachments(userDataPath); + const orphanedAttachments = await sql.removeKnownAttachments(allAttachments); + await attachments.deleteAll({ + userDataPath, + attachments: orphanedAttachments, + }); + ready = true; autoUpdate.initialize(getMainWindow, locale.messages); diff --git a/package.json b/package.json index af002244ff..b404abe05a 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "firstline": "^1.2.1", "form-data": "^2.3.2", "fs-extra": "^5.0.0", + "glob": "^7.1.2", "google-libphonenumber": "^3.0.7", "got": "^8.2.0", "intl-tel-input": "^12.1.15", @@ -119,7 +120,6 @@ "eslint-plugin-mocha": "^4.12.1", "eslint-plugin-more": "^0.3.1", "extract-zip": "^1.6.6", - "glob": "^7.1.2", "grunt": "^1.0.1", "grunt-cli": "^1.2.0", "grunt-contrib-concat": "^1.0.1",