Co-authored-by: scott@signal.org
Co-authored-by: ken@signal.org
This commit is contained in:
Ken Powers 2019-05-16 15:32:11 -07:00 committed by Scott Nonnenberg
parent 8c8856785b
commit 29de50c12a
100 changed files with 7572 additions and 693 deletions

View file

@ -1,6 +1,14 @@
/* global Whisper, Signal, setTimeout, clearTimeout, MessageController */
/* global
ConversationController,
Whisper,
Signal,
setTimeout,
clearTimeout,
MessageController
*/
const { isFunction, isNumber, omit } = require('lodash');
const { computeHash } = require('./types/conversation');
const getGuid = require('uuid/v4');
const {
getMessageById,
@ -356,17 +364,41 @@ async function _addAttachmentToMessage(message, attachment, { type, index }) {
}
if (type === 'group-avatar') {
const group = message.get('group');
if (!group) {
throw new Error("_addAttachmentToMessage: group didn't exist");
const conversationId = message.get('conversationid');
const conversation = ConversationController.get(conversationId);
if (!conversation) {
logger.warn("_addAttachmentToMessage: conversation didn't exist");
}
const existingAvatar = group.avatar;
const existingAvatar = conversation.get('avatar');
if (existingAvatar && existingAvatar.path) {
await Signal.Migrations.deleteAttachmentData(existingAvatar.path);
}
_replaceAttachment(group, 'avatar', attachment, logPrefix);
const data = await Signal.Migrations.loadAttachmentData(attachment.path);
conversation.set({
avatar: {
...attachment,
hash: await computeHash(data),
},
});
await Signal.Data.updateConversation(
conversationId,
conversation.attributes,
{
Conversation: Whisper.Conversation,
}
);
return;
}
if (type === 'sticker') {
const sticker = message.get('sticker');
if (!sticker) {
throw new Error("_addAttachmentToMessage: sticker didn't exist");
}
_replaceAttachment(sticker, 'data', attachment, logPrefix);
return;
}

View file

@ -7,6 +7,7 @@ module.exports = {
arrayBufferToBase64,
typedArrayToArrayBuffer,
base64ToArrayBuffer,
bytesFromHexString,
bytesFromString,
concatenateBytes,
constantTimeEqual,
@ -16,6 +17,7 @@ module.exports = {
decryptFile,
decryptSymmetric,
deriveAccessKey,
deriveStickerPackKey,
encryptAesCtr,
encryptDeviceName,
encryptAttachment,
@ -25,8 +27,10 @@ module.exports = {
getAccessKeyVerifier,
getFirstBytes,
getRandomBytes,
getRandomValue,
getViewOfArrayBuffer,
getZeroes,
hexFromBytes,
highBitsToInt,
hmacSha256,
intsToByteHighAndLow,
@ -58,6 +62,25 @@ function bytesFromString(string) {
function stringFromBytes(buffer) {
return dcodeIO.ByteBuffer.wrap(buffer).toString('utf8');
}
function hexFromBytes(buffer) {
return dcodeIO.ByteBuffer.wrap(buffer).toString('hex');
}
function bytesFromHexString(string) {
return dcodeIO.ByteBuffer.wrap(string, 'hex').toArrayBuffer();
}
async function deriveStickerPackKey(packKey) {
const salt = getZeroes(32);
const info = bytesFromString('Sticker Pack');
const [part1, part2] = await libsignal.HKDF.deriveSecrets(
packKey,
salt,
info
);
return concatenateBytes(part1, part2);
}
// High-level Operations
@ -366,6 +389,16 @@ function getRandomBytes(n) {
return bytes;
}
function getRandomValue(low, high) {
const diff = high - low;
const bytes = new Uint32Array(1);
window.crypto.getRandomValues(bytes);
// Because high and low are inclusive
const mod = diff + 1;
return bytes[0] % mod + low;
}
function getZeroes(n) {
const result = new Uint8Array(n);

18
js/modules/data.d.ts vendored
View file

@ -1,2 +1,20 @@
export function searchMessages(query: string): Promise<Array<any>>;
export function searchConversations(query: string): Promise<Array<any>>;
export function updateStickerLastUsed(
packId: string,
stickerId: number,
time: number
): Promise<void>;
export function updateStickerPackStatus(
packId: string,
status: 'advertised' | 'installed' | 'error' | 'pending',
options?: { timestamp: number }
): Promise<void>;
export function getRecentStickers(): Promise<
Array<{
id: number;
packId: string;
}>
>;

View file

@ -27,6 +27,7 @@ const DATABASE_UPDATE_TIMEOUT = 2 * 60 * 1000; // two minutes
const SQL_CHANNEL_KEY = 'sql-channel';
const ERASE_SQL_KEY = 'erase-sql-key';
const ERASE_ATTACHMENTS_KEY = 'erase-attachments';
const ERASE_STICKERS_KEY = 'erase-stickers';
const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments';
const _jobs = Object.create(null);
@ -138,6 +139,17 @@ module.exports = {
removeAttachmentDownloadJob,
removeAllAttachmentDownloadJobs,
createOrUpdateStickerPack,
updateStickerPackStatus,
createOrUpdateSticker,
updateStickerLastUsed,
addStickerPackReference,
deleteStickerPackReference,
deleteStickerPack,
getAllStickerPacks,
getAllStickers,
getRecentStickers,
removeAll,
removeAllConfiguration,
@ -884,6 +896,44 @@ async function removeAllAttachmentDownloadJobs() {
await channels.removeAllAttachmentDownloadJobs();
}
// Stickers
async function createOrUpdateStickerPack(pack) {
await channels.createOrUpdateStickerPack(pack);
}
async function updateStickerPackStatus(packId, status, options) {
await channels.updateStickerPackStatus(packId, status, options);
}
async function createOrUpdateSticker(sticker) {
await channels.createOrUpdateSticker(sticker);
}
async function updateStickerLastUsed(packId, stickerId, timestamp) {
await channels.updateStickerLastUsed(packId, stickerId, timestamp);
}
async function addStickerPackReference(messageId, packId) {
await channels.addStickerPackReference(messageId, packId);
}
async function deleteStickerPackReference(messageId, packId) {
const paths = await channels.deleteStickerPackReference(messageId, packId);
return paths;
}
async function deleteStickerPack(packId) {
const paths = await channels.deleteStickerPack(packId);
return paths;
}
async function getAllStickerPacks() {
const packs = await channels.getAllStickerPacks();
return packs;
}
async function getAllStickers() {
const stickers = await channels.getAllStickers();
return stickers;
}
async function getRecentStickers() {
const recentStickers = await channels.getRecentStickers();
return recentStickers;
}
// Other
async function removeAll() {
@ -903,6 +953,7 @@ async function removeOtherData() {
await Promise.all([
callChannel(ERASE_SQL_KEY),
callChannel(ERASE_ATTACHMENTS_KEY),
callChannel(ERASE_STICKERS_KEY),
]);
}

View file

@ -18,6 +18,7 @@ module.exports = {
isLinkInWhitelist,
isMediaLinkInWhitelist,
isLinkSneaky,
isStickerPack,
};
const SUPPORTED_DOMAINS = [
@ -37,7 +38,9 @@ const SUPPORTED_DOMAINS = [
'pinterest.com',
'www.pinterest.com',
'pin.it',
'signal.org',
];
function isLinkInWhitelist(link) {
try {
const url = new URL(link);
@ -61,6 +64,10 @@ function isLinkInWhitelist(link) {
}
}
function isStickerPack(link) {
return (link || '').startsWith('https://signal.org/addstickers/');
}
const SUPPORTED_MEDIA_DOMAINS = /^([^.]+\.)*(ytimg.com|cdninstagram.com|redd.it|imgur.com|fbcdn.net|pinimg.com)$/i;
function isMediaLinkInWhitelist(link) {
try {
@ -138,28 +145,33 @@ function getDomain(url) {
const MB = 1024 * 1024;
const KB = 1024;
function getChunkPattern(size) {
function getChunkPattern(size, initialOffset) {
if (size > MB) {
return _getRequestPattern(size, MB);
return _getRequestPattern(size, MB, initialOffset);
} else if (size > 500 * KB) {
return _getRequestPattern(size, 500 * KB);
return _getRequestPattern(size, 500 * KB, initialOffset);
} else if (size > 100 * KB) {
return _getRequestPattern(size, 100 * KB);
return _getRequestPattern(size, 100 * KB, initialOffset);
} else if (size > 50 * KB) {
return _getRequestPattern(size, 50 * KB);
return _getRequestPattern(size, 50 * KB, initialOffset);
} else if (size > 10 * KB) {
return _getRequestPattern(size, 10 * KB);
return _getRequestPattern(size, 10 * KB, initialOffset);
} else if (size > KB) {
return _getRequestPattern(size, KB);
return _getRequestPattern(size, KB, initialOffset);
}
throw new Error(`getChunkPattern: Unsupported size: ${size}`);
return {
start: {
start: initialOffset,
end: size - 1,
},
};
}
function _getRequestPattern(size, increment) {
function _getRequestPattern(size, increment, initialOffset) {
const results = [];
let offset = 0;
let offset = initialOffset || 0;
while (size - offset > increment) {
results.push({
start: offset,

View file

@ -9,6 +9,7 @@ const Emoji = require('../../ts/util/emoji');
const IndexedDB = require('./indexeddb');
const Notifications = require('../../ts/notifications');
const OS = require('../../ts/OS');
const Stickers = require('./stickers');
const Settings = require('./settings');
const Util = require('../../ts/util');
const { migrateToSQL } = require('./migrate_to_sql');
@ -69,8 +70,20 @@ const {
// State
const { createLeftPane } = require('../../ts/state/roots/createLeftPane');
const {
createStickerButton,
} = require('../../ts/state/roots/createStickerButton');
const {
createStickerManager,
} = require('../../ts/state/roots/createStickerManager');
const {
createStickerPreviewModal,
} = require('../../ts/state/roots/createStickerPreviewModal');
const { createStore } = require('../../ts/state/createStore');
const conversationsDuck = require('../../ts/state/ducks/conversations');
const itemsDuck = require('../../ts/state/ducks/items');
const stickersDuck = require('../../ts/state/ducks/stickers');
const userDuck = require('../../ts/state/ducks/user');
// Migrations
@ -112,6 +125,7 @@ function initializeMigrations({
}
const {
getPath,
getStickersPath,
createReader,
createAbsolutePathGetter,
createWriterForNew,
@ -130,25 +144,40 @@ function initializeMigrations({
const loadAttachmentData = Type.loadData(readAttachmentData);
const loadPreviewData = MessageType.loadPreviewData(loadAttachmentData);
const loadQuoteData = MessageType.loadQuoteData(loadAttachmentData);
const loadStickerData = MessageType.loadStickerData(loadAttachmentData);
const getAbsoluteAttachmentPath = createAbsolutePathGetter(attachmentsPath);
const deleteOnDisk = Attachments.createDeleter(attachmentsPath);
const writeNewAttachmentData = createWriterForNew(attachmentsPath);
const copyIntoAttachmentsDirectory = Attachments.copyIntoAttachmentsDirectory(
attachmentsPath
);
const stickersPath = getStickersPath(userDataPath);
const writeNewStickerData = createWriterForNew(stickersPath);
const getAbsoluteStickerPath = createAbsolutePathGetter(stickersPath);
const deleteSticker = Attachments.createDeleter(stickersPath);
const readStickerData = createReader(stickersPath);
return {
attachmentsPath,
copyIntoAttachmentsDirectory,
deleteAttachmentData: deleteOnDisk,
deleteExternalMessageFiles: MessageType.deleteAllExternalFiles({
deleteAttachmentData: Type.deleteData(deleteOnDisk),
deleteOnDisk,
}),
deleteSticker,
getAbsoluteAttachmentPath,
getAbsoluteStickerPath,
getPlaceholderMigrations,
getCurrentVersion,
loadAttachmentData,
loadMessage: MessageType.createAttachmentLoader(loadAttachmentData),
loadPreviewData,
loadQuoteData,
loadStickerData,
readAttachmentData,
readStickerData,
run,
processNewAttachment: attachment =>
MessageType.processNewAttachment(attachment, {
@ -161,6 +190,13 @@ function initializeMigrations({
makeVideoScreenshot,
logger,
}),
processNewSticker: stickerData =>
MessageType.processNewSticker(stickerData, {
writeNewStickerData,
getAbsoluteStickerPath,
getImageDimensions,
logger,
}),
upgradeMessageSchema: (message, options = {}) => {
const { maxVersion } = options;
@ -227,10 +263,15 @@ exports.setup = (options = {}) => {
const Roots = {
createLeftPane,
createStickerButton,
createStickerManager,
createStickerPreviewModal,
};
const Ducks = {
conversations: conversationsDuck,
items: itemsDuck,
user: userDuck,
stickers: stickersDuck,
};
const State = {
bindActionCreators,
@ -278,6 +319,7 @@ exports.setup = (options = {}) => {
RefreshSenderCertificate,
Settings,
State,
Stickers,
Types,
Util,
Views,

1
js/modules/stickers.d.ts vendored Normal file
View file

@ -0,0 +1 @@
export function maybeDeletePack(packId: string): Promise<void>;

495
js/modules/stickers.js Normal file
View file

@ -0,0 +1,495 @@
/* global
textsecure,
Signal,
log,
reduxStore,
reduxActions,
URL
*/
const BLESSED_PACKS = {};
const { isNumber, pick, reject, groupBy } = require('lodash');
const pMap = require('p-map');
const Queue = require('p-queue');
const qs = require('qs');
const { makeLookup } = require('../../ts/util/makeLookup');
const { base64ToArrayBuffer, deriveStickerPackKey } = require('./crypto');
const {
addStickerPackReference,
createOrUpdateSticker,
createOrUpdateStickerPack,
deleteStickerPack,
deleteStickerPackReference,
getAllStickerPacks,
getAllStickers,
getRecentStickers,
updateStickerPackStatus,
} = require('./data');
module.exports = {
BLESSED_PACKS,
copyStickerToAttachments,
deletePack,
deletePackReference,
downloadStickerPack,
getDataFromLink,
getInitialState,
getInstalledStickerPacks,
getSticker,
getStickerPack,
getStickerPackStatus,
load,
maybeDeletePack,
downloadQueuedPacks,
redactPackId,
};
let initialState = null;
let packsToDownload = null;
const downloadQueue = new Queue({ concurrency: 1 });
async function load() {
const [packs, recentStickers] = await Promise.all([
getPacksForRedux(),
getRecentStickersForRedux(),
]);
initialState = {
packs,
recentStickers,
blessedPacks: BLESSED_PACKS,
};
packsToDownload = capturePacksToDownload(packs);
}
function getDataFromLink(link) {
const { hash } = new URL(link);
if (!hash) {
return null;
}
const data = hash.slice(1);
const params = qs.parse(data);
return {
id: params.pack_id,
key: params.pack_key,
};
}
function getInstalledStickerPacks() {
const state = reduxStore.getState();
const { stickers } = state;
const { packs } = stickers;
if (!packs) {
return [];
}
const values = Object.values(packs);
return values.filter(pack => pack.status === 'installed');
}
function downloadQueuedPacks() {
const ids = Object.keys(packsToDownload);
ids.forEach(id => {
const { key, status } = packsToDownload[id];
// The queuing is done inside this function, no need to await here
downloadStickerPack(id, key, { finalStatus: status });
});
packsToDownload = {};
}
function capturePacksToDownload(existingPackLookup) {
const toDownload = Object.create(null);
// First, ensure that blessed packs are in good shape
const blessedIds = Object.keys(BLESSED_PACKS);
blessedIds.forEach(id => {
const existing = existingPackLookup[id];
if (
!existing ||
(existing.status !== 'advertised' && existing.status !== 'installed')
) {
toDownload[id] = {
id,
...BLESSED_PACKS[id],
};
}
});
// Then, find error cases in packs we already know about
const existingIds = Object.keys(existingPackLookup);
existingIds.forEach(id => {
if (toDownload[id]) {
return;
}
const existing = existingPackLookup[id];
if (doesPackNeedDownload(existing)) {
toDownload[id] = {
id,
key: existing.key,
status: existing.attemptedStatus,
};
}
});
return toDownload;
}
function doesPackNeedDownload(pack) {
if (!pack) {
return true;
}
const stickerCount = Object.keys(pack.stickers || {}).length;
return (
!pack.status ||
pack.status === 'error' ||
pack.status === 'pending' ||
!pack.stickerCount ||
stickerCount < pack.stickerCount
);
}
async function getPacksForRedux() {
const [packs, stickers] = await Promise.all([
getAllStickerPacks(),
getAllStickers(),
]);
const stickersByPack = groupBy(stickers, sticker => sticker.packId);
const fullSet = packs.map(pack => ({
...pack,
stickers: makeLookup(stickersByPack[pack.id] || [], 'id'),
}));
return makeLookup(fullSet, 'id');
}
async function getRecentStickersForRedux() {
const recent = await getRecentStickers();
return recent.map(sticker => ({
packId: sticker.packId,
stickerId: sticker.id,
}));
}
function getInitialState() {
return initialState;
}
function redactPackId(packId) {
return `[REDACTED]${packId.slice(-3)}`;
}
function getReduxStickerActions() {
const actions = reduxActions;
if (actions && actions.stickers) {
return actions.stickers;
}
return {};
}
async function decryptSticker(packKey, ciphertext) {
const binaryKey = base64ToArrayBuffer(packKey);
const derivedKey = await deriveStickerPackKey(binaryKey);
const plaintext = await textsecure.crypto.decryptAttachment(
ciphertext,
derivedKey
);
return plaintext;
}
async function downloadSticker(packId, packKey, proto) {
const ciphertext = await textsecure.messaging.getSticker(packId, proto.id);
const plaintext = await decryptSticker(packKey, ciphertext);
const sticker = await Signal.Migrations.processNewSticker(plaintext);
return {
...pick(proto, ['id', 'emoji']),
...sticker,
packId,
};
}
async function downloadStickerPack(packId, packKey, options = {}) {
// This will ensure that only one download process is in progress at any given time
return downloadQueue.add(async () => {
try {
await doDownloadStickerPack(packId, packKey, options);
} catch (error) {
log.error(
'doDownloadStickerPack threw an error:',
error && error.stack ? error.stack : error
);
}
});
}
async function doDownloadStickerPack(packId, packKey, options = {}) {
const { messageId, fromSync } = options;
const {
stickerAdded,
stickerPackAdded,
stickerPackUpdated,
installStickerPack,
} = getReduxStickerActions();
const finalStatus = options.finalStatus || 'advertised';
const existing = getStickerPack(packId);
if (!doesPackNeedDownload(existing)) {
log.warn(
`Download for pack ${redactPackId(
packId
)} requested, but it does not need re-download. Skipping.`
);
return;
}
const downloadAttempts = (existing ? existing.downloadAttempts || 0 : 0) + 1;
if (downloadAttempts > 3) {
log.warn(
`Refusing to attempt another download for pack ${redactPackId(
packId
)}, attempt number ${downloadAttempts}`
);
if (existing.status !== 'error') {
await updateStickerPackStatus(packId, 'error');
stickerPackUpdated(packId, {
status: 'error',
});
}
return;
}
let coverProto;
let coverStickerId;
let coverIncludedInList;
let nonCoverStickers;
try {
const ciphertext = await textsecure.messaging.getStickerPackManifest(
packId
);
const plaintext = await decryptSticker(packKey, ciphertext);
const proto = textsecure.protobuf.StickerPack.decode(plaintext);
const firstStickerProto = proto.stickers ? proto.stickers[0] : null;
const stickerCount = proto.stickers.length;
coverProto = proto.cover || firstStickerProto;
coverStickerId = coverProto ? coverProto.id : null;
if (!coverProto || !isNumber(coverStickerId)) {
throw new Error(
`Sticker pack ${redactPackId(
packId
)} is malformed - it has no cover, and no stickers`
);
}
nonCoverStickers = reject(
proto.stickers,
sticker => !isNumber(sticker.id) || sticker.id === coverStickerId
);
coverIncludedInList = nonCoverStickers.length < stickerCount;
// status can be:
// - 'pending'
// - 'advertised'
// - 'error'
// - 'installed'
const pack = {
id: packId,
key: packKey,
attemptedStatus: finalStatus,
coverStickerId,
downloadAttempts,
stickerCount,
status: 'pending',
...pick(proto, ['title', 'author']),
};
await createOrUpdateStickerPack(pack);
stickerPackAdded(pack);
if (messageId) {
await addStickerPackReference(messageId, packId);
}
} catch (error) {
log.error(
`Error downloading manifest for sticker pack ${redactPackId(packId)}:`,
error && error.stack ? error.stack : error
);
const pack = {
id: packId,
key: packKey,
attemptedStatus: finalStatus,
downloadAttempts,
status: 'error',
};
await createOrUpdateStickerPack(pack);
stickerPackAdded(pack);
return;
}
// We have a separate try/catch here because we're starting to download stickers here
// and we want to preserve more of the pack on an error.
try {
const downloadStickerJob = async stickerProto => {
const stickerInfo = await downloadSticker(packId, packKey, stickerProto);
const sticker = {
...stickerInfo,
isCoverOnly: !coverIncludedInList && stickerInfo.id === coverStickerId,
};
await createOrUpdateSticker(sticker);
stickerAdded(sticker);
};
// Download the cover first
await downloadStickerJob(coverProto);
// Then the rest
await pMap(nonCoverStickers, downloadStickerJob, { concurrency: 3 });
if (finalStatus === 'installed') {
await installStickerPack(packId, packKey, { fromSync });
} else {
// Mark the pack as complete
await updateStickerPackStatus(packId, finalStatus);
stickerPackUpdated(packId, {
status: finalStatus,
});
}
} catch (error) {
log.error(
`Error downloading stickers for sticker pack ${redactPackId(packId)}:`,
error && error.stack ? error.stack : error
);
const errorState = 'error';
await updateStickerPackStatus(packId, errorState);
if (stickerPackUpdated) {
stickerPackUpdated(packId, {
state: errorState,
});
}
}
}
function getStickerPack(packId) {
const state = reduxStore.getState();
const { stickers } = state;
const { packs } = stickers;
if (!packs) {
return null;
}
return packs[packId];
}
function getStickerPackStatus(packId) {
const pack = getStickerPack(packId);
if (!pack) {
return null;
}
return pack.status;
}
function getSticker(packId, stickerId) {
const state = reduxStore.getState();
const { stickers } = state;
const { packs } = stickers;
const pack = packs[packId];
if (!pack || !pack.stickers) {
return null;
}
return pack.stickers[stickerId];
}
async function copyStickerToAttachments(packId, stickerId) {
const sticker = getSticker(packId, stickerId);
if (!sticker) {
return null;
}
const { path } = sticker;
const absolutePath = Signal.Migrations.getAbsoluteStickerPath(path);
const newPath = await Signal.Migrations.copyIntoAttachmentsDirectory(
absolutePath
);
return {
...sticker,
path: newPath,
};
}
// In the case where a sticker pack is uninstalled, we want to delete it if there are no
// more references left. We'll delete a nonexistent reference, then check if there are
// any references left, just like usual.
async function maybeDeletePack(packId) {
// This hardcoded string is fine because message ids are GUIDs
await deletePackReference('NOT-USED', packId);
}
// We don't generally delete packs outright; we just remove references to them, and if
// the last reference is deleted, we finally then remove the pack itself from database
// and from disk.
async function deletePackReference(messageId, packId) {
const isBlessed = Boolean(BLESSED_PACKS[packId]);
if (isBlessed) {
return;
}
// This call uses locking to prevent race conditions with other reference removals,
// or an incoming message creating a new message->pack reference
const paths = await deleteStickerPackReference(messageId, packId);
// If we don't get a list of paths back, then the sticker pack was not deleted
if (!paths) {
return;
}
const { removeStickerPack } = getReduxStickerActions();
removeStickerPack(packId);
await pMap(paths, Signal.Migrations.deleteSticker, {
concurrency: 3,
});
}
// The override; doesn't honor our ref-counting scheme - just deletes it all.
async function deletePack(packId) {
const isBlessed = Boolean(BLESSED_PACKS[packId]);
if (isBlessed) {
return;
}
// This call uses locking to prevent race conditions with other reference removals,
// or an incoming message creating a new message->pack reference
const paths = await deleteStickerPack(packId);
const { removeStickerPack } = getReduxStickerActions();
removeStickerPack(packId);
await pMap(paths, Signal.Migrations.deleteSticker, {
concurrency: 3,
});
}

View file

@ -179,7 +179,7 @@ exports.loadData = readAttachmentData => {
}
const data = await readAttachmentData(attachment.path);
return Object.assign({}, attachment, { data });
return Object.assign({}, attachment, { data, size: data.byteLength });
};
};

View file

@ -140,11 +140,12 @@ async function deleteExternalFiles(conversation, options = {}) {
}
module.exports = {
deleteExternalFiles,
migrateConversation,
maybeUpdateAvatar,
maybeUpdateProfileAvatar,
createLastMessageUpdate,
arrayBufferToBase64,
base64ToArrayBuffer,
computeHash,
createLastMessageUpdate,
deleteExternalFiles,
maybeUpdateAvatar,
maybeUpdateProfileAvatar,
migrateConversation,
};

View file

@ -308,7 +308,33 @@ const toVersion9 = exports._withSchemaVersion({
});
const toVersion10 = exports._withSchemaVersion({
schemaVersion: 10,
upgrade: exports._mapPreviewAttachments(Attachment.migrateDataToFileSystem),
upgrade: async (message, context) => {
const processPreviews = exports._mapPreviewAttachments(
Attachment.migrateDataToFileSystem
);
const processSticker = async (stickerMessage, stickerContext) => {
const { sticker } = stickerMessage;
if (!sticker || !sticker.data || !sticker.data.data) {
return stickerMessage;
}
return {
...stickerMessage,
sticker: {
...sticker,
data: await Attachment.migrateDataToFileSystem(
sticker.data,
stickerContext
),
},
};
};
const previewProcessed = await processPreviews(message, context);
const stickerProcessed = await processSticker(previewProcessed, context);
return stickerProcessed;
},
});
const VERSIONS = [
@ -462,6 +488,44 @@ exports.processNewAttachment = async (
return finalAttachment;
};
exports.processNewSticker = async (
stickerData,
{
writeNewStickerData,
getAbsoluteStickerPath,
getImageDimensions,
logger,
} = {}
) => {
if (!isFunction(writeNewStickerData)) {
throw new TypeError('context.writeNewStickerData is required');
}
if (!isFunction(getAbsoluteStickerPath)) {
throw new TypeError('context.getAbsoluteStickerPath is required');
}
if (!isFunction(getImageDimensions)) {
throw new TypeError('context.getImageDimensions is required');
}
if (!isObject(logger)) {
throw new TypeError('context.logger is required');
}
const path = await writeNewStickerData(stickerData);
const absolutePath = await getAbsoluteStickerPath(path);
const { width, height } = await getImageDimensions({
objectUrl: absolutePath,
logger,
});
return {
contentType: 'image/webp',
path,
width,
height,
};
};
exports.createAttachmentLoader = loadAttachmentData => {
if (!isFunction(loadAttachmentData)) {
throw new TypeError(
@ -532,6 +596,23 @@ exports.loadPreviewData = loadAttachmentData => {
};
};
exports.loadStickerData = loadAttachmentData => {
if (!isFunction(loadAttachmentData)) {
throw new TypeError('loadStickerData: loadAttachmentData is required');
}
return async sticker => {
if (!sticker || !sticker.data) {
return null;
}
return {
...sticker,
data: await loadAttachmentData(sticker.data),
};
};
};
exports.deleteAllExternalFiles = ({ deleteAttachmentData, deleteOnDisk }) => {
if (!isFunction(deleteAttachmentData)) {
throw new TypeError(
@ -546,7 +627,7 @@ exports.deleteAllExternalFiles = ({ deleteAttachmentData, deleteOnDisk }) => {
}
return async message => {
const { attachments, quote, contact, preview } = message;
const { attachments, quote, contact, preview, sticker } = message;
if (attachments && attachments.length) {
await Promise.all(attachments.map(deleteAttachmentData));
@ -590,6 +671,14 @@ exports.deleteAllExternalFiles = ({ deleteAttachmentData, deleteOnDisk }) => {
})
);
}
if (sticker && sticker.data && sticker.data.path) {
await deleteOnDisk(sticker.data.path);
if (sticker.data.thumbnail && sticker.data.thumbnail.path) {
await deleteOnDisk(sticker.data.thumbnail.path);
}
}
};
};

View file

@ -4,8 +4,9 @@ const ProxyAgent = require('proxy-agent');
const { Agent } = require('https');
const is = require('@sindresorhus/is');
const { redactPackId } = require('./stickers');
/* global Buffer, setTimeout, log, _, getGuid */
/* global Signal, Buffer, setTimeout, log, _, getGuid */
/* eslint-disable more/no-then, no-bitwise, no-nested-ternary */
@ -175,16 +176,12 @@ function getContentType(response) {
function _promiseAjax(providedUrl, options) {
return new Promise((resolve, reject) => {
const url = providedUrl || `${options.host}/${options.path}`;
if (options.disableLogs) {
log.info(
`${options.type} [REDACTED_URL]${
options.unauthenticated ? ' (unauth)' : ''
}`
);
const unauthLabel = options.unauthenticated ? ' (unauth)' : '';
if (options.redactUrl) {
log.info(`${options.type} ${options.redactUrl(url)}${unauthLabel}`);
} else {
log.info(
`${options.type} ${url}${options.unauthenticated ? ' (unauth)' : ''}`
);
log.info(`${options.type} ${url}${unauthLabel}`);
}
const timeout =
@ -282,10 +279,10 @@ function _promiseAjax(providedUrl, options) {
if (options.responseType === 'json') {
if (options.validateResponse) {
if (!_validateResponse(result, options.validateResponse)) {
if (options.disableLogs) {
if (options.redactUrl) {
log.info(
options.type,
'[REDACTED_URL]',
options.redactUrl(url),
response.status,
'Error'
);
@ -304,10 +301,10 @@ function _promiseAjax(providedUrl, options) {
}
}
if (response.status >= 0 && response.status < 400) {
if (options.disableLogs) {
if (options.redactUrl) {
log.info(
options.type,
'[REDACTED_URL]',
options.redactUrl(url),
response.status,
'Success'
);
@ -324,8 +321,13 @@ function _promiseAjax(providedUrl, options) {
return resolve(result, response.status);
}
if (options.disableLogs) {
log.info(options.type, '[REDACTED_URL]', response.status, 'Error');
if (options.redactUrl) {
log.info(
options.type,
options.redactUrl(url),
response.status,
'Error'
);
} else {
log.error(options.type, url, response.status, 'Error');
}
@ -340,8 +342,8 @@ function _promiseAjax(providedUrl, options) {
});
})
.catch(e => {
if (options.disableLogs) {
log.error(options.type, '[REDACTED_URL]', 0, 'Error');
if (options.redactUrl) {
log.error(options.type, options.redactUrl(url), 0, 'Error');
} else {
log.error(options.type, url, 0, 'Error');
}
@ -435,6 +437,7 @@ function initialize({
function connect({ username: initialUsername, password: initialPassword }) {
let username = initialUsername;
let password = initialPassword;
const PARSE_RANGE_HEADER = /\/(\d+)$/;
// Thanks, function hoisting!
return {
@ -449,8 +452,9 @@ function initialize({
getProfile,
getProfileUnauth,
getProvisioningSocket,
getProxiedSize,
getSenderCertificate,
getSticker,
getStickerPackManifest,
makeProxiedRequest,
putAttachment,
registerKeys,
@ -834,6 +838,33 @@ function initialize({
});
}
function redactStickerUrl(stickerUrl) {
return stickerUrl.replace(
/(\/stickers\/)([^/]+)(\/)/,
(match, begin, packId, end) => `${begin}${redactPackId(packId)}${end}`
);
}
function getSticker(packId, stickerId) {
return _outerAjax(`${cdnUrl}/stickers/${packId}/full/${stickerId}`, {
certificateAuthority,
proxyUrl,
responseType: 'arraybuffer',
type: 'GET',
redactUrl: redactStickerUrl,
});
}
function getStickerPackManifest(packId) {
return _outerAjax(`${cdnUrl}/stickers/${packId}/manifest.proto`, {
certificateAuthority,
proxyUrl,
responseType: 'arraybuffer',
type: 'GET',
redactUrl: redactStickerUrl,
});
}
async function getAttachment(id) {
// This is going to the CDN, not the service, so we use _outerAjax
return _outerAjax(`${cdnUrl}/attachments/${id}`, {
@ -918,45 +949,64 @@ function initialize({
return attachmentIdString;
}
// eslint-disable-next-line no-shadow
async function getProxiedSize(url) {
const result = await _outerAjax(url, {
processData: false,
responseType: 'arraybufferwithdetails',
proxyUrl: contentProxyUrl,
type: 'HEAD',
disableLogs: true,
});
function getHeaderPadding() {
const length = Signal.Crypto.getRandomValue(1, 64);
let characters = '';
const { response } = result;
if (!response.headers || !response.headers.get) {
throw new Error('getProxiedSize: Problem retrieving header value');
for (let i = 0, max = length; i < max; i += 1) {
characters += String.fromCharCode(
Signal.Crypto.getRandomValue(65, 122)
);
}
const size = response.headers.get('content-length');
return parseInt(size, 10);
return characters;
}
// eslint-disable-next-line no-shadow
function makeProxiedRequest(url, options = {}) {
async function makeProxiedRequest(url, options = {}) {
const { returnArrayBuffer, start, end } = options;
let headers;
const headers = {
'X-SignalPadding': getHeaderPadding(),
};
if (_.isNumber(start) && _.isNumber(end)) {
headers = {
Range: `bytes=${start}-${end}`,
};
headers.Range = `bytes=${start}-${end}`;
}
return _outerAjax(url, {
const result = await _outerAjax(url, {
processData: false,
responseType: returnArrayBuffer ? 'arraybufferwithdetails' : null,
proxyUrl: contentProxyUrl,
type: 'GET',
redirect: 'follow',
disableLogs: true,
redactUrl: () => '[REDACTED_URL]',
headers,
});
if (!returnArrayBuffer) {
return result;
}
const { response } = result;
if (!response.headers || !response.headers.get) {
throw new Error('makeProxiedRequest: Problem retrieving header value');
}
const range = response.headers.get('content-range');
const match = PARSE_RANGE_HEADER.exec(range);
if (!match || !match[1]) {
throw new Error(
`makeProxiedRequest: Unable to parse total size from ${range}`
);
}
const totalSize = parseInt(match[1], 10);
return {
totalSize,
result,
};
}
function getMessageSocket() {