Sync group stories through storage service

protos/SignalStorage.proto

@@ -101,6 +101,12 @@ message GroupV1Record {
 }
 
 message GroupV2Record {
+  enum StorySendMode {
+    DEFAULT = 0;
+    DISABLED = 1;
+    ENABLED = 2;
+  }
+
   optional bytes masterKey = 1;
   optional bool blocked = 2;
   optional bool whitelisted = 3;
@@ -109,6 +115,8 @@ message GroupV2Record {
   optional uint64 mutedUntilTimestamp = 6;
   optional bool dontNotifyForMentionsIfMuted = 7;
   optional bool hideStory = 8;
+  reserved /* storySendEnabled */ 9; // removed
+  optional StorySendMode storySendMode = 10;
 }
 
 message AccountRecord {

ts/background.ts

@@ -51,6 +51,7 @@ import {
 import { senderCertificateService } from './services/senderCertificate';
 import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher';
 import * as KeyboardLayout from './services/keyboardLayout';
+import * as StorageService from './services/storage';
 import { RoutineProfileRefresher } from './routineProfileRefresh';
 import { isMoreRecentThan, isOlderThan, toDayMillis } from './util/timestamp';
 import { isValidReactionEmoji } from './reactions/isValidReactionEmoji';
@@ -786,7 +787,7 @@ export async function startApp(): Promise<void> {
         window.isBeforeVersion(lastVersion, 'v1.36.0-beta.1') &&
         window.isAfterVersion(lastVersion, 'v1.35.0-beta.1')
       ) {
-        await window.Signal.Services.eraseAllStorageServiceState();
+        await StorageService.eraseAllStorageServiceState();
       }
 
       if (window.isBeforeVersion(lastVersion, 'v5.2.0')) {
@@ -846,6 +847,8 @@ export async function startApp(): Promise<void> {
 
       // Don't block on the following operation
       window.Signal.Data.ensureFilePermissions();
+
+      StorageService.reprocessUnknownFields();
     }
 
     try {
@@ -1737,7 +1740,7 @@ export async function startApp(): Promise<void> {
   });
 
   async function runStorageService() {
-    window.Signal.Services.enableStorageService();
+    StorageService.enableStorageService();
 
     if (window.ConversationController.areWePrimaryDevice()) {
       log.warn(
@@ -3548,7 +3551,7 @@ export async function startApp(): Promise<void> {
       }
       case FETCH_LATEST_ENUM.STORAGE_MANIFEST:
         log.info('onFetchLatestSync: fetching latest manifest');
-        await window.Signal.Services.runStorageServiceSyncJob();
+        await StorageService.runStorageServiceSyncJob();
         break;
       case FETCH_LATEST_ENUM.SUBSCRIPTION_STATUS:
         log.info('onFetchLatestSync: fetching latest subscription status');
@@ -3582,12 +3585,12 @@ export async function startApp(): Promise<void> {
           'onKeysSync: updated storage service key, erasing state and fetching'
         );
         await window.storage.put('storageKey', storageServiceKeyBase64);
-        await window.Signal.Services.eraseAllStorageServiceState({
+        await StorageService.eraseAllStorageServiceState({
           keepUnknownFields: true,
         });
       }
 
-      await window.Signal.Services.runStorageServiceSyncJob();
+      await StorageService.runStorageServiceSyncJob();
     }
   }
 

ts/components/SendStoryModal.tsx

@@ -444,7 +444,7 @@ export const SendStoryModal = ({
       <>
         <div className="SendStoryModal__selected-lists">{selectedNames}</div>
         <button
-          aria-label={i18n('SendStoryModal__ok')}
+          aria-label={i18n('ok')}
           className="SendStoryModal__ok"
           disabled={!chosenGroupIds.size}
           onClick={() => {

ts/groups.ts

@@ -18,6 +18,7 @@ import {
   getCheckedCredentialsForToday,
   maybeFetchNewCredentials,
 } from './services/groupCredentialFetcher';
+import { storageServiceUploadJob } from './services/storage';
 import dataInterface from './sql/Client';
 import { toWebSafeBase64, fromWebSafeBase64 } from './util/webSafeBase64';
 import { assertDev, strictAssert } from './util/assert';
@@ -1933,7 +1934,7 @@ export async function createGroupV2(
   );
 
   await conversation.queueJob('storageServiceUploadJob', async () => {
-    await window.Signal.Services.storageServiceUploadJob();
+    await storageServiceUploadJob();
   });
 
   const timestamp = Date.now();

ts/model-types.d.ts

@@ -30,6 +30,7 @@ import type { GiftBadgeStates } from './components/conversation/Message';
 import type { LinkPreviewType } from './types/message/LinkPreviews';
 
 import type { StickerType } from './types/Stickers';
+import type { StorySendMode } from './types/Stories';
 import type { MIMEType } from './types/MIME';
 
 import AccessRequiredEnum = Proto.AccessControl.AccessRequired;
@@ -349,7 +350,7 @@ export type ConversationAttributesType = {
   //   to leave a group.
   left?: boolean;
   groupVersion?: number;
-  isGroupStorySendReady?: boolean;
+  storySendMode?: StorySendMode;
 
   // GroupV1 only
   members?: Array<string>;

ts/models/conversations.ts

@@ -28,6 +28,7 @@ import * as EmbeddedContact from '../types/EmbeddedContact';
 import * as Conversation from '../types/Conversation';
 import type { StickerType, StickerWithHydratedData } from '../types/Stickers';
 import * as Stickers from '../types/Stickers';
+import { StorySendMode } from '../types/Stories';
 import type {
   ContactWithHydratedAvatar,
   GroupV1InfoType,
@@ -69,6 +70,7 @@ import { migrateColor } from '../util/migrateColor';
 import { isNotNil } from '../util/isNotNil';
 import { dropNull } from '../util/dropNull';
 import { notificationService } from '../services/notifications';
+import { storageServiceUploadJob } from '../services/storage';
 import { getSendOptions } from '../util/getSendOptions';
 import { isConversationAccepted } from '../util/isConversationAccepted';
 import { markConversationRead } from '../util/markConversationRead';
@@ -132,7 +134,7 @@ import { validateConversation } from '../util/validateConversation';
 /* eslint-disable more/no-then */
 window.Whisper = window.Whisper || {};
 
-const { Services, Util } = window.Signal;
+const { Util } = window.Signal;
 const { Message } = window.Signal.Types;
 const {
   deleteAttachmentData,
@@ -1868,7 +1870,7 @@ export class ConversationModel extends window.Backbone
       groupVersion,
       groupId: this.get('groupId'),
       groupLink: this.getGroupLink(),
-      isGroupStorySendReady: Boolean(this.get('isGroupStorySendReady')),
+      storySendMode: this.getStorySendMode(),
       hideStory: Boolean(this.get('hideStory')),
       inboxPosition,
       isArchived: this.get('isArchived'),
@@ -5182,7 +5184,7 @@ export class ConversationModel extends window.Backbone
     this.set({ needsStorageServiceSync: true });
 
     this.queueJob('captureChange', async () => {
-      Services.storageServiceUploadJob();
+      storageServiceUploadJob();
     });
   }
 
@@ -5498,6 +5500,14 @@ export class ConversationModel extends window.Backbone
     }
     return window.textsecure.storage.protocol.signAlternateIdentity();
   }
+
+  getStorySendMode(): StorySendMode | undefined {
+    if (!isGroup(this.attributes)) {
+      return undefined;
+    }
+
+    return this.get('storySendMode') ?? StorySendMode.IfActive;
+  }
 }
 
 window.Whisper.Conversation = ConversationModel;

ts/services/storage.ts

@@ -39,6 +39,7 @@ import { BackOff } from '../util/BackOff';
 import { storageJobQueue } from '../util/JobQueue';
 import { sleep } from '../util/sleep';
 import { isMoreRecentThan } from '../util/timestamp';
+import { map, filter } from '../util/iterables';
 import { ourProfileKeyService } from './ourProfileKey';
 import {
   ConversationTypes,
@@ -61,6 +62,7 @@ import type {
   UninstalledStickerPackType,
 } from '../sql/Interface';
 import { MY_STORIES_ID } from '../types/Stories';
+import { isNotNil } from '../util/isNotNil';
 
 type IManifestRecordIdentifier = Proto.ManifestRecord.IIdentifier;
 
@@ -119,7 +121,7 @@ function encryptRecord(
   const storageItem = new Proto.StorageItem();
 
   const storageKeyBuffer = storageID
-    ? Bytes.fromBase64(String(storageID))
+    ? Bytes.fromBase64(storageID)
     : generateStorageID();
 
   const storageKeyBase64 = window.storage.get('storageKey');
@@ -149,9 +151,9 @@ function generateStorageID(): Uint8Array {
 
 type GeneratedManifestType = {
   postUploadUpdateFunctions: Array<() => unknown>;
-  deleteKeys: Array<Uint8Array>;
+  recordsByID: Map<string, MergeableItemType | RemoteRecord>;
-  newItems: Set<Proto.IStorageItem>;
+  insertKeys: Set<string>;
-  storageManifest: Proto.IStorageManifest;
+  deleteKeys: Set<string>;
 };
 
 async function generateManifest(
@@ -169,10 +171,9 @@ async function generateManifest(
   const ITEM_TYPE = Proto.ManifestRecord.Identifier.Type;
 
   const postUploadUpdateFunctions: Array<() => unknown> = [];
-  const insertKeys: Array<string> = [];
+  const insertKeys = new Set<string>();
-  const deleteKeys: Array<Uint8Array> = [];
+  const deleteKeys = new Set<string>();
-  const manifestRecordKeys: Set<IManifestRecordIdentifier> = new Set();
+  const recordsByID = new Map<string, MergeableItemType | RemoteRecord>();
-  const newItems: Set<Proto.IStorageItem> = new Set();
 
   function processStorageRecord({
     conversation,
@@ -189,9 +190,6 @@ async function generateManifest(
     storageNeedsSync: boolean;
     storageRecord: Proto.IStorageRecord;
   }) {
-    const identifier = new Proto.ManifestRecord.Identifier();
-    identifier.type = identifierType;
-
     const currentRedactedID = currentStorageID
       ? redactStorageID(currentStorageID, currentStorageVersion)
       : undefined;
@@ -202,24 +200,16 @@ async function generateManifest(
       ? Bytes.toBase64(generateStorageID())
       : currentStorageID;
 
-    let storageItem;
+    recordsByID.set(storageID, {
-    try {
+      itemType: identifierType,
-      storageItem = encryptRecord(storageID, storageRecord);
+      storageID,
-    } catch (err) {
+      storageRecord,
-      log.error(
+    });
-        `storageService.upload(${version}): encrypt record failed:`,
-        Errors.toLogFormat(err)
-      );
-      throw err;
-    }
-    identifier.raw = storageItem.key;
 
     // When a client needs to update a given record it should create it
     // under a new key and delete the existing key.
     if (isNewItem) {
-      newItems.add(storageItem);
+      insertKeys.add(storageID);
-
-      insertKeys.push(storageID);
       const newRedactedID = redactStorageID(storageID, version, conversation);
       if (currentStorageID) {
         log.info(
@@ -227,7 +217,7 @@ async function generateManifest(
             `updating from=${currentRedactedID} ` +
             `to=${newRedactedID}`
         );
-        deleteKeys.push(Bytes.fromBase64(currentStorageID));
+        deleteKeys.add(currentStorageID);
       } else {
         log.info(
           `storageService.upload(${version}): adding key=${newRedactedID}`
@@ -235,8 +225,6 @@ async function generateManifest(
       }
     }
 
-    manifestRecordKeys.add(identifier);
-
     return {
       isNewItem,
       storageID,
@@ -293,7 +281,7 @@ async function generateManifest(
             `due to ${dropReason}`
         );
         conversation.unset('storageID');
-        deleteKeys.push(Bytes.fromBase64(droppedID));
+        deleteKeys.add(droppedID);
         continue;
       }
 
@@ -468,11 +456,7 @@ async function generateManifest(
   // When updating the manifest, ensure all "unknown" keys are added to the
   // new manifest, so we don't inadvertently delete something we don't understand
   unknownRecordsArray.forEach((record: UnknownRecord) => {
-    const identifier = new Proto.ManifestRecord.Identifier();
+    recordsByID.set(record.storageID, record);
-    identifier.type = record.itemType;
-    identifier.raw = Bytes.fromBase64(record.storageID);
-
-    manifestRecordKeys.add(identifier);
   });
 
   const recordsWithErrors: ReadonlyArray<UnknownRecord> = window.storage.get(
@@ -489,11 +473,7 @@ async function generateManifest(
   // These records failed to merge in the previous fetchManifest, but we still
   // need to include them so that the manifest is complete
   recordsWithErrors.forEach((record: UnknownRecord) => {
-    const identifier = new Proto.ManifestRecord.Identifier();
+    recordsByID.set(record.storageID, record);
-    identifier.type = record.itemType;
-    identifier.raw = Bytes.fromBase64(record.storageID);
-
-    manifestRecordKeys.add(identifier);
   });
 
   // Delete keys that we wanted to drop during the processing of the manifest.
@@ -511,83 +491,73 @@ async function generateManifest(
   );
 
   for (const { storageID } of storedPendingDeletes) {
-    deleteKeys.push(Bytes.fromBase64(storageID));
+    deleteKeys.add(storageID);
   }
 
   // Validate before writing
 
-  const rawDuplicates = new Set();
+  const duplicates = new Set<string>();
-  const typeRawDuplicates = new Set();
+  const typeDuplicates = new Set();
   let hasAccountType = false;
-  manifestRecordKeys.forEach(identifier => {
+  for (const [storageID, { itemType }] of recordsByID) {
     // Ensure there are no duplicate StorageIdentifiers in your manifest
     //   This can be broken down into two parts:
     //     There are no duplicate type+raw pairs
     //     There are no duplicate raw bytes
-    strictAssert(identifier.raw, 'manifest record key without raw identifier');
+    const typeAndID = `${itemType}+${storageID}`;
-    const storageID = Bytes.toBase64(identifier.raw);
+    if (duplicates.has(storageID) || typeDuplicates.has(typeAndID)) {
-    const typeAndRaw = `${identifier.type}+${storageID}`;
-    if (
-      rawDuplicates.has(identifier.raw) ||
-      typeRawDuplicates.has(typeAndRaw)
-    ) {
       log.warn(
         `storageService.upload(${version}): removing from duplicate item ` +
           'from the manifest',
         redactStorageID(storageID),
-        identifier.type
+        itemType
       );
-      manifestRecordKeys.delete(identifier);
+      recordsByID.delete(storageID);
     }
-    rawDuplicates.add(identifier.raw);
+    duplicates.add(storageID);
-    typeRawDuplicates.add(typeAndRaw);
+    typeDuplicates.add(typeAndID);
 
     // Ensure all deletes are not present in the manifest
-    const hasDeleteKey = deleteKeys.find(
+    const hasDeleteKey = deleteKeys.has(storageID);
-      key => Bytes.toBase64(key) === storageID
-    );
     if (hasDeleteKey) {
       log.warn(
         `storageService.upload(${version}): removing key which has been deleted`,
         redactStorageID(storageID),
-        identifier.type
+        itemType
       );
-      manifestRecordKeys.delete(identifier);
+      recordsByID.delete(storageID);
     }
 
     // Ensure that there is *exactly* one Account type in the manifest
-    if (identifier.type === ITEM_TYPE.ACCOUNT) {
+    if (itemType === ITEM_TYPE.ACCOUNT) {
       if (hasAccountType) {
         log.warn(
           `storageService.upload(${version}): removing duplicate account`,
           redactStorageID(storageID)
         );
-        manifestRecordKeys.delete(identifier);
+        recordsByID.delete(storageID);
       }
       hasAccountType = true;
     }
-  });
+  }
 
-  rawDuplicates.clear();
+  duplicates.clear();
-  typeRawDuplicates.clear();
+  typeDuplicates.clear();
 
   const storageKeyDuplicates = new Set<string>();
 
-  newItems.forEach(storageItem => {
+  for (const storageID of insertKeys) {
     // Ensure there are no duplicate StorageIdentifiers in your list of inserts
-    strictAssert(storageItem.key, 'New storage item without key');
-
-    const storageID = Bytes.toBase64(storageItem.key);
     if (storageKeyDuplicates.has(storageID)) {
       log.warn(
         `storageService.upload(${version}): ` +
           'removing duplicate identifier from inserts',
         redactStorageID(storageID)
       );
-      newItems.delete(storageItem);
+      insertKeys.delete(storageID);
     }
     storageKeyDuplicates.add(storageID);
-  });
+  }
 
   storageKeyDuplicates.clear();
 
@@ -608,15 +578,13 @@ async function generateManifest(
     );
 
     const localKeys: Set<string> = new Set();
-    manifestRecordKeys.forEach((identifier: IManifestRecordIdentifier) => {
+    for (const storageID of recordsByID.keys()) {
-      strictAssert(identifier.raw, 'Identifier without raw field');
-      const storageID = Bytes.toBase64(identifier.raw);
       localKeys.add(storageID);
 
       if (!remoteKeys.has(storageID)) {
         pendingInserts.add(storageID);
       }
-    });
+    }
 
     remoteKeys.forEach(storageID => {
       if (!localKeys.has(storageID)) {
@@ -624,9 +592,9 @@ async function generateManifest(
       }
     });
 
-    if (deleteKeys.length !== pendingDeletes.size) {
+    if (deleteKeys.size !== pendingDeletes.size) {
-      const localDeletes = deleteKeys.map(key =>
+      const localDeletes = Array.from(deleteKeys).map(key =>
-        redactStorageID(Bytes.toBase64(key))
+        redactStorageID(key)
       );
       const remoteDeletes: Array<string> = [];
       pendingDeletes.forEach(id => remoteDeletes.push(redactStorageID(id)));
@@ -639,24 +607,77 @@ async function generateManifest(
       );
       throw new Error('invalid write delete keys length do not match');
     }
-    if (newItems.size !== pendingInserts.size) {
+    if (insertKeys.size !== pendingInserts.size) {
       throw new Error('invalid write insert items length do not match');
     }
-    deleteKeys.forEach(key => {
+    for (const storageID of deleteKeys) {
-      const storageID = Bytes.toBase64(key);
       if (!pendingDeletes.has(storageID)) {
         throw new Error(
           'invalid write delete key missing from pending deletes'
         );
       }
-    });
+    }
-    insertKeys.forEach(storageID => {
+    for (const storageID of insertKeys) {
       if (!pendingInserts.has(storageID)) {
         throw new Error(
           'invalid write insert key missing from pending inserts'
         );
       }
+    }
+  }
+
+  return {
+    postUploadUpdateFunctions,
+    recordsByID,
+    insertKeys,
+    deleteKeys,
+  };
+}
+
+type EncryptManifestOptionsType = {
+  recordsByID: Map<string, MergeableItemType | RemoteRecord>;
+  insertKeys: Set<string>;
+};
+
+type EncryptedManifestType = {
+  newItems: Set<Proto.IStorageItem>;
+  storageManifest: Proto.IStorageManifest;
+};
+
+async function encryptManifest(
+  version: number,
+  { recordsByID, insertKeys }: EncryptManifestOptionsType
+): Promise<EncryptedManifestType> {
+  const manifestRecordKeys: Set<IManifestRecordIdentifier> = new Set();
+  const newItems: Set<Proto.IStorageItem> = new Set();
+
+  for (const [storageID, { itemType, storageRecord }] of recordsByID) {
+    const identifier = new Proto.ManifestRecord.Identifier({
+      type: itemType,
+      raw: Bytes.fromBase64(storageID),
     });
+
+    manifestRecordKeys.add(identifier);
+
+    if (insertKeys.has(storageID)) {
+      strictAssert(
+        storageRecord !== undefined,
+        'Inserted items must have an associated record'
+      );
+
+      let storageItem;
+      try {
+        storageItem = encryptRecord(storageID, storageRecord);
+      } catch (err) {
+        log.error(
+          `storageService.upload(${version}): encrypt record failed:`,
+          Errors.toLogFormat(err)
+        );
+        throw err;
+      }
+
+      newItems.add(storageItem);
+    }
   }
 
   const manifestRecord = new Proto.ManifestRecord();
@@ -683,8 +704,6 @@ async function generateManifest(
   storageManifest.value = encryptedManifest;
 
   return {
-    postUploadUpdateFunctions,
-    deleteKeys,
     newItems,
     storageManifest,
   };
@@ -692,18 +711,14 @@ async function generateManifest(
 
 async function uploadManifest(
   version: number,
-  {
+  { postUploadUpdateFunctions, deleteKeys }: GeneratedManifestType,
-    postUploadUpdateFunctions,
+  { newItems, storageManifest }: EncryptedManifestType
-    deleteKeys,
-    newItems,
-    storageManifest,
-  }: GeneratedManifestType
 ): Promise<void> {
   if (!window.textsecure.messaging) {
     throw new Error('storageService.uploadManifest: We are offline!');
   }
 
-  if (newItems.size === 0 && deleteKeys.length === 0) {
+  if (newItems.size === 0 && deleteKeys.size === 0) {
     log.info(`storageService.upload(${version}): nothing to upload`);
     return;
   }
@@ -712,13 +727,15 @@ async function uploadManifest(
   try {
     log.info(
       `storageService.upload(${version}): inserting=${newItems.size} ` +
-        `deleting=${deleteKeys.length}`
+        `deleting=${deleteKeys.size}`
     );
 
     const writeOperation = new Proto.WriteOperation();
     writeOperation.manifest = storageManifest;
     writeOperation.insertItem = Array.from(newItems);
-    writeOperation.deleteKey = deleteKeys;
+    writeOperation.deleteKey = Array.from(deleteKeys).map(storageID =>
+      Bytes.fromBase64(storageID)
+    );
 
     await window.textsecure.messaging.modifyStorageRecords(
       Proto.WriteOperation.encode(writeOperation).finish(),
@@ -813,16 +830,19 @@ async function createNewManifest() {
 
   const version = window.storage.get('manifestVersion', 0);
 
-  const { postUploadUpdateFunctions, newItems, storageManifest } =
+  const generatedManifest = await generateManifest(version, undefined, true);
-    await generateManifest(version, undefined, true);
 
-  await uploadManifest(version, {
+  const encryptedManifest = await encryptManifest(version, generatedManifest);
-    postUploadUpdateFunctions,
+
+  await uploadManifest(
+    version,
+    {
+      ...generatedManifest,
       // we have created a new manifest, there should be no keys to delete
-    deleteKeys: [],
+      deleteKeys: new Set(),
-    newItems,
+    },
-    storageManifest,
+    encryptedManifest
-  });
+  );
 }
 
 async function decryptManifest(
@@ -1158,7 +1178,8 @@ async function processManifest(
 
   let conflictCount = 0;
   if (remoteOnlyRecords.size) {
-    conflictCount = await processRemoteRecords(version, remoteOnlyRecords);
+    const fetchResult = await fetchRemoteRecords(version, remoteOnlyRecords);
+    conflictCount = await processRemoteRecords(version, fetchResult);
   }
 
   // Post-merge, if our local records contain any storage IDs that were not
@@ -1302,10 +1323,15 @@ async function processManifest(
   return conflictCount;
 }
 
-async function processRemoteRecords(
+export type FetchRemoteRecordsResultType = Readonly<{
+  missingKeys: Set<string>;
+  decryptedItems: ReadonlyArray<MergeableItemType>;
+}>;
+
+async function fetchRemoteRecords(
   storageVersion: number,
   remoteOnlyRecords: Map<string, RemoteRecord>
-): Promise<number> {
+): Promise<FetchRemoteRecordsResultType> {
   const storageKeyBase64 = window.storage.get('storageKey');
   if (!storageKeyBase64) {
     throw new Error('No storage key');
@@ -1318,8 +1344,8 @@ async function processRemoteRecords(
   const storageKey = Bytes.fromBase64(storageKeyBase64);
 
   log.info(
-    `storageService.process(${storageVersion}): fetching remote keys ` +
+    `storageService.fetchRemoteRecords(${storageVersion}): ` +
-      `count=${remoteOnlyRecords.size}`
+      `fetching remote keys count=${remoteOnlyRecords.size}`
   );
 
   const credentials = window.storage.get('storageCredentials');
@@ -1349,7 +1375,7 @@ async function processRemoteRecords(
 
   const missingKeys = new Set<string>(remoteOnlyRecords.keys());
 
-  const decryptedStorageItems = await pMap(
+  const decryptedItems = await pMap(
     storageItems,
     async (
       storageRecordWrapper: Proto.IStorageItem
@@ -1410,17 +1436,24 @@ async function processRemoteRecords(
   );
 
   log.info(
-    `storageService.process(${storageVersion}): missing remote ` +
+    `storageService.fetchRemoteRecords(${storageVersion}): missing remote ` +
       `keys=${JSON.stringify(redactedMissingKeys)} ` +
       `count=${missingKeys.size}`
   );
 
+  return { decryptedItems, missingKeys };
+}
+
+async function processRemoteRecords(
+  storageVersion: number,
+  { decryptedItems, missingKeys }: FetchRemoteRecordsResultType
+): Promise<number> {
   const ITEM_TYPE = Proto.ManifestRecord.Identifier.Type;
   const droppedKeys = new Set<string>();
 
   // Drop all GV1 records for which we have GV2 record in the same manifest
   const masterKeys = new Map<string, string>();
-  for (const { itemType, storageID, storageRecord } of decryptedStorageItems) {
+  for (const { itemType, storageID, storageRecord } of decryptedItems) {
     if (itemType === ITEM_TYPE.GROUPV2 && storageRecord.groupV2?.masterKey) {
       masterKeys.set(
         Bytes.toBase64(storageRecord.groupV2.masterKey),
@@ -1431,7 +1464,7 @@ async function processRemoteRecords(
 
   let accountItem: MergeableItemType | undefined;
 
-  const prunedStorageItems = decryptedStorageItems.filter(item => {
+  const prunedStorageItems = decryptedItems.filter(item => {
     const { itemType, storageID, storageRecord } = item;
     if (itemType === ITEM_TYPE.ACCOUNT) {
       if (accountItem !== undefined) {
@@ -1775,7 +1808,8 @@ async function upload(fromSync = false): Promise<void> {
       previousManifest,
       false
     );
-    await uploadManifest(version, generatedManifest);
+    const encryptedManifest = await encryptManifest(version, generatedManifest);
+    await uploadManifest(version, generatedManifest, encryptedManifest);
 
     // Clear pending delete keys after successful upload
     await window.storage.put('storage-service-pending-deletes', []);
@@ -1819,6 +1853,67 @@ export async function eraseAllStorageServiceState({
   log.info('storageService.eraseAllStorageServiceState: complete');
 }
 
+export async function reprocessUnknownFields(): Promise<void> {
+  ourProfileKeyService.blockGetWithPromise(
+    storageJobQueue(async () => {
+      const version = window.storage.get('manifestVersion') ?? 0;
+
+      log.info(`storageService.reprocessUnknownFields(${version}): starting`);
+
+      const { recordsByID, insertKeys } = await generateManifest(
+        version,
+        undefined,
+        true
+      );
+
+      const newRecords = Array.from(
+        filter(
+          map(recordsByID, ([key, item]): MergeableItemType | undefined => {
+            if (!insertKeys.has(key)) {
+              return undefined;
+            }
+            strictAssert(
+              item.storageRecord !== undefined,
+              'Inserted records must have storageRecord'
+            );
+
+            if (!item.storageRecord.__unknownFields?.length) {
+              return undefined;
+            }
+
+            return {
+              ...item,
+
+              storageRecord: Proto.StorageRecord.decode(
+                Proto.StorageRecord.encode(item.storageRecord).finish()
+              ),
+            };
+          }),
+          isNotNil
+        )
+      );
+
+      const conflictCount = await processRemoteRecords(version, {
+        decryptedItems: newRecords,
+        missingKeys: new Set(),
+      });
+
+      log.info(
+        `storageService.reprocessUnknownFields(${version}): done, ` +
+          `conflictCount=${conflictCount}`
+      );
+
+      const hasConflicts = conflictCount !== 0;
+      if (hasConflicts) {
+        log.info(
+          `storageService.reprocessUnknownFields(${version}): uploading`
+        );
+        await upload();
+      }
+    })
+  );
+}
+
 export const storageServiceUploadJob = debounce(() => {
   if (!storageServiceEnabled) {
     log.info('storageService.storageServiceUploadJob: called before enabled');

ts/services/storageRecordOps.ts

@@ -50,7 +50,7 @@ import type {
   StickerPackInfoType,
 } from '../sql/Interface';
 import dataInterface from '../sql/Client';
-import { MY_STORIES_ID } from '../types/Stories';
+import { MY_STORIES_ID, StorySendMode } from '../types/Stories';
 import * as RemoteConfig from '../RemoteConfig';
 
 const MY_STORIES_BYTES = uuidToBytes(MY_STORIES_ID);
@@ -406,6 +406,18 @@ export function toGroupV2Record(
     conversation.get('dontNotifyForMentionsIfMuted')
   );
   groupV2Record.hideStory = Boolean(conversation.get('hideStory'));
+  const storySendMode = conversation.get('storySendMode');
+  if (storySendMode !== undefined) {
+    if (storySendMode === StorySendMode.IfActive) {
+      groupV2Record.storySendMode = Proto.GroupV2Record.StorySendMode.DEFAULT;
+    } else if (storySendMode === StorySendMode.Never) {
+      groupV2Record.storySendMode = Proto.GroupV2Record.StorySendMode.DISABLED;
+    } else if (storySendMode === StorySendMode.Always) {
+      groupV2Record.storySendMode = Proto.GroupV2Record.StorySendMode.ENABLED;
+    } else {
+      throw missingCaseError(storySendMode);
+    }
+  }
 
   applyUnknownFields(groupV2Record, conversation);
 
@@ -785,6 +797,23 @@ export async function mergeGroupV2Record(
   const oldStorageID = conversation.get('storageID');
   const oldStorageVersion = conversation.get('storageVersion');
 
+  const recordStorySendMode =
+    groupV2Record.storySendMode ?? Proto.GroupV2Record.StorySendMode.DEFAULT;
+  let storySendMode: StorySendMode;
+  if (recordStorySendMode === Proto.GroupV2Record.StorySendMode.DEFAULT) {
+    storySendMode = StorySendMode.IfActive;
+  } else if (
+    recordStorySendMode === Proto.GroupV2Record.StorySendMode.DISABLED
+  ) {
+    storySendMode = StorySendMode.Never;
+  } else if (
+    recordStorySendMode === Proto.GroupV2Record.StorySendMode.ENABLED
+  ) {
+    storySendMode = StorySendMode.Always;
+  } else {
+    throw missingCaseError(recordStorySendMode);
+  }
+
   conversation.set({
     hideStory: Boolean(groupV2Record.hideStory),
     isArchived: Boolean(groupV2Record.archived),
@@ -794,6 +823,7 @@ export async function mergeGroupV2Record(
     ),
     storageID,
     storageVersion,
+    storySendMode,
   });
 
   conversation.setMuteExpiration(

ts/signal.ts

@@ -69,12 +69,7 @@ import { initializeGroupCredentialFetcher } from './services/groupCredentialFetc
 import { initializeNetworkObserver } from './services/networkObserver';
 import { initializeUpdateListener } from './services/updateListener';
 import { calling } from './services/calling';
-import {
+import * as storage from './services/storage';
-  enableStorageService,
-  eraseAllStorageServiceState,
-  runStorageServiceSyncJob,
-  storageServiceUploadJob,
-} from './services/storage';
 
 import type { LoggerType } from './types/Logging';
 import type {
@@ -449,13 +444,12 @@ export const setup = (options: {
 
   const Services = {
     calling,
-    enableStorageService,
-    eraseAllStorageServiceState,
     initializeGroupCredentialFetcher,
     initializeNetworkObserver,
     initializeUpdateListener,
-    runStorageServiceSyncJob,
+
-    storageServiceUploadJob,
+    // Testing
+    storage,
   };
 
   const State = {

ts/state/ducks/conversations.ts

@@ -49,6 +49,7 @@ import type { BodyRangeType } from '../../types/Util';
 import { CallMode } from '../../types/Calling';
 import type { MediaItemType } from '../../types/MediaItem';
 import type { UUIDStringType } from '../../types/UUID';
+import { StorySendMode } from '../../types/Stories';
 import {
   getGroupSizeRecommendedLimit,
   getGroupSizeHardLimit,
@@ -215,7 +216,7 @@ export type ConversationType = {
   groupVersion?: 1 | 2;
   groupId?: string;
   groupLink?: string;
-  isGroupStorySendReady?: boolean;
+  storySendMode?: StorySendMode;
   messageRequestsEnabled?: boolean;
   acceptedMessageRequest: boolean;
   secretParams?: string;
@@ -2098,10 +2099,17 @@ function toggleGroupsForStorySend(
           return;
         }
 
+        const oldStorySendMode = conversation.getStorySendMode();
+        const newStorySendMode =
+          oldStorySendMode === StorySendMode.Always
+            ? StorySendMode.Never
+            : StorySendMode.Always;
+
         conversation.set({
-          isGroupStorySendReady: !conversation.get('isGroupStorySendReady'),
+          storySendMode: newStorySendMode,
         });
         await window.Signal.Data.updateConversation(conversation.attributes);
+        conversation.captureChange('storySendMode');
       })
     );
 

ts/state/selectors/conversations.ts

@@ -17,6 +17,7 @@ import type {
   MessagesByConversationType,
   PreJoinConversationType,
 } from '../ducks/conversations';
+import type { StoriesStateType } from '../ducks/stories';
 import type { UsernameSaveState } from '../ducks/conversationsEnums';
 import {
   ComposerStep,
@@ -43,6 +44,7 @@ import {
   isGroupV1,
   isGroupV2,
 } from '../../util/whatTypeOfConversation';
+import { isGroupInStoryMode } from '../../util/isGroupInStoryMode';
 
 import {
   getIntl,
@@ -531,18 +533,37 @@ export const getComposableGroups = createSelector(
     )
 );
 
+const getConversationIdsWithStories = createSelector(
+  (state: StateType): StoriesStateType => state.stories,
+  (stories: StoriesStateType): Set<string> => {
+    return new Set(stories.stories.map(({ conversationId }) => conversationId));
+  }
+);
+
 export const getNonGroupStories = createSelector(
   getComposableGroups,
-  (groups: Array<ConversationType>): Array<ConversationType> =>
+  getConversationIdsWithStories,
-    groups.filter(group => !group.isGroupStorySendReady)
+  (
+    groups: Array<ConversationType>,
+    conversationIdsWithStories: Set<string>
+  ): Array<ConversationType> => {
+    return groups.filter(
+      group => !isGroupInStoryMode(group, conversationIdsWithStories)
+    );
+  }
 );
 
 export const getGroupStories = createSelector(
   getConversationLookup,
-  (conversationLookup: ConversationLookupType): Array<ConversationType> =>
+  getConversationIdsWithStories,
-    Object.values(conversationLookup).filter(
+  (
-      conversation => conversation.isGroupStorySendReady
+    conversationLookup: ConversationLookupType,
-    )
+    conversationIdsWithStories: Set<string>
+  ): Array<ConversationType> => {
+    return Object.values(conversationLookup).filter(conversation =>
+      isGroupInStoryMode(conversation, conversationIdsWithStories)
+    );
+  }
 );
 
 const getNormalizedComposerConversationSearchTerm = createSelector(

ts/types/StorageService.d.ts

@@ -8,6 +8,9 @@ export type ExtendedStorageID = {
 
 export type RemoteRecord = ExtendedStorageID & {
   itemType: number;
+
+  // For compatibility with MergeableItemType
+  storageRecord?: void;
 };
 
 export type UnknownRecord = RemoteRecord;

ts/types/Stories.ts

@@ -145,6 +145,12 @@ export enum HasStories {
   Unread = 'Unread',
 }
 
+export enum StorySendMode {
+  IfActive = 'IfActive',
+  Always = 'Always',
+  Never = 'Never',
+}
+
 const getStoriesAvailable = () =>
   isEnabled('desktop.stories') ||
   isEnabled('desktop.internalUser') ||

ts/util/isGroupInStoryMode.ts

@@ -0,0 +1,23 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { ConversationType } from '../state/ducks/conversations';
+import { StorySendMode } from '../types/Stories';
+import { assertDev } from './assert';
+
+export function isGroupInStoryMode(
+  { id, type, storySendMode }: ConversationType,
+  conversationIdsWithStories: Set<string>
+): boolean {
+  if (type !== 'group') {
+    return false;
+  }
+  assertDev(
+    storySendMode !== undefined,
+    'isGroupInStoryMode: groups must have storySendMode field'
+  );
+  if (storySendMode === StorySendMode.IfActive) {
+    return conversationIdsWithStories.has(id);
+  }
+  return storySendMode === StorySendMode.Always;
+}

ts/window.d.ts

@@ -5,7 +5,6 @@
 
 /* eslint-disable max-classes-per-file */
 
-import type { Cancelable } from 'lodash';
 import type { Store } from 'redux';
 import type * as Backbone from 'backbone';
 import type * as Underscore from 'underscore';
@@ -29,6 +28,7 @@ import type {
 } from './challenge';
 import type { WebAPIConnectType } from './textsecure/WebAPI';
 import type { CallingClass } from './services/calling';
+import type * as StorageService from './services/storage';
 import type * as Groups from './groups';
 import type * as Crypto from './Crypto';
 import type * as Curve from './Curve';
@@ -143,17 +143,12 @@ export type SignalCoreType = {
   RemoteConfig: typeof RemoteConfig;
   Services: {
     calling: CallingClass;
-    enableStorageService: () => void;
-    eraseAllStorageServiceState: (options?: {
-      keepUnknownFields?: boolean | undefined;
-    }) => Promise<void>;
     initializeGroupCredentialFetcher: () => Promise<void>;
     initializeNetworkObserver: (network: ReduxActions['network']) => void;
     initializeUpdateListener: (updates: ReduxActions['updates']) => void;
     retryPlaceholders?: Util.RetryPlaceholders;
     lightSessionResetQueue?: PQueue;
-    runStorageServiceSyncJob: (() => void) & Cancelable;
+    storage: typeof StorageService;
-    storageServiceUploadJob: (() => void) & Cancelable;
   };
   Migrations: ReturnType<typeof initializeMigrations>;
   Types: {