Save group send endorsements

package.json

@@ -211,7 +211,7 @@
     "@formatjs/intl": "2.6.7",
     "@indutny/rezip-electron": "1.3.1",
     "@mixer/parallel-prettier": "2.0.3",
-    "@signalapp/mock-server": "6.4.2",
+    "@signalapp/mock-server": "6.4.3",
     "@storybook/addon-a11y": "7.4.5",
     "@storybook/addon-actions": "7.4.5",
     "@storybook/addon-controls": "7.4.5",

ts/groups.ts

@@ -96,6 +96,10 @@ import { SeenStatus } from './MessageSeenStatus';
 import { incrementMessageCounter } from './util/incrementMessageCounter';
 import { sleep } from './util/sleep';
 import { groupInvitesRoute } from './util/signalRoutes';
+import {
+  decodeGroupSendEndorsementResponse,
+  isGroupSendEndorsementResponseEmpty,
+} from './util/groupSendEndorsements';
 
 type AccessRequiredEnum = Proto.AccessControl.AccessRequired;
 
@@ -1440,29 +1444,26 @@ export function buildPromoteMemberChange({
 
 async function uploadGroupChange({
   actions,
-  group,
+  groupId,
+  groupPublicParamsBase64,
+  groupSecretParamsBase64,
   inviteLinkPassword,
 }: {
   actions: Proto.GroupChange.IActions;
-  group: ConversationAttributesType;
+  groupId: string;
+  groupPublicParamsBase64: string;
+  groupSecretParamsBase64: string;
   inviteLinkPassword?: string;
 }): Promise<Proto.IGroupChangeResponse> {
-  const logId = idForLogging(group.groupId);
+  const logId = idForLogging(groupId);
 
   // Ensure we have the credentials we need before attempting GroupsV2 operations
   await maybeFetchNewCredentials();
 
-  if (!group.secretParams) {
-    throw new Error('uploadGroupChange: group was missing secretParams!');
-  }
-  if (!group.publicParams) {
-    throw new Error('uploadGroupChange: group was missing publicParams!');
-  }
-
   return makeRequestWithCredentials({
     logId: `uploadGroupChange/${logId}`,
-    publicParams: group.publicParams,
+    publicParams: groupPublicParamsBase64,
-    secretParams: group.secretParams,
+    secretParams: groupSecretParamsBase64,
     request: (sender, options) =>
       sender.modifyGroup(actions, options, inviteLinkPassword),
   });
@@ -1551,14 +1552,26 @@ export async function modifyGroupV2({
           );
         }
 
+        const { groupId, secretParams, publicParams } = conversation.attributes;
+        strictAssert(groupId, 'modifyGroupV2: missing groupId');
+        strictAssert(secretParams, 'modifyGroupV2: missing secretParams');
+        strictAssert(publicParams, 'modifyGroupV2: missing publicParams');
+
         // Upload. If we don't have permission, the server will return an error here.
         const groupChangeResponse = await uploadGroupChange({
           actions,
+          groupId,
+          groupPublicParamsBase64: publicParams,
+          groupSecretParamsBase64: secretParams,
           inviteLinkPassword,
-          group: conversation.attributes,
         });
-        const { groupChange } = groupChangeResponse;
+        const { groupChange, groupSendEndorsementResponse } =
-        strictAssert(groupChange, 'missing groupChange');
+          groupChangeResponse;
+        strictAssert(groupChange, 'modifyGroupV2: missing groupChange');
+        strictAssert(
+          groupSendEndorsementResponse,
+          'modifyGroupV2: missing groupSendEndorsementResponse'
+        );
 
         const groupChangeBuffer =
           Proto.GroupChange.encode(groupChange).finish();
@@ -1588,6 +1601,21 @@ export async function modifyGroupV2({
           recipients: syncMessageOnly ? [] : groupV2Info.members.slice(),
           revision: groupV2Info.revision,
         });
+
+        // Read this after `maybeUpdateGroup` because it may have been updated
+        const { membersV2 } = conversation.attributes;
+        strictAssert(membersV2, 'modifyGroupV2: missing membersV2');
+
+        const groupEndorsementData = decodeGroupSendEndorsementResponse({
+          groupId,
+          groupSendEndorsementResponse,
+          groupSecretParamsBase64: secretParams,
+          groupMembersV2: membersV2,
+        });
+
+        await dataInterface.replaceAllEndorsementsForGroup(
+          groupEndorsementData
+        );
       });
 
       // If we've gotten here with no error, we exit!
@@ -1875,20 +1903,35 @@ export async function createGroupV2(
     pendingMembersV2,
   };
 
-  const groupProto = await buildGroupProto({
+  const groupProto = buildGroupProto({
     id: groupId,
     avatarUrl: uploadedAvatar?.key,
     ...protoAndConversationAttributes,
   });
 
   try {
-    await makeRequestWithCredentials({
+    const groupResponse = await makeRequestWithCredentials({
       logId: `createGroupV2/${logId}`,
       publicParams,
       secretParams,
       request: (sender, requestOptions) =>
         sender.createGroup(groupProto, requestOptions),
     });
+
+    const { groupSendEndorsementResponse } = groupResponse;
+    strictAssert(
+      groupSendEndorsementResponse,
+      'missing groupSendEndorsementResponse'
+    );
+
+    const groupEndorsementData = decodeGroupSendEndorsementResponse({
+      groupId,
+      groupSendEndorsementResponse,
+      groupSecretParamsBase64: secretParams,
+      groupMembersV2: membersV2,
+    });
+
+    await dataInterface.replaceAllEndorsementsForGroup(groupEndorsementData);
   } catch (error) {
     if (!(error instanceof HTTPError)) {
       throw error;
@@ -2394,13 +2437,17 @@ export async function initiateMigrationToGroupV2(
         avatarUrl: avatarAttribute?.url,
       });
 
+      let groupSendEndorsementResponse: Uint8Array | null | undefined;
       try {
-        await makeRequestWithCredentials({
+        const groupResponse = await makeRequestWithCredentials({
           logId: `createGroup/${logId}`,
           publicParams,
           secretParams,
           request: (sender, options) => sender.createGroup(groupProto, options),
         });
+
+        groupSendEndorsementResponse =
+          groupResponse.groupSendEndorsementResponse;
       } catch (error) {
         log.error(
           `initiateMigrationToGroupV2/${logId}: Error creating group:`,
@@ -2442,6 +2489,20 @@ export async function initiateMigrationToGroupV2(
 
       // Save these most recent updates to conversation
       updateConversation(conversation.attributes);
+
+      strictAssert(
+        groupSendEndorsementResponse,
+        'missing groupSendEndorsementResponse'
+      );
+
+      const groupEndorsementData = decodeGroupSendEndorsementResponse({
+        groupId,
+        groupSendEndorsementResponse,
+        groupSecretParamsBase64: secretParams,
+        groupMembersV2: membersV2,
+      });
+
+      await dataInterface.replaceAllEndorsementsForGroup(groupEndorsementData);
     });
   } catch (error) {
     const logId = conversation.idForLogging();
@@ -2714,6 +2775,7 @@ export async function respondToGroupV2Migration({
   };
 
   let firstGroupState: Proto.IGroup | null | undefined;
+  let groupSendEndorsementResponse: Uint8Array | null | undefined;
 
   try {
     const response: GroupLogResponseType = await makeRequestWithCredentials({
@@ -2727,6 +2789,7 @@ export async function respondToGroupV2Migration({
             includeFirstState: true,
             includeLastState: false,
             maxSupportedChangeEpoch: SUPPORTED_CHANGE_EPOCH,
+            cachedEndorsementsExpiration: null, // we won't have them here
           },
           options
         ),
@@ -2734,6 +2797,7 @@ export async function respondToGroupV2Migration({
 
     // Attempt to start with the first group state, only later processing future updates
     firstGroupState = response?.changes?.groupChanges?.[0]?.groupState;
+    groupSendEndorsementResponse = response.groupSendEndorsementResponse;
   } catch (error) {
     if (error.code === GROUP_ACCESS_DENIED_CODE) {
       log.info(
@@ -2746,7 +2810,10 @@ export async function respondToGroupV2Migration({
           secretParams,
           request: (sender, options) => sender.getGroup(options),
         });
+
         firstGroupState = groupResponse.group;
+        groupSendEndorsementResponse =
+          groupResponse.groupSendEndorsementResponse;
       } catch (secondError) {
         if (secondError.code === GROUP_ACCESS_DENIED_CODE) {
           log.info(
@@ -2910,6 +2977,20 @@ export async function respondToGroupV2Migration({
     receivedAt,
     sentAt,
   });
+
+  if (!isGroupSendEndorsementResponseEmpty(groupSendEndorsementResponse)) {
+    const { membersV2 } = conversation.attributes;
+    strictAssert(membersV2, 'missing membersV2');
+
+    const groupEndorsementData = decodeGroupSendEndorsementResponse({
+      groupId,
+      groupSendEndorsementResponse,
+      groupSecretParamsBase64: secretParams,
+      groupMembersV2: membersV2,
+    });
+
+    await dataInterface.replaceAllEndorsementsForGroup(groupEndorsementData);
+  }
 }
 
 // Fetching and applying group changes
@@ -3648,12 +3729,15 @@ async function updateGroupViaState({
 }): Promise<UpdatesResultType> {
   const logId = idForLogging(group.groupId);
   const { publicParams, secretParams } = group;
-  if (!secretParams) {
+
-    throw new Error('updateGroupViaState: group was missing secretParams!');
+  strictAssert(
-  }
+    secretParams,
-  if (!publicParams) {
+    'updateGroupViaState: group was missing secretParams!'
-    throw new Error('updateGroupViaState: group was missing publicParams!');
+  );
-  }
+  strictAssert(
+    publicParams,
+    'updateGroupViaState: group was missing publicParams!'
+  );
 
   const groupResponse = await makeRequestWithCredentials({
     logId: `getGroup/${logId}`,
@@ -3662,8 +3746,12 @@ async function updateGroupViaState({
     request: (sender, requestOptions) => sender.getGroup(requestOptions),
   });
 
-  const groupState = groupResponse.group;
+  const { group: groupState, groupSendEndorsementResponse } = groupResponse;
-  strictAssert(groupState, 'Group state must be present');
+  strictAssert(groupState, 'updateGroupViaState: Group state must be present');
+  strictAssert(
+    groupSendEndorsementResponse,
+    'updateGroupViaState: Endorsement must be present'
+  );
 
   const decryptedGroupState = decryptGroupState(
     groupState,
@@ -3681,6 +3769,26 @@ async function updateGroupViaState({
     groupState: decryptedGroupState,
   });
 
+  // If we're not in the group, we won't receive endorsements
+  if (
+    groupSendEndorsementResponse != null &&
+    groupSendEndorsementResponse.byteLength > 0
+  ) {
+    // Use the latest state of the group after applying changes
+    const { groupId, membersV2 } = newAttributes;
+    strictAssert(groupId, 'updateGroupViaState: Group must have groupId');
+    strictAssert(membersV2, 'updateGroupViaState: Group must have membersV2');
+
+    const groupEndorsementData = decodeGroupSendEndorsementResponse({
+      groupId,
+      groupSendEndorsementResponse,
+      groupSecretParamsBase64: secretParams,
+      groupMembersV2: membersV2,
+    });
+
+    await dataInterface.replaceAllEndorsementsForGroup(groupEndorsementData);
+  }
+
   return {
     newAttributes,
     groupChangeMessages: extractDiffs({
@@ -3798,7 +3906,14 @@ async function updateGroupViaLogs({
   // `integrateGroupChanges`.
   let revisionToFetch = isNumber(currentRevision) ? currentRevision : undefined;
 
+  const { groupId } = group;
+  strictAssert(groupId != null, 'Group must have groupId');
+
+  let cachedEndorsementsExpiration =
+    await dataInterface.getGroupSendCombinedEndorsementExpiration(groupId);
+
   let response: GroupLogResponseType;
+  let groupSendEndorsementResponse: Uint8Array | null = null;
   const changes: Array<Proto.IGroupChanges> = [];
   do {
     // eslint-disable-next-line no-await-in-loop
@@ -3815,29 +3930,71 @@ async function updateGroupViaLogs({
             includeFirstState,
             includeLastState: true,
             maxSupportedChangeEpoch: SUPPORTED_CHANGE_EPOCH,
+            cachedEndorsementsExpiration,
           },
           requestOptions
         ),
     });
 
+    // When the log is long enough that it needs to be paginated, the server is
+    // not stateful enough to only give us endorsements when we need them.
+    // In this case we need to delete all endorsements and send `0` to get
+    // endorsements from the next page.
+    if (response.paginated && cachedEndorsementsExpiration != null) {
+      log.info(
+        'updateGroupViaLogs: Received paginated response, deleting group endorsements'
+      );
+      // eslint-disable-next-line no-await-in-loop
+      await dataInterface.deleteAllEndorsementsForGroup(groupId);
+      cachedEndorsementsExpiration = null; // gets sent as 0 in header
+    }
+
+    // Note: We should only get this on the final page
+    if (response.groupSendEndorsementResponse != null) {
+      groupSendEndorsementResponse = response.groupSendEndorsementResponse;
+    }
+
     changes.push(response.changes);
-    if (response.end) {
+    if (response.paginated && response.end) {
       revisionToFetch = response.end + 1;
     }
 
     includeFirstState = false;
   } while (
+    response.paginated &&
     response.end &&
     (newRevision === undefined || response.end < newRevision)
   );
 
   // Would be nice to cache the unused groupChanges here, to reduce server roundtrips
 
-  return integrateGroupChanges({
+  const updates = await integrateGroupChanges({
     changes,
     group,
     newRevision,
   });
+
+  // If we're not in the group, we won't receive endorsements
+  if (!isGroupSendEndorsementResponseEmpty(groupSendEndorsementResponse)) {
+    log.info('updateGroupViaLogs: Saving group endorsements');
+    // Use the latest state of the group after applying changes
+    const { membersV2 } = updates.newAttributes;
+    strictAssert(
+      membersV2 != null,
+      'updateGroupViaLogs: Group must have membersV2'
+    );
+
+    const groupEndorsementData = decodeGroupSendEndorsementResponse({
+      groupId,
+      groupSendEndorsementResponse,
+      groupMembersV2: membersV2,
+      groupSecretParamsBase64: secretParams,
+    });
+
+    await dataInterface.replaceAllEndorsementsForGroup(groupEndorsementData);
+  }
+
+  return updates;
 }
 
 async function generateLeftGroupChanges(
@@ -5914,6 +6071,7 @@ function decryptGroupChange(
         addMember.added,
         'decryptGroupChange: AddMember was missing added field!'
       );
+
       const decrypted = decryptMember(
         clientZkGroupCipher,
         addMember.added,

ts/sql/Interface.ts

@@ -31,6 +31,7 @@ import type {
 } from '../types/CallDisposition';
 import type { CallLinkStateType, CallLinkType } from '../types/CallLink';
 import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
+import type { GroupSendEndorsementsData } from '../types/GroupSendEndorsements';
 
 export type AdjacentMessagesByConversationOptionsType = Readonly<{
   conversationId: string;
@@ -526,6 +527,14 @@ export type DataInterface = {
     serviceId: ServiceIdString
   ) => Promise<Array<ConversationType>>;
 
+  replaceAllEndorsementsForGroup: (
+    data: GroupSendEndorsementsData
+  ) => Promise<void>;
+  deleteAllEndorsementsForGroup: (groupId: string) => Promise<void>;
+  getGroupSendCombinedEndorsementExpiration: (
+    groupId: string
+  ) => Promise<number | null>;
+
   getMessageCount: (conversationId?: string) => Promise<number>;
   getStoryCount: (conversationId: string) => Promise<number>;
   saveMessage: (

ts/sql/Server.ts

@@ -174,6 +174,11 @@ import {
   updateCallLinkAdminKeyByRoomId,
   updateCallLinkState,
 } from './server/callLinks';
+import {
+  replaceAllEndorsementsForGroup,
+  deleteAllEndorsementsForGroup,
+  getGroupSendCombinedEndorsementExpiration,
+} from './server/groupEndorsements';
 import { CallMode } from '../types/Calling';
 import {
   attachmentDownloadJobSchema,
@@ -286,6 +291,10 @@ const dataInterface: ServerInterface = {
   getAllConversationIds,
   getAllGroupsInvolvingServiceId,
 
+  replaceAllEndorsementsForGroup,
+  deleteAllEndorsementsForGroup,
+  getGroupSendCombinedEndorsementExpiration,
+
   searchMessages,
 
   getMessageCount,
@@ -4530,9 +4539,9 @@ function getAttachmentDownloadJob(
     SELECT * FROM attachment_downloads
     WHERE
       messageId = ${job.messageId}
-    AND 
+    AND
       attachmentType = ${job.attachmentType}
-    AND 
+    AND
       digest = ${job.digest};
   `;
 
@@ -4570,7 +4579,7 @@ async function getNextAttachmentDownloadJobs({
         })
       AND
         messageId IN (${sqlJoin(prioritizeMessageIds)})
-      -- for priority messages, let's load them oldest first; this helps, e.g. for stories where we 
+      -- for priority messages, let's load them oldest first; this helps, e.g. for stories where we
       -- want the oldest one first
       ORDER BY receivedAt ASC
       LIMIT ${limit}
@@ -4681,11 +4690,11 @@ function removeAttachmentDownloadJobSync(
 ): void {
   const [query, params] = sql`
     DELETE FROM attachment_downloads
-    WHERE 
+    WHERE
       messageId = ${job.messageId}
     AND
-      attachmentType = ${job.attachmentType} 
+      attachmentType = ${job.attachmentType}
-    AND 
+    AND
       digest = ${job.digest};
   `;
 
@@ -6003,6 +6012,8 @@ async function removeAll(): Promise<void> {
       DELETE FROM conversations;
       DELETE FROM emojis;
       DELETE FROM groupCallRingCancellations;
+      DELETE FROM groupSendCombinedEndorsement;
+      DELETE FROM groupSendMemberEndorsement;
       DELETE FROM identityKeys;
       DELETE FROM items;
       DELETE FROM jobs;
@@ -6053,6 +6064,8 @@ async function removeAllConfiguration(): Promise<void> {
   db.transaction(() => {
     db.exec(
       `
+      DELETE FROM groupSendCombinedEndorsement;
+      DELETE FROM groupSendMemberEndorsement;
       DELETE FROM identityKeys;
       DELETE FROM jobs;
       DELETE FROM kyberPreKeys;

ts/sql/migrations/1050-group-send-endorsements.ts

@@ -0,0 +1,50 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from '@signalapp/better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+import { sql } from '../util';
+
+export const version = 1050;
+
+export function updateToSchemaVersion1050(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 1050) {
+    return;
+  }
+
+  db.transaction(() => {
+    const [createTables] = sql`
+      DROP TABLE IF EXISTS groupSendCombinedEndorsement;
+      DROP TABLE IF EXISTS groupSendMemberEndorsement;
+
+      -- From GroupSendEndorsementsResponse->ReceivedEndorsements in libsignal
+      -- this is the combined endorsement for all group members
+      CREATE TABLE groupSendCombinedEndorsement (
+        groupId TEXT NOT NULL PRIMARY KEY, -- Only one endorsement per group
+        expiration INTEGER NOT NULL, -- Unix timestamp in seconds
+        endorsement BLOB NOT NULL
+      ) STRICT;
+
+      -- From GroupSendEndorsementsResponse->ReceivedEndorsements in libsignal
+      -- these are the individual endorsements for each group member
+      CREATE TABLE groupSendMemberEndorsement (
+        groupId TEXT NOT NULL,
+        memberAci TEXT NOT NULL,
+        expiration INTEGER NOT NULL, -- Unix timestamp in seconds
+        endorsement BLOB NOT NULL,
+        PRIMARY KEY (groupId, memberAci) -- Only one endorsement per group member
+      ) STRICT;
+    `;
+
+    db.exec(createTables);
+
+    db.pragma('user_version = 1050');
+  })();
+
+  logger.info('updateToSchemaVersion1050: success!');
+}

ts/sql/migrations/index.ts

@@ -79,10 +79,11 @@ import { updateToSchemaVersion1000 } from './1000-mark-unread-call-history-messa
 import { updateToSchemaVersion1010 } from './1010-call-links-table';
 import { updateToSchemaVersion1020 } from './1020-self-merges';
 import { updateToSchemaVersion1030 } from './1030-unblock-event';
+import { updateToSchemaVersion1040 } from './1040-undownloaded-backed-up-media';
 import {
-  updateToSchemaVersion1040,
+  updateToSchemaVersion1050,
   version as MAX_VERSION,
-} from './1040-undownloaded-backed-up-media';
+} from './1050-group-send-endorsements';
 
 function updateToSchemaVersion1(
   currentVersion: number,
@@ -2029,6 +2030,7 @@ export const SCHEMA_VERSIONS = [
   updateToSchemaVersion1020,
   updateToSchemaVersion1030,
   updateToSchemaVersion1040,
+  updateToSchemaVersion1050,
 ];
 
 export class DBVersionFromFutureError extends Error {

ts/sql/server/groupEndorsements.ts

@@ -0,0 +1,87 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from '@signalapp/better-sqlite3';
+import type {
+  GroupSendCombinedEndorsementRecord,
+  GroupSendEndorsementsData,
+  GroupSendMemberEndorsementRecord,
+} from '../../types/GroupSendEndorsements';
+import { groupSendEndorsementExpirationSchema } from '../../types/GroupSendEndorsements';
+import { getReadonlyInstance, getWritableInstance, prepare } from '../Server';
+import { sql } from '../util';
+
+/**
+ * We don't need to store more than one endorsement per group or per member.
+ */
+export async function replaceAllEndorsementsForGroup(
+  data: GroupSendEndorsementsData
+): Promise<void> {
+  const db = await getWritableInstance();
+  db.transaction(() => {
+    const { combinedEndorsement, memberEndorsements } = data;
+    _replaceCombinedEndorsement(db, combinedEndorsement);
+    _replaceMemberEndorsements(db, memberEndorsements);
+  })();
+}
+
+function _replaceCombinedEndorsement(
+  db: Database,
+  combinedEndorsement: GroupSendCombinedEndorsementRecord
+): void {
+  const { groupId, expiration, endorsement } = combinedEndorsement;
+  const [insertCombined, insertCombinedParams] = sql`
+    INSERT OR REPLACE INTO groupSendCombinedEndorsement
+    (groupId, expiration, endorsement)
+    VALUES (${groupId}, ${expiration}, ${endorsement});
+  `;
+  prepare<Array<unknown>>(db, insertCombined).run(insertCombinedParams);
+}
+
+function _replaceMemberEndorsements(
+  db: Database,
+  memberEndorsements: ReadonlyArray<GroupSendMemberEndorsementRecord>
+) {
+  for (const memberEndorsement of memberEndorsements) {
+    const { groupId, memberAci, expiration, endorsement } = memberEndorsement;
+    const [replaceMember, replaceMemberParams] = sql`
+      INSERT OR REPLACE INTO groupSendMemberEndorsement
+      (groupId, memberAci, expiration, endorsement)
+      VALUES (${groupId}, ${memberAci}, ${expiration}, ${endorsement});
+    `;
+    prepare<Array<unknown>>(db, replaceMember).run(replaceMemberParams);
+  }
+}
+
+export async function deleteAllEndorsementsForGroup(
+  groupId: string
+): Promise<void> {
+  const db = await getWritableInstance();
+  db.transaction(() => {
+    const [deleteCombined, deleteCombinedParams] = sql`
+      DELETE FROM groupSendCombinedEndorsement
+      WHERE groupId = ${groupId};
+    `;
+    const [deleteMembers, deleteMembersParams] = sql`
+      DELETE FROM groupSendMemberEndorsement
+      WHERE groupId = ${groupId};
+    `;
+    prepare<Array<unknown>>(db, deleteCombined).run(deleteCombinedParams);
+    prepare<Array<unknown>>(db, deleteMembers).run(deleteMembersParams);
+  })();
+}
+
+export async function getGroupSendCombinedEndorsementExpiration(
+  groupId: string
+): Promise<number | null> {
+  const db = getReadonlyInstance();
+  const [selectGroup, selectGroupParams] = sql`
+    SELECT expiration FROM groupSendCombinedEndorsement
+    WHERE groupId = ${groupId};
+  `;
+  const value = prepare(db, selectGroup).pluck().get(selectGroupParams);
+  if (value == null) {
+    return null;
+  }
+  return groupSendEndorsementExpirationSchema.parse(value);
+}

ts/sql/util.ts

@@ -36,7 +36,12 @@ export function jsonToObject<T>(json: string): T {
   return JSON.parse(json);
 }
 
-export type QueryTemplateParam = string | number | null | undefined;
+export type QueryTemplateParam =
+  | Uint8Array
+  | string
+  | number
+  | null
+  | undefined;
 export type QueryFragmentValue = QueryFragment | QueryTemplateParam;
 
 export type QueryFragment = [
@@ -148,7 +153,7 @@ export type QueryTemplate = [string, ReadonlyArray<QueryTemplateParam>];
  */
 export function sql(
   strings: TemplateStringsArray,
-  ...values: ReadonlyArray<QueryFragment | QueryTemplateParam>
+  ...values: Array<QueryFragment | QueryTemplateParam>
 ): QueryTemplate {
   const [{ fragment }, params] = sqlFragment(strings, ...values);
   return [fragment, params];

ts/textsecure/WebAPI.ts

@@ -727,14 +727,22 @@ export type GetGroupLogOptionsType = Readonly<{
   includeFirstState: boolean;
   includeLastState: boolean;
   maxSupportedChangeEpoch: number;
+  cachedEndorsementsExpiration: number | null; // seconds
 }>;
 export type GroupLogResponseType = {
-  currentRevision?: number;
-  start?: number;
-  end?: number;
   changes: Proto.GroupChanges;
   groupSendEndorsementResponse: Uint8Array | null;
-};
+} & (
+  | {
+      paginated: false;
+    }
+  | {
+      paginated: true;
+      currentRevision: number;
+      start: number;
+      end: number;
+    }
+);
 
 export type ProfileRequestDataType = {
   about: string | null;
@@ -3901,6 +3909,7 @@ export function initialize({
         includeFirstState,
         includeLastState,
         maxSupportedChangeEpoch,
+        cachedEndorsementsExpiration,
       } = options;
 
       // If we don't know starting revision - fetch it from the server
@@ -3935,8 +3944,7 @@ export function initialize({
         httpType: 'GET',
         responseType: 'byteswithdetails',
         headers: {
-          // TODO(jamie): To be implmented in DESKTOP-699
+          'Cached-Send-Endorsements': String(cachedEndorsementsExpiration ?? 0),
-          'Cached-Send-Endorsements': '0',
         },
         urlParameters:
           `/${startVersion}?` +
@@ -3963,6 +3971,7 @@ export function initialize({
           isNumber(currentRevision)
         ) {
           return {
+            paginated: true,
             changes,
             start,
             end,
@@ -3973,6 +3982,7 @@ export function initialize({
       }
 
       return {
+        paginated: false,
         changes,
         groupSendEndorsementResponse,
       };

ts/types/GroupSendEndorsements.ts

@@ -0,0 +1,79 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+import { z } from 'zod';
+import { aciSchema, type AciString } from './ServiceId';
+import * as Bytes from '../Bytes';
+
+const GROUPV2_ID_LENGTH = 32; // 32 bytes
+
+/**
+ * The endorsement for a single group member.
+ * Used when sending a group message.
+ */
+export type GroupSendCombinedEndorsementRecord = Readonly<{
+  groupId: string;
+  /** Unix timestamp in seconds */
+  expiration: number;
+  endorsement: Uint8Array;
+}>;
+
+/**
+ * The endorsement for a single group member.
+ *
+ * Used when:
+ * - Sending a group message to an individual member[1]
+ * - Fetching a pre-key bundle for an individual member[1]
+ * - Fetching a unversioned profile for an individual member[1]
+ *
+ * [1]: As a fallback when the access key comes back unauthorized.
+ */
+export type GroupSendMemberEndorsementRecord = Readonly<{
+  groupId: string;
+  memberAci: AciString;
+  /** Unix timestamp in seconds */
+  expiration: number;
+  endorsement: Uint8Array;
+}>;
+
+/**
+ * Deserialized data from a group send endorsements response.
+ * Used for updating the database in a single transaction.
+ */
+export type GroupSendEndorsementsData = Readonly<{
+  combinedEndorsement: GroupSendCombinedEndorsementRecord;
+  memberEndorsements: ReadonlyArray<GroupSendMemberEndorsementRecord>;
+}>;
+
+const groupIdSchema = z.string().refine(value => {
+  return Bytes.fromBase64(value).byteLength === GROUPV2_ID_LENGTH;
+});
+
+/**
+ * Unix timestamp in seconds.
+ * Used to trigger a group refresh when the expiration is less than 2 hours away.
+ */
+export const groupSendEndorsementExpirationSchema = z.number().int().positive(); // not 0
+
+export const groupSendEndorsementSchema = z
+  .instanceof(Uint8Array)
+  .refine(array => {
+    return array.byteLength > 0; // not empty
+  });
+
+export const groupSendCombinedEndorsementSchema = z.object({
+  groupId: groupIdSchema,
+  expiration: groupSendEndorsementExpirationSchema,
+  endorsement: groupSendEndorsementSchema,
+});
+
+export const groupSendMemberEndorsementSchema = z.object({
+  groupId: groupIdSchema,
+  memberAci: aciSchema,
+  expiration: groupSendEndorsementExpirationSchema,
+  endorsement: groupSendEndorsementSchema,
+});
+
+export const groupSendEndorsementsDataSchema = z.object({
+  combinedEndorsement: groupSendCombinedEndorsementSchema,
+  memberEndorsements: z.array(groupSendMemberEndorsementSchema).min(1),
+});

ts/util/groupSendEndorsements.ts

@@ -0,0 +1,112 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+import { Aci } from '@signalapp/libsignal-client';
+import {
+  groupSendEndorsementsDataSchema,
+  type GroupSendEndorsementsData,
+} from '../types/GroupSendEndorsements';
+import { strictAssert } from './assert';
+import {
+  GroupSecretParams,
+  GroupSendEndorsementsResponse,
+  ServerPublicParams,
+} from './zkgroup';
+import { fromAciObject } from '../types/ServiceId';
+import * as log from '../logging/log';
+import type { GroupV2MemberType } from '../model-types';
+
+/**
+ * Despite being optional, the protobufs decoding will create an empty uint8array
+ */
+export function isGroupSendEndorsementResponseEmpty(
+  value: Uint8Array | null | undefined
+): value is null | undefined {
+  return value == null || value.byteLength === 0;
+}
+
+export function decodeGroupSendEndorsementResponse({
+  groupId,
+  groupSendEndorsementResponse,
+  groupSecretParamsBase64,
+  groupMembersV2,
+}: {
+  groupId: string;
+  groupSendEndorsementResponse: Uint8Array;
+  groupSecretParamsBase64: string;
+  groupMembersV2: ReadonlyArray<GroupV2MemberType>;
+}): GroupSendEndorsementsData {
+  const idForLogging = `groupV2(${groupId})`;
+
+  strictAssert(
+    groupSendEndorsementResponse != null,
+    'Missing groupSendEndorsementResponse'
+  );
+
+  strictAssert(
+    groupSendEndorsementResponse.byteLength > 0,
+    'Received empty groupSendEndorsementResponse'
+  );
+
+  const response = new GroupSendEndorsementsResponse(
+    Buffer.from(groupSendEndorsementResponse)
+  );
+
+  const expiration = response.getExpiration().getTime();
+
+  const localUser = Aci.parseFromServiceIdString(
+    window.textsecure.storage.user.getCheckedAci()
+  );
+
+  const groupSecretParams = new GroupSecretParams(
+    Buffer.from(groupSecretParamsBase64, 'base64')
+  );
+
+  const serverPublicParams = new ServerPublicParams(
+    Buffer.from(window.getServerPublicParams(), 'base64')
+  );
+
+  const groupMembers = groupMembersV2.map(member => {
+    return Aci.parseFromServiceIdString(member.aci);
+  });
+
+  const receivedEndorsements = response.receiveWithServiceIds(
+    groupMembers,
+    localUser,
+    groupSecretParams,
+    serverPublicParams
+  );
+
+  const { combinedEndorsement, endorsements } = receivedEndorsements;
+
+  strictAssert(
+    endorsements.length === groupMembers.length,
+    `Member endorsements must match input array (expected: ${groupMembers.length}, actual: ${endorsements.length})`
+  );
+
+  log.info(
+    `decodeGroupSendEndorsementResponse: Received endorsements (group: ${idForLogging}, expiration: ${expiration}, members: ${groupMembers.length})`
+  );
+
+  const groupEndorsementData: GroupSendEndorsementsData = {
+    combinedEndorsement: {
+      groupId,
+      expiration,
+      endorsement: combinedEndorsement.getContents(),
+    },
+    memberEndorsements: groupMembers.map((groupMember, index) => {
+      const endorsement = endorsements.at(index);
+      strictAssert(
+        endorsement != null,
+        `Missing endorsement at index ${index}`
+      );
+      return {
+        groupId,
+        memberAci: fromAciObject(groupMember),
+        expiration,
+        endorsement: endorsement.getContents(),
+      };
+    }),
+  };
+
+  return groupSendEndorsementsDataSchema.parse(groupEndorsementData);
+}

yarn.lock

@@ -3992,21 +3992,21 @@
     type-fest "^3.5.0"
     uuid "^8.3.0"
 
-"@signalapp/libsignal-client@^0.42.0":
+"@signalapp/libsignal-client@^0.45.0":
-  version "0.42.0"
+  version "0.45.1"
-  resolved "https://registry.yarnpkg.com/@signalapp/libsignal-client/-/libsignal-client-0.42.0.tgz#259d87233f1e065ae93cf8fe758bcc2461e3e814"
+  resolved "https://registry.yarnpkg.com/@signalapp/libsignal-client/-/libsignal-client-0.45.1.tgz#83b0b3880ad9da522e2d948ffe6cfd1c5b8b8003"
-  integrity sha512-03lr1LmMTSy3lto8lbdaQMvuvwqs7+fatNP3Kp6dHAnR/OoXh6Y1l493U5X86Z87XGdM0gfGntxZwZ+Qju9Dpg==
+  integrity sha512-jKNGLD8QQkLEopX7Fb5XG7LlIe559TgqfC1UCgUV9YW4pPpvM+RPbW4ndL1v8WO/Toff4nVXJXJV6kzYiK2lDA==
   dependencies:
     node-gyp-build "^4.2.3"
     type-fest "^3.5.0"
     uuid "^8.3.0"
 
-"@signalapp/mock-server@6.4.2":
+"@signalapp/mock-server@6.4.3":
-  version "6.4.2"
+  version "6.4.3"
-  resolved "https://registry.yarnpkg.com/@signalapp/mock-server/-/mock-server-6.4.2.tgz#9c0ccabaf7d9a8728503245d2fa2b4d7da6a5ccd"
+  resolved "https://registry.yarnpkg.com/@signalapp/mock-server/-/mock-server-6.4.3.tgz#5672fb2c0123998ece49b68232debb07715c55a9"
-  integrity sha512-qL5wUGkbquZA6mKieuSOwlX51UyUFlLeQq+Z/F+gX910l8aYVV0niwtR1hYNPgvgxakPPXJ3VhIWE4qMgQRkrw==
+  integrity sha512-6EnR4o349f+BO7fTCfkMzOYSzJcyhGvS7JExmp7YmKWQ+/YI5dgVQfsThw9uy/GUNDtfbVzUjMkPKI2kdDm2eA==
   dependencies:
-    "@signalapp/libsignal-client" "^0.42.0"
+    "@signalapp/libsignal-client" "^0.45.0"
     debug "^4.3.2"
     long "^4.0.0"
     micro "^9.3.4"
@@ -7577,7 +7577,7 @@ caniuse-lite@^1.0.30001349, caniuse-lite@^1.0.30001541:
   resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001541.tgz"
   integrity sha512-bLOsqxDgTqUBkzxbNlSBt8annkDpQB9NdzdTbO2ooJ+eC/IQcvDspDc058g84ejCelF7vHUx57KIOjEecOHXaw==
 
-canvas@^2.6.1, "canvas@https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz":
+canvas@^2.6.1, "canvas@https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz", dmg-license@^1.0.11, "dmg-license@https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz", jsdom@^15.2.1, "jsdom@https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz":
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz#cb46cf7e01574aa6390858149f66897afe53c9ca"
 
@@ -8955,10 +8955,6 @@ dmg-builder@24.6.3:
   optionalDependencies:
     dmg-license "^1.0.11"
 
-dmg-license@^1.0.11, "dmg-license@https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz":
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz#cb46cf7e01574aa6390858149f66897afe53c9ca"
-
 dns-equal@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/dns-equal/-/dns-equal-1.0.0.tgz#b39e7f1da6eb0a75ba9c17324b34753c47e0654d"
@@ -13387,10 +13383,6 @@ jsdoc@^4.0.0:
     strip-json-comments "^3.1.0"
     underscore "~1.13.2"
 
-jsdom@^15.2.1, "jsdom@https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz":
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/nop/-/nop-1.0.0.tgz#cb46cf7e01574aa6390858149f66897afe53c9ca"
-
 jsesc@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b"
@@ -18454,7 +18446,7 @@ string-length@^5.0.1:
     char-regex "^2.0.0"
     strip-ansi "^7.0.1"
 
-"string-width-cjs@npm:string-width@^4.2.0":
+"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -18497,15 +18489,6 @@ string-width@^4.1.0, string-width@^4.2.0:
     is-fullwidth-code-point "^3.0.0"
     strip-ansi "^6.0.0"
 
-string-width@^4.2.3:
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
 string-width@^5.0.1, string-width@^5.1.2:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794"
@@ -18586,7 +18569,7 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -18620,13 +18603,6 @@ strip-ansi@^6.0.0:
   dependencies:
     ansi-regex "^5.0.0"
 
-strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.1:
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45"
@@ -20244,7 +20220,7 @@ workerpool@6.2.1:
   resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343"
   integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -20278,15 +20254,6 @@ wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"