From a404539996b24332e66070ba5cf4aa97a513f427 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 23 Jan 2025 10:17:21 +0000 Subject: [PATCH 001/124] WIP doodles on MembershipManager test cases --- spec/unit/matrixrtc/MembeshipManager.spec.ts | 244 +++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 spec/unit/matrixrtc/MembeshipManager.spec.ts diff --git a/spec/unit/matrixrtc/MembeshipManager.spec.ts b/spec/unit/matrixrtc/MembeshipManager.spec.ts new file mode 100644 index 00000000000..bc047c44f2e --- /dev/null +++ b/spec/unit/matrixrtc/MembeshipManager.spec.ts @@ -0,0 +1,244 @@ +/* +Copyright 2025 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixClient, Room } from "../../../src"; +import { MatrixRTCSession } from "../../../src/matrixrtc/MatrixRTCSession"; +import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; +import { makeMockRoom, membershipTemplate } from "./mocks"; + +describe("MatrixRTCSession", () => { + describe("LegacyMembershipManager", () => { + let client: MatrixClient; + let sess: MatrixRTCSession | undefined; + let room: Room; + + beforeEach(() => { + client = new MatrixClient({ baseUrl: "base_url" }); + client.getUserId = jest.fn().mockReturnValue("@alice:example.org"); + client.getDeviceId = jest.fn().mockReturnValue("AAAAAAA"); + room = makeMockRoom(membershipTemplate); + }); + + afterEach(() => { + client.stopClient(); + client.matrixRTC.stop(); + if (sess) sess.stop(); + sess = undefined; + }); + + describe("isJoined()", () => { + it("defaults to false", () => { + const manager = new LegacyMembershipManager({}, room, client, () => undefined); + expect(manager.isJoined()).toEqual(false); + }); + + it("returns true after join()", () => { + const manager = new LegacyMembershipManager({}, room, client, () => undefined); + manager.join([]); + expect(manager.isJoined()).toEqual(true); + }); + }); + + describe("join()", () => { + describe("sends a membership event", () => { + it("sends a membership event with session payload when joining a call", async () => {}); + + it("does not prefix the state key with _ for rooms that support user-owned state events", async () => {}); + + // const realSetTimeout = setTimeout; + // jest.useFakeTimers(); + // sess!.joinRoomSession([mockFocus], mockFocus); + // await Promise.race([sentStateEvent, new Promise((resolve) => realSetTimeout(resolve, 500))]); + // expect(client.sendStateEvent).toHaveBeenCalledWith( + // mockRoom!.roomId, + // EventType.GroupCallMemberPrefix, + // { + // application: "m.call", + // scope: "m.room", + // call_id: "", + // device_id: "AAAAAAA", + // expires: DEFAULT_EXPIRE_DURATION, + // foci_preferred: [mockFocus], + // focus_active: { + // focus_selection: "oldest_membership", + // type: "livekit", + // }, + // }, + // "_@alice:example.org_AAAAAAA", + // ); + // await Promise.race([sentDelayedState, new Promise((resolve) => realSetTimeout(resolve, 500))]); + // // Because we actually want to send the state + // expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + // // For checking if the delayed event is still there or got removed while sending the state. + // expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); + // // For scheduling the delayed event + // expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + // // This returns no error so we do not check if we reschedule the event again. this is done in another test. + + // jest.useRealTimers(); + }); + + describe("schedules a delayed leave event if server supports it", () => {}); + + it("uses membershipExpiryTimeout from config", async () => { + // const realSetTimeout = setTimeout; + // jest.useFakeTimers(); + // sess!.joinRoomSession([mockFocus], mockFocus, { membershipExpiryTimeout: 60000 }); + // await Promise.race([sentStateEvent, new Promise((resolve) => realSetTimeout(resolve, 500))]); + // expect(client.sendStateEvent).toHaveBeenCalledWith( + // mockRoom!.roomId, + // EventType.GroupCallMemberPrefix, + // { + // application: "m.call", + // scope: "m.room", + // call_id: "", + // device_id: "AAAAAAA", + // expires: 60000, + // foci_preferred: [mockFocus], + // focus_active: { + // focus_selection: "oldest_membership", + // type: "livekit", + // }, + // }, + // "_@alice:example.org_AAAAAAA", + // ); + // await Promise.race([sentDelayedState, new Promise((resolve) => realSetTimeout(resolve, 500))]); + // expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + // jest.useRealTimers(); + }); + + it("does nothing if join called when already joined", async () => { + // const realSetTimeout = setTimeout; + // jest.useFakeTimers(); + // sess!.joinRoomSession([mockFocus], mockFocus); + // await Promise.race([sentStateEvent, new Promise((resolve) => realSetTimeout(resolve, 500))]); + // expect(client.sendStateEvent).toHaveBeenCalledWith( + // mockRoom!.roomId, + // EventType.GroupCallMemberPrefix, + // { + // application: "m.call", + // scope: "m.room", + // call_id: "", + // device_id: "AAAAAAA", + // expires: DEFAULT_EXPIRE_DURATION, + // foci_preferred: [mockFocus], + // focus_active: { + // focus_selection: "oldest_membership", + // type: "livekit", + // }, + // }, + // "_@alice:example.org_AAAAAAA", + // ); + // await Promise.race([sentDelayedState, new Promise((resolve) => realSetTimeout(resolve, 500))]); + // expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + // jest.useRealTimers(); + }); + }); + + describe("leave()", () => { + it("does nothing if not joined", () => { + // const manager = new LegacyMembershipManager({}, room, client, () => undefined); + // manager.leave(); + }); + }); + + describe("getOldestMembership", () => { + it("returns the oldest membership event", () => { + jest.useFakeTimers(); + jest.setSystemTime(4000); + const mockRoom = makeMockRoom([ + Object.assign({}, membershipTemplate, { device_id: "foo", created_ts: 3000 }), + Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), + Object.assign({}, membershipTemplate, { device_id: "bar", created_ts: 2000 }), + ]); + + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess.getOldestMembership()!.deviceId).toEqual("old"); + jest.useRealTimers(); + }); + }); + + describe("getsActiveFocus", () => { + const firstPreferredFocus = { + type: "livekit", + livekit_service_url: "https://active.url", + livekit_alias: "!active:active.url", + }; + it("gets the correct active focus with oldest_membership", () => { + // jest.useFakeTimers(); + // jest.setSystemTime(3000); + // const mockRoom = makeMockRoom([ + // Object.assign({}, membershipTemplate, { + // device_id: "foo", + // created_ts: 500, + // foci_preferred: [firstPreferredFocus], + // }), + // Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), + // Object.assign({}, membershipTemplate, { device_id: "bar", created_ts: 2000 }), + // ]); + // sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + // sess.joinRoomSession([{ type: "livekit", livekit_service_url: "htts://test.org" }], { + // type: "livekit", + // focus_selection: "oldest_membership", + // }); + // expect(sess.getActiveFocus()).toBe(firstPreferredFocus); + // jest.useRealTimers(); + }); + it("does not provide focus if the selection method is unknown", () => { + // const mockRoom = makeMockRoom([ + // Object.assign({}, membershipTemplate, { + // device_id: "foo", + // created_ts: 500, + // foci_preferred: [firstPreferredFocus], + // }), + // Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), + // Object.assign({}, membershipTemplate, { device_id: "bar", created_ts: 2000 }), + // ]); + // sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + // sess.joinRoomSession([{ type: "livekit", livekit_service_url: "htts://test.org" }], { + // type: "livekit", + // focus_selection: "unknown", + // }); + // expect(sess.getActiveFocus()).toBe(undefined); + }); + }); + + describe("onRTCSessionMemberUpdate()", () => { + it("does nothing if not joined", () => {}); + it("does nothing if own membership still present", () => {}); + it("recreates membership if it is missing", () => {}); + }); + + // TODO: not sure about this name + describe("background timers", () => { + it("sends keep-alive for delayed leave event where supported", () => {}); + + it("extends `expires` when call still active", () => {}); + }); + + describe("server error handling", () => { + describe("retries sending delayed leave event", () => { + it("sends it if delayed leave event is still valid at time of retry", () => {}); + it("abandons it if delayed leave event is no longer valid at time of retry", () => {}); + }); + + describe("retries sending membership event", () => { + it("sends it if still joined at time of retry", () => {}); + it("abandons it if call no longer joined at time of retry", () => {}); + }); + }); + }); +}); From 834d461ff51db4d44281a4a54da7f3fbb473deab Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 23 Jan 2025 10:21:47 +0000 Subject: [PATCH 002/124] . --- spec/unit/matrixrtc/MembeshipManager.spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/unit/matrixrtc/MembeshipManager.spec.ts b/spec/unit/matrixrtc/MembeshipManager.spec.ts index bc047c44f2e..9ebedfc66bd 100644 --- a/spec/unit/matrixrtc/MembeshipManager.spec.ts +++ b/spec/unit/matrixrtc/MembeshipManager.spec.ts @@ -227,12 +227,16 @@ describe("MatrixRTCSession", () => { it("sends keep-alive for delayed leave event where supported", () => {}); it("extends `expires` when call still active", () => {}); + + it("does not send more than once per `membershipKeepAlivePeriod`", () => {}); }); describe("server error handling", () => { describe("retries sending delayed leave event", () => { it("sends it if delayed leave event is still valid at time of retry", () => {}); - it("abandons it if delayed leave event is no longer valid at time of retry", () => {}); + it("abandons it if delayed leave event is no longer valid at time of retry", () => { + // I think this will break on LegacyMembershipManager + }); }); describe("retries sending membership event", () => { From 6594860a2d8aceae48c6ad5af308498f1b1a2403 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 17 Feb 2025 17:19:56 +0700 Subject: [PATCH 003/124] initial membership manager test setup. --- package.json | 3 +- spec/unit/matrixrtc/MatrixRTCSession.spec.ts | 114 ---- spec/unit/matrixrtc/MembeshipManager.spec.ts | 647 ++++++++++++++----- spec/unit/matrixrtc/mocks.ts | 40 +- spec/unit/matrixrtc/testEnvironment.ts | 58 ++ src/matrixrtc/MatrixRTCSession.ts | 43 +- src/matrixrtc/MembershipManager.ts | 34 +- yarn.lock | 5 + 8 files changed, 644 insertions(+), 300 deletions(-) create mode 100644 spec/unit/matrixrtc/testEnvironment.ts diff --git a/package.json b/package.json index d46142d821b..17b87a1e3ab 100644 --- a/package.json +++ b/package.json @@ -124,7 +124,8 @@ "typedoc-plugin-coverage": "^3.0.0", "typedoc-plugin-mdn-links": "^4.0.0", "typedoc-plugin-missing-exports": "^3.0.0", - "typescript": "^5.4.2" + "typescript": "^5.4.2", + "wait-for-expect": "^3.0.2" }, "@casualbot/jest-sonar-reporter": { "outputDirectory": "coverage", diff --git a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts index 1afca35e681..963d7c5fede 100644 --- a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts +++ b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts @@ -432,120 +432,6 @@ describe("MatrixRTCSession", () => { expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); jest.useRealTimers(); }); - - describe("calls", () => { - const activeFocusConfig = { type: "livekit", livekit_service_url: "https://active.url" }; - const activeFocus = { type: "livekit", focus_selection: "oldest_membership" }; - - async function testJoin(useOwnedStateEvents: boolean): Promise { - if (useOwnedStateEvents) { - mockRoom.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); - } - - jest.useFakeTimers(); - - // preparing the delayed disconnect should handle the delay being too long - const sendDelayedStateExceedAttempt = new Promise((resolve) => { - const error = new MatrixError({ - "errcode": "M_UNKNOWN", - "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", - "org.matrix.msc4140.max_delay": 7500, - }); - sendDelayedStateMock.mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); - }); - - const userStateKey = `${!useOwnedStateEvents ? "_" : ""}@alice:example.org_AAAAAAA`; - // preparing the delayed disconnect should handle ratelimiting - const sendDelayedStateAttempt = new Promise((resolve) => { - const error = new MatrixError({ errcode: "M_LIMIT_EXCEEDED" }); - sendDelayedStateMock.mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); - }); - - // setting the membership state should handle ratelimiting (also with a retry-after value) - const sendStateEventAttempt = new Promise((resolve) => { - const error = new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ); - sendStateEventMock.mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); - }); - - sess!.joinRoomSession([activeFocusConfig], activeFocus, { - membershipServerSideExpiryTimeout: 9000, - }); - - await sendDelayedStateExceedAttempt.then(); // needed to resolve after the send attempt catches - await sendDelayedStateAttempt; - const callProps = (d: number) => { - return [mockRoom!.roomId, { delay: d }, "org.matrix.msc3401.call.member", {}, userStateKey]; - }; - expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(1, ...callProps(9000)); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(2, ...callProps(7500)); - - jest.advanceTimersByTime(5000); - - await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches - jest.advanceTimersByTime(1000); - - await sentStateEvent; - expect(client.sendStateEvent).toHaveBeenCalledWith( - mockRoom!.roomId, - EventType.GroupCallMemberPrefix, - { - application: "m.call", - scope: "m.room", - call_id: "", - expires: 14400000, - device_id: "AAAAAAA", - foci_preferred: [activeFocusConfig], - focus_active: activeFocus, - } satisfies SessionMembershipData, - userStateKey, - ); - await sentDelayedState; - - // should have prepared the heartbeat to keep delaying the leave event while still connected - await updatedDelayedEvent; - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - - // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. - await flushPromises(); - jest.advanceTimersByTime(5000); - // should update delayed disconnect - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); - - jest.useRealTimers(); - } - - it("sends a membership event with session payload when joining a call", async () => { - await testJoin(false); - }); - - it("does not prefix the state key with _ for rooms that support user-owned state events", async () => { - await testJoin(true); - }); - }); - - it("does nothing if join called when already joined", async () => { - sess!.joinRoomSession([mockFocus], mockFocus); - await sentStateEvent; - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - - sess!.joinRoomSession([mockFocus], mockFocus); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - }); }); describe("onMembershipsChanged", () => { diff --git a/spec/unit/matrixrtc/MembeshipManager.spec.ts b/spec/unit/matrixrtc/MembeshipManager.spec.ts index 9ebedfc66bd..c4c12d195fc 100644 --- a/spec/unit/matrixrtc/MembeshipManager.spec.ts +++ b/spec/unit/matrixrtc/MembeshipManager.spec.ts @@ -1,3 +1,6 @@ +/** + * @jest-environment ./spec/unit/matrixrtc/testEnvironment.ts + */ /* Copyright 2025 The Matrix.org Foundation C.I.C. @@ -14,39 +17,77 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { MatrixClient, Room } from "../../../src"; -import { MatrixRTCSession } from "../../../src/matrixrtc/MatrixRTCSession"; +import { Mock } from "jest-mock"; +import waitForExpect from "wait-for-expect"; + +import { EventType, HTTPError, MatrixError, Room } from "../../../src"; +import { Focus, LivekitFocusActive, SessionMembershipData } from "../../../src/matrixrtc"; import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; -import { makeMockRoom, membershipTemplate } from "./mocks"; +import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, MockClient } from "./mocks"; +import { flushPromises } from "../../test-utils/flushPromises"; +function waitForMockCall(method: any, returnVal?: any) { + return new Promise((resolve) => { + (method as Mock).mockImplementation(() => { + resolve(); + return returnVal; + }); + }); +} -describe("MatrixRTCSession", () => { - describe("LegacyMembershipManager", () => { - let client: MatrixClient; - let sess: MatrixRTCSession | undefined; +function createAsyncHandle(method: any) { + const handle: { resolve?: (...args: unknown[]) => void; reject?: (...args: any[]) => void } = {}; + (method as Mock).mockImplementation(() => { + return new Promise((resolve, reject) => { + handle.reject = reject; + handle.resolve = resolve; + }); + }); + return handle; +} + +/** + * Tests different MembershipManager implementations. Some tests don't apply to `LegacyMembershipManager` + * use !SkipForLegacy to skip those. See: testEnvironment for more details. + */ +describe("MembershipManager", () => { + describe.each([ + { TestMembershipManager: LegacyMembershipManager, description: "LegacyMembershipManager" }, + // Here we will add the new implementation of the MembershipManager. + // It is not yet tested since it would currently fail all tests. Adding the MembershipManger looks like this: + // { TestMembershipManager: MembershipManager, description: "MembershipManager" }, + ])("$description", ({ TestMembershipManager }) => { + let client: MockClient; let room: Room; + const focusActive: LivekitFocusActive = { + focus_selection: "oldest_membership", + type: "livekit", + }; + const focus: Focus = { + type: "livekit", + livekit_service_url: "https://active.url", + livekit_alias: "!active:active.url", + }; beforeEach(() => { - client = new MatrixClient({ baseUrl: "base_url" }); - client.getUserId = jest.fn().mockReturnValue("@alice:example.org"); - client.getDeviceId = jest.fn().mockReturnValue("AAAAAAA"); + // default to fake timers + jest.useFakeTimers(); + client = makeMockClient("@alice:example.org", "AAAAAAA"); room = makeMockRoom(membershipTemplate); }); afterEach(() => { - client.stopClient(); - client.matrixRTC.stop(); - if (sess) sess.stop(); - sess = undefined; + jest.useRealTimers(); + // no need to clean up mocks since we will recreate the client }); describe("isJoined()", () => { it("defaults to false", () => { - const manager = new LegacyMembershipManager({}, room, client, () => undefined); + const manager = new TestMembershipManager({}, room, client, () => undefined); expect(manager.isJoined()).toEqual(false); }); it("returns true after join()", () => { - const manager = new LegacyMembershipManager({}, room, client, () => undefined); + const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([]); expect(manager.isJoined()).toEqual(true); }); @@ -54,195 +95,457 @@ describe("MatrixRTCSession", () => { describe("join()", () => { describe("sends a membership event", () => { - it("sends a membership event with session payload when joining a call", async () => {}); - - it("does not prefix the state key with _ for rooms that support user-owned state events", async () => {}); - - // const realSetTimeout = setTimeout; - // jest.useFakeTimers(); - // sess!.joinRoomSession([mockFocus], mockFocus); - // await Promise.race([sentStateEvent, new Promise((resolve) => realSetTimeout(resolve, 500))]); - // expect(client.sendStateEvent).toHaveBeenCalledWith( - // mockRoom!.roomId, - // EventType.GroupCallMemberPrefix, - // { - // application: "m.call", - // scope: "m.room", - // call_id: "", - // device_id: "AAAAAAA", - // expires: DEFAULT_EXPIRE_DURATION, - // foci_preferred: [mockFocus], - // focus_active: { - // focus_selection: "oldest_membership", - // type: "livekit", - // }, - // }, - // "_@alice:example.org_AAAAAAA", - // ); - // await Promise.race([sentDelayedState, new Promise((resolve) => realSetTimeout(resolve, 500))]); - // // Because we actually want to send the state - // expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - // // For checking if the delayed event is still there or got removed while sending the state. - // expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - // // For scheduling the delayed event - // expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - // // This returns no error so we do not check if we reschedule the event again. this is done in another test. - - // jest.useRealTimers(); - }); - - describe("schedules a delayed leave event if server supports it", () => {}); + it("sends a membership event with session payload when joining a call", async () => { + // Spys/Mocks + + // eslint-disable-next-line camelcase + const _unstable_updateDelayedEventHandle = createAsyncHandle( + client._unstable_updateDelayedEvent as Mock, + ); + + // Test + const memberManager = new TestMembershipManager(undefined, room, client, () => undefined); + memberManager.join([focus], focusActive); + + // expects + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledWith( + room.roomId, + "org.matrix.msc3401.call.member", + { + application: "m.call", + call_id: "", + device_id: "AAAAAAA", + expires: 14400000, + foci_preferred: [focus], + focus_active: focusActive, + scope: "m.room", + }, + "_@alice:example.org_AAAAAAA", + ); + // eslint-disable-next-line camelcase + _unstable_updateDelayedEventHandle.resolve?.({ delay_id: "myId" }); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( + room.roomId, + { delay: 8000 }, + "org.matrix.msc3401.call.member", + {}, + "_@alice:example.org_AAAAAAA", + ); + }); + + describe("does not prefix the state key with _ for rooms that support user-owned state events", () => { + async function testJoin(useOwnedStateEvents: boolean): Promise { + if (useOwnedStateEvents) { + room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); + } + + const sentStateEvent = waitForMockCall(client.sendStateEvent); + const updatedDelayedEvent = waitForMockCall(client._unstable_updateDelayedEvent); + const sentDelayedState = waitForMockCall(client._unstable_sendDelayedStateEvent, { + delay_id: "id", + }); + + // preparing the delayed disconnect should handle the delay being too long + const sendDelayedStateExceedAttempt = new Promise((resolve) => { + const error = new MatrixError({ + "errcode": "M_UNKNOWN", + "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", + "org.matrix.msc4140.max_delay": 7500, + }); + (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); + }); + }); + + const userStateKey = `${!useOwnedStateEvents ? "_" : ""}@alice:example.org_AAAAAAA`; + // preparing the delayed disconnect should handle ratelimiting + const sendDelayedStateAttempt = new Promise((resolve) => { + const error = new MatrixError({ errcode: "M_LIMIT_EXCEEDED" }); + (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); + }); + }); + + // setting the membership state should handle ratelimiting (also with a retry-after value) + const sendStateEventAttempt = new Promise((resolve) => { + const error = new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ); + (client.sendStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); + }); + }); + const manager = new TestMembershipManager( + { + membershipServerSideExpiryTimeout: 9000, + }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); + + await sendDelayedStateExceedAttempt.then(); // needed to resolve after the send attempt catches + await sendDelayedStateAttempt; + const callProps = (d: number) => { + return [room!.roomId, { delay: d }, "org.matrix.msc3401.call.member", {}, userStateKey]; + }; + expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(1, ...callProps(9000)); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(2, ...callProps(7500)); + + jest.advanceTimersByTime(5000); + + await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches + jest.advanceTimersByTime(1000); + + await sentStateEvent; + expect(client.sendStateEvent).toHaveBeenCalledWith( + room!.roomId, + EventType.GroupCallMemberPrefix, + { + application: "m.call", + scope: "m.room", + call_id: "", + expires: 14400000, + device_id: "AAAAAAA", + foci_preferred: [focus], + focus_active: focusActive, + } satisfies SessionMembershipData, + userStateKey, + ); + await sentDelayedState; + + // should have prepared the heartbeat to keep delaying the leave event while still connected + await updatedDelayedEvent; + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); + + // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. + await flushPromises(); + jest.advanceTimersByTime(5000); + // should update delayed disconnect + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + } + + it("sends a membership event with session payload when joining a call", async () => { + await testJoin(false); + }); + + it("does not prefix the state key with _ for rooms that support user-owned state events", async () => { + await testJoin(true); + }); + }); + }); + + describe("delayed leave event", () => { + it("does not try again to schedule a delayed leave event if not supported", () => { + const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + delayedHandle.reject?.(Error("Server does not support the delayed events API")); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + }); + it("does try to schedule a delayed leave event again if rate limited", () => { + const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); + waitForExpect(() => { + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + }); + }); + it("uses membershipServerSideExpiryTimeout from config", async () => { + const manager = new TestMembershipManager( + { membershipServerSideExpiryTimeout: 123456 }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( + room.roomId, + { delay: 123456 }, + "org.matrix.msc3401.call.member", + {}, + "_@alice:example.org_AAAAAAA", + ); + }); + }); it("uses membershipExpiryTimeout from config", async () => { - // const realSetTimeout = setTimeout; - // jest.useFakeTimers(); - // sess!.joinRoomSession([mockFocus], mockFocus, { membershipExpiryTimeout: 60000 }); - // await Promise.race([sentStateEvent, new Promise((resolve) => realSetTimeout(resolve, 500))]); - // expect(client.sendStateEvent).toHaveBeenCalledWith( - // mockRoom!.roomId, - // EventType.GroupCallMemberPrefix, - // { - // application: "m.call", - // scope: "m.room", - // call_id: "", - // device_id: "AAAAAAA", - // expires: 60000, - // foci_preferred: [mockFocus], - // focus_active: { - // focus_selection: "oldest_membership", - // type: "livekit", - // }, - // }, - // "_@alice:example.org_AAAAAAA", - // ); - // await Promise.race([sentDelayedState, new Promise((resolve) => realSetTimeout(resolve, 500))]); - // expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - // jest.useRealTimers(); + const manager = new TestMembershipManager( + { membershipExpiryTimeout: 1234567 }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledWith( + room.roomId, + EventType.GroupCallMemberPrefix, + { + application: "m.call", + scope: "m.room", + call_id: "", + device_id: "AAAAAAA", + expires: 1234567, + foci_preferred: [focus], + focus_active: { + focus_selection: "oldest_membership", + type: "livekit", + }, + }, + "_@alice:example.org_AAAAAAA", + ); }); it("does nothing if join called when already joined", async () => { - // const realSetTimeout = setTimeout; - // jest.useFakeTimers(); - // sess!.joinRoomSession([mockFocus], mockFocus); - // await Promise.race([sentStateEvent, new Promise((resolve) => realSetTimeout(resolve, 500))]); - // expect(client.sendStateEvent).toHaveBeenCalledWith( - // mockRoom!.roomId, - // EventType.GroupCallMemberPrefix, - // { - // application: "m.call", - // scope: "m.room", - // call_id: "", - // device_id: "AAAAAAA", - // expires: DEFAULT_EXPIRE_DURATION, - // foci_preferred: [mockFocus], - // focus_active: { - // focus_selection: "oldest_membership", - // type: "livekit", - // }, - // }, - // "_@alice:example.org_AAAAAAA", - // ); - // await Promise.race([sentDelayedState, new Promise((resolve) => realSetTimeout(resolve, 500))]); - // expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - // jest.useRealTimers(); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + manager.join([focus], focusActive); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); }); describe("leave()", () => { - it("does nothing if not joined", () => { - // const manager = new LegacyMembershipManager({}, room, client, () => undefined); - // manager.leave(); - }); - }); - - describe("getOldestMembership", () => { - it("returns the oldest membership event", () => { - jest.useFakeTimers(); - jest.setSystemTime(4000); - const mockRoom = makeMockRoom([ - Object.assign({}, membershipTemplate, { device_id: "foo", created_ts: 3000 }), - Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), - Object.assign({}, membershipTemplate, { device_id: "bar", created_ts: 2000 }), - ]); - - sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); - expect(sess.getOldestMembership()!.deviceId).toEqual("old"); - jest.useRealTimers(); + it("does nothing if not joined !FailsForLegacy", () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.leave(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); }); }); describe("getsActiveFocus", () => { - const firstPreferredFocus = { - type: "livekit", - livekit_service_url: "https://active.url", - livekit_alias: "!active:active.url", - }; - it("gets the correct active focus with oldest_membership", () => { - // jest.useFakeTimers(); - // jest.setSystemTime(3000); - // const mockRoom = makeMockRoom([ - // Object.assign({}, membershipTemplate, { - // device_id: "foo", - // created_ts: 500, - // foci_preferred: [firstPreferredFocus], - // }), - // Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), - // Object.assign({}, membershipTemplate, { device_id: "bar", created_ts: 2000 }), - // ]); - // sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); - // sess.joinRoomSession([{ type: "livekit", livekit_service_url: "htts://test.org" }], { - // type: "livekit", - // focus_selection: "oldest_membership", - // }); - // expect(sess.getActiveFocus()).toBe(firstPreferredFocus); - // jest.useRealTimers(); + it("gets the correct active focus with oldest_membership !FailsForLegacy", () => { + const getOlderstMembership = jest.fn(); + const manager = new TestMembershipManager({}, room, client, getOlderstMembership); + // Before joining the active focus should be undefined (see FocusInUse on MatrixRTCSession) + expect(manager.getActiveFocus()).toBe(undefined); + manager.join([focus], focusActive); + // after joining we want our own focus to be the once we select + expect(manager.getActiveFocus()).toBe(focus); + getOlderstMembership.mockReturnValue( + mockCallMembership( + Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), + room.roomId, + ), + ); + // If there is an older member we use its focus. + expect(manager.getActiveFocus()).toBe(membershipTemplate.foci_preferred[0]); }); + it("does not provide focus if the selection method is unknown", () => { - // const mockRoom = makeMockRoom([ - // Object.assign({}, membershipTemplate, { - // device_id: "foo", - // created_ts: 500, - // foci_preferred: [firstPreferredFocus], - // }), - // Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), - // Object.assign({}, membershipTemplate, { device_id: "bar", created_ts: 2000 }), - // ]); - // sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); - // sess.joinRoomSession([{ type: "livekit", livekit_service_url: "htts://test.org" }], { - // type: "livekit", - // focus_selection: "unknown", - // }); - // expect(sess.getActiveFocus()).toBe(undefined); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], Object.assign(focusActive, { type: "unknown_type" })); + expect(manager.getActiveFocus()).toBe(undefined); }); }); describe("onRTCSessionMemberUpdate()", () => { - it("does nothing if not joined", () => {}); - it("does nothing if own membership still present", () => {}); - it("recreates membership if it is missing", () => {}); + it("does nothing if not joined", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await flushPromises(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); + }); + it("does nothing if own membership still present", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + + await waitForMockCall(client.sendStateEvent); + const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; + // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` + (client.sendStateEvent as Mock).mockReset(); + (client._unstable_updateDelayedEvent as Mock).mockReset(); + (client._unstable_sendDelayedStateEvent as Mock).mockReset(); + + manager.onRTCSessionMemberUpdate([ + mockCallMembership(membershipTemplate, room.roomId), + mockCallMembership( + myMembership as SessionMembershipData, + room.roomId, + client.getUserId() ?? undefined, + ), + ]); + await flushPromises(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); + }); + it("recreates membership if it is missing", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + waitForMockCall(client._unstable_sendDelayedStateEvent, { delay_id: "id" }); + await flushPromises(); + // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` + (client.sendStateEvent as Mock).mockClear(); + (client._unstable_updateDelayedEvent as Mock).mockClear(); + (client._unstable_sendDelayedStateEvent as Mock).mockClear(); + waitForMockCall(client._unstable_sendDelayedStateEvent, { delay_id: "id" }); + + manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await flushPromises(); + expect(client.sendStateEvent).toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalled(); + + expect(client._unstable_updateDelayedEvent).toHaveBeenCalled(); + }); }); // TODO: not sure about this name describe("background timers", () => { - it("sends keep-alive for delayed leave event where supported", () => {}); + it("sends only one keep-alive for delayed leave event per `membershipKeepAlivePeriod`", async () => { + const manager = new TestMembershipManager( + { membershipKeepAlivePeriod: 10_000 }, + room, + client, + () => undefined, + ); + waitForMockCall(client._unstable_sendDelayedStateEvent, { + delay_id: "id", + }); + manager.join([focus], focusActive); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - it("extends `expires` when call still active", () => {}); + // The first call is from checking id the server deleted the delayed event + // so it does not need a `advanceTimersByTime` + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - it("does not send more than once per `membershipKeepAlivePeriod`", () => {}); + for (let i = 2; i <= 12; i++) { + // flush promises before advancing the timers to make sure schdulers are setup + await flushPromises(); + jest.advanceTimersByTime(10_000); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); + } + }); + + it("extends `expires` when call still active !FailsForLegacy", async () => { + const manager = new TestMembershipManager( + { membershipExpiryTimeout: 10_000 }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + // TODO make this check the expire ts and also check it after each update. + const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; + expect(sentMembership.expires).toBe(10_000); + for (let i = 2; i <= 12; i++) { + // flush promises before advancing the timers to make sure schdulers are setup + await flushPromises(); + jest.advanceTimersByTime(10_000); + expect(client.sendStateEvent).toHaveBeenCalledTimes(i); + const sentMembership = (client.sendStateEvent as Mock).mock.calls[i][2] as SessionMembershipData; + expect(sentMembership.expires).toBe(10_000 * i); + } + }); }); describe("server error handling", () => { + // Those tests might have been targeted at sending delayed restart events? describe("retries sending delayed leave event", () => { - it("sends it if delayed leave event is still valid at time of retry", () => {}); - it("abandons it if delayed leave event is no longer valid at time of retry", () => { - // I think this will break on LegacyMembershipManager + it("sends retry if call membership event is still valid at time of retry", async () => { + const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); + + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + + handle.reject?.( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); + await flushPromises(); + jest.advanceTimersByTime(1000); + await flushPromises(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + }); + it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { + const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); + + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + handle.reject?.( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); + await flushPromises(); + + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + // Remove our own membership so that there is no reason the send the delayed leave anymore. + manager.onRTCSessionMemberUpdate([]); + // Wait for all timers to be setup + await flushPromises(); + jest.advanceTimersByTime(1000); + // We should send a new own membership and a new delayed event after the rate limit timeout. + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + expect(client.sendStateEvent).toHaveBeenCalledTimes(2); }); }); + describe("retries sending delayed leave event update", () => { + it("resends the initial check delayed update event", async () => { + (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + const handle = createAsyncHandle(client._unstable_updateDelayedEvent); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + await flushPromises(); + handle.reject?.( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); + // Hit rate limit + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); + await flushPromises(); + jest.advanceTimersByTime(1000); + await flushPromises(); - describe("retries sending membership event", () => { - it("sends it if still joined at time of retry", () => {}); - it("abandons it if call no longer joined at time of retry", () => {}); + // hit second rate limit. + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + const handleSuccess = createAsyncHandle(client._unstable_sendDelayedStateEvent); + await flushPromises(); + handleSuccess.resolve?.(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + jest.advanceTimersByTime(1000); + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + }); }); + + // describe: "retries sending membership event" + // it: "sends it if still joined at time of retry" + // // TODO: see what this is doing and how its different to: "recreates membership if it is missing" + // it: "abandons it if call no longer joined at time of retry" }); }); }); diff --git a/spec/unit/matrixrtc/mocks.ts b/spec/unit/matrixrtc/mocks.ts index 428014384ab..988b937376e 100644 --- a/spec/unit/matrixrtc/mocks.ts +++ b/spec/unit/matrixrtc/mocks.ts @@ -14,8 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { EventType, type MatrixEvent, type Room } from "../../../src"; -import { type SessionMembershipData } from "../../../src/matrixrtc/CallMembership"; +import { EventType, type MatrixClient, type MatrixEvent, type Room } from "../../../src"; +import { CallMembership, type SessionMembershipData } from "../../../src/matrixrtc/CallMembership"; import { secureRandomString } from "../../../src/randomstring"; type MembershipData = SessionMembershipData[] | SessionMembershipData | {}; @@ -40,6 +40,31 @@ export const membershipTemplate: SessionMembershipData = { ], }; +export type MockClient = Pick< + MatrixClient, + | "getUserId" + | "getDeviceId" + | "sendEvent" + | "sendStateEvent" + | "_unstable_sendDelayedStateEvent" + | "_unstable_updateDelayedEvent" + | "cancelPendingEvent" +>; +/** + * Mocks a object that has all required methods for a matrixRTC session client. + */ +export function makeMockClient(userId: string, deviceId: string): MockClient { + return { + getDeviceId: () => deviceId, + getUserId: () => userId, + sendEvent: jest.fn(), + sendStateEvent: jest.fn(), + cancelPendingEvent: jest.fn(), + _unstable_updateDelayedEvent: jest.fn(), + _unstable_sendDelayedStateEvent: jest.fn(), + }; +} + export function makeMockRoom(membershipData: MembershipData): Room { const roomId = secureRandomString(8); // Caching roomState here so it does not get recreated when calling `getLiveTimeline.getState()` @@ -88,16 +113,17 @@ export function makeMockRoomState(membershipData: MembershipData, roomId: string }; } -export function mockRTCEvent(membershipData: MembershipData, roomId: string): MatrixEvent { +export function mockRTCEvent(membershipData: MembershipData, roomId: string, customSender?: string): MatrixEvent { + const sender = customSender ?? "@mock:user.example"; return { getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix), getContent: jest.fn().mockReturnValue(membershipData), - getSender: jest.fn().mockReturnValue("@mock:user.example"), + getSender: jest.fn().mockReturnValue(sender), getTs: jest.fn().mockReturnValue(Date.now()), getRoomId: jest.fn().mockReturnValue(roomId), - sender: { - userId: "@mock:user.example", - }, isDecryptionFailure: jest.fn().mockReturnValue(false), } as unknown as MatrixEvent; } +export function mockCallMembership(membershipData: MembershipData, roomId: string, sender?: string): CallMembership { + return new CallMembership(mockRTCEvent(membershipData, roomId, sender), membershipData); +} diff --git a/spec/unit/matrixrtc/testEnvironment.ts b/spec/unit/matrixrtc/testEnvironment.ts new file mode 100644 index 00000000000..efda067ee4f --- /dev/null +++ b/spec/unit/matrixrtc/testEnvironment.ts @@ -0,0 +1,58 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +This file adds a custom test environment for the MembershipManager.spec.ts +It can be used with the comment at the top of the file: + +@jest-environment ./spec/unit/matrixrtc/testEnvironment.ts + +It is very specific to the MembershipManager.spec.ts file and introduces the following behaviour: + - The describe each block in the MembershipManager.spec.ts will go through describe block names `LegacyMembershipManager` and `MembershipManager` + - It will check all tests that are a child or indirect child of the `LegacyMembershipManager` block and skip the ones which include "!FailsForLegacy" + in their test name. +*/ + +import { TestEnvironment } from "jest-environment-node"; +import { JestEnvironmentConfig, EnvironmentContext } from "@jest/environment"; + +import { logger } from "../../../src/logger"; + +class CustomEnvironment extends TestEnvironment { + constructor(config: JestEnvironmentConfig, context: EnvironmentContext) { + super(config, context); + } + + async handleTestEvent(event: any) { + if (event.name === "test_start" && event.test.name.includes("!FailsForLegacy")) { + let parent = event.test.parent; + let isLegacy = false; + while (parent) { + if (parent.name === "LegacyMembershipManager") { + isLegacy = true; + break; + } else { + parent = parent.parent; + } + } + if (isLegacy) { + logger.log("skip test: ", event.test.name); + event.test.mode = "skip"; + } + } + } +} +module.exports = CustomEnvironment; diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 64672500abb..407ef6e0f53 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -55,33 +55,46 @@ export type MatrixRTCSessionEventHandlerMap = { }; export interface MembershipConfig { + + // The proposed name changes follow the following pattern: + // - use membershipEvent for the join event + // - use timeout for anything that is a destructive time period + // - use duration for anything that is a constructive time period + // - use delayedLeaveEvent for anything related to the delayed event login + // - dont use `delay` for anything that is not related to the delayed event login + /** * The timeout (in milliseconds) after we joined the call, that our membership should expire * unless we have explicitly updated it. */ + // membershipEventExpiryTimeout membershipExpiryTimeout?: number; /** * The period (in milliseconds) with which we check that our membership event still exists on the * server. If it is not found we create it again. */ + // This is currently not used. I also think we do not need it since this information should come down via sync? memberEventCheckPeriod?: number; /** * The minimum delay (in milliseconds) after which we will retry sending the membership event if it * failed to send. */ + // rename to: membershipEventMinimumRetryDuration callMemberEventRetryDelayMinimum?: number; /** * The timeout (in milliseconds) with which the deleayed leave event on the server is configured. * After this time the server will set the event to the disconnected stat if it has not received a keep-alive from the client. */ + // I would like to rename this to `delayedLeaveEventTimeout` (having the word delayed, event, and leave is helpful i think) membershipServerSideExpiryTimeout?: number; /** * The interval (in milliseconds) in which the client will send membership keep-alives to the server. */ + // rename to: delayedLeaveEventRestartDuration membershipKeepAlivePeriod?: number; /** @@ -153,7 +166,9 @@ export class MatrixRTCSession extends TypedEventEmitter, + ): CallMembership[] { const roomState = room.getLiveTimeline().getState(EventTimeline.FORWARDS); if (!roomState) { logger.warn("Couldn't get state for room " + room.roomId); @@ -225,9 +240,29 @@ export class MatrixRTCSession extends TypedEventEmitter, + public readonly room: Pick, public memberships: CallMembership[], ) { super(); diff --git a/src/matrixrtc/MembershipManager.ts b/src/matrixrtc/MembershipManager.ts index f398afb7067..ff83c3a67e4 100644 --- a/src/matrixrtc/MembershipManager.ts +++ b/src/matrixrtc/MembershipManager.ts @@ -38,7 +38,7 @@ export interface IMembershipManager { * @returns It resolves with true in case the leave was sent successfully. * It resolves with false in case we hit the timeout before sending successfully. */ - leave(timeout: number | undefined): Promise; + leave(timeout?: number): Promise; /** * Call this if the MatrixRTC session members have changed. */ @@ -50,6 +50,36 @@ export interface IMembershipManager { getActiveFocus(): Focus | undefined; } +export class MembershipManager implements IMembershipManager { + public constructor( + private joinConfig: MembershipConfig | undefined, + private room: Pick, + private client: Pick< + MatrixClient, + | "getUserId" + | "getDeviceId" + | "sendStateEvent" + | "_unstable_sendDelayedStateEvent" + | "_unstable_updateDelayedEvent" + >, + private getOldestMembership: () => CallMembership | undefined, + ) {} + public isJoined(): boolean { + throw new Error("Method not implemented."); + } + public join(fociPreferred: Focus[], fociActive?: Focus): void { + throw new Error("Method not implemented."); + } + public leave(timeout?: number): Promise { + throw new Error("Method not implemented."); + } + public onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { + throw new Error("Method not implemented."); + } + public getActiveFocus(): Focus | undefined { + throw new Error("Method not implemented."); + } +} /** * This internal class is used by the MatrixRTCSession to manage the local user's own membership of the session. * @@ -117,7 +147,6 @@ export class LegacyMembershipManager implements IMembershipManager { | "getUserId" | "getDeviceId" | "sendStateEvent" - | "_unstable_sendDelayedEvent" | "_unstable_sendDelayedStateEvent" | "_unstable_updateDelayedEvent" >, @@ -312,6 +341,7 @@ export class LegacyMembershipManager implements IMembershipManager { if (this.disconnectDelayId !== undefined) { this.scheduleDelayDisconnection(); } + // TODO throw or log an error if this.disconnectDelayId === undefined } else { // Not joined let sentDelayedDisconnect = false; diff --git a/yarn.lock b/yarn.lock index d2e8e2334d5..c1abc8ea062 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6413,6 +6413,11 @@ w3c-xmlserializer@^4.0.0: dependencies: xml-name-validator "^4.0.0" +wait-for-expect@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/wait-for-expect/-/wait-for-expect-3.0.2.tgz#d2f14b2f7b778c9b82144109c8fa89ceaadaa463" + integrity sha512-cfS1+DZxuav1aBYbaO/kE06EOS8yRw7qOFoD3XtjTkYvCvh3zUvNST8DXK/nPaeqIzIv3P3kL3lRJn8iwOiSag== + walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" From 738016fddb678ca29b58f6b80adbcdc59f8c826d Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Mon, 17 Feb 2025 12:19:58 +0000 Subject: [PATCH 004/124] Updates from discussion --- spec/unit/matrixrtc/MembeshipManager.spec.ts | 37 +++++++++++++++++++- src/client.ts | 7 +++- src/matrixrtc/MatrixRTCSession.ts | 17 +++++---- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/spec/unit/matrixrtc/MembeshipManager.spec.ts b/spec/unit/matrixrtc/MembeshipManager.spec.ts index c4c12d195fc..115709cacd7 100644 --- a/spec/unit/matrixrtc/MembeshipManager.spec.ts +++ b/spec/unit/matrixrtc/MembeshipManager.spec.ts @@ -396,6 +396,7 @@ describe("MembershipManager", () => { (client._unstable_sendDelayedStateEvent as Mock).mockClear(); waitForMockCall(client._unstable_sendDelayedStateEvent, { delay_id: "id" }); + // our own membership is removed: manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); await flushPromises(); expect(client.sendStateEvent).toHaveBeenCalled(); @@ -409,7 +410,7 @@ describe("MembershipManager", () => { describe("background timers", () => { it("sends only one keep-alive for delayed leave event per `membershipKeepAlivePeriod`", async () => { const manager = new TestMembershipManager( - { membershipKeepAlivePeriod: 10_000 }, + { membershipKeepAlivePeriod: 10_000, membershipServerSideExpiryTimeout: 30_000 }, room, client, () => undefined, @@ -424,12 +425,16 @@ describe("MembershipManager", () => { // so it does not need a `advanceTimersByTime` await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); + // TODO: check that update delayed event is called with the correct HTTP request timeout + // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); for (let i = 2; i <= 12; i++) { // flush promises before advancing the timers to make sure schdulers are setup await flushPromises(); jest.advanceTimersByTime(10_000); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); + // TODO: check that update delayed event is called with the correct HTTP request timeout + // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); } }); @@ -458,6 +463,7 @@ describe("MembershipManager", () => { }); describe("server error handling", () => { + // Types of server error: 429 rate limit with no retry-after header, 429 with retry-after, 50x server error (maybe retry every second), connection/socket timeout // Those tests might have been targeted at sending delayed restart events? describe("retries sending delayed leave event", () => { it("sends retry if call membership event is still valid at time of retry", async () => { @@ -499,6 +505,7 @@ describe("MembershipManager", () => { expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); // Remove our own membership so that there is no reason the send the delayed leave anymore. + // the membership is no longer present on the homeserver manager.onRTCSessionMemberUpdate([]); // Wait for all timers to be setup await flushPromises(); @@ -507,6 +514,34 @@ describe("MembershipManager", () => { expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); expect(client.sendStateEvent).toHaveBeenCalledTimes(2); }); + it("abandons retry loop if leave() was called", async () => { + const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); + + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + handle.reject?.( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); + await flushPromises(); + + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + // the user terminated the call locally + manager.leave(); + + // Wait for all timers to be setup + await flushPromises(); + jest.advanceTimersByTime(1000); + + // No new events should have been sent: + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + }); }); describe("retries sending delayed leave event update", () => { it("resends the initial check delayed update event", async () => { diff --git a/src/client.ts b/src/client.ts index 972d3341d47..89ec7bcec91 100644 --- a/src/client.ts +++ b/src/client.ts @@ -3414,7 +3414,11 @@ export class MatrixClient extends TypedEventEmitter { + public async _unstable_updateDelayedEvent( + delayId: string, + action: UpdateDelayedEventAction, + requestOptions: IRequestOpts = {}, + ): Promise { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { throw Error("Server does not support the delayed events API"); } @@ -3426,6 +3430,7 @@ export class MatrixClient extends TypedEventEmitter Date: Mon, 17 Feb 2025 21:12:37 +0700 Subject: [PATCH 005/124] revert renaming comments --- src/matrixrtc/MatrixRTCSession.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 53ec87568bf..63f2090ca9e 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -55,56 +55,43 @@ export type MatrixRTCSessionEventHandlerMap = { }; export interface MembershipConfig { - // The proposed name changes follow the following pattern: - // - use membershipEvent for the join event - // - use timeout for anything that is a destructive time period - // - use duration for anything that is a constructive time period - // - use delayedLeaveEvent for anything related to the delayed event login - // - dont use `delay` for anything that is not related to the delayed event login - /** * The timeout (in milliseconds) after we joined the call, that our membership should expire * unless we have explicitly updated it. * * This is what goes into the m.rtc.member event expiry field. */ - // membershipEventExpiryTTL membershipExpiryTimeout?: number; // hours /** * The period (in milliseconds) with which we check that our membership event still exists on the * server. If it is not found we create it again. */ - // This is currently not used. I also think we do not need it since this information should come down via sync? memberEventCheckPeriod?: number; /** * The minimum delay (in milliseconds) after which we will retry sending the membership event if it * failed to send. */ - // rename to: membershipEventMinimumRetryDuration - // membershipEventRateLimit callMemberEventRetryDelayMinimum?: number; /** * The timeout (in milliseconds) with which the deleayed leave event on the server is configured. * After this time the server will set the event to the disconnected stat if it has not received a keep-alive from the client. */ - // I would like to rename this to `delayedLeaveEventDelay` (having the word delayed, event, and leave is helpful i think) - membershipServerSideExpiryTimeout?: number; // 15s + membershipServerSideExpiryTimeout?: number; /** * The interval (in milliseconds) in which the client will send membership keep-alives to the server. */ - // rename to: delayedLeaveEventRestartPeriod - membershipKeepAlivePeriod?: number; // 5s + membershipKeepAlivePeriod?: number; /** * @deprecated It should be possible to make it stable without this. */ - // membershipEventJitter callMemberEventRetryJitter?: number; } + export interface EncryptionConfig { /** * If true, generate and share a media key for this participant, From 8c8e97ed9f288badb5d24359c990d1e71f15bdf1 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 17 Feb 2025 21:18:01 +0700 Subject: [PATCH 006/124] remove unused import --- spec/unit/matrixrtc/MatrixRTCSession.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts index 963d7c5fede..4becc689eb0 100644 --- a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts +++ b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts @@ -20,7 +20,6 @@ import { DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "../../../sr import { MatrixRTCSession, MatrixRTCSessionEvent } from "../../../src/matrixrtc/MatrixRTCSession"; import { type EncryptionKeysEventContent } from "../../../src/matrixrtc/types"; import { secureRandomString } from "../../../src/randomstring"; -import { flushPromises } from "../../test-utils/flushPromises"; import { makeMockRoom, makeMockRoomState, membershipTemplate } from "./mocks"; const mockFocus = { type: "mock" }; From d9192f30575bf11fcd72e9dabcb5b35e21b7425f Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 17 Feb 2025 21:50:36 +0700 Subject: [PATCH 007/124] fix leave delayed event resend test. It was missing a flush. --- spec/unit/matrixrtc/MembeshipManager.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/unit/matrixrtc/MembeshipManager.spec.ts b/spec/unit/matrixrtc/MembeshipManager.spec.ts index 115709cacd7..014a87a896e 100644 --- a/spec/unit/matrixrtc/MembeshipManager.spec.ts +++ b/spec/unit/matrixrtc/MembeshipManager.spec.ts @@ -514,7 +514,7 @@ describe("MembershipManager", () => { expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); expect(client.sendStateEvent).toHaveBeenCalledTimes(2); }); - it("abandons retry loop if leave() was called", async () => { + it("abandons retry loop if leave() was called !FailsForLegacy", async () => { const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); const manager = new TestMembershipManager({}, room, client, () => undefined); @@ -537,10 +537,10 @@ describe("MembershipManager", () => { // Wait for all timers to be setup await flushPromises(); jest.advanceTimersByTime(1000); + await flushPromises(); // No new events should have been sent: expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); }); describe("retries sending delayed leave event update", () => { From 0444816ebf2e2523d38b6eae3f081b25eff4b041 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 17 Feb 2025 23:22:28 +0700 Subject: [PATCH 008/124] comment out and remove unused variables --- spec/unit/matrixrtc/MatrixRTCSession.spec.ts | 7 +-- src/matrixrtc/MembershipManager.ts | 61 ++++++++++---------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts index 4becc689eb0..b006b247195 100644 --- a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts +++ b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts @@ -321,11 +321,9 @@ describe("MatrixRTCSession", () => { let sendStateEventMock: jest.Mock; let sendDelayedStateMock: jest.Mock; let sendEventMock: jest.Mock; - let updateDelayedEventMock: jest.Mock; let sentStateEvent: Promise; let sentDelayedState: Promise; - let updatedDelayedEvent: Promise; beforeEach(() => { sentStateEvent = new Promise((resolve) => { @@ -339,15 +337,12 @@ describe("MatrixRTCSession", () => { }; }); }); - updatedDelayedEvent = new Promise((r) => { - updateDelayedEventMock = jest.fn(r); - }); sendEventMock = jest.fn(); client.sendStateEvent = sendStateEventMock; client._unstable_sendDelayedStateEvent = sendDelayedStateMock; client.sendEvent = sendEventMock; - client._unstable_updateDelayedEvent = updateDelayedEventMock; + client._unstable_updateDelayedEvent = jest.fn(); mockRoom = makeMockRoom([]); sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); diff --git a/src/matrixrtc/MembershipManager.ts b/src/matrixrtc/MembershipManager.ts index ff83c3a67e4..74fec962204 100644 --- a/src/matrixrtc/MembershipManager.ts +++ b/src/matrixrtc/MembershipManager.ts @@ -50,36 +50,37 @@ export interface IMembershipManager { getActiveFocus(): Focus | undefined; } -export class MembershipManager implements IMembershipManager { - public constructor( - private joinConfig: MembershipConfig | undefined, - private room: Pick, - private client: Pick< - MatrixClient, - | "getUserId" - | "getDeviceId" - | "sendStateEvent" - | "_unstable_sendDelayedStateEvent" - | "_unstable_updateDelayedEvent" - >, - private getOldestMembership: () => CallMembership | undefined, - ) {} - public isJoined(): boolean { - throw new Error("Method not implemented."); - } - public join(fociPreferred: Focus[], fociActive?: Focus): void { - throw new Error("Method not implemented."); - } - public leave(timeout?: number): Promise { - throw new Error("Method not implemented."); - } - public onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { - throw new Error("Method not implemented."); - } - public getActiveFocus(): Focus | undefined { - throw new Error("Method not implemented."); - } -} +// export class MembershipManager implements IMembershipManager { +// public constructor( +// private joinConfig: MembershipConfig | undefined, +// private room: Pick, +// private client: Pick< +// MatrixClient, +// | "getUserId" +// | "getDeviceId" +// | "sendStateEvent" +// | "_unstable_sendDelayedStateEvent" +// | "_unstable_updateDelayedEvent" +// >, +// private getOldestMembership: () => CallMembership | undefined, +// ) {} +// public isJoined(): boolean { +// throw new Error("Method not implemented."); +// } +// public join(fociPreferred: Focus[], fociActive?: Focus): void { +// throw new Error("Method not implemented."); +// } +// public leave(timeout?: number): Promise { +// throw new Error("Method not implemented."); +// } +// public onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { +// throw new Error("Method not implemented."); +// } +// public getActiveFocus(): Focus | undefined { +// throw new Error("Method not implemented."); +// } +// } + /** * This internal class is used by the MatrixRTCSession to manage the local user's own membership of the session. * From 7125d45e4396bbd92af4db28930be4228cf6acaa Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 17 Feb 2025 23:44:02 +0700 Subject: [PATCH 009/124] es lint --- spec/unit/matrixrtc/MatrixRTCSession.spec.ts | 2 +- spec/unit/matrixrtc/MembeshipManager.spec.ts | 8 ++++---- spec/unit/matrixrtc/testEnvironment.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts index b006b247195..85af425e39b 100644 --- a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts +++ b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { encodeBase64, EventType, MatrixClient, MatrixError, type MatrixEvent, type Room } from "../../../src"; +import { encodeBase64, EventType, MatrixClient, type MatrixError, type MatrixEvent, type Room } from "../../../src"; import { KnownMembership } from "../../../src/@types/membership"; import { DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "../../../src/matrixrtc/CallMembership"; import { MatrixRTCSession, MatrixRTCSessionEvent } from "../../../src/matrixrtc/MatrixRTCSession"; diff --git a/spec/unit/matrixrtc/MembeshipManager.spec.ts b/spec/unit/matrixrtc/MembeshipManager.spec.ts index 014a87a896e..585c484ea79 100644 --- a/spec/unit/matrixrtc/MembeshipManager.spec.ts +++ b/spec/unit/matrixrtc/MembeshipManager.spec.ts @@ -17,13 +17,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { Mock } from "jest-mock"; +import { type Mock } from "jest-mock"; import waitForExpect from "wait-for-expect"; -import { EventType, HTTPError, MatrixError, Room } from "../../../src"; -import { Focus, LivekitFocusActive, SessionMembershipData } from "../../../src/matrixrtc"; +import { EventType, HTTPError, MatrixError, type Room } from "../../../src"; +import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; -import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, MockClient } from "./mocks"; +import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; import { flushPromises } from "../../test-utils/flushPromises"; function waitForMockCall(method: any, returnVal?: any) { return new Promise((resolve) => { diff --git a/spec/unit/matrixrtc/testEnvironment.ts b/spec/unit/matrixrtc/testEnvironment.ts index efda067ee4f..dbc870dc09f 100644 --- a/spec/unit/matrixrtc/testEnvironment.ts +++ b/spec/unit/matrixrtc/testEnvironment.ts @@ -27,7 +27,7 @@ It is very specific to the MembershipManager.spec.ts file and introduces the fol */ import { TestEnvironment } from "jest-environment-node"; -import { JestEnvironmentConfig, EnvironmentContext } from "@jest/environment"; +import { type JestEnvironmentConfig, type EnvironmentContext } from "@jest/environment"; import { logger } from "../../../src/logger"; From 3d46d050ecc1e72a123e5420f0e6773ec26b5ef9 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 17 Feb 2025 23:51:05 +0700 Subject: [PATCH 010/124] use jsdom instead of node test environment --- package.json | 1 + .../{MembeshipManager.spec.ts => MembershipManager.spec.ts} | 1 + spec/unit/matrixrtc/testEnvironment.ts | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) rename spec/unit/matrixrtc/{MembeshipManager.spec.ts => MembershipManager.spec.ts} (99%) diff --git a/package.json b/package.json index 17b87a1e3ab..785278ed016 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,7 @@ "@babel/preset-env": "^7.12.11", "@babel/preset-typescript": "^7.12.7", "@casualbot/jest-sonar-reporter": "2.2.7", + "@jest/environment": "^29.7.0", "@peculiar/webcrypto": "^1.4.5", "@stylistic/eslint-plugin": "^3.0.0", "@types/bs58": "^4.0.1", diff --git a/spec/unit/matrixrtc/MembeshipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts similarity index 99% rename from spec/unit/matrixrtc/MembeshipManager.spec.ts rename to spec/unit/matrixrtc/MembershipManager.spec.ts index 585c484ea79..43571fc22cc 100644 --- a/spec/unit/matrixrtc/MembeshipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -25,6 +25,7 @@ import { type Focus, type LivekitFocusActive, type SessionMembershipData } from import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; import { flushPromises } from "../../test-utils/flushPromises"; + function waitForMockCall(method: any, returnVal?: any) { return new Promise((resolve) => { (method as Mock).mockImplementation(() => { diff --git a/spec/unit/matrixrtc/testEnvironment.ts b/spec/unit/matrixrtc/testEnvironment.ts index dbc870dc09f..3898aa9cde7 100644 --- a/spec/unit/matrixrtc/testEnvironment.ts +++ b/spec/unit/matrixrtc/testEnvironment.ts @@ -26,7 +26,7 @@ It is very specific to the MembershipManager.spec.ts file and introduces the fol in their test name. */ -import { TestEnvironment } from "jest-environment-node"; +import { TestEnvironment } from "jest-environment-jsdom"; import { type JestEnvironmentConfig, type EnvironmentContext } from "@jest/environment"; import { logger } from "../../../src/logger"; From b0492ca51ddd6876bec8f2b29e090af644b3e809 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 18 Feb 2025 00:04:29 +0700 Subject: [PATCH 011/124] remove unused variables --- src/matrixrtc/MembershipManager.ts | 48 +++++++++++------------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/src/matrixrtc/MembershipManager.ts b/src/matrixrtc/MembershipManager.ts index 74fec962204..795fbee5d17 100644 --- a/src/matrixrtc/MembershipManager.ts +++ b/src/matrixrtc/MembershipManager.ts @@ -50,36 +50,24 @@ export interface IMembershipManager { getActiveFocus(): Focus | undefined; } -// export class MembershipManager implements IMembershipManager { -// public constructor( -// private joinConfig: MembershipConfig | undefined, -// private room: Pick, -// private client: Pick< -// MatrixClient, -// | "getUserId" -// | "getDeviceId" -// | "sendStateEvent" -// | "_unstable_sendDelayedStateEvent" -// | "_unstable_updateDelayedEvent" -// >, -// private getOldestMembership: () => CallMembership | undefined, -// ) {} -// public isJoined(): boolean { -// throw new Error("Method not implemented."); -// } -// public join(fociPreferred: Focus[], fociActive?: Focus): void { -// throw new Error("Method not implemented."); -// } -// public leave(timeout?: number): Promise { -// throw new Error("Method not implemented."); -// } -// public onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { -// throw new Error("Method not implemented."); -// } -// public getActiveFocus(): Focus | undefined { -// throw new Error("Method not implemented."); -// } -// } +export class MembershipManager implements IMembershipManager { + public constructor() {} + public isJoined(): boolean { + throw new Error("Method not implemented."); + } + public join(fociPreferred: Focus[], fociActive?: Focus): void { + throw new Error("Method not implemented."); + } + public leave(timeout?: number): Promise { + throw new Error("Method not implemented."); + } + public onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { + throw new Error("Method not implemented."); + } + public getActiveFocus(): Focus | undefined { + throw new Error("Method not implemented."); + } +} /** * This internal class is used by the MatrixRTCSession to manage the local user's own membership of the session. From c59f04d848c996fd61498ebc5ca5bab8c91f2e46 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 18 Feb 2025 00:07:13 +0700 Subject: [PATCH 012/124] remove unused export --- src/matrixrtc/MembershipManager.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/matrixrtc/MembershipManager.ts b/src/matrixrtc/MembershipManager.ts index 795fbee5d17..577e52dc3f7 100644 --- a/src/matrixrtc/MembershipManager.ts +++ b/src/matrixrtc/MembershipManager.ts @@ -50,25 +50,6 @@ export interface IMembershipManager { getActiveFocus(): Focus | undefined; } -export class MembershipManager implements IMembershipManager { - public constructor() {} - public isJoined(): boolean { - throw new Error("Method not implemented."); - } - public join(fociPreferred: Focus[], fociActive?: Focus): void { - throw new Error("Method not implemented."); - } - public leave(timeout?: number): Promise { - throw new Error("Method not implemented."); - } - public onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { - throw new Error("Method not implemented."); - } - public getActiveFocus(): Focus | undefined { - throw new Error("Method not implemented."); - } -} - /** * This internal class is used by the MatrixRTCSession to manage the local user's own membership of the session. * From e1fbdcd964e5cd4fc73af670e2572b2a0fcaad36 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 18 Feb 2025 14:57:09 +0700 Subject: [PATCH 013/124] temp --- spec/unit/matrixrtc/MembershipManager.spec.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 43571fc22cc..7ffb66a6ebb 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -449,7 +449,6 @@ describe("MembershipManager", () => { manager.join([focus], focusActive); await waitForMockCall(client.sendStateEvent); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - // TODO make this check the expire ts and also check it after each update. const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; expect(sentMembership.expires).toBe(10_000); for (let i = 2; i <= 12; i++) { @@ -465,7 +464,6 @@ describe("MembershipManager", () => { describe("server error handling", () => { // Types of server error: 429 rate limit with no retry-after header, 429 with retry-after, 50x server error (maybe retry every second), connection/socket timeout - // Those tests might have been targeted at sending delayed restart events? describe("retries sending delayed leave event", () => { it("sends retry if call membership event is still valid at time of retry", async () => { const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); @@ -544,7 +542,7 @@ describe("MembershipManager", () => { expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); }); }); - describe("retries sending delayed leave event update", () => { + describe("retries sending update delayed leave event restart", () => { it("resends the initial check delayed update event", async () => { (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); const handle = createAsyncHandle(client._unstable_updateDelayedEvent); @@ -560,20 +558,23 @@ describe("MembershipManager", () => { new Headers({ "Retry-After": "1" }), ), ); + await flushPromises(); + // Hit rate limit expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - await flushPromises(); jest.advanceTimersByTime(1000); - await flushPromises(); - // hit second rate limit. - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); - const handleSuccess = createAsyncHandle(client._unstable_sendDelayedStateEvent); + // Hit second rate limit. await flushPromises(); - handleSuccess.resolve?.(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + + // Setup resolve + const handleSuccess = createAsyncHandle(client._unstable_updateDelayedEvent); + await flushPromises(); jest.advanceTimersByTime(1000); await flushPromises(); + handleSuccess.resolve?.(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); }); }); From cd1321b351259c79a249dc049654dfd80fe355f2 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 20 Feb 2025 00:26:04 +0700 Subject: [PATCH 014/124] review --- spec/unit/matrixrtc/MembershipManager.spec.ts | 17 +++++++++-------- spec/unit/matrixrtc/mocks.ts | 2 +- src/matrixrtc/MatrixRTCSession.ts | 16 ++++++++-------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 7ffb66a6ebb..d6e82f9d7e4 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -96,7 +96,7 @@ describe("MembershipManager", () => { describe("join()", () => { describe("sends a membership event", () => { - it("sends a membership event with session payload when joining a call", async () => { + it("sends a membership event and schedules delayed leave when joining a call", async () => { // Spys/Mocks // eslint-disable-next-line camelcase @@ -235,7 +235,7 @@ describe("MembershipManager", () => { expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); } - it("sends a membership event with session payload when joining a call", async () => { + it("sends a membership event after rate limits during delayed event setup when joining a call", async () => { await testJoin(false); }); @@ -319,6 +319,7 @@ describe("MembershipManager", () => { }); describe("leave()", () => { + // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed it("does nothing if not joined !FailsForLegacy", () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.leave(); @@ -334,9 +335,7 @@ describe("MembershipManager", () => { // Before joining the active focus should be undefined (see FocusInUse on MatrixRTCSession) expect(manager.getActiveFocus()).toBe(undefined); manager.join([focus], focusActive); - // after joining we want our own focus to be the once we select - expect(manager.getActiveFocus()).toBe(focus); - getOlderstMembership.mockReturnValue( + // After joining we want our own focus to be the one we select. mockCallMembership( Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), room.roomId, @@ -430,7 +429,7 @@ describe("MembershipManager", () => { // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); for (let i = 2; i <= 12; i++) { - // flush promises before advancing the timers to make sure schdulers are setup + // flush promises before advancing the timers to make sure schedulers are setup await flushPromises(); jest.advanceTimersByTime(10_000); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); @@ -486,6 +485,7 @@ describe("MembershipManager", () => { await flushPromises(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); + // FailsForLegacy as implementation does not re-check membership before retrying it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); @@ -509,10 +509,11 @@ describe("MembershipManager", () => { // Wait for all timers to be setup await flushPromises(); jest.advanceTimersByTime(1000); - // We should send a new own membership and a new delayed event after the rate limit timeout. + // We should send the first own membership and a new delayed event after the rate limit timeout. expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); - expect(client.sendStateEvent).toHaveBeenCalledTimes(2); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); + // FailsForLegacy as implementation does not re-check membership before retrying it("abandons retry loop if leave() was called !FailsForLegacy", async () => { const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); diff --git a/spec/unit/matrixrtc/mocks.ts b/spec/unit/matrixrtc/mocks.ts index 988b937376e..dc3949d9b94 100644 --- a/spec/unit/matrixrtc/mocks.ts +++ b/spec/unit/matrixrtc/mocks.ts @@ -51,7 +51,7 @@ export type MockClient = Pick< | "cancelPendingEvent" >; /** - * Mocks a object that has all required methods for a matrixRTC session client. + * Mocks a object that has all required methods for a MatrixRTC session client. */ export function makeMockClient(userId: string, deviceId: string): MockClient { return { diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 63f2090ca9e..fe92dfd5082 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -59,9 +59,9 @@ export interface MembershipConfig { * The timeout (in milliseconds) after we joined the call, that our membership should expire * unless we have explicitly updated it. * - * This is what goes into the m.rtc.member event expiry field. + * This is what goes into the m.rtc.member event expiry field and is typically set to a number of hours. */ - membershipExpiryTimeout?: number; // hours + membershipExpiryTimeout?: number; /** * The period (in milliseconds) with which we check that our membership event still exists on the @@ -231,15 +231,15 @@ export class MatrixRTCSession extends TypedEventEmitter Date: Thu, 20 Feb 2025 00:26:14 +0700 Subject: [PATCH 015/124] fixup tests --- spec/unit/matrixrtc/MembershipManager.spec.ts | 86 +++++++++++-------- 1 file changed, 52 insertions(+), 34 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index d6e82f9d7e4..ad493f6e7b7 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -25,6 +25,7 @@ import { type Focus, type LivekitFocusActive, type SessionMembershipData } from import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; import { flushPromises } from "../../test-utils/flushPromises"; +// import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; function waitForMockCall(method: any, returnVal?: any) { return new Promise((resolve) => { @@ -48,7 +49,7 @@ function createAsyncHandle(method: any) { /** * Tests different MembershipManager implementations. Some tests don't apply to `LegacyMembershipManager` - * use !SkipForLegacy to skip those. See: testEnvironment for more details. + * use !FailsForLegacy to skip those. See: testEnvironment for more details. */ describe("MembershipManager", () => { describe.each([ @@ -74,6 +75,8 @@ describe("MembershipManager", () => { jest.useFakeTimers(); client = makeMockClient("@alice:example.org", "AAAAAAA"); room = makeMockRoom(membershipTemplate); + // provide a default mock that is like the default "non error" server behaviour. + (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); }); afterEach(() => { @@ -87,7 +90,7 @@ describe("MembershipManager", () => { expect(manager.isJoined()).toEqual(false); }); - it("returns true after join()", () => { + it("returns true after join()", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([]); expect(manager.isJoined()).toEqual(true); @@ -125,7 +128,7 @@ describe("MembershipManager", () => { "_@alice:example.org_AAAAAAA", ); // eslint-disable-next-line camelcase - _unstable_updateDelayedEventHandle.resolve?.({ delay_id: "myId" }); + _unstable_updateDelayedEventHandle.resolve?.(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( room.roomId, { delay: 8000 }, @@ -141,7 +144,6 @@ describe("MembershipManager", () => { room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); } - const sentStateEvent = waitForMockCall(client.sendStateEvent); const updatedDelayedEvent = waitForMockCall(client._unstable_updateDelayedEvent); const sentDelayedState = waitForMockCall(client._unstable_sendDelayedStateEvent, { delay_id: "id", @@ -207,7 +209,6 @@ describe("MembershipManager", () => { await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches jest.advanceTimersByTime(1000); - await sentStateEvent; expect(client.sendStateEvent).toHaveBeenCalledWith( room!.roomId, EventType.GroupCallMemberPrefix, @@ -231,6 +232,7 @@ describe("MembershipManager", () => { // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. await flushPromises(); jest.advanceTimersByTime(5000); + await flushPromises(); // should update delayed disconnect expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); } @@ -287,6 +289,7 @@ describe("MembershipManager", () => { client, () => undefined, ); + manager.join([focus], focusActive); await waitForMockCall(client.sendStateEvent); expect(client.sendStateEvent).toHaveBeenCalledWith( @@ -329,13 +332,33 @@ describe("MembershipManager", () => { }); describe("getsActiveFocus", () => { - it("gets the correct active focus with oldest_membership !FailsForLegacy", () => { - const getOlderstMembership = jest.fn(); - const manager = new TestMembershipManager({}, room, client, getOlderstMembership); + it("gets the correct active focus with oldest_membership", () => { + const getOldestMembership = jest.fn(); + const manager = new TestMembershipManager({}, room, client, getOldestMembership); // Before joining the active focus should be undefined (see FocusInUse on MatrixRTCSession) expect(manager.getActiveFocus()).toBe(undefined); manager.join([focus], focusActive); // After joining we want our own focus to be the one we select. + getOldestMembership.mockReturnValue( + mockCallMembership( + { + ...membershipTemplate, + foci_preferred: [ + { + livekit_alias: "!active:active.url", + livekit_service_url: "https://active.url", + type: "livekit", + }, + ], + device_id: client.getDeviceId(), + created_ts: 1000, + }, + room.roomId, + client.getUserId()!, + ), + ); + expect(manager.getActiveFocus()).toStrictEqual(focus); + getOldestMembership.mockReturnValue( mockCallMembership( Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), room.roomId, @@ -366,6 +389,7 @@ describe("MembershipManager", () => { manager.join([focus], focusActive); await waitForMockCall(client.sendStateEvent); + await flushPromises(); const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` (client.sendStateEvent as Mock).mockReset(); @@ -388,13 +412,11 @@ describe("MembershipManager", () => { it("recreates membership if it is missing", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - waitForMockCall(client._unstable_sendDelayedStateEvent, { delay_id: "id" }); await flushPromises(); // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` (client.sendStateEvent as Mock).mockClear(); (client._unstable_updateDelayedEvent as Mock).mockClear(); (client._unstable_sendDelayedStateEvent as Mock).mockClear(); - waitForMockCall(client._unstable_sendDelayedStateEvent, { delay_id: "id" }); // our own membership is removed: manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); @@ -415,9 +437,6 @@ describe("MembershipManager", () => { client, () => undefined, ); - waitForMockCall(client._unstable_sendDelayedStateEvent, { - delay_id: "id", - }); manager.join([focus], focusActive); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); @@ -432,6 +451,8 @@ describe("MembershipManager", () => { // flush promises before advancing the timers to make sure schedulers are setup await flushPromises(); jest.advanceTimersByTime(10_000); + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); // TODO: check that update delayed event is called with the correct HTTP request timeout // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); @@ -451,11 +472,12 @@ describe("MembershipManager", () => { const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; expect(sentMembership.expires).toBe(10_000); for (let i = 2; i <= 12; i++) { - // flush promises before advancing the timers to make sure schdulers are setup + // flush promises before advancing the timers to make sure schedulers are setup await flushPromises(); jest.advanceTimersByTime(10_000); + await flushPromises(); expect(client.sendStateEvent).toHaveBeenCalledTimes(i); - const sentMembership = (client.sendStateEvent as Mock).mock.calls[i][2] as SessionMembershipData; + const sentMembership = (client.sendStateEvent as Mock).mock.lastCall![2] as SessionMembershipData; expect(sentMembership.expires).toBe(10_000 * i); } }); @@ -487,11 +509,7 @@ describe("MembershipManager", () => { }); // FailsForLegacy as implementation does not re-check membership before retrying it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { - const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); - - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - handle.reject?.( + (client._unstable_sendDelayedStateEvent as any).mockRejectedValue( new MatrixError( { errcode: "M_LIMIT_EXCEEDED" }, 429, @@ -500,9 +518,14 @@ describe("MembershipManager", () => { new Headers({ "Retry-After": "1" }), ), ); - await flushPromises(); + const manager = new TestMembershipManager({}, room, client, () => undefined); + // Should call _unstable_sendDelayedStateEvent but not sendStateEvent because of the + // RateLimit error. + manager.join([focus], focusActive); + await flushPromises(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); // Remove our own membership so that there is no reason the send the delayed leave anymore. // the membership is no longer present on the homeserver manager.onRTCSessionMemberUpdate([]); @@ -544,13 +567,8 @@ describe("MembershipManager", () => { }); }); describe("retries sending update delayed leave event restart", () => { - it("resends the initial check delayed update event", async () => { - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); - const handle = createAsyncHandle(client._unstable_updateDelayedEvent); - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - await flushPromises(); - handle.reject?.( + it("resends the initial check delayed update event !FailsForLegacy", async () => { + (client._unstable_updateDelayedEvent as any).mockRejectedValue( new MatrixError( { errcode: "M_LIMIT_EXCEEDED" }, 429, @@ -559,24 +577,24 @@ describe("MembershipManager", () => { new Headers({ "Retry-After": "1" }), ), ); - await flushPromises(); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); // Hit rate limit + await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - jest.advanceTimersByTime(1000); // Hit second rate limit. + jest.advanceTimersByTime(1000); await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); // Setup resolve - const handleSuccess = createAsyncHandle(client._unstable_updateDelayedEvent); - await flushPromises(); + (client._unstable_updateDelayedEvent as Mock).mockImplementation(() => {}); jest.advanceTimersByTime(1000); await flushPromises(); - handleSuccess.resolve?.(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); }); From a21c6e5f55cf5c9188dae8872472918aace1aa17 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 20 Feb 2025 00:31:26 +0700 Subject: [PATCH 016/124] more review --- spec/unit/matrixrtc/MembershipManager.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index ad493f6e7b7..65410a7d53c 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -459,6 +459,10 @@ describe("MembershipManager", () => { } }); + // The expires logic was removed for the legacy call manager. + // Delayed events should replace it entirely but before they have wide adoption + // the expiration logic still makes sense. + // TODO: add git commit when we removed it. it("extends `expires` when call still active !FailsForLegacy", async () => { const manager = new TestMembershipManager( { membershipExpiryTimeout: 10_000 }, From 8df56bb7356925410607134eede9d90656476c14 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 13:17:42 +0100 Subject: [PATCH 017/124] remove wait for expect dependency --- package.json | 3 +-- spec/unit/matrixrtc/MembershipManager.spec.ts | 10 +++++----- spec/unit/matrixrtc/testEnvironment.ts | 2 +- yarn.lock | 5 ----- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 785278ed016..ea39a75e25d 100644 --- a/package.json +++ b/package.json @@ -125,8 +125,7 @@ "typedoc-plugin-coverage": "^3.0.0", "typedoc-plugin-mdn-links": "^4.0.0", "typedoc-plugin-missing-exports": "^3.0.0", - "typescript": "^5.4.2", - "wait-for-expect": "^3.0.2" + "typescript": "^5.4.2" }, "@casualbot/jest-sonar-reporter": { "outputDirectory": "coverage", diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 65410a7d53c..d789b425d65 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -18,7 +18,6 @@ limitations under the License. */ import { type Mock } from "jest-mock"; -import waitForExpect from "wait-for-expect"; import { EventType, HTTPError, MatrixError, type Room } from "../../../src"; import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; @@ -255,14 +254,15 @@ describe("MembershipManager", () => { delayedHandle.reject?.(Error("Server does not support the delayed events API")); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); }); - it("does try to schedule a delayed leave event again if rate limited", () => { + it("does try to schedule a delayed leave event again if rate limited", async () => { const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); - waitForExpect(() => { - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); - }); + await flushPromises(); + jest.advanceTimersByTime(5000); + await flushPromises(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); it("uses membershipServerSideExpiryTimeout from config", async () => { const manager = new TestMembershipManager( diff --git a/spec/unit/matrixrtc/testEnvironment.ts b/spec/unit/matrixrtc/testEnvironment.ts index 3898aa9cde7..759b00fa13f 100644 --- a/spec/unit/matrixrtc/testEnvironment.ts +++ b/spec/unit/matrixrtc/testEnvironment.ts @@ -1,5 +1,5 @@ /* -Copyright 2023 The Matrix.org Foundation C.I.C. +Copyright 2025 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/yarn.lock b/yarn.lock index c1abc8ea062..d2e8e2334d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6413,11 +6413,6 @@ w3c-xmlserializer@^4.0.0: dependencies: xml-name-validator "^4.0.0" -wait-for-expect@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/wait-for-expect/-/wait-for-expect-3.0.2.tgz#d2f14b2f7b778c9b82144109c8fa89ceaadaa463" - integrity sha512-cfS1+DZxuav1aBYbaO/kE06EOS8yRw7qOFoD3XtjTkYvCvh3zUvNST8DXK/nPaeqIzIv3P3kL3lRJn8iwOiSag== - walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" From 73bed3b72d250357b94fa57f96a85f229d7455f4 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 15:57:43 +0100 Subject: [PATCH 018/124] temp --- spec/unit/matrixrtc/MembershipManager.spec.ts | 31 ++++++++++--------- ...ent.ts => memberManagerTestEnvironment.ts} | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) rename spec/unit/matrixrtc/{testEnvironment.ts => memberManagerTestEnvironment.ts} (96%) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index d789b425d65..134528f1437 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -1,5 +1,5 @@ /** - * @jest-environment ./spec/unit/matrixrtc/testEnvironment.ts + * @jest-environment ./spec/unit/matrixrtc/memberManagerTestEnvironment.ts */ /* Copyright 2025 The Matrix.org Foundation C.I.C. @@ -17,27 +17,28 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { type Mock } from "jest-mock"; +import { type MockedFunction, type Mock } from "jest-mock"; import { EventType, HTTPError, MatrixError, type Room } from "../../../src"; import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; import { flushPromises } from "../../test-utils/flushPromises"; +import { defer } from "../../../src/utils"; // import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; -function waitForMockCall(method: any, returnVal?: any) { +function waitForMockCall(method: MockedFunction, returnVal?: any) { return new Promise((resolve) => { - (method as Mock).mockImplementation(() => { + method.mockImplementation(() => { resolve(); return returnVal; }); }); } - -function createAsyncHandle(method: any) { +defer +function createAsyncHandle(method: MockedFunction) { const handle: { resolve?: (...args: unknown[]) => void; reject?: (...args: any[]) => void } = {}; - (method as Mock).mockImplementation(() => { + method.mockImplementation(() => { return new Promise((resolve, reject) => { handle.reject = reject; handle.resolve = resolve; @@ -70,11 +71,11 @@ describe("MembershipManager", () => { }; beforeEach(() => { - // default to fake timers + // Default to fake timers jest.useFakeTimers(); client = makeMockClient("@alice:example.org", "AAAAAAA"); room = makeMockRoom(membershipTemplate); - // provide a default mock that is like the default "non error" server behaviour. + // Provide a default mock. Representing the default "non error" server behaviour. (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); }); @@ -101,10 +102,7 @@ describe("MembershipManager", () => { it("sends a membership event and schedules delayed leave when joining a call", async () => { // Spys/Mocks - // eslint-disable-next-line camelcase - const _unstable_updateDelayedEventHandle = createAsyncHandle( - client._unstable_updateDelayedEvent as Mock, - ); + const updateDelayedEventHandle = createAsyncHandle(client._unstable_updateDelayedEvent as Mock); // Test const memberManager = new TestMembershipManager(undefined, room, client, () => undefined); @@ -126,8 +124,7 @@ describe("MembershipManager", () => { }, "_@alice:example.org_AAAAAAA", ); - // eslint-disable-next-line camelcase - _unstable_updateDelayedEventHandle.resolve?.(); + updateDelayedEventHandle.resolve?.(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( room.roomId, { delay: 8000 }, @@ -139,6 +136,10 @@ describe("MembershipManager", () => { describe("does not prefix the state key with _ for rooms that support user-owned state events", () => { async function testJoin(useOwnedStateEvents: boolean): Promise { + // TODO: this test does quiet a bit. Its more a like a test story summarizing to: + // - send delay with too long timeout and get server error (test delayedEventTimeout gets overwritten) + // - run into rate limit for sending delayed event + // - run into rate limit when setting membership state. if (useOwnedStateEvents) { room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); } diff --git a/spec/unit/matrixrtc/testEnvironment.ts b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts similarity index 96% rename from spec/unit/matrixrtc/testEnvironment.ts rename to spec/unit/matrixrtc/memberManagerTestEnvironment.ts index 759b00fa13f..727fe12644b 100644 --- a/spec/unit/matrixrtc/testEnvironment.ts +++ b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts @@ -18,7 +18,7 @@ limitations under the License. This file adds a custom test environment for the MembershipManager.spec.ts It can be used with the comment at the top of the file: -@jest-environment ./spec/unit/matrixrtc/testEnvironment.ts +@jest-environment ./spec/unit/matrixrtc/membermanagerTestEnvironment.ts It is very specific to the MembershipManager.spec.ts file and introduces the following behaviour: - The describe each block in the MembershipManager.spec.ts will go through describe block names `LegacyMembershipManager` and `MembershipManager` From 8db2d0af4daa09524cdf7e2478f524382a3940ee Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 19 Feb 2025 22:49:30 +0700 Subject: [PATCH 019/124] fix wrong mocked meberhsip template --- spec/unit/matrixrtc/mocks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/matrixrtc/mocks.ts b/spec/unit/matrixrtc/mocks.ts index dc3949d9b94..0c8cb2ade4e 100644 --- a/spec/unit/matrixrtc/mocks.ts +++ b/spec/unit/matrixrtc/mocks.ts @@ -25,7 +25,7 @@ export const membershipTemplate: SessionMembershipData = { call_id: "", device_id: "AAAAAAA", scope: "m.room", - focus_active: { type: "livekit", livekit_service_url: "https://lk.url" }, + focus_active: { type: "livekit", focus_selection: "oldest_membership" }, foci_preferred: [ { livekit_alias: "!alias:something.org", From 7e4636b57d9ecb1ed7b0bb5ca8e870be3242cd39 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 19 Feb 2025 22:51:12 +0700 Subject: [PATCH 020/124] rename MembershipManager -> LegacyMembershipManager And remove the IMembershipManager from it --- ...pManager.ts => LegacyMembershipManager.ts} | 39 +------------------ 1 file changed, 1 insertion(+), 38 deletions(-) rename src/matrixrtc/{MembershipManager.ts => LegacyMembershipManager.ts} (90%) diff --git a/src/matrixrtc/MembershipManager.ts b/src/matrixrtc/LegacyMembershipManager.ts similarity index 90% rename from src/matrixrtc/MembershipManager.ts rename to src/matrixrtc/LegacyMembershipManager.ts index 577e52dc3f7..843463b4e4e 100644 --- a/src/matrixrtc/MembershipManager.ts +++ b/src/matrixrtc/LegacyMembershipManager.ts @@ -11,44 +11,7 @@ import { type Focus } from "./focus.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { type MembershipConfig } from "./MatrixRTCSession.ts"; import { type EmptyObject } from "../@types/common.ts"; -/** - * This interface defines what a MembershipManager uses and exposes. - * This interface is what we use to write tests and allows to change the actual implementation - * Without breaking tests because of some internal method renaming. - * - * @internal - */ -export interface IMembershipManager { - /** - * If we are trying to join the session. - * It does not reflect if the room state is already configures to represent us being joined. - * It only means that the Manager is running. - * @returns true if we intend to be participating in the MatrixRTC session - */ - isJoined(): boolean; - /** - * Start sending all necessary events to make this user participant in the RTC session. - * @param fociPreferred the list of preferred foci to use in the joined RTC membership event. - * @param fociActive the active focus to use in the joined RTC membership event. - */ - join(fociPreferred: Focus[], fociActive?: Focus): void; - /** - * Send all necessary events to make this user leave the RTC session. - * @param timeout the maximum duration in ms until the promise is forced to resolve. - * @returns It resolves with true in case the leave was sent successfully. - * It resolves with false in case we hit the timeout before sending successfully. - */ - leave(timeout?: number): Promise; - /** - * Call this if the MatrixRTC session members have changed. - */ - onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise; - /** - * The used active focus in the currently joined session. - * @returns the used active focus in the currently joined session or undefined if not joined. - */ - getActiveFocus(): Focus | undefined; -} +import { type IMembershipManager } from "./NewMembershipManager.ts"; /** * This internal class is used by the MatrixRTCSession to manage the local user's own membership of the session. From 412ee18c59f1bcd0ea69878aaa1af1224de2a00a Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 19 Feb 2025 22:51:36 +0700 Subject: [PATCH 021/124] Add new memberhsip manager --- src/matrixrtc/MatrixRTCSession.ts | 3 +- src/matrixrtc/NewMembershipManager.ts | 736 ++++++++++++++++++++++++++ 2 files changed, 738 insertions(+), 1 deletion(-) create mode 100644 src/matrixrtc/NewMembershipManager.ts diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index fe92dfd5082..1c576c21c15 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -25,8 +25,9 @@ import { RoomStateEvent } from "../models/room-state.ts"; import { type Focus } from "./focus.ts"; import { KnownMembership } from "../@types/membership.ts"; import { type MatrixEvent } from "../models/event.ts"; -import { LegacyMembershipManager, type IMembershipManager } from "./MembershipManager.ts"; +import { type IMembershipManager } from "./NewMembershipManager.ts"; import { EncryptionManager, type IEncryptionManager, type Statistics } from "./EncryptionManager.ts"; +import { LegacyMembershipManager } from "./LegacyMembershipManager.ts"; const logger = rootLogger.getChild("MatrixRTCSession"); diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts new file mode 100644 index 00000000000..5141db68119 --- /dev/null +++ b/src/matrixrtc/NewMembershipManager.ts @@ -0,0 +1,736 @@ +import { EventType } from "../@types/event.ts"; +import { UpdateDelayedEventAction } from "../@types/requests.ts"; +import type { MatrixClient } from "../client.ts"; +import { HTTPError, MatrixError } from "../http-api/errors.ts"; +import { logger } from "../logger.ts"; +import { type Room } from "../models/room.ts"; +import { sleep } from "../utils.ts"; +import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts"; +import { type Focus } from "./focus.ts"; +import { isLivekitFocusActive } from "./LivekitFocus.ts"; +import { type MembershipConfig } from "./MatrixRTCSession.ts"; + +/** + * This interface defines what a MembershipManager uses and exposes. + * This interface is what we use to write tests and allows to change the actual implementation + * Without breaking tests because of some internal method renaming. + * + * @internal + */ +export interface IMembershipManager { + /** + * If we are trying to join the session. + * It does not reflect if the room state is already configures to represent us being joined. + * It only means that the Manager is running. + * @returns true if we intend to be participating in the MatrixRTC session + */ + isJoined(): boolean; + /** + * Start sending all necessary events to make this user participant in the RTC session. + * @param fociPreferred the list of preferred foci to use in the joined RTC membership event. + * @param fociActive the active focus to use in the joined RTC membership event. + */ + join(fociPreferred: Focus[], fociActive?: Focus): void; + /** + * Send all necessary events to make this user leave the RTC session. + * @param timeout the maximum duration in ms until the promise is forced to resolve. + * @returns It resolves with true in case the leave was sent successfully. + * It resolves with false in case we hit the timeout before sending successfully. + */ + leave(timeout?: number): Promise; + /** + * Call this if the MatrixRTC session members have changed. + */ + onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise; + /** + * The used active focus in the currently joined session. + * @returns the used active focus in the currently joined session or undefined if not joined. + */ + getActiveFocus(): Focus | undefined; +} + +// CONFIG: +const ALLOWED_MEMBERSHIP_SEND_RETRIES = 5; +const ALLOWED_DELAYED_EVENT_UPDATE_RETRIES = 5; +const ALLOWED_DELAYED_EVENT_SEND_RETRIES = 5; + +// SCHEDULER TYPES: +enum MembershipActionType { + SendJoinEvent = "SendJoinEvent", // -> DelayedLeaveActionType.SendFirstDelayedEvent + Update = "Update", // -> MembershipActionType.Update + SendLeaveEvent = "SendLeaveEvent", +} +function isMembershipActionType(val: any): val is MembershipActionType { + return val in MembershipActionType; +} + +enum DelayedLeaveActionType { + SendFirstDelayedEvent = "SendFirstDelayedEvent", // -> MembershipActionType.SendJoinEvent + SendMainDelayedEvent = "SendMainDelayedEvent", // -> DelayedLeaveActionType.RestartDelayedEvent + RestartDelayedEvent = "RestartDelayedEvent", // -> DelayedLeaveActionType.SendJoinEvent, DelayedLeaveActionType.RestartDelayedEvent + SendScheduledDelayedLeaveEvent = "SendScheduledDelayedLeaveEvent", // -> MembershipActionType.SendLeaveEvent +} + +function isDelayedLeaveActionType(val: any): val is DelayedLeaveActionType { + return val in DelayedLeaveActionType; +} + +enum DirectMemberhsipManagerActions { + Join = DelayedLeaveActionType.SendFirstDelayedEvent, + Leave = DelayedLeaveActionType.SendScheduledDelayedLeaveEvent, +} +interface ActionSchedulerState { + delayId?: string; + nextRelativeExpiry: number; + running: boolean; + hasMemberStateEvent: boolean; + sendMembershipRetries: number; + sendDelayedEventRetries: number; + updateDelayedEventRetries: number; +} + +interface Action { + /** + * When this action should be executed + */ + ts: number; + /** + * The state of the different loops + * can also be thought of as the type of the action + */ + type: DelayedLeaveActionType | MembershipActionType | DirectMemberhsipManagerActions; + // An id to reference the action, + // that can be used in case there is a chance this action should be altered or removed. + // TODO: can we drop this? + id?: string; +} + +/** + * @internal + */ +class ActionScheduler { + public state: ActionSchedulerState; + + public constructor( + state: ActionSchedulerState, + private manager: Pick, + ) { + this.state = state; + } + // state variables for a wakeup mechanism (in case we add some action externally and need to leave the current sleep) + private wakeupPromise?: Promise; + private wakeup?: (value: void | PromiseLike) => void; + private didWakeUp = false; + + private actions: Action[] = []; + private insertions: Action[] = []; + private resetWith?: Action[]; + /** + * This starts the main loop of the memberhsip manager that handles event sending, delayed event sending and delayed event restarting. + * @param initialActions The initial actions the manager will start with. It should be enough to pass: DelayedLeaveActionType.Initial + * @throws This throws an error only if the memberhsip cannot run anymore. For example it reached the maximum retires. + * In most other error cases the manager will try to handle any server errors by itself. + */ + public async startWithActions(initialActions: Action[]): Promise { + this.actions = initialActions; + + while (this.actions.length > 0) { + this.actions.sort((a, b) => a.ts - b.ts); + logger.debug("Current MembershipManager action queue: ", this.actions, "\nDate.now: ", +Date.now()); + const nextAction = this.actions[0]; + + this.wakeupPromise = new Promise((resolve) => { + this.wakeup = resolve; + }); + if (nextAction.ts > Date.now()) await Promise.race([this.wakeupPromise, sleep(nextAction.ts - Date.now())]); + if (this.didWakeUp) { + // In case of a wakeup we do not want to run the next action because the next action now might be sth different. + // Instead we recompute the actions array and do another iteration. + this.didWakeUp = false; + } else { + try { + if (isDelayedLeaveActionType(nextAction.type)) { + await this.manager.delayedLeaveLoopHandler(this.state, nextAction.type); + } else if (isMembershipActionType(nextAction.type)) { + await this.manager.membershipLoopHandler(this.state, nextAction.type); + } + } catch (e) { + throw Error("The MemberhsipManager has to shut down because of the end condition: " + e); + } + } + + this.actions = this.actions.filter((a) => a !== nextAction); + this.actions.push(...this.insertions); + this.insertions = []; + + if (this.resetWith) { + this.actions = this.resetWith; + this.resetWith = undefined; + } + } + } + + public addAction(action: Action): void { + // Dont add any other actions if we have a leave scheduled + if (this.actions.some((a) => a.type === DirectMemberhsipManagerActions.Leave)) return; + this.insertions.push(action); + if (this.actions[0].ts > action.ts) { + this.didWakeUp = true; + this.wakeup?.(); + } + } + public resetActions(actions: Action[]): void { + this.resetWith = actions; + const nextTs = this.actions[0]?.ts; + const newestTs = actions.map((a) => a.ts).sort((a, b) => a - b)[0]; + if (nextTs && newestTs && nextTs > newestTs) { + this.didWakeUp = true; + this.wakeup?.(); + } + } + public hasAction(condition: (action: Action) => boolean): boolean { + return this.actions.some(condition); + } +} +/** + * This Class takes care of the membership management. + * It has the following tasks: + * - Send the users leave delayed event before sending the memberhsip + * - Sent the users membership if the state machine is started + * - Check if the delayed event was canceled due to sending the membership + * - update the delayed event (`restart`) + * - Update the state event every ~5h = `DEFAULT_EXPIRE_DURATION` (so it does not get treated as expired) + * - When the state machine is stopped: + * - Disconnect the member + * - Stop the timer for the delay refresh + * - Stop the timer for updateint the state event + */ + +export class MembershipManager implements IMembershipManager { + public isJoined(): boolean { + return this.scheduler.state.running; + } + /** + * @throws can throw if it exceeds a configured maximum retry. + * @param fociPreferred + * @param focusActive + */ + public join(fociPreferred: Focus[], focusActive?: Focus): void { + this.fociPreferred = fociPreferred; + this.focusActive = focusActive; + if (!this.scheduler.state.running) { + this.scheduler.state.running = true; + this.scheduler.startWithActions([{ ts: Date.now(), type: DirectMemberhsipManagerActions.Join }]); + } + } + + public leave(timeout?: number): Promise { + this.scheduler.state.running = false; + + if (this.leavePromise && this.scheduler.hasAction((a) => a.type === DirectMemberhsipManagerActions.Leave)) { + return this.leavePromise; + } + + // reset scheduled actions so we will not do any new actions. + this.scheduler.resetActions([{ type: DirectMemberhsipManagerActions.Leave, ts: Date.now() }]); + return new Promise((resolve, reject) => { + this.leavePromiseHandle.reject = reject; + this.leavePromiseHandle.resolve = resolve; + if (timeout) setTimeout(() => resolve(false), timeout); + }); + } + private leavePromise?: Promise; + private leavePromiseHandle: { + reject?: (reason: any) => void; + resolve?: (didSendLeaveEvent: boolean) => void; + } = {}; + + public async onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { + const isMyMembership = (m: CallMembership): boolean => + m.sender === this.client.getUserId() && m.deviceId === this.client.getDeviceId(); + + if (this.isJoined() && !memberships.some(isMyMembership)) { + logger.warn("Missing own membership: force re-join"); + this.scheduler.addAction({ ts: Date.now(), type: DirectMemberhsipManagerActions.Join }); + } + } + + public getActiveFocus(): Focus | undefined { + if (this.focusActive) { + // A livekit active focus + if (isLivekitFocusActive(this.focusActive)) { + if (this.focusActive.focus_selection === "oldest_membership") { + const oldestMembership = this.getOldestMembership(); + return oldestMembership?.getPreferredFoci()[0]; + } + } else { + logger.warn("Unknown own ActiveFocus type. This makes it impossible to connect to an SFU."); + } + } else { + // We do not understand the membership format (could be legacy). We default to oldestMembership + // Once there are other methods this is a hard error! + const oldestMembership = this.getOldestMembership(); + return oldestMembership?.getPreferredFoci()[0]; + } + } + + /** + * @throws if the client does not return user or device id. + * @param joinConfig + * @param room + * @param client + * @param getOldestMembership + */ + public constructor( + private joinConfig: MembershipConfig | undefined, + private room: Pick, + private client: Pick< + MatrixClient, + | "getUserId" + | "getDeviceId" + | "sendStateEvent" + | "_unstable_sendDelayedStateEvent" + | "_unstable_updateDelayedEvent" + >, + private getOldestMembership: () => CallMembership | undefined, + ) { + const [userId, deviceId] = [this.client.getUserId(), this.client.getDeviceId()]; + if (userId === null) throw Error("Missing userId in client"); + if (deviceId === null) throw Error("Missing deviceId in client"); + this.deviceId = deviceId; + this.userId = userId; + this.stateKey = this.makeMembershipStateKey(userId, deviceId); + } + + // Membership Event parameters: + private userId: string; + private deviceId: string; + private stateKey: string; + private fociPreferred?: Focus[]; + private focusActive?: Focus; + + // Config: + private membershipServerSideExpiryTimeoutOverride?: number; + + private get callMemberEventRetryDelayMinimum(): number { + return this.joinConfig?.callMemberEventRetryDelayMinimum ?? 3_000; + } + private get membershipExpiryTimeout(): number { + return this.joinConfig?.membershipExpiryTimeout ?? DEFAULT_EXPIRE_DURATION; + } + private get membershipServerSideExpiryTimeout(): number { + return ( + this.membershipServerSideExpiryTimeoutOverride ?? + this.joinConfig?.membershipServerSideExpiryTimeout ?? + 8_000 + ); + } + private get membershipKeepAlivePeriod(): number { + return this.joinConfig?.membershipKeepAlivePeriod ?? 5_000; + } + + // Scheduler: + private scheduler = new ActionScheduler( + { + hasMemberStateEvent: false, + running: false, + nextRelativeExpiry: this.membershipExpiryTimeout, + delayId: undefined, + sendMembershipRetries: 0, + sendDelayedEventRetries: 0, + updateDelayedEventRetries: 0, + }, + this, + ); + + // Loop Handlers: + public async delayedLeaveLoopHandler(state: ActionSchedulerState, type: DelayedLeaveActionType): Promise { + switch (type) { + case DelayedLeaveActionType.SendFirstDelayedEvent: + // Remove all running updates and restarts + // Before we start we check if we come from a state where we have a delay id. + if (state.delayId) { + try { + await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel); + state.delayId = undefined; + state.updateDelayedEventRetries = 0; + this.scheduler.addAction({ + ts: Date.now(), + type: DelayedLeaveActionType.SendFirstDelayedEvent, + }); + } catch (e) { + this.handleRateLimitError( + e, + ALLOWED_DELAYED_EVENT_UPDATE_RETRIES, + state.updateDelayedEventRetries, + (retryIn) => { + state.updateDelayedEventRetries++; + this.scheduler.addAction({ + ts: Date.now() + retryIn, + type: DelayedLeaveActionType.SendFirstDelayedEvent, + }); + }, + () => { + throw Error( + "Exceeded maximum delayed event update (cancel) attempts (client._unstable_updateDelayedEvent): " + + e, + ); + }, + ); + this.handleNotFoundError(e, () => { + // If we get a M_NOT_FOUND we know that the delayed event got already removed. + // This means we are good and can set it to undefined and run this again. + state.delayId = undefined; + this.scheduler.addAction({ + ts: Date.now(), + type: DelayedLeaveActionType.SendFirstDelayedEvent, + }); + }); + } + return; + } + try { + const response = await this.client._unstable_sendDelayedStateEvent( + this.room.roomId, + { + delay: this.membershipServerSideExpiryTimeout, + }, + EventType.GroupCallMemberPrefix, + {}, // leave event + this.stateKey, + ); + // Success we reset retires and set delayId. + state.sendDelayedEventRetries = 0; + state.delayId = response.delay_id; + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); + } catch (e) { + this.handleMaxDelayeExceededError(e, () => { + this.scheduler.addAction({ + ts: Date.now(), + type: DelayedLeaveActionType.SendFirstDelayedEvent, + }); + }); + this.handleRateLimitError( + e, + ALLOWED_DELAYED_EVENT_SEND_RETRIES, + state.sendDelayedEventRetries, + (retryIn) => { + logger.warn("Retry sending delayed disconnection due to rate limit:", e); + state.sendDelayedEventRetries++; + this.scheduler.addAction({ + ts: Date.now() + retryIn, + type: DelayedLeaveActionType.SendFirstDelayedEvent, + }); + }, + () => { + throw Error( + "Exceeded maximum delayed event send attempts (client._unstable_sendDelayedStateEvent): " + + e, + ); + }, + ); + } + break; + case DelayedLeaveActionType.RestartDelayedEvent: + if (!state.delayId) { + // Delay id got reset. This action was used to check if the hs canceled the delayed event when the join state got sent. + this.scheduler.addAction({ + ts: Date.now(), + type: state.hasMemberStateEvent + ? DelayedLeaveActionType.SendMainDelayedEvent + : DelayedLeaveActionType.SendFirstDelayedEvent, + }); + break; + } + try { + await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart); + this.scheduler.addAction({ + ts: Date.now() + this.membershipKeepAlivePeriod, + type: DelayedLeaveActionType.RestartDelayedEvent, + }); + } catch (e) { + // TODO this also needs a test: get rate limit while checking id delayed event is scheduled + this.handleNotFoundError(e, () => { + state.delayId = undefined; + this.scheduler.addAction({ ts: Date.now(), type: DelayedLeaveActionType.SendMainDelayedEvent }); + }); + this.handleRateLimitError( + e, + ALLOWED_DELAYED_EVENT_UPDATE_RETRIES, + state.updateDelayedEventRetries, + (retryIn) => { + this.scheduler.addAction({ + ts: Date.now() + retryIn, + type: DelayedLeaveActionType.RestartDelayedEvent, + }); + }, + () => { + throw Error( + "Exceeded maximum restart delayed event update attempts (client._unstable_updateDelayedEvent): " + + e, + ); + }, + ); + } + break; + case DelayedLeaveActionType.SendMainDelayedEvent: + try { + const response = await this.client._unstable_sendDelayedStateEvent( + this.room.roomId, + { + delay: this.membershipServerSideExpiryTimeout, + }, + EventType.GroupCallMemberPrefix, + {}, // leave event + this.stateKey, + ); + state.delayId = response.delay_id; + this.scheduler.addAction({ + ts: Date.now() + this.membershipKeepAlivePeriod, + type: DelayedLeaveActionType.RestartDelayedEvent, + }); + } catch (e) { + this.handleMaxDelayeExceededError(e, () => { + this.scheduler.addAction({ + ts: Date.now(), + type: DelayedLeaveActionType.SendMainDelayedEvent, + }); + }); + this.handleRateLimitError( + e, + state.sendMembershipRetries, + ALLOWED_MEMBERSHIP_SEND_RETRIES, + (retryIn) => { + logger.warn("Retry sending delayed disconnection due to rate limit:", e); + state.sendMembershipRetries++; + this.scheduler.addAction({ + ts: Date.now() + Math.max(retryIn, this.callMemberEventRetryDelayMinimum), + type: MembershipActionType.Update, + }); + }, + () => { + throw Error( + "Exceeded maximum own Membership state update attempts (client.sendStateEvent): " + e, + ); + }, + ); + } + break; + case DelayedLeaveActionType.SendScheduledDelayedLeaveEvent: + if (state.delayId) { + try { + await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send); + state.hasMemberStateEvent = false; + this.leavePromiseHandle.resolve?.(true); + } catch (e) { + const notFoundHandled = this.handleNotFoundError(e, () => { + state.delayId = undefined; + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); + }); + const rateLimitHandled = this.handleRateLimitError( + e, + ALLOWED_DELAYED_EVENT_UPDATE_RETRIES, + state.updateDelayedEventRetries, + (retryIn) => { + this.scheduler.addAction({ + ts: Date.now() + retryIn, + type: DelayedLeaveActionType.SendScheduledDelayedLeaveEvent, + }); + }, + () => { + this.leavePromiseHandle.reject?.(e); + }, + ); + if (!(notFoundHandled || rateLimitHandled)) { + this.leavePromiseHandle.reject?.(e); + } + } + } else { + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); + } + break; + } + } + + public async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { + switch (type) { + case MembershipActionType.Update: + try { + await this.client.sendStateEvent( + this.room.roomId, + EventType.GroupCallMemberPrefix, + this.makeMyMembership(state.nextRelativeExpiry), + this.stateKey, + ); + state.nextRelativeExpiry += this.membershipExpiryTimeout; + // Success, we reset retries and schedule update. + state.sendMembershipRetries = 0; + this.scheduler.addAction({ + ts: Date.now() + this.membershipExpiryTimeout, + type: MembershipActionType.Update, + }); + } catch (e) { + const rateLimitHandled = this.handleRateLimitError( + e, + state.sendMembershipRetries, + ALLOWED_MEMBERSHIP_SEND_RETRIES, + (retryIn) => { + logger.warn("Retry sending membership state event due to rate limit:", e); + state.sendMembershipRetries++; + this.scheduler.addAction({ + ts: Date.now() + Math.max(retryIn, this.callMemberEventRetryDelayMinimum), + type: MembershipActionType.Update, + }); + }, + () => { + throw Error( + "Exceeded maximum own Membership state update attempts (client.sendStateEvent): " + e, + ); + }, + ); + if (!rateLimitHandled) { + this.scheduler.addAction({ + ts: Date.now() + this.callMemberEventRetryDelayMinimum, + type: MembershipActionType.Update, + }); + } + } + break; + case MembershipActionType.SendJoinEvent: + try { + await this.client.sendStateEvent( + this.room.roomId, + EventType.GroupCallMemberPrefix, + this.makeMyMembership(state.nextRelativeExpiry), + this.stateKey, + ); + state.nextRelativeExpiry += this.membershipExpiryTimeout; + state.hasMemberStateEvent = true; + this.scheduler.addAction({ ts: Date.now(), type: DelayedLeaveActionType.RestartDelayedEvent }); + this.scheduler.addAction({ + ts: Date.now() + this.membershipExpiryTimeout, + type: MembershipActionType.Update, + }); + } catch (e) { + this.handleRateLimitError( + e, + ALLOWED_MEMBERSHIP_SEND_RETRIES, + state.sendMembershipRetries, + (resendDelay) => { + state.sendMembershipRetries++; + this.scheduler.addAction({ + ts: Date.now() + resendDelay, + type: DelayedLeaveActionType.RestartDelayedEvent, + }); + }, + () => { + throw Error( + "Exceeded maximum Own Membership state update attempts (client.sendStateEvent): " + e, + ); + }, + ); + } + break; + case MembershipActionType.SendLeaveEvent: + // we are good already + if (!state.hasMemberStateEvent) return; + + // This is only a fallback in case we do not have working delayed events support. + // first we should try to just send the scheduled leave event + try { + this.client.sendStateEvent( + this.room.roomId, + EventType.GroupCallMemberPrefix, + {}, + this.makeMembershipStateKey(this.userId, this.deviceId), + ); + state.hasMemberStateEvent = false; + } catch {} + } + } + + // HELPERS + private makeMembershipStateKey(localUserId: string, localDeviceId: string): string { + const stateKey = `${localUserId}_${localDeviceId}`; + if (/^org\.matrix\.msc(3757|3779)\b/.exec(this.room.getVersion())) { + return stateKey; + } else { + return `_${stateKey}`; + } + } + /** + * Constructs our own membership + */ + private makeMyMembership(expires: number): SessionMembershipData { + return { + call_id: "", + scope: "m.room", + application: "m.call", + device_id: this.deviceId, + expires, + focus_active: { type: "livekit", focus_selection: "oldest_membership" }, + foci_preferred: this.fociPreferred ?? [], + }; + } + private handleNotFoundError(e: unknown, onNotFound: () => void): boolean { + if (e instanceof MatrixError && e.errcode === "M_NOT_FOUND") { + onNotFound(); + return true; + } + return false; + } + private handleMaxDelayeExceededError(e: unknown, didSetupDelayTimeout: () => void): boolean { + if ( + e instanceof MatrixError && + e.errcode === "M_UNKNOWN" && + e.data["org.matrix.msc4140.errcode"] === "M_MAX_DELAY_EXCEEDED" + ) { + const maxDelayAllowed = e.data["org.matrix.msc4140.max_delay"]; + if (typeof maxDelayAllowed === "number" && this.membershipServerSideExpiryTimeout > maxDelayAllowed) { + this.membershipServerSideExpiryTimeoutOverride = maxDelayAllowed; + } + didSetupDelayTimeout(); + logger.warn("Retry sending delayed disconnection event due to server timeout limitations:", e); + return true; + } + return false; + } + + /** + * + * @param e + * @param allowedRetries + * @param currentRetries + * @param onRetry + * @param onAbort + * @returns Returns true if it did anything. + */ + private handleRateLimitError( + e: unknown, + allowedRetries: number, + currentRetries: number, + onRetry: (retryIn: number) => void, + onAbort: () => void, + ): boolean { + if (currentRetries < allowedRetries && e instanceof HTTPError && e.isRateLimitError()) { + let resendDelay: number; + const defaultMs = 5000; + try { + resendDelay = e.getRetryAfterMs() ?? defaultMs; + logger.info(`Rate limited by server, retrying in ${resendDelay}ms`); + } catch (e) { + logger.warn( + `Error while retrieving a rate-limit retry delay, retrying after default delay of ${defaultMs}`, + e, + ); + resendDelay = defaultMs; + } + onRetry(resendDelay); + return true; + } else if (e instanceof HTTPError && e.isRateLimitError()) { + onAbort(); + return true; + } + return false; + } +} From cd87cb439f4dc5f5f9dced748cb4d57da15f7c75 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 19 Feb 2025 22:52:36 +0700 Subject: [PATCH 022/124] fix tests to be compatible with old and new membership manager --- spec/unit/matrixrtc/MembershipManager.spec.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 134528f1437..8382e2c5ddc 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -21,11 +21,10 @@ import { type MockedFunction, type Mock } from "jest-mock"; import { EventType, HTTPError, MatrixError, type Room } from "../../../src"; import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; -import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; +import { LegacyMembershipManager } from "../../../src/matrixrtc/LegacyMembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; import { flushPromises } from "../../test-utils/flushPromises"; -import { defer } from "../../../src/utils"; -// import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; +import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; function waitForMockCall(method: MockedFunction, returnVal?: any) { return new Promise((resolve) => { @@ -35,7 +34,7 @@ function waitForMockCall(method: MockedFunction, returnVal?: any) { }); }); } -defer + function createAsyncHandle(method: MockedFunction) { const handle: { resolve?: (...args: unknown[]) => void; reject?: (...args: any[]) => void } = {}; method.mockImplementation(() => { @@ -56,7 +55,7 @@ describe("MembershipManager", () => { { TestMembershipManager: LegacyMembershipManager, description: "LegacyMembershipManager" }, // Here we will add the new implementation of the MembershipManager. // It is not yet tested since it would currently fail all tests. Adding the MembershipManger looks like this: - // { TestMembershipManager: MembershipManager, description: "MembershipManager" }, + { TestMembershipManager: MembershipManager, description: "MembershipManager" }, ])("$description", ({ TestMembershipManager }) => { let client: MockClient; let room: Room; From 3ac28c23106a37eabe49c0fa3c1b17bac3e6f80f Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 20 Feb 2025 01:02:34 +0700 Subject: [PATCH 023/124] Comment cleanup --- spec/unit/matrixrtc/MembershipManager.spec.ts | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 8382e2c5ddc..217d952f942 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -53,8 +53,6 @@ function createAsyncHandle(method: MockedFunction) { describe("MembershipManager", () => { describe.each([ { TestMembershipManager: LegacyMembershipManager, description: "LegacyMembershipManager" }, - // Here we will add the new implementation of the MembershipManager. - // It is not yet tested since it would currently fail all tests. Adding the MembershipManger looks like this: { TestMembershipManager: MembershipManager, description: "MembershipManager" }, ])("$description", ({ TestMembershipManager }) => { let client: MockClient; @@ -70,7 +68,7 @@ describe("MembershipManager", () => { }; beforeEach(() => { - // Default to fake timers + // Default to fake timers. jest.useFakeTimers(); client = makeMockClient("@alice:example.org", "AAAAAAA"); room = makeMockRoom(membershipTemplate); @@ -80,7 +78,7 @@ describe("MembershipManager", () => { afterEach(() => { jest.useRealTimers(); - // no need to clean up mocks since we will recreate the client + // There is no need to clean up mocks since we will recreate the client. }); describe("isJoined()", () => { @@ -418,7 +416,7 @@ describe("MembershipManager", () => { (client._unstable_updateDelayedEvent as Mock).mockClear(); (client._unstable_sendDelayedStateEvent as Mock).mockClear(); - // our own membership is removed: + // Our own membership is removed: manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); await flushPromises(); expect(client.sendStateEvent).toHaveBeenCalled(); @@ -428,7 +426,7 @@ describe("MembershipManager", () => { }); }); - // TODO: not sure about this name + // TODO: Not sure about this name describe("background timers", () => { it("sends only one keep-alive for delayed leave event per `membershipKeepAlivePeriod`", async () => { const manager = new TestMembershipManager( @@ -444,7 +442,7 @@ describe("MembershipManager", () => { // so it does not need a `advanceTimersByTime` await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - // TODO: check that update delayed event is called with the correct HTTP request timeout + // TODO: Check that update delayed event is called with the correct HTTP request timeout // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); for (let i = 2; i <= 12; i++) { @@ -454,7 +452,7 @@ describe("MembershipManager", () => { await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); - // TODO: check that update delayed event is called with the correct HTTP request timeout + // TODO: Check that update delayed event is called with the correct HTTP request timeout // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); } }); @@ -462,7 +460,7 @@ describe("MembershipManager", () => { // The expires logic was removed for the legacy call manager. // Delayed events should replace it entirely but before they have wide adoption // the expiration logic still makes sense. - // TODO: add git commit when we removed it. + // TODO: Add git commit when we removed it. it("extends `expires` when call still active !FailsForLegacy", async () => { const manager = new TestMembershipManager( { membershipExpiryTimeout: 10_000 }, @@ -511,7 +509,7 @@ describe("MembershipManager", () => { await flushPromises(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); - // FailsForLegacy as implementation does not re-check membership before retrying + // FailsForLegacy as implementation does not re-check membership before retrying. it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { (client._unstable_sendDelayedStateEvent as any).mockRejectedValue( new MatrixError( @@ -540,7 +538,7 @@ describe("MembershipManager", () => { expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); - // FailsForLegacy as implementation does not re-check membership before retrying + // FailsForLegacy as implementation does not re-check membership before retrying. it("abandons retry loop if leave() was called !FailsForLegacy", async () => { const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); @@ -601,11 +599,6 @@ describe("MembershipManager", () => { expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); }); - - // describe: "retries sending membership event" - // it: "sends it if still joined at time of retry" - // // TODO: see what this is doing and how its different to: "recreates membership if it is missing" - // it: "abandons it if call no longer joined at time of retry" }); }); }); From a81106ea67acfbf9d3b2ec765c8f80154d9f81de Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 20 Feb 2025 02:25:59 +0700 Subject: [PATCH 024/124] Allow join to throw - Add tests for throwing cases - Fixs based on tests --- spec/unit/matrixrtc/MembershipManager.spec.ts | 46 ++++++++++++++++++- src/matrixrtc/LegacyMembershipManager.ts | 2 +- src/matrixrtc/MatrixRTCSession.ts | 9 +++- src/matrixrtc/NewMembershipManager.ts | 37 ++++++--------- 4 files changed, 69 insertions(+), 25 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 217d952f942..d3628765238 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -570,7 +570,7 @@ describe("MembershipManager", () => { }); describe("retries sending update delayed leave event restart", () => { it("resends the initial check delayed update event !FailsForLegacy", async () => { - (client._unstable_updateDelayedEvent as any).mockRejectedValue( + (client._unstable_updateDelayedEvent as Mock).mockRejectedValue( new MatrixError( { errcode: "M_LIMIT_EXCEEDED" }, 429, @@ -600,5 +600,49 @@ describe("MembershipManager", () => { }); }); }); + describe("unrecoverable errors", () => { + // !FailsForLegacy because legacy does not have a retry limit and no mechanism to communicate unrecoverable errors. + it("throws, when reaching maximum number of retires for initial delayed event creation !FailsForLegacy", async () => { + const delayEventSendError = jest.fn(); + (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "2" }), + ), + ); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive).catch(delayEventSendError); + await flushPromises(); + for (let i = 0; i < 5; i++) { + jest.advanceTimersByTime(2000); + await flushPromises(); + } + expect(delayEventSendError).toHaveBeenCalled(); + }); + // !FailsForLegacy because legacy does not have a retry limit and no mechanism to communicate unrecoverable errors. + it("throws, when reaching maximum number of retires !FailsForLegacy", async () => { + const delayEventRestartError = jest.fn(); + (client._unstable_updateDelayedEvent as Mock).mockRejectedValue( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive).catch(delayEventRestartError); + await flushPromises(); + for (let i = 0; i < 5; i++) { + jest.advanceTimersByTime(1000); + await flushPromises(); + } + expect(delayEventRestartError).toHaveBeenCalled(); + }, 5000); + }); }); }); diff --git a/src/matrixrtc/LegacyMembershipManager.ts b/src/matrixrtc/LegacyMembershipManager.ts index 843463b4e4e..dc64f9c53ba 100644 --- a/src/matrixrtc/LegacyMembershipManager.ts +++ b/src/matrixrtc/LegacyMembershipManager.ts @@ -90,7 +90,7 @@ export class LegacyMembershipManager implements IMembershipManager { return this.relativeExpiry !== undefined; } - public join(fociPreferred: Focus[], fociActive?: Focus): void { + public async join(fociPreferred: Focus[], fociActive?: Focus): Promise { this.ownFocusActive = fociActive; this.ownFociPreferred = fociPreferred; this.relativeExpiry = this.membershipExpiryTimeout; diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 1c576c21c15..7e7125f0f26 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -91,6 +91,10 @@ export interface MembershipConfig { * @deprecated It should be possible to make it stable without this. */ callMemberEventRetryJitter?: number; + /** + * The maximum rate limit retries the manager will do for delayed event sending/updating and state event sending. + */ + maximumRateLimitRetryCount?: number; } export interface EncryptionConfig { @@ -319,7 +323,10 @@ export class MatrixRTCSession extends TypedEventEmitter + // TODO: Consider exposing this as a signal from the RTCSession so it can be used in the UI. + logger.error("MembershipManager encountered an unrecoverable error: ", e), + ); this.encryptionManager!.join(joinConfig); this.emit(MatrixRTCSessionEvent.JoinStateChanged, true); diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 5141db68119..6ed86be5a4b 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -29,8 +29,9 @@ export interface IMembershipManager { * Start sending all necessary events to make this user participant in the RTC session. * @param fociPreferred the list of preferred foci to use in the joined RTC membership event. * @param fociActive the active focus to use in the joined RTC membership event. + * @throws can throw if it exceeds a configured maximum retry. */ - join(fociPreferred: Focus[], fociActive?: Focus): void; + join(fociPreferred: Focus[], fociActive?: Focus): Promise; /** * Send all necessary events to make this user leave the RTC session. * @param timeout the maximum duration in ms until the promise is forced to resolve. @@ -49,11 +50,6 @@ export interface IMembershipManager { getActiveFocus(): Focus | undefined; } -// CONFIG: -const ALLOWED_MEMBERSHIP_SEND_RETRIES = 5; -const ALLOWED_DELAYED_EVENT_UPDATE_RETRIES = 5; -const ALLOWED_DELAYED_EVENT_SEND_RETRIES = 5; - // SCHEDULER TYPES: enum MembershipActionType { SendJoinEvent = "SendJoinEvent", // -> DelayedLeaveActionType.SendFirstDelayedEvent @@ -99,10 +95,6 @@ interface Action { * can also be thought of as the type of the action */ type: DelayedLeaveActionType | MembershipActionType | DirectMemberhsipManagerActions; - // An id to reference the action, - // that can be used in case there is a chance this action should be altered or removed. - // TODO: can we drop this? - id?: string; } /** @@ -215,13 +207,14 @@ export class MembershipManager implements IMembershipManager { * @param fociPreferred * @param focusActive */ - public join(fociPreferred: Focus[], focusActive?: Focus): void { + public join(fociPreferred: Focus[], focusActive?: Focus): Promise { this.fociPreferred = fociPreferred; this.focusActive = focusActive; if (!this.scheduler.state.running) { this.scheduler.state.running = true; - this.scheduler.startWithActions([{ ts: Date.now(), type: DirectMemberhsipManagerActions.Join }]); + return this.scheduler.startWithActions([{ ts: Date.now(), type: DirectMemberhsipManagerActions.Join }]); } + return Promise.resolve(); } public leave(timeout?: number): Promise { @@ -328,6 +321,9 @@ export class MembershipManager implements IMembershipManager { private get membershipKeepAlivePeriod(): number { return this.joinConfig?.membershipKeepAlivePeriod ?? 5_000; } + private get maximumRateLimitRetryCount(): number { + return this.joinConfig?.maximumRateLimitRetryCount ?? 5; + } // Scheduler: private scheduler = new ActionScheduler( @@ -361,7 +357,6 @@ export class MembershipManager implements IMembershipManager { } catch (e) { this.handleRateLimitError( e, - ALLOWED_DELAYED_EVENT_UPDATE_RETRIES, state.updateDelayedEventRetries, (retryIn) => { state.updateDelayedEventRetries++; @@ -412,7 +407,6 @@ export class MembershipManager implements IMembershipManager { }); this.handleRateLimitError( e, - ALLOWED_DELAYED_EVENT_SEND_RETRIES, state.sendDelayedEventRetries, (retryIn) => { logger.warn("Retry sending delayed disconnection due to rate limit:", e); @@ -444,6 +438,7 @@ export class MembershipManager implements IMembershipManager { } try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart); + state.updateDelayedEventRetries = 0; this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, type: DelayedLeaveActionType.RestartDelayedEvent, @@ -456,9 +451,9 @@ export class MembershipManager implements IMembershipManager { }); this.handleRateLimitError( e, - ALLOWED_DELAYED_EVENT_UPDATE_RETRIES, state.updateDelayedEventRetries, (retryIn) => { + state.updateDelayedEventRetries++; this.scheduler.addAction({ ts: Date.now() + retryIn, type: DelayedLeaveActionType.RestartDelayedEvent, @@ -499,7 +494,6 @@ export class MembershipManager implements IMembershipManager { this.handleRateLimitError( e, state.sendMembershipRetries, - ALLOWED_MEMBERSHIP_SEND_RETRIES, (retryIn) => { logger.warn("Retry sending delayed disconnection due to rate limit:", e); state.sendMembershipRetries++; @@ -521,6 +515,7 @@ export class MembershipManager implements IMembershipManager { try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send); state.hasMemberStateEvent = false; + state.updateDelayedEventRetries = 0; this.leavePromiseHandle.resolve?.(true); } catch (e) { const notFoundHandled = this.handleNotFoundError(e, () => { @@ -529,9 +524,9 @@ export class MembershipManager implements IMembershipManager { }); const rateLimitHandled = this.handleRateLimitError( e, - ALLOWED_DELAYED_EVENT_UPDATE_RETRIES, state.updateDelayedEventRetries, (retryIn) => { + state.updateDelayedEventRetries++; this.scheduler.addAction({ ts: Date.now() + retryIn, type: DelayedLeaveActionType.SendScheduledDelayedLeaveEvent, @@ -573,7 +568,6 @@ export class MembershipManager implements IMembershipManager { const rateLimitHandled = this.handleRateLimitError( e, state.sendMembershipRetries, - ALLOWED_MEMBERSHIP_SEND_RETRIES, (retryIn) => { logger.warn("Retry sending membership state event due to rate limit:", e); state.sendMembershipRetries++; @@ -606,6 +600,7 @@ export class MembershipManager implements IMembershipManager { ); state.nextRelativeExpiry += this.membershipExpiryTimeout; state.hasMemberStateEvent = true; + state.sendMembershipRetries = 0; this.scheduler.addAction({ ts: Date.now(), type: DelayedLeaveActionType.RestartDelayedEvent }); this.scheduler.addAction({ ts: Date.now() + this.membershipExpiryTimeout, @@ -614,13 +609,12 @@ export class MembershipManager implements IMembershipManager { } catch (e) { this.handleRateLimitError( e, - ALLOWED_MEMBERSHIP_SEND_RETRIES, state.sendMembershipRetries, (resendDelay) => { state.sendMembershipRetries++; this.scheduler.addAction({ ts: Date.now() + resendDelay, - type: DelayedLeaveActionType.RestartDelayedEvent, + type: MembershipActionType.SendJoinEvent, }); }, () => { @@ -707,12 +701,11 @@ export class MembershipManager implements IMembershipManager { */ private handleRateLimitError( e: unknown, - allowedRetries: number, currentRetries: number, onRetry: (retryIn: number) => void, onAbort: () => void, ): boolean { - if (currentRetries < allowedRetries && e instanceof HTTPError && e.isRateLimitError()) { + if (currentRetries < this.maximumRateLimitRetryCount && e instanceof HTTPError && e.isRateLimitError()) { let resendDelay: number; const defaultMs = 5000; try { From 6c2a5d1f796ca1326088bf4506b6d98d560730dc Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 20 Feb 2025 08:03:26 +0300 Subject: [PATCH 025/124] introduce membershipExpiryTimeoutSlack --- src/matrixrtc/MatrixRTCSession.ts | 9 +++++++++ src/matrixrtc/NewMembershipManager.ts | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 7e7125f0f26..981ee4125b8 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -64,6 +64,15 @@ export interface MembershipConfig { */ membershipExpiryTimeout?: number; + /** + * The slack in (in milliseconds) which the manager will leave of the meberhsip `expires` time to make sure it + * sends the updated state event early enough. + * + * A slack of 1000ms and a `membershipExpiryTimeout` of 10000ms would result in a memberhsip event update every 9s and + * a memberhsip event that would be considered expired after 10s. + */ + membershipExpiryTimeoutSlack?: number; + /** * The period (in milliseconds) with which we check that our membership event still exists on the * server. If it is not found we create it again. diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 6ed86be5a4b..6fe360248b7 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -311,6 +311,9 @@ export class MembershipManager implements IMembershipManager { private get membershipExpiryTimeout(): number { return this.joinConfig?.membershipExpiryTimeout ?? DEFAULT_EXPIRE_DURATION; } + private get membershipExpiryTimeoutSlack(): number { + return this.joinConfig?.membershipExpiryTimeoutSlack ?? 10_000; + } private get membershipServerSideExpiryTimeout(): number { return ( this.membershipServerSideExpiryTimeoutOverride ?? @@ -561,7 +564,7 @@ export class MembershipManager implements IMembershipManager { // Success, we reset retries and schedule update. state.sendMembershipRetries = 0; this.scheduler.addAction({ - ts: Date.now() + this.membershipExpiryTimeout, + ts: Date.now() + this.membershipExpiryTimeout - this.membershipExpiryTimeoutSlack, type: MembershipActionType.Update, }); } catch (e) { From 061285fc49052fb031d392fac344b173a839cde0 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 20 Feb 2025 08:03:42 +0300 Subject: [PATCH 026/124] more detailed comments and cleanup --- src/matrixrtc/NewMembershipManager.ts | 149 +++++++++++++++++--------- 1 file changed, 97 insertions(+), 52 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 6fe360248b7..c6c9965b781 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -52,34 +52,59 @@ export interface IMembershipManager { // SCHEDULER TYPES: enum MembershipActionType { - SendJoinEvent = "SendJoinEvent", // -> DelayedLeaveActionType.SendFirstDelayedEvent - Update = "Update", // -> MembershipActionType.Update - SendLeaveEvent = "SendLeaveEvent", + SendJoinEvent = "SendJoinEvent", + // -> MembershipActionType.SendJoinEvent if we run in a rate limit and need to retry + // -> MembershipActionType.Update if we successfully send it to schedule the expire event update + // -> DelayedLeaveActionType.RestartDelayedEvent to recheck the delayed event + Update = "Update", + // -> MembershipActionType.Update if the timeout has passed so the next update is required. + SendLeaveEvent = "SendLeaveEvent", // -> MembershipActionType.SendLeaveEvent } function isMembershipActionType(val: any): val is MembershipActionType { return val in MembershipActionType; } enum DelayedLeaveActionType { - SendFirstDelayedEvent = "SendFirstDelayedEvent", // -> MembershipActionType.SendJoinEvent - SendMainDelayedEvent = "SendMainDelayedEvent", // -> DelayedLeaveActionType.RestartDelayedEvent - RestartDelayedEvent = "RestartDelayedEvent", // -> DelayedLeaveActionType.SendJoinEvent, DelayedLeaveActionType.RestartDelayedEvent - SendScheduledDelayedLeaveEvent = "SendScheduledDelayedLeaveEvent", // -> MembershipActionType.SendLeaveEvent + SendFirstDelayedEvent = "SendFirstDelayedEvent", + // -> MembershipActionType.SendJoinEvent if successful + // -> DelayedLeaveActionType.SendFirstDelayedEvent on error, retry sending the first delayed event. + SendMainDelayedEvent = "SendMainDelayedEvent", + // -> DelayedLeaveActionType.RestartDelayedEvent on success start updating the delayed event + // -> DelayedLeaveActionType.SendMainDelayedEvent on error try again + RestartDelayedEvent = "RestartDelayedEvent", + // -> DelayedLeaveActionType.SendMainDelayedEvent on missing delay id but there is a rtc state event + // -> DelayedLeaveActionType.SendFirstDelayedEvent on missing delay id and there is no state event + // -> DelayedLeaveActionType.RestartDelayedEvent on success we schedule the next restart + SendScheduledDelayedLeaveEvent = "SendScheduledDelayedLeaveEvent", + // -> MembershipActionType.SendLeaveEvent on failiour (not found) we need to send the leave manually and cannot use the scheduled delayed event + // -> DelayedLeaveActionType.SendScheduledDelayedLeaveEvent on error we try again. } function isDelayedLeaveActionType(val: any): val is DelayedLeaveActionType { return val in DelayedLeaveActionType; } +/** + * Actions that are supposed to be used from outside the main handle methods. + */ enum DirectMemberhsipManagerActions { Join = DelayedLeaveActionType.SendFirstDelayedEvent, Leave = DelayedLeaveActionType.SendScheduledDelayedLeaveEvent, } interface ActionSchedulerState { + /** The delayId we got when successfully sending the delayed leave event. + * Gets set to undefined if the server claims it cannot find the delayed event anymore. */ delayId?: string; + /** Stores the value we want to use for the `expires` field in the next own membership update. */ nextRelativeExpiry: number; + /** Flag that gets set once join is called. + * The manager tries its best to get the user into the call. + * Does not imply the user is actually joined via room state. */ running: boolean; + /** The manager is in the state where its actually connected to the session. */ hasMemberStateEvent: boolean; + // Retry counters that get used to limit the maximum rate limit retires we want to do. + // They get reused for each rate limit loop we run into and reset to 0 on unrecoverable failiour or success. sendMembershipRetries: number; sendDelayedEventRetries: number; updateDelayedEventRetries: number; @@ -244,6 +269,7 @@ export class MembershipManager implements IMembershipManager { if (this.isJoined() && !memberships.some(isMyMembership)) { logger.warn("Missing own membership: force re-join"); + this.scheduler.state.hasMemberStateEvent = false; this.scheduler.addAction({ ts: Date.now(), type: DirectMemberhsipManagerActions.Join }); } } @@ -327,7 +353,6 @@ export class MembershipManager implements IMembershipManager { private get maximumRateLimitRetryCount(): number { return this.joinConfig?.maximumRateLimitRetryCount ?? 5; } - // Scheduler: private scheduler = new ActionScheduler( { @@ -347,6 +372,7 @@ export class MembershipManager implements IMembershipManager { switch (type) { case DelayedLeaveActionType.SendFirstDelayedEvent: // Remove all running updates and restarts + this.scheduler.resetActions([]); // Before we start we check if we come from a state where we have a delay id. if (state.delayId) { try { @@ -385,47 +411,47 @@ export class MembershipManager implements IMembershipManager { }); }); } - return; - } - try { - const response = await this.client._unstable_sendDelayedStateEvent( - this.room.roomId, - { - delay: this.membershipServerSideExpiryTimeout, - }, - EventType.GroupCallMemberPrefix, - {}, // leave event - this.stateKey, - ); - // Success we reset retires and set delayId. - state.sendDelayedEventRetries = 0; - state.delayId = response.delay_id; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); - } catch (e) { - this.handleMaxDelayeExceededError(e, () => { - this.scheduler.addAction({ - ts: Date.now(), - type: DelayedLeaveActionType.SendFirstDelayedEvent, - }); - }); - this.handleRateLimitError( - e, - state.sendDelayedEventRetries, - (retryIn) => { - logger.warn("Retry sending delayed disconnection due to rate limit:", e); - state.sendDelayedEventRetries++; + } else { + try { + const response = await this.client._unstable_sendDelayedStateEvent( + this.room.roomId, + { + delay: this.membershipServerSideExpiryTimeout, + }, + EventType.GroupCallMemberPrefix, + {}, // leave event + this.stateKey, + ); + // Success we reset retires and set delayId. + state.sendDelayedEventRetries = 0; + state.delayId = response.delay_id; + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); + } catch (e) { + this.handleMaxDelayeExceededError(e, () => { this.scheduler.addAction({ - ts: Date.now() + retryIn, + ts: Date.now(), type: DelayedLeaveActionType.SendFirstDelayedEvent, }); - }, - () => { - throw Error( - "Exceeded maximum delayed event send attempts (client._unstable_sendDelayedStateEvent): " + - e, - ); - }, - ); + }); + this.handleRateLimitError( + e, + state.sendDelayedEventRetries, + (retryIn) => { + logger.warn("Retry sending delayed disconnection due to rate limit:", e); + state.sendDelayedEventRetries++; + this.scheduler.addAction({ + ts: Date.now() + retryIn, + type: DelayedLeaveActionType.SendFirstDelayedEvent, + }); + }, + () => { + throw Error( + "Exceeded maximum delayed event send attempts (client._unstable_sendDelayedStateEvent): " + + e, + ); + }, + ); + } } break; case DelayedLeaveActionType.RestartDelayedEvent: @@ -502,12 +528,13 @@ export class MembershipManager implements IMembershipManager { state.sendMembershipRetries++; this.scheduler.addAction({ ts: Date.now() + Math.max(retryIn, this.callMemberEventRetryDelayMinimum), - type: MembershipActionType.Update, + type: DelayedLeaveActionType.SendMainDelayedEvent, }); }, () => { throw Error( - "Exceeded maximum own Membership state update attempts (client.sendStateEvent): " + e, + "Exceeded maximum send delayed event attempts (client._unstable_sendDelayedStateEvent): " + + e, ); }, ); @@ -519,6 +546,7 @@ export class MembershipManager implements IMembershipManager { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send); state.hasMemberStateEvent = false; state.updateDelayedEventRetries = 0; + this.scheduler.resetActions([]); this.leavePromiseHandle.resolve?.(true); } catch (e) { const notFoundHandled = this.handleNotFoundError(e, () => { @@ -536,11 +564,11 @@ export class MembershipManager implements IMembershipManager { }); }, () => { - this.leavePromiseHandle.reject?.(e); + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); }, ); if (!(notFoundHandled || rateLimitHandled)) { - this.leavePromiseHandle.reject?.(e); + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } } } else { @@ -629,7 +657,7 @@ export class MembershipManager implements IMembershipManager { } break; case MembershipActionType.SendLeaveEvent: - // we are good already + // We are good already if (!state.hasMemberStateEvent) return; // This is only a fallback in case we do not have working delayed events support. @@ -641,8 +669,25 @@ export class MembershipManager implements IMembershipManager { {}, this.makeMembershipStateKey(this.userId, this.deviceId), ); + state.updateDelayedEventRetries = 0; + this.scheduler.resetActions([]); + this.leavePromiseHandle.resolve?.(true); state.hasMemberStateEvent = false; - } catch {} + } catch (e) { + this.handleRateLimitError( + e, + state.sendMembershipRetries, + (retryIn) => { + this.scheduler.addAction({ + ts: Date.now() + retryIn, + type: MembershipActionType.SendLeaveEvent, + }); + }, + () => { + throw Error("could not send final leave event due to max rate limit retries" + e); + }, + ); + } } } From d116b539cc00fd9a0009e277d73caddb4fd41370 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 12:52:55 +0100 Subject: [PATCH 027/124] warn if slack is misconfigured and use default values instead --- src/matrixrtc/NewMembershipManager.ts | 35 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index c6c9965b781..fe4ed3bf2a3 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -334,11 +334,29 @@ export class MembershipManager implements IMembershipManager { private get callMemberEventRetryDelayMinimum(): number { return this.joinConfig?.callMemberEventRetryDelayMinimum ?? 3_000; } - private get membershipExpiryTimeout(): number { + private get membershipEventExpiryTimeout(): number { return this.joinConfig?.membershipExpiryTimeout ?? DEFAULT_EXPIRE_DURATION; } - private get membershipExpiryTimeoutSlack(): number { - return this.joinConfig?.membershipExpiryTimeoutSlack ?? 10_000; + private get membershipTimerExpiryTimeout(): number { + let expiryTimeout = this.membershipEventExpiryTimeout; + const expiryTimeoutSlack = this.joinConfig?.membershipExpiryTimeoutSlack; + if (expiryTimeoutSlack) { + if (expiryTimeout > expiryTimeoutSlack) { + expiryTimeout = expiryTimeout - expiryTimeoutSlack; + } else { + logger.warn( + "The membershipExpiryTimeoutSlack is misconfigured. It cannot be less than the membershipExpiryTimeout", + "membershipExpiryTimeout:", + expiryTimeout, + "membershipExpiryTimeoutSlack:", + expiryTimeoutSlack, + ); + } + } else { + // Default Slack + expiryTimeout -= 5_000; + } + return Math.max(expiryTimeout, 1000); } private get membershipServerSideExpiryTimeout(): number { return ( @@ -358,7 +376,7 @@ export class MembershipManager implements IMembershipManager { { hasMemberStateEvent: false, running: false, - nextRelativeExpiry: this.membershipExpiryTimeout, + nextRelativeExpiry: this.membershipEventExpiryTimeout, delayId: undefined, sendMembershipRetries: 0, sendDelayedEventRetries: 0, @@ -588,11 +606,12 @@ export class MembershipManager implements IMembershipManager { this.makeMyMembership(state.nextRelativeExpiry), this.stateKey, ); - state.nextRelativeExpiry += this.membershipExpiryTimeout; + state.nextRelativeExpiry += this.membershipEventExpiryTimeout; // Success, we reset retries and schedule update. state.sendMembershipRetries = 0; + this.scheduler.addAction({ - ts: Date.now() + this.membershipExpiryTimeout - this.membershipExpiryTimeoutSlack, + ts: Date.now() + this.membershipTimerExpiryTimeout, type: MembershipActionType.Update, }); } catch (e) { @@ -629,12 +648,12 @@ export class MembershipManager implements IMembershipManager { this.makeMyMembership(state.nextRelativeExpiry), this.stateKey, ); - state.nextRelativeExpiry += this.membershipExpiryTimeout; + state.nextRelativeExpiry += this.membershipEventExpiryTimeout; state.hasMemberStateEvent = true; state.sendMembershipRetries = 0; this.scheduler.addAction({ ts: Date.now(), type: DelayedLeaveActionType.RestartDelayedEvent }); this.scheduler.addAction({ - ts: Date.now() + this.membershipExpiryTimeout, + ts: Date.now() + this.membershipTimerExpiryTimeout, type: MembershipActionType.Update, }); } catch (e) { From c3a02db417dbc7798c385e6b022645c3bd976d21 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 12:53:13 +0100 Subject: [PATCH 028/124] fix action resets. --- src/matrixrtc/NewMembershipManager.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index fe4ed3bf2a3..8b299f6b303 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -176,14 +176,14 @@ class ActionScheduler { } } - this.actions = this.actions.filter((a) => a !== nextAction); - this.actions.push(...this.insertions); - this.insertions = []; - if (this.resetWith) { this.actions = this.resetWith; this.resetWith = undefined; } + + this.actions = this.actions.filter((a) => a !== nextAction); + this.actions.push(...this.insertions); + this.insertions = []; } } @@ -191,7 +191,8 @@ class ActionScheduler { // Dont add any other actions if we have a leave scheduled if (this.actions.some((a) => a.type === DirectMemberhsipManagerActions.Leave)) return; this.insertions.push(action); - if (this.actions[0].ts > action.ts) { + const nextTs = this.actions[0]?.ts; + if (!nextTs || nextTs > action.ts) { this.didWakeUp = true; this.wakeup?.(); } @@ -389,10 +390,10 @@ export class MembershipManager implements IMembershipManager { public async delayedLeaveLoopHandler(state: ActionSchedulerState, type: DelayedLeaveActionType): Promise { switch (type) { case DelayedLeaveActionType.SendFirstDelayedEvent: - // Remove all running updates and restarts - this.scheduler.resetActions([]); // Before we start we check if we come from a state where we have a delay id. if (state.delayId) { + // Remove all running updates and restarts + this.scheduler.resetActions([]); try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel); state.delayId = undefined; From 023665734268ee5e8f16a10500f241e6bd8dc3fa Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 12:53:27 +0100 Subject: [PATCH 029/124] flatten MembershipManager.spec.ts --- spec/unit/matrixrtc/MembershipManager.spec.ts | 1014 ++++++++--------- 1 file changed, 504 insertions(+), 510 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index d3628765238..2f60c44d599 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -50,581 +50,521 @@ function createAsyncHandle(method: MockedFunction) { * Tests different MembershipManager implementations. Some tests don't apply to `LegacyMembershipManager` * use !FailsForLegacy to skip those. See: testEnvironment for more details. */ -describe("MembershipManager", () => { - describe.each([ - { TestMembershipManager: LegacyMembershipManager, description: "LegacyMembershipManager" }, - { TestMembershipManager: MembershipManager, description: "MembershipManager" }, - ])("$description", ({ TestMembershipManager }) => { - let client: MockClient; - let room: Room; - const focusActive: LivekitFocusActive = { - focus_selection: "oldest_membership", - type: "livekit", - }; - const focus: Focus = { - type: "livekit", - livekit_service_url: "https://active.url", - livekit_alias: "!active:active.url", - }; - - beforeEach(() => { - // Default to fake timers. - jest.useFakeTimers(); - client = makeMockClient("@alice:example.org", "AAAAAAA"); - room = makeMockRoom(membershipTemplate); - // Provide a default mock. Representing the default "non error" server behaviour. - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); - }); - afterEach(() => { - jest.useRealTimers(); - // There is no need to clean up mocks since we will recreate the client. - }); +describe.each([ + { TestMembershipManager: LegacyMembershipManager, description: "LegacyMembershipManager" }, + { TestMembershipManager: MembershipManager, description: "MembershipManager" }, +])("$description", ({ TestMembershipManager }) => { + let client: MockClient; + let room: Room; + const focusActive: LivekitFocusActive = { + focus_selection: "oldest_membership", + type: "livekit", + }; + const focus: Focus = { + type: "livekit", + livekit_service_url: "https://active.url", + livekit_alias: "!active:active.url", + }; + + beforeEach(() => { + // Fefault to fake timers. + jest.useFakeTimers(); + client = makeMockClient("@alice:example.org", "AAAAAAA"); + room = makeMockRoom(membershipTemplate); + // Provide a default mock that is like the default "non error" server behaviour. + (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + }); - describe("isJoined()", () => { - it("defaults to false", () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - expect(manager.isJoined()).toEqual(false); - }); + afterEach(() => { + jest.useRealTimers(); + // There is no need to clean up mocks since we will recreate the client. + }); - it("returns true after join()", async () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([]); - expect(manager.isJoined()).toEqual(true); - }); + describe("isJoined()", () => { + it("defaults to false", () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + expect(manager.isJoined()).toEqual(false); }); - describe("join()", () => { - describe("sends a membership event", () => { - it("sends a membership event and schedules delayed leave when joining a call", async () => { - // Spys/Mocks + it("returns true after join()", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([]); + expect(manager.isJoined()).toEqual(true); + }); + }); - const updateDelayedEventHandle = createAsyncHandle(client._unstable_updateDelayedEvent as Mock); + describe("join()", () => { + describe("sends a membership event", () => { + it("sends a membership event and schedules delayed leave when joining a call", async () => { + // Spys/Mocks - // Test - const memberManager = new TestMembershipManager(undefined, room, client, () => undefined); - memberManager.join([focus], focusActive); + // eslint-disable-next-line camelcase + const _unstable_updateDelayedEventHandle = createAsyncHandle( + client._unstable_updateDelayedEvent as Mock, + ); - // expects - await waitForMockCall(client.sendStateEvent); - expect(client.sendStateEvent).toHaveBeenCalledWith( - room.roomId, - "org.matrix.msc3401.call.member", - { - application: "m.call", - call_id: "", - device_id: "AAAAAAA", - expires: 14400000, - foci_preferred: [focus], - focus_active: focusActive, - scope: "m.room", - }, - "_@alice:example.org_AAAAAAA", - ); - updateDelayedEventHandle.resolve?.(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( - room.roomId, - { delay: 8000 }, - "org.matrix.msc3401.call.member", - {}, - "_@alice:example.org_AAAAAAA", - ); - }); + // Test + const memberManager = new TestMembershipManager(undefined, room, client, () => undefined); + memberManager.join([focus], focusActive); + // await flushPromises(); + // expects + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledWith( + room.roomId, + "org.matrix.msc3401.call.member", + { + application: "m.call", + call_id: "", + device_id: "AAAAAAA", + expires: 14400000, + foci_preferred: [focus], + focus_active: focusActive, + scope: "m.room", + }, + "_@alice:example.org_AAAAAAA", + ); + // eslint-disable-next-line camelcase + _unstable_updateDelayedEventHandle.resolve?.(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( + room.roomId, + { delay: 8000 }, + "org.matrix.msc3401.call.member", + {}, + "_@alice:example.org_AAAAAAA", + ); + }); - describe("does not prefix the state key with _ for rooms that support user-owned state events", () => { - async function testJoin(useOwnedStateEvents: boolean): Promise { - // TODO: this test does quiet a bit. Its more a like a test story summarizing to: - // - send delay with too long timeout and get server error (test delayedEventTimeout gets overwritten) - // - run into rate limit for sending delayed event - // - run into rate limit when setting membership state. - if (useOwnedStateEvents) { - room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); - } - - const updatedDelayedEvent = waitForMockCall(client._unstable_updateDelayedEvent); - const sentDelayedState = waitForMockCall(client._unstable_sendDelayedStateEvent, { - delay_id: "id", - }); + describe("does not prefix the state key with _ for rooms that support user-owned state events", () => { + async function testJoin(useOwnedStateEvents: boolean): Promise { + if (useOwnedStateEvents) { + room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); + } - // preparing the delayed disconnect should handle the delay being too long - const sendDelayedStateExceedAttempt = new Promise((resolve) => { - const error = new MatrixError({ - "errcode": "M_UNKNOWN", - "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", - "org.matrix.msc4140.max_delay": 7500, - }); - (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); - }); + const updatedDelayedEvent = waitForMockCall(client._unstable_updateDelayedEvent); + const sentDelayedState = waitForMockCall(client._unstable_sendDelayedStateEvent, { + delay_id: "id", + }); - const userStateKey = `${!useOwnedStateEvents ? "_" : ""}@alice:example.org_AAAAAAA`; - // preparing the delayed disconnect should handle ratelimiting - const sendDelayedStateAttempt = new Promise((resolve) => { - const error = new MatrixError({ errcode: "M_LIMIT_EXCEEDED" }); - (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); + // preparing the delayed disconnect should handle the delay being too long + const sendDelayedStateExceedAttempt = new Promise((resolve) => { + const error = new MatrixError({ + "errcode": "M_UNKNOWN", + "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", + "org.matrix.msc4140.max_delay": 7500, + }); + (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); }); + }); - // setting the membership state should handle ratelimiting (also with a retry-after value) - const sendStateEventAttempt = new Promise((resolve) => { - const error = new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ); - (client.sendStateEvent as Mock).mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); + const userStateKey = `${!useOwnedStateEvents ? "_" : ""}@alice:example.org_AAAAAAA`; + // preparing the delayed disconnect should handle ratelimiting + const sendDelayedStateAttempt = new Promise((resolve) => { + const error = new MatrixError({ errcode: "M_LIMIT_EXCEEDED" }); + (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); }); - const manager = new TestMembershipManager( - { - membershipServerSideExpiryTimeout: 9000, - }, - room, - client, - () => undefined, - ); - manager.join([focus], focusActive); + }); - await sendDelayedStateExceedAttempt.then(); // needed to resolve after the send attempt catches - await sendDelayedStateAttempt; - const callProps = (d: number) => { - return [room!.roomId, { delay: d }, "org.matrix.msc3401.call.member", {}, userStateKey]; - }; - expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(1, ...callProps(9000)); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(2, ...callProps(7500)); + // setting the membership state should handle ratelimiting (also with a retry-after value) + const sendStateEventAttempt = new Promise((resolve) => { + const error = new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ); + (client.sendStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); + }); + }); + const manager = new TestMembershipManager( + { + membershipServerSideExpiryTimeout: 9000, + }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); - jest.advanceTimersByTime(5000); + await sendDelayedStateExceedAttempt.then(); // needed to resolve after the send attempt catches + await sendDelayedStateAttempt; + const callProps = (d: number) => { + return [room!.roomId, { delay: d }, "org.matrix.msc3401.call.member", {}, userStateKey]; + }; + expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(1, ...callProps(9000)); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(2, ...callProps(7500)); - await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches - jest.advanceTimersByTime(1000); + jest.advanceTimersByTime(5000); - expect(client.sendStateEvent).toHaveBeenCalledWith( - room!.roomId, - EventType.GroupCallMemberPrefix, - { - application: "m.call", - scope: "m.room", - call_id: "", - expires: 14400000, - device_id: "AAAAAAA", - foci_preferred: [focus], - focus_active: focusActive, - } satisfies SessionMembershipData, - userStateKey, - ); - await sentDelayedState; - - // should have prepared the heartbeat to keep delaying the leave event while still connected - await updatedDelayedEvent; - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - - // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. - await flushPromises(); - jest.advanceTimersByTime(5000); - await flushPromises(); - // should update delayed disconnect - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); - } + await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches + jest.advanceTimersByTime(1000); - it("sends a membership event after rate limits during delayed event setup when joining a call", async () => { - await testJoin(false); - }); + expect(client.sendStateEvent).toHaveBeenCalledWith( + room!.roomId, + EventType.GroupCallMemberPrefix, + { + application: "m.call", + scope: "m.room", + call_id: "", + expires: 14400000, + device_id: "AAAAAAA", + foci_preferred: [focus], + focus_active: focusActive, + } satisfies SessionMembershipData, + userStateKey, + ); + await sentDelayedState; - it("does not prefix the state key with _ for rooms that support user-owned state events", async () => { - await testJoin(true); - }); - }); - }); + // should have prepared the heartbeat to keep delaying the leave event while still connected + await updatedDelayedEvent; + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - describe("delayed leave event", () => { - it("does not try again to schedule a delayed leave event if not supported", () => { - const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - delayedHandle.reject?.(Error("Server does not support the delayed events API")); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - }); - it("does try to schedule a delayed leave event again if rate limited", async () => { - const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); + // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. await flushPromises(); jest.advanceTimersByTime(5000); await flushPromises(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + // should update delayed disconnect + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + } + + it("sends a membership event after rate limits during delayed event setup when joining a call", async () => { + await testJoin(false); }); - it("uses membershipServerSideExpiryTimeout from config", async () => { - const manager = new TestMembershipManager( - { membershipServerSideExpiryTimeout: 123456 }, - room, - client, - () => undefined, - ); - manager.join([focus], focusActive); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( - room.roomId, - { delay: 123456 }, - "org.matrix.msc3401.call.member", - {}, - "_@alice:example.org_AAAAAAA", - ); + + it("does not prefix the state key with _ for rooms that support user-owned state events", async () => { + await testJoin(true); }); }); + }); - it("uses membershipExpiryTimeout from config", async () => { + describe("delayed leave event", () => { + it("does not try again to schedule a delayed leave event if not supported", () => { + const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + delayedHandle.reject?.(Error("Server does not support the delayed events API")); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + }); + it("does try to schedule a delayed leave event again if rate limited", () => { + const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); + waitForExpect(() => { + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + }); + }); + it("uses membershipServerSideExpiryTimeout from config", async () => { const manager = new TestMembershipManager( - { membershipExpiryTimeout: 1234567 }, + { membershipServerSideExpiryTimeout: 123456 }, room, client, () => undefined, ); - manager.join([focus], focusActive); - await waitForMockCall(client.sendStateEvent); - expect(client.sendStateEvent).toHaveBeenCalledWith( + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( room.roomId, - EventType.GroupCallMemberPrefix, - { - application: "m.call", - scope: "m.room", - call_id: "", - device_id: "AAAAAAA", - expires: 1234567, - foci_preferred: [focus], - focus_active: { - focus_selection: "oldest_membership", - type: "livekit", - }, - }, + { delay: 123456 }, + "org.matrix.msc3401.call.member", + {}, "_@alice:example.org_AAAAAAA", ); }); + }); - it("does nothing if join called when already joined", async () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - await waitForMockCall(client.sendStateEvent); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - manager.join([focus], focusActive); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - }); + it("uses membershipExpiryTimeout from config", async () => { + const manager = new TestMembershipManager( + { membershipExpiryTimeout: 1234567 }, + room, + client, + () => undefined, + ); + + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledWith( + room.roomId, + EventType.GroupCallMemberPrefix, + { + application: "m.call", + scope: "m.room", + call_id: "", + device_id: "AAAAAAA", + expires: 1234567, + foci_preferred: [focus], + focus_active: { + focus_selection: "oldest_membership", + type: "livekit", + }, + }, + "_@alice:example.org_AAAAAAA", + ); }); - describe("leave()", () => { - // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed - it("does nothing if not joined !FailsForLegacy", () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.leave(); - expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); - expect(client.sendStateEvent).not.toHaveBeenCalled(); - }); + it("does nothing if join called when already joined", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + manager.join([focus], focusActive); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); + }); - describe("getsActiveFocus", () => { - it("gets the correct active focus with oldest_membership", () => { - const getOldestMembership = jest.fn(); - const manager = new TestMembershipManager({}, room, client, getOldestMembership); - // Before joining the active focus should be undefined (see FocusInUse on MatrixRTCSession) - expect(manager.getActiveFocus()).toBe(undefined); - manager.join([focus], focusActive); - // After joining we want our own focus to be the one we select. - getOldestMembership.mockReturnValue( - mockCallMembership( - { - ...membershipTemplate, - foci_preferred: [ - { - livekit_alias: "!active:active.url", - livekit_service_url: "https://active.url", - type: "livekit", - }, - ], - device_id: client.getDeviceId(), - created_ts: 1000, - }, - room.roomId, - client.getUserId()!, - ), - ); - expect(manager.getActiveFocus()).toStrictEqual(focus); - getOldestMembership.mockReturnValue( - mockCallMembership( - Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), - room.roomId, - ), - ); - // If there is an older member we use its focus. - expect(manager.getActiveFocus()).toBe(membershipTemplate.foci_preferred[0]); - }); + describe("leave()", () => { + // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed + it("does nothing if not joined !FailsForLegacy", () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.leave(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + }); + }); - it("does not provide focus if the selection method is unknown", () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], Object.assign(focusActive, { type: "unknown_type" })); - expect(manager.getActiveFocus()).toBe(undefined); - }); + describe("getsActiveFocus", () => { + it("gets the correct active focus with oldest_membership", () => { + const getOldestMembership = jest.fn(); + const manager = new TestMembershipManager({}, room, client, getOldestMembership); + // Before joining the active focus should be undefined (see FocusInUse on MatrixRTCSession) + expect(manager.getActiveFocus()).toBe(undefined); + manager.join([focus], focusActive); + // After joining we want our own focus to be the one we select. + getOldestMembership.mockReturnValue( + mockCallMembership( + { + ...membershipTemplate, + foci_preferred: [ + { + livekit_alias: "!active:active.url", + livekit_service_url: "https://active.url", + type: "livekit", + }, + ], + device_id: client.getDeviceId(), + created_ts: 1000, + }, + room.roomId, + client.getUserId()!, + ), + ); + expect(manager.getActiveFocus()).toStrictEqual(focus); + getOldestMembership.mockReturnValue( + mockCallMembership( + Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), + room.roomId, + ), + ); + // If there is an older member we use its focus. + expect(manager.getActiveFocus()).toBe(membershipTemplate.foci_preferred[0]); }); - describe("onRTCSessionMemberUpdate()", () => { - it("does nothing if not joined", async () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); - await flushPromises(); - expect(client.sendStateEvent).not.toHaveBeenCalled(); - expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); - expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); - }); - it("does nothing if own membership still present", async () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); + it("does not provide focus if the selection method is unknown", () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], Object.assign(focusActive, { type: "unknown_type" })); + expect(manager.getActiveFocus()).toBe(undefined); + }); + }); - await waitForMockCall(client.sendStateEvent); - await flushPromises(); - const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; - // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` - (client.sendStateEvent as Mock).mockReset(); - (client._unstable_updateDelayedEvent as Mock).mockReset(); - (client._unstable_sendDelayedStateEvent as Mock).mockReset(); - - manager.onRTCSessionMemberUpdate([ - mockCallMembership(membershipTemplate, room.roomId), - mockCallMembership( - myMembership as SessionMembershipData, - room.roomId, - client.getUserId() ?? undefined, - ), - ]); + describe("onRTCSessionMemberUpdate()", () => { + it("does nothing if not joined", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await flushPromises(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); + }); + it("does nothing if own membership still present", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + + await waitForMockCall(client.sendStateEvent); + await flushPromises(); + const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; + // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` + (client.sendStateEvent as Mock).mockReset(); + (client._unstable_updateDelayedEvent as Mock).mockReset(); + (client._unstable_sendDelayedStateEvent as Mock).mockReset(); + + manager.onRTCSessionMemberUpdate([ + mockCallMembership(membershipTemplate, room.roomId), + mockCallMembership(myMembership as SessionMembershipData, room.roomId, client.getUserId() ?? undefined), + ]); + await flushPromises(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); + }); + it("recreates membership if it is missing", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + await flushPromises(); + // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` + (client.sendStateEvent as Mock).mockClear(); + (client._unstable_updateDelayedEvent as Mock).mockClear(); + (client._unstable_sendDelayedStateEvent as Mock).mockClear(); + + // Our own membership is removed: + manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await flushPromises(); + expect(client.sendStateEvent).toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalled(); + + expect(client._unstable_updateDelayedEvent).toHaveBeenCalled(); + }); + }); + + // TODO: Not sure about this name + describe("background timers", () => { + it("sends only one keep-alive for delayed leave event per `membershipKeepAlivePeriod`", async () => { + const manager = new TestMembershipManager( + { membershipKeepAlivePeriod: 10_000, membershipServerSideExpiryTimeout: 30_000 }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + + // The first call is from checking id the server deleted the delayed event + // so it does not need a `advanceTimersByTime` + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); + // TODO: Check that update delayed event is called with the correct HTTP request timeout + // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); + + for (let i = 2; i <= 12; i++) { + // flush promises before advancing the timers to make sure schedulers are setup await flushPromises(); - expect(client.sendStateEvent).not.toHaveBeenCalled(); - expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); - expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); - }); - it("recreates membership if it is missing", async () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); + jest.advanceTimersByTime(10_000); await flushPromises(); - // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` - (client.sendStateEvent as Mock).mockClear(); - (client._unstable_updateDelayedEvent as Mock).mockClear(); - (client._unstable_sendDelayedStateEvent as Mock).mockClear(); - // Our own membership is removed: - manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); - await flushPromises(); - expect(client.sendStateEvent).toHaveBeenCalled(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalled(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); + // TODO: Check that update delayed event is called with the correct HTTP request timeout + // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); + } + }); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalled(); - }); + // The expires logic was removed for the legacy call manager. + // Delayed events should replace it entirely but before they have wide adoption + // the expiration logic still makes sense. + // TODO: Add git commit when we removed it. + it("extends `expires` when call still active !FailsForLegacy", async () => { + const manager = new TestMembershipManager( + { membershipExpiryTimeout: 10_000 }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; + expect(sentMembership.expires).toBe(10_000); + for (let i = 2; i <= 12; i++) { + // flush promises before advancing the timers to make sure schedulers are setup + await flushPromises(); + jest.advanceTimersByTime(10_000); + await flushPromises(); + expect(client.sendStateEvent).toHaveBeenCalledTimes(i); + const sentMembership = (client.sendStateEvent as Mock).mock.lastCall![2] as SessionMembershipData; + expect(sentMembership.expires).toBe(10_000 * i); + } }); + }); - // TODO: Not sure about this name - describe("background timers", () => { - it("sends only one keep-alive for delayed leave event per `membershipKeepAlivePeriod`", async () => { - const manager = new TestMembershipManager( - { membershipKeepAlivePeriod: 10_000, membershipServerSideExpiryTimeout: 30_000 }, - room, - client, - () => undefined, - ); + describe("server error handling", () => { + // Types of server error: 429 rate limit with no retry-after header, 429 with retry-after, 50x server error (maybe retry every second), connection/socket timeout + describe("retries sending delayed leave event", () => { + it("sends retry if call membership event is still valid at time of retry", async () => { + const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); + + const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - // The first call is from checking id the server deleted the delayed event - // so it does not need a `advanceTimersByTime` + handle.reject?.( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); await flushPromises(); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - // TODO: Check that update delayed event is called with the correct HTTP request timeout - // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); - - for (let i = 2; i <= 12; i++) { - // flush promises before advancing the timers to make sure schedulers are setup - await flushPromises(); - jest.advanceTimersByTime(10_000); - await flushPromises(); - - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); - // TODO: Check that update delayed event is called with the correct HTTP request timeout - // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); - } + jest.advanceTimersByTime(1000); + await flushPromises(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); - - // The expires logic was removed for the legacy call manager. - // Delayed events should replace it entirely but before they have wide adoption - // the expiration logic still makes sense. - // TODO: Add git commit when we removed it. - it("extends `expires` when call still active !FailsForLegacy", async () => { - const manager = new TestMembershipManager( - { membershipExpiryTimeout: 10_000 }, - room, - client, - () => undefined, + // FailsForLegacy as implementation does not re-check membership before retrying. + it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { + (client._unstable_sendDelayedStateEvent as any).mockRejectedValue( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), ); + const manager = new TestMembershipManager({}, room, client, () => undefined); + // Should call _unstable_sendDelayedStateEvent but not sendStateEvent because of the + // RateLimit error. manager.join([focus], focusActive); - await waitForMockCall(client.sendStateEvent); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; - expect(sentMembership.expires).toBe(10_000); - for (let i = 2; i <= 12; i++) { - // flush promises before advancing the timers to make sure schedulers are setup - await flushPromises(); - jest.advanceTimersByTime(10_000); - await flushPromises(); - expect(client.sendStateEvent).toHaveBeenCalledTimes(i); - const sentMembership = (client.sendStateEvent as Mock).mock.lastCall![2] as SessionMembershipData; - expect(sentMembership.expires).toBe(10_000 * i); - } - }); - }); - - describe("server error handling", () => { - // Types of server error: 429 rate limit with no retry-after header, 429 with retry-after, 50x server error (maybe retry every second), connection/socket timeout - describe("retries sending delayed leave event", () => { - it("sends retry if call membership event is still valid at time of retry", async () => { - const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); - - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - - handle.reject?.( - new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ), - ); - await flushPromises(); - jest.advanceTimersByTime(1000); - await flushPromises(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); - }); - // FailsForLegacy as implementation does not re-check membership before retrying. - it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { - (client._unstable_sendDelayedStateEvent as any).mockRejectedValue( - new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ), - ); - const manager = new TestMembershipManager({}, room, client, () => undefined); - // Should call _unstable_sendDelayedStateEvent but not sendStateEvent because of the - // RateLimit error. - manager.join([focus], focusActive); - - await flushPromises(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); - // Remove our own membership so that there is no reason the send the delayed leave anymore. - // the membership is no longer present on the homeserver - manager.onRTCSessionMemberUpdate([]); - // Wait for all timers to be setup - await flushPromises(); - jest.advanceTimersByTime(1000); - // We should send the first own membership and a new delayed event after the rate limit timeout. - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - }); - // FailsForLegacy as implementation does not re-check membership before retrying. - it("abandons retry loop if leave() was called !FailsForLegacy", async () => { - const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); - - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - handle.reject?.( - new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ), - ); - await flushPromises(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - // the user terminated the call locally - manager.leave(); - - // Wait for all timers to be setup - await flushPromises(); - jest.advanceTimersByTime(1000); - await flushPromises(); - - // No new events should have been sent: - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - }); + await flushPromises(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + // Remove our own membership so that there is no reason the send the delayed leave anymore. + // the membership is no longer present on the homeserver + manager.onRTCSessionMemberUpdate([]); + // Wait for all timers to be setup + await flushPromises(); + jest.advanceTimersByTime(1000); + // We should send the first own membership and a new delayed event after the rate limit timeout. + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); - describe("retries sending update delayed leave event restart", () => { - it("resends the initial check delayed update event !FailsForLegacy", async () => { - (client._unstable_updateDelayedEvent as Mock).mockRejectedValue( - new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ), - ); - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - - // Hit rate limit - await flushPromises(); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); + // FailsForLegacy as implementation does not re-check membership before retrying. + it("abandons retry loop if leave() was called !FailsForLegacy", async () => { + const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); - // Hit second rate limit. - jest.advanceTimersByTime(1000); - await flushPromises(); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); - - // Setup resolve - (client._unstable_updateDelayedEvent as Mock).mockImplementation(() => {}); - jest.advanceTimersByTime(1000); - await flushPromises(); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - }); - }); - }); - describe("unrecoverable errors", () => { - // !FailsForLegacy because legacy does not have a retry limit and no mechanism to communicate unrecoverable errors. - it("throws, when reaching maximum number of retires for initial delayed event creation !FailsForLegacy", async () => { - const delayEventSendError = jest.fn(); - (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue( + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + handle.reject?.( new MatrixError( { errcode: "M_LIMIT_EXCEEDED" }, 429, undefined, undefined, - new Headers({ "Retry-After": "2" }), + new Headers({ "Retry-After": "1" }), ), ); - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive).catch(delayEventSendError); await flushPromises(); - for (let i = 0; i < 5; i++) { - jest.advanceTimersByTime(2000); - await flushPromises(); - } - expect(delayEventSendError).toHaveBeenCalled(); + + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + // the user terminated the call locally + manager.leave(); + + // Wait for all timers to be setup + await flushPromises(); + jest.advanceTimersByTime(1000); + await flushPromises(); + + // No new events should have been sent: + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); }); - // !FailsForLegacy because legacy does not have a retry limit and no mechanism to communicate unrecoverable errors. - it("throws, when reaching maximum number of retires !FailsForLegacy", async () => { - const delayEventRestartError = jest.fn(); + }); + describe("retries sending update delayed leave event restart", () => { + it("resends the initial check delayed update event !FailsForLegacy", async () => { (client._unstable_updateDelayedEvent as Mock).mockRejectedValue( new MatrixError( { errcode: "M_LIMIT_EXCEEDED" }, @@ -635,14 +575,68 @@ describe("MembershipManager", () => { ), ); const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive).catch(delayEventRestartError); + manager.join([focus], focusActive); + + // Hit rate limit await flushPromises(); - for (let i = 0; i < 5; i++) { - jest.advanceTimersByTime(1000); - await flushPromises(); - } - expect(delayEventRestartError).toHaveBeenCalled(); - }, 5000); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); + + // Hit second rate limit. + jest.advanceTimersByTime(1000); + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + + // Setup resolve + (client._unstable_updateDelayedEvent as Mock).mockImplementation(() => {}); + jest.advanceTimersByTime(1000); + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + }); }); }); + describe("unrecoverable errors", () => { + // !FailsForLegacy because legacy does not have a retry limit and no mechanism to communicate unrecoverable errors. + it("throws, when reaching maximum number of retires for initial delayed event creation !FailsForLegacy", async () => { + const delayEventSendError = jest.fn(); + (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "2" }), + ), + ); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive).catch(delayEventSendError); + await flushPromises(); + for (let i = 0; i < 5; i++) { + jest.advanceTimersByTime(2000); + await flushPromises(); + } + expect(delayEventSendError).toHaveBeenCalled(); + }); + // !FailsForLegacy because legacy does not have a retry limit and no mechanism to communicate unrecoverable errors. + it("throws, when reaching maximum number of retires !FailsForLegacy", async () => { + const delayEventRestartError = jest.fn(); + (client._unstable_updateDelayedEvent as Mock).mockRejectedValue( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive).catch(delayEventRestartError); + await flushPromises(); + for (let i = 0; i < 5; i++) { + jest.advanceTimersByTime(1000); + await flushPromises(); + } + expect(delayEventRestartError).toHaveBeenCalled(); + }, 5000); + }); }); From 1cf0caa796c0f215dc9c6d2f1f15f1e112c61136 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 16:19:27 +0100 Subject: [PATCH 030/124] rename testEnvironment to memberManagerTestEnvironment --- spec/unit/matrixrtc/memberManagerTestEnvironment.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/matrixrtc/memberManagerTestEnvironment.ts b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts index 727fe12644b..9c6f83400dc 100644 --- a/spec/unit/matrixrtc/memberManagerTestEnvironment.ts +++ b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts @@ -18,7 +18,7 @@ limitations under the License. This file adds a custom test environment for the MembershipManager.spec.ts It can be used with the comment at the top of the file: -@jest-environment ./spec/unit/matrixrtc/membermanagerTestEnvironment.ts +@jest-environment ./spec/unit/matrixrtc/memberManagerTestEnvironment.ts It is very specific to the MembershipManager.spec.ts file and introduces the following behaviour: - The describe each block in the MembershipManager.spec.ts will go through describe block names `LegacyMembershipManager` and `MembershipManager` From a16f1ca77b8957a3ca4dde3a941522ea3496a4fc Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 16:20:26 +0100 Subject: [PATCH 031/124] allow configuring Legacy manager in the matrixRTC session --- src/matrixrtc/MatrixRTCSession.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 981ee4125b8..1adf3987163 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -25,7 +25,7 @@ import { RoomStateEvent } from "../models/room-state.ts"; import { type Focus } from "./focus.ts"; import { KnownMembership } from "../@types/membership.ts"; import { type MatrixEvent } from "../models/event.ts"; -import { type IMembershipManager } from "./NewMembershipManager.ts"; +import { MembershipManager, type IMembershipManager } from "./NewMembershipManager.ts"; import { EncryptionManager, type IEncryptionManager, type Statistics } from "./EncryptionManager.ts"; import { LegacyMembershipManager } from "./LegacyMembershipManager.ts"; @@ -56,6 +56,12 @@ export type MatrixRTCSessionEventHandlerMap = { }; export interface MembershipConfig { + /** + * Use Legacy Manager + * @deprecated + */ + useLegacyMembershipManager?: boolean; + /** * The timeout (in milliseconds) after we joined the call, that our membership should expire * unless we have explicitly updated it. @@ -321,14 +327,20 @@ export class MatrixRTCSession extends TypedEventEmitter - this.getOldestMembership(), - ); + // Create MembershipManager + if (joinConfig?.useLegacyMembershipManager ?? true) { + this.membershipManager = new LegacyMembershipManager(joinConfig, this.room, this.client, () => + this.getOldestMembership(), + ); + } else { + this.membershipManager = new MembershipManager(joinConfig, this.room, this.client, () => + this.getOldestMembership(), + ); + } } // Join! From 2f90d9ce1f49d321c97830de07d301adf49474d4 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 16:20:38 +0100 Subject: [PATCH 032/124] deprecate LegacyMembershipManager --- src/matrixrtc/LegacyMembershipManager.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/LegacyMembershipManager.ts b/src/matrixrtc/LegacyMembershipManager.ts index dc64f9c53ba..01e0b5d58b6 100644 --- a/src/matrixrtc/LegacyMembershipManager.ts +++ b/src/matrixrtc/LegacyMembershipManager.ts @@ -27,7 +27,8 @@ import { type IMembershipManager } from "./NewMembershipManager.ts"; * * It is recommended to only use this interface for testing to allow replacing this class. * - * @internal + * @internal + * @deprecated */ export class LegacyMembershipManager implements IMembershipManager { private relativeExpiry: number | undefined; From 83f0af1cef2617823b9850e69a6ddb58b379b8e4 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 16:27:10 +0100 Subject: [PATCH 033/124] remove usage of waitForExpect --- spec/unit/matrixrtc/MembershipManager.spec.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 2f60c44d599..4b4c8339fff 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -252,14 +252,15 @@ describe.each([ delayedHandle.reject?.(Error("Server does not support the delayed events API")); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); }); - it("does try to schedule a delayed leave event again if rate limited", () => { + it("does try to schedule a delayed leave event again if rate limited", async () => { const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); - waitForExpect(() => { - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); - }); + await flushPromises(); + jest.advanceTimersByTime(5000); + await flushPromises(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); it("uses membershipServerSideExpiryTimeout from config", async () => { const manager = new TestMembershipManager( From 87bb5978cf0c13576f67c8845a3605b454b9a7e1 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Feb 2025 16:31:42 +0100 Subject: [PATCH 034/124] flatten tests and add comments --- spec/unit/matrixrtc/MembershipManager.spec.ts | 970 +++++++++--------- ...ent.ts => memberManagerTestEnvironment.ts} | 2 +- 2 files changed, 483 insertions(+), 489 deletions(-) rename spec/unit/matrixrtc/{testEnvironment.ts => memberManagerTestEnvironment.ts} (96%) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index d789b425d65..0019b24fd71 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -1,5 +1,5 @@ /** - * @jest-environment ./spec/unit/matrixrtc/testEnvironment.ts + * @jest-environment ./spec/unit/matrixrtc/memberManagerTestEnvironment.ts */ /* Copyright 2025 The Matrix.org Foundation C.I.C. @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { type Mock } from "jest-mock"; +import { type MockedFunction, type Mock } from "jest-mock"; import { EventType, HTTPError, MatrixError, type Room } from "../../../src"; import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; @@ -26,18 +26,18 @@ import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, t import { flushPromises } from "../../test-utils/flushPromises"; // import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; -function waitForMockCall(method: any, returnVal?: any) { +function waitForMockCall(method: MockedFunction, returnVal?: any) { return new Promise((resolve) => { - (method as Mock).mockImplementation(() => { + method.mockImplementation(() => { resolve(); return returnVal; }); }); } -function createAsyncHandle(method: any) { +function createAsyncHandle(method: MockedFunction) { const handle: { resolve?: (...args: unknown[]) => void; reject?: (...args: any[]) => void } = {}; - (method as Mock).mockImplementation(() => { + method.mockImplementation(() => { return new Promise((resolve, reject) => { handle.reject = reject; handle.resolve = resolve; @@ -50,562 +50,556 @@ function createAsyncHandle(method: any) { * Tests different MembershipManager implementations. Some tests don't apply to `LegacyMembershipManager` * use !FailsForLegacy to skip those. See: testEnvironment for more details. */ -describe("MembershipManager", () => { - describe.each([ - { TestMembershipManager: LegacyMembershipManager, description: "LegacyMembershipManager" }, - // Here we will add the new implementation of the MembershipManager. - // It is not yet tested since it would currently fail all tests. Adding the MembershipManger looks like this: - // { TestMembershipManager: MembershipManager, description: "MembershipManager" }, - ])("$description", ({ TestMembershipManager }) => { - let client: MockClient; - let room: Room; - const focusActive: LivekitFocusActive = { - focus_selection: "oldest_membership", - type: "livekit", - }; - const focus: Focus = { - type: "livekit", - livekit_service_url: "https://active.url", - livekit_alias: "!active:active.url", - }; - - beforeEach(() => { - // default to fake timers - jest.useFakeTimers(); - client = makeMockClient("@alice:example.org", "AAAAAAA"); - room = makeMockRoom(membershipTemplate); - // provide a default mock that is like the default "non error" server behaviour. - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); - }); +describe.each([ + { TestMembershipManager: LegacyMembershipManager, description: "LegacyMembershipManager" }, + // Here we will add the new implementation of the MembershipManager. + // It is not yet tested since it would currently fail all tests. Adding the MembershipManger looks like this: + // { TestMembershipManager: MembershipManager, description: "MembershipManager" }, +])("$description", ({ TestMembershipManager }) => { + let client: MockClient; + let room: Room; + const focusActive: LivekitFocusActive = { + focus_selection: "oldest_membership", + type: "livekit", + }; + const focus: Focus = { + type: "livekit", + livekit_service_url: "https://active.url", + livekit_alias: "!active:active.url", + }; + + beforeEach(() => { + // Default to fake timers + jest.useFakeTimers(); + client = makeMockClient("@alice:example.org", "AAAAAAA"); + room = makeMockRoom(membershipTemplate); + // Provide a default mock. Representing the default "non error" server behaviour. + (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + }); - afterEach(() => { - jest.useRealTimers(); - // no need to clean up mocks since we will recreate the client - }); + afterEach(() => { + jest.useRealTimers(); + // no need to clean up mocks since we will recreate the client + }); - describe("isJoined()", () => { - it("defaults to false", () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - expect(manager.isJoined()).toEqual(false); - }); + describe("isJoined()", () => { + it("defaults to false", () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + expect(manager.isJoined()).toEqual(false); + }); - it("returns true after join()", async () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([]); - expect(manager.isJoined()).toEqual(true); - }); + it("returns true after join()", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([]); + expect(manager.isJoined()).toEqual(true); }); + }); - describe("join()", () => { - describe("sends a membership event", () => { - it("sends a membership event and schedules delayed leave when joining a call", async () => { - // Spys/Mocks + describe("join()", () => { + describe("sends a membership event", () => { + it("sends a membership event and schedules delayed leave when joining a call", async () => { + // Spys/Mocks - // eslint-disable-next-line camelcase - const _unstable_updateDelayedEventHandle = createAsyncHandle( - client._unstable_updateDelayedEvent as Mock, - ); + const updateDelayedEventHandle = createAsyncHandle(client._unstable_updateDelayedEvent as Mock); - // Test - const memberManager = new TestMembershipManager(undefined, room, client, () => undefined); - memberManager.join([focus], focusActive); + // Test + const memberManager = new TestMembershipManager(undefined, room, client, () => undefined); + memberManager.join([focus], focusActive); - // expects - await waitForMockCall(client.sendStateEvent); - expect(client.sendStateEvent).toHaveBeenCalledWith( - room.roomId, - "org.matrix.msc3401.call.member", - { - application: "m.call", - call_id: "", - device_id: "AAAAAAA", - expires: 14400000, - foci_preferred: [focus], - focus_active: focusActive, - scope: "m.room", - }, - "_@alice:example.org_AAAAAAA", - ); - // eslint-disable-next-line camelcase - _unstable_updateDelayedEventHandle.resolve?.(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( - room.roomId, - { delay: 8000 }, - "org.matrix.msc3401.call.member", - {}, - "_@alice:example.org_AAAAAAA", - ); - }); + // expects + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledWith( + room.roomId, + "org.matrix.msc3401.call.member", + { + application: "m.call", + call_id: "", + device_id: "AAAAAAA", + expires: 14400000, + foci_preferred: [focus], + focus_active: focusActive, + scope: "m.room", + }, + "_@alice:example.org_AAAAAAA", + ); + updateDelayedEventHandle.resolve?.(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( + room.roomId, + { delay: 8000 }, + "org.matrix.msc3401.call.member", + {}, + "_@alice:example.org_AAAAAAA", + ); + }); - describe("does not prefix the state key with _ for rooms that support user-owned state events", () => { - async function testJoin(useOwnedStateEvents: boolean): Promise { - if (useOwnedStateEvents) { - room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); - } + describe("does not prefix the state key with _ for rooms that support user-owned state events", () => { + async function testJoin(useOwnedStateEvents: boolean): Promise { + // TODO: this test does quiet a bit. Its more a like a test story summarizing to: + // - send delay with too long timeout and get server error (test delayedEventTimeout gets overwritten) + // - run into rate limit for sending delayed event + // - run into rate limit when setting membership state. + if (useOwnedStateEvents) { + room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); + } - const updatedDelayedEvent = waitForMockCall(client._unstable_updateDelayedEvent); - const sentDelayedState = waitForMockCall(client._unstable_sendDelayedStateEvent, { - delay_id: "id", - }); + const updatedDelayedEvent = waitForMockCall(client._unstable_updateDelayedEvent); + const sentDelayedState = waitForMockCall(client._unstable_sendDelayedStateEvent, { + delay_id: "id", + }); - // preparing the delayed disconnect should handle the delay being too long - const sendDelayedStateExceedAttempt = new Promise((resolve) => { - const error = new MatrixError({ - "errcode": "M_UNKNOWN", - "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", - "org.matrix.msc4140.max_delay": 7500, - }); - (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); + // preparing the delayed disconnect should handle the delay being too long + const sendDelayedStateExceedAttempt = new Promise((resolve) => { + const error = new MatrixError({ + "errcode": "M_UNKNOWN", + "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", + "org.matrix.msc4140.max_delay": 7500, }); - - const userStateKey = `${!useOwnedStateEvents ? "_" : ""}@alice:example.org_AAAAAAA`; - // preparing the delayed disconnect should handle ratelimiting - const sendDelayedStateAttempt = new Promise((resolve) => { - const error = new MatrixError({ errcode: "M_LIMIT_EXCEEDED" }); - (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); + (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); }); + }); - // setting the membership state should handle ratelimiting (also with a retry-after value) - const sendStateEventAttempt = new Promise((resolve) => { - const error = new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ); - (client.sendStateEvent as Mock).mockImplementationOnce(() => { - resolve(); - return Promise.reject(error); - }); + const userStateKey = `${!useOwnedStateEvents ? "_" : ""}@alice:example.org_AAAAAAA`; + // preparing the delayed disconnect should handle ratelimiting + const sendDelayedStateAttempt = new Promise((resolve) => { + const error = new MatrixError({ errcode: "M_LIMIT_EXCEEDED" }); + (client._unstable_sendDelayedStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); }); - const manager = new TestMembershipManager( - { - membershipServerSideExpiryTimeout: 9000, - }, - room, - client, - () => undefined, - ); - manager.join([focus], focusActive); + }); - await sendDelayedStateExceedAttempt.then(); // needed to resolve after the send attempt catches - await sendDelayedStateAttempt; - const callProps = (d: number) => { - return [room!.roomId, { delay: d }, "org.matrix.msc3401.call.member", {}, userStateKey]; - }; - expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(1, ...callProps(9000)); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(2, ...callProps(7500)); + // setting the membership state should handle ratelimiting (also with a retry-after value) + const sendStateEventAttempt = new Promise((resolve) => { + const error = new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ); + (client.sendStateEvent as Mock).mockImplementationOnce(() => { + resolve(); + return Promise.reject(error); + }); + }); + const manager = new TestMembershipManager( + { + membershipServerSideExpiryTimeout: 9000, + }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); - jest.advanceTimersByTime(5000); + await sendDelayedStateExceedAttempt.then(); // needed to resolve after the send attempt catches + await sendDelayedStateAttempt; + const callProps = (d: number) => { + return [room!.roomId, { delay: d }, "org.matrix.msc3401.call.member", {}, userStateKey]; + }; + expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(1, ...callProps(9000)); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(2, ...callProps(7500)); - await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches - jest.advanceTimersByTime(1000); + jest.advanceTimersByTime(5000); - expect(client.sendStateEvent).toHaveBeenCalledWith( - room!.roomId, - EventType.GroupCallMemberPrefix, - { - application: "m.call", - scope: "m.room", - call_id: "", - expires: 14400000, - device_id: "AAAAAAA", - foci_preferred: [focus], - focus_active: focusActive, - } satisfies SessionMembershipData, - userStateKey, - ); - await sentDelayedState; - - // should have prepared the heartbeat to keep delaying the leave event while still connected - await updatedDelayedEvent; - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - - // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. - await flushPromises(); - jest.advanceTimersByTime(5000); - await flushPromises(); - // should update delayed disconnect - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); - } + await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches + jest.advanceTimersByTime(1000); - it("sends a membership event after rate limits during delayed event setup when joining a call", async () => { - await testJoin(false); - }); + expect(client.sendStateEvent).toHaveBeenCalledWith( + room!.roomId, + EventType.GroupCallMemberPrefix, + { + application: "m.call", + scope: "m.room", + call_id: "", + expires: 14400000, + device_id: "AAAAAAA", + foci_preferred: [focus], + focus_active: focusActive, + } satisfies SessionMembershipData, + userStateKey, + ); + await sentDelayedState; - it("does not prefix the state key with _ for rooms that support user-owned state events", async () => { - await testJoin(true); - }); - }); - }); + // should have prepared the heartbeat to keep delaying the leave event while still connected + await updatedDelayedEvent; + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - describe("delayed leave event", () => { - it("does not try again to schedule a delayed leave event if not supported", () => { - const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - delayedHandle.reject?.(Error("Server does not support the delayed events API")); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - }); - it("does try to schedule a delayed leave event again if rate limited", async () => { - const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); + // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. await flushPromises(); jest.advanceTimersByTime(5000); await flushPromises(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + // should update delayed disconnect + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); + } + + it("sends a membership event after rate limits during delayed event setup when joining a call", async () => { + await testJoin(false); }); - it("uses membershipServerSideExpiryTimeout from config", async () => { - const manager = new TestMembershipManager( - { membershipServerSideExpiryTimeout: 123456 }, - room, - client, - () => undefined, - ); - manager.join([focus], focusActive); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( - room.roomId, - { delay: 123456 }, - "org.matrix.msc3401.call.member", - {}, - "_@alice:example.org_AAAAAAA", - ); + + it("does not prefix the state key with _ for rooms that support user-owned state events", async () => { + await testJoin(true); }); }); + }); - it("uses membershipExpiryTimeout from config", async () => { + describe("delayed leave event", () => { + it("does not try again to schedule a delayed leave event if not supported", () => { + const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + delayedHandle.reject?.(Error("Server does not support the delayed events API")); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + }); + it("does try to schedule a delayed leave event again if rate limited", async () => { + const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); + await flushPromises(); + jest.advanceTimersByTime(5000); + await flushPromises(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + }); + it("uses membershipServerSideExpiryTimeout from config", async () => { const manager = new TestMembershipManager( - { membershipExpiryTimeout: 1234567 }, + { membershipServerSideExpiryTimeout: 123456 }, room, client, () => undefined, ); - manager.join([focus], focusActive); - await waitForMockCall(client.sendStateEvent); - expect(client.sendStateEvent).toHaveBeenCalledWith( + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledWith( room.roomId, - EventType.GroupCallMemberPrefix, - { - application: "m.call", - scope: "m.room", - call_id: "", - device_id: "AAAAAAA", - expires: 1234567, - foci_preferred: [focus], - focus_active: { - focus_selection: "oldest_membership", - type: "livekit", - }, - }, + { delay: 123456 }, + "org.matrix.msc3401.call.member", + {}, "_@alice:example.org_AAAAAAA", ); }); + }); - it("does nothing if join called when already joined", async () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - await waitForMockCall(client.sendStateEvent); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - manager.join([focus], focusActive); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - }); + it("uses membershipExpiryTimeout from config", async () => { + const manager = new TestMembershipManager( + { membershipExpiryTimeout: 1234567 }, + room, + client, + () => undefined, + ); + + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledWith( + room.roomId, + EventType.GroupCallMemberPrefix, + { + application: "m.call", + scope: "m.room", + call_id: "", + device_id: "AAAAAAA", + expires: 1234567, + foci_preferred: [focus], + focus_active: { + focus_selection: "oldest_membership", + type: "livekit", + }, + }, + "_@alice:example.org_AAAAAAA", + ); }); - describe("leave()", () => { - // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed - it("does nothing if not joined !FailsForLegacy", () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.leave(); - expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); - expect(client.sendStateEvent).not.toHaveBeenCalled(); - }); + it("does nothing if join called when already joined", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + manager.join([focus], focusActive); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); + }); - describe("getsActiveFocus", () => { - it("gets the correct active focus with oldest_membership", () => { - const getOldestMembership = jest.fn(); - const manager = new TestMembershipManager({}, room, client, getOldestMembership); - // Before joining the active focus should be undefined (see FocusInUse on MatrixRTCSession) - expect(manager.getActiveFocus()).toBe(undefined); - manager.join([focus], focusActive); - // After joining we want our own focus to be the one we select. - getOldestMembership.mockReturnValue( - mockCallMembership( - { - ...membershipTemplate, - foci_preferred: [ - { - livekit_alias: "!active:active.url", - livekit_service_url: "https://active.url", - type: "livekit", - }, - ], - device_id: client.getDeviceId(), - created_ts: 1000, - }, - room.roomId, - client.getUserId()!, - ), - ); - expect(manager.getActiveFocus()).toStrictEqual(focus); - getOldestMembership.mockReturnValue( - mockCallMembership( - Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), - room.roomId, - ), - ); - // If there is an older member we use its focus. - expect(manager.getActiveFocus()).toBe(membershipTemplate.foci_preferred[0]); - }); + describe("leave()", () => { + // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed + it("does nothing if not joined !FailsForLegacy", () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.leave(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + }); + }); - it("does not provide focus if the selection method is unknown", () => { - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], Object.assign(focusActive, { type: "unknown_type" })); - expect(manager.getActiveFocus()).toBe(undefined); - }); + describe("getsActiveFocus", () => { + it("gets the correct active focus with oldest_membership", () => { + const getOldestMembership = jest.fn(); + const manager = new TestMembershipManager({}, room, client, getOldestMembership); + // Before joining the active focus should be undefined (see FocusInUse on MatrixRTCSession) + expect(manager.getActiveFocus()).toBe(undefined); + manager.join([focus], focusActive); + // After joining we want our own focus to be the one we select. + getOldestMembership.mockReturnValue( + mockCallMembership( + { + ...membershipTemplate, + foci_preferred: [ + { + livekit_alias: "!active:active.url", + livekit_service_url: "https://active.url", + type: "livekit", + }, + ], + device_id: client.getDeviceId(), + created_ts: 1000, + }, + room.roomId, + client.getUserId()!, + ), + ); + expect(manager.getActiveFocus()).toStrictEqual(focus); + getOldestMembership.mockReturnValue( + mockCallMembership( + Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), + room.roomId, + ), + ); + // If there is an older member we use its focus. + expect(manager.getActiveFocus()).toBe(membershipTemplate.foci_preferred[0]); + }); + + it("does not provide focus if the selection method is unknown", () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], Object.assign(focusActive, { type: "unknown_type" })); + expect(manager.getActiveFocus()).toBe(undefined); + }); + }); + + describe("onRTCSessionMemberUpdate()", () => { + it("does nothing if not joined", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await flushPromises(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); + }); + it("does nothing if own membership still present", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + + await waitForMockCall(client.sendStateEvent); + await flushPromises(); + const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; + // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` + (client.sendStateEvent as Mock).mockReset(); + (client._unstable_updateDelayedEvent as Mock).mockReset(); + (client._unstable_sendDelayedStateEvent as Mock).mockReset(); + + manager.onRTCSessionMemberUpdate([ + mockCallMembership(membershipTemplate, room.roomId), + mockCallMembership(myMembership as SessionMembershipData, room.roomId, client.getUserId() ?? undefined), + ]); + await flushPromises(); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); + expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); + }); + it("recreates membership if it is missing", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + await flushPromises(); + // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` + (client.sendStateEvent as Mock).mockClear(); + (client._unstable_updateDelayedEvent as Mock).mockClear(); + (client._unstable_sendDelayedStateEvent as Mock).mockClear(); + + // our own membership is removed: + manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await flushPromises(); + expect(client.sendStateEvent).toHaveBeenCalled(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalled(); + + expect(client._unstable_updateDelayedEvent).toHaveBeenCalled(); + }); + }); + + // TODO: not sure about this name + describe("background timers", () => { + it("sends only one keep-alive for delayed leave event per `membershipKeepAlivePeriod`", async () => { + const manager = new TestMembershipManager( + { membershipKeepAlivePeriod: 10_000, membershipServerSideExpiryTimeout: 30_000 }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + + // The first call is from checking id the server deleted the delayed event + // so it does not need a `advanceTimersByTime` + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); + // TODO: check that update delayed event is called with the correct HTTP request timeout + // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); + + for (let i = 2; i <= 12; i++) { + // flush promises before advancing the timers to make sure schedulers are setup + await flushPromises(); + jest.advanceTimersByTime(10_000); + await flushPromises(); + + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); + // TODO: check that update delayed event is called with the correct HTTP request timeout + // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); + } + }); + + // The expires logic was removed for the legacy call manager. + // Delayed events should replace it entirely but before they have wide adoption + // the expiration logic still makes sense. + // TODO: add git commit when we removed it. + it("extends `expires` when call still active !FailsForLegacy", async () => { + const manager = new TestMembershipManager( + { membershipExpiryTimeout: 10_000 }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive); + await waitForMockCall(client.sendStateEvent); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; + expect(sentMembership.expires).toBe(10_000); + for (let i = 2; i <= 12; i++) { + // flush promises before advancing the timers to make sure schedulers are setup + await flushPromises(); + jest.advanceTimersByTime(10_000); + await flushPromises(); + expect(client.sendStateEvent).toHaveBeenCalledTimes(i); + const sentMembership = (client.sendStateEvent as Mock).mock.lastCall![2] as SessionMembershipData; + expect(sentMembership.expires).toBe(10_000 * i); + } }); + }); + + describe("server error handling", () => { + // Types of server error: 429 rate limit with no retry-after header, 429 with retry-after, 50x server error (maybe retry every second), connection/socket timeout + describe("retries sending delayed leave event", () => { + it("sends retry if call membership event is still valid at time of retry", async () => { + const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); - describe("onRTCSessionMemberUpdate()", () => { - it("does nothing if not joined", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + manager.join([focus], focusActive); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + + handle.reject?.( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); await flushPromises(); - expect(client.sendStateEvent).not.toHaveBeenCalled(); - expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); - expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); + jest.advanceTimersByTime(1000); + await flushPromises(); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); - it("does nothing if own membership still present", async () => { + // FailsForLegacy as implementation does not re-check membership before retrying + it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { + (client._unstable_sendDelayedStateEvent as any).mockRejectedValue( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); const manager = new TestMembershipManager({}, room, client, () => undefined); + // Should call _unstable_sendDelayedStateEvent but not sendStateEvent because of the + // RateLimit error. manager.join([focus], focusActive); - await waitForMockCall(client.sendStateEvent); await flushPromises(); - const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; - // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` - (client.sendStateEvent as Mock).mockReset(); - (client._unstable_updateDelayedEvent as Mock).mockReset(); - (client._unstable_sendDelayedStateEvent as Mock).mockReset(); - - manager.onRTCSessionMemberUpdate([ - mockCallMembership(membershipTemplate, room.roomId), - mockCallMembership( - myMembership as SessionMembershipData, - room.roomId, - client.getUserId() ?? undefined, - ), - ]); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + // Remove our own membership so that there is no reason the send the delayed leave anymore. + // the membership is no longer present on the homeserver + manager.onRTCSessionMemberUpdate([]); + // Wait for all timers to be setup await flushPromises(); - expect(client.sendStateEvent).not.toHaveBeenCalled(); - expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); - expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); + jest.advanceTimersByTime(1000); + // We should send the first own membership and a new delayed event after the rate limit timeout. + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); - it("recreates membership if it is missing", async () => { + // FailsForLegacy as implementation does not re-check membership before retrying + it("abandons retry loop if leave() was called !FailsForLegacy", async () => { + const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); + const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); + handle.reject?.( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), + ); await flushPromises(); - // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` - (client.sendStateEvent as Mock).mockClear(); - (client._unstable_updateDelayedEvent as Mock).mockClear(); - (client._unstable_sendDelayedStateEvent as Mock).mockClear(); - // our own membership is removed: - manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); + // the user terminated the call locally + manager.leave(); + + // Wait for all timers to be setup + await flushPromises(); + jest.advanceTimersByTime(1000); await flushPromises(); - expect(client.sendStateEvent).toHaveBeenCalled(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalled(); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalled(); + // No new events should have been sent: + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); }); }); - - // TODO: not sure about this name - describe("background timers", () => { - it("sends only one keep-alive for delayed leave event per `membershipKeepAlivePeriod`", async () => { - const manager = new TestMembershipManager( - { membershipKeepAlivePeriod: 10_000, membershipServerSideExpiryTimeout: 30_000 }, - room, - client, - () => undefined, + describe("retries sending update delayed leave event restart", () => { + it("resends the initial check delayed update event !FailsForLegacy", async () => { + (client._unstable_updateDelayedEvent as any).mockRejectedValue( + new MatrixError( + { errcode: "M_LIMIT_EXCEEDED" }, + 429, + undefined, + undefined, + new Headers({ "Retry-After": "1" }), + ), ); + const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - // The first call is from checking id the server deleted the delayed event - // so it does not need a `advanceTimersByTime` + // Hit rate limit await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - // TODO: check that update delayed event is called with the correct HTTP request timeout - // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); - for (let i = 2; i <= 12; i++) { - // flush promises before advancing the timers to make sure schedulers are setup - await flushPromises(); - jest.advanceTimersByTime(10_000); - await flushPromises(); - - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); - // TODO: check that update delayed event is called with the correct HTTP request timeout - // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); - } - }); + // Hit second rate limit. + jest.advanceTimersByTime(1000); + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); - // The expires logic was removed for the legacy call manager. - // Delayed events should replace it entirely but before they have wide adoption - // the expiration logic still makes sense. - // TODO: add git commit when we removed it. - it("extends `expires` when call still active !FailsForLegacy", async () => { - const manager = new TestMembershipManager( - { membershipExpiryTimeout: 10_000 }, - room, - client, - () => undefined, - ); - manager.join([focus], focusActive); - await waitForMockCall(client.sendStateEvent); + // Setup resolve + (client._unstable_updateDelayedEvent as Mock).mockImplementation(() => {}); + jest.advanceTimersByTime(1000); + await flushPromises(); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; - expect(sentMembership.expires).toBe(10_000); - for (let i = 2; i <= 12; i++) { - // flush promises before advancing the timers to make sure schedulers are setup - await flushPromises(); - jest.advanceTimersByTime(10_000); - await flushPromises(); - expect(client.sendStateEvent).toHaveBeenCalledTimes(i); - const sentMembership = (client.sendStateEvent as Mock).mock.lastCall![2] as SessionMembershipData; - expect(sentMembership.expires).toBe(10_000 * i); - } }); }); - describe("server error handling", () => { - // Types of server error: 429 rate limit with no retry-after header, 429 with retry-after, 50x server error (maybe retry every second), connection/socket timeout - describe("retries sending delayed leave event", () => { - it("sends retry if call membership event is still valid at time of retry", async () => { - const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); - - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - - handle.reject?.( - new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ), - ); - await flushPromises(); - jest.advanceTimersByTime(1000); - await flushPromises(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); - }); - // FailsForLegacy as implementation does not re-check membership before retrying - it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { - (client._unstable_sendDelayedStateEvent as any).mockRejectedValue( - new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ), - ); - const manager = new TestMembershipManager({}, room, client, () => undefined); - // Should call _unstable_sendDelayedStateEvent but not sendStateEvent because of the - // RateLimit error. - manager.join([focus], focusActive); - - await flushPromises(); - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); - // Remove our own membership so that there is no reason the send the delayed leave anymore. - // the membership is no longer present on the homeserver - manager.onRTCSessionMemberUpdate([]); - // Wait for all timers to be setup - await flushPromises(); - jest.advanceTimersByTime(1000); - // We should send the first own membership and a new delayed event after the rate limit timeout. - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - }); - // FailsForLegacy as implementation does not re-check membership before retrying - it("abandons retry loop if leave() was called !FailsForLegacy", async () => { - const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); - - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - handle.reject?.( - new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ), - ); - await flushPromises(); - - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - // the user terminated the call locally - manager.leave(); - - // Wait for all timers to be setup - await flushPromises(); - jest.advanceTimersByTime(1000); - await flushPromises(); - - // No new events should have been sent: - expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - }); - }); - describe("retries sending update delayed leave event restart", () => { - it("resends the initial check delayed update event !FailsForLegacy", async () => { - (client._unstable_updateDelayedEvent as any).mockRejectedValue( - new MatrixError( - { errcode: "M_LIMIT_EXCEEDED" }, - 429, - undefined, - undefined, - new Headers({ "Retry-After": "1" }), - ), - ); - const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive); - - // Hit rate limit - await flushPromises(); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - - // Hit second rate limit. - jest.advanceTimersByTime(1000); - await flushPromises(); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); - - // Setup resolve - (client._unstable_updateDelayedEvent as Mock).mockImplementation(() => {}); - jest.advanceTimersByTime(1000); - await flushPromises(); - expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); - expect(client.sendStateEvent).toHaveBeenCalledTimes(1); - }); - }); - - // describe: "retries sending membership event" - // it: "sends it if still joined at time of retry" - // // TODO: see what this is doing and how its different to: "recreates membership if it is missing" - // it: "abandons it if call no longer joined at time of retry" - }); + // describe: "retries sending membership event" + // it: "sends it if still joined at time of retry" + // // TODO: see what this is doing and how its different to: "recreates membership if it is missing" + // it: "abandons it if call no longer joined at time of retry" }); }); diff --git a/spec/unit/matrixrtc/testEnvironment.ts b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts similarity index 96% rename from spec/unit/matrixrtc/testEnvironment.ts rename to spec/unit/matrixrtc/memberManagerTestEnvironment.ts index 759b00fa13f..9c6f83400dc 100644 --- a/spec/unit/matrixrtc/testEnvironment.ts +++ b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts @@ -18,7 +18,7 @@ limitations under the License. This file adds a custom test environment for the MembershipManager.spec.ts It can be used with the comment at the top of the file: -@jest-environment ./spec/unit/matrixrtc/testEnvironment.ts +@jest-environment ./spec/unit/matrixrtc/memberManagerTestEnvironment.ts It is very specific to the MembershipManager.spec.ts file and introduces the following behaviour: - The describe each block in the MembershipManager.spec.ts will go through describe block names `LegacyMembershipManager` and `MembershipManager` From c7017b9384b31191e53b4d483a94cb61353983c8 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 11:45:00 +0100 Subject: [PATCH 035/124] clean up leave logic branch --- src/matrixrtc/NewMembershipManager.ts | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 8b299f6b303..a0232653493 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -188,8 +188,6 @@ class ActionScheduler { } public addAction(action: Action): void { - // Dont add any other actions if we have a leave scheduled - if (this.actions.some((a) => a.type === DirectMemberhsipManagerActions.Leave)) return; this.insertions.push(action); const nextTs = this.actions[0]?.ts; if (!nextTs || nextTs > action.ts) { @@ -206,9 +204,6 @@ class ActionScheduler { this.wakeup?.(); } } - public hasAction(condition: (action: Action) => boolean): boolean { - return this.actions.some(condition); - } } /** * This Class takes care of the membership management. @@ -246,17 +241,17 @@ export class MembershipManager implements IMembershipManager { public leave(timeout?: number): Promise { this.scheduler.state.running = false; - if (this.leavePromise && this.scheduler.hasAction((a) => a.type === DirectMemberhsipManagerActions.Leave)) { - return this.leavePromise; + if (!this.leavePromise) { + // reset scheduled actions so we will not do any new actions. + this.scheduler.resetActions([{ type: DirectMemberhsipManagerActions.Leave, ts: Date.now() }]); + this.leavePromise = new Promise((resolve, reject) => { + this.leavePromiseHandle.reject = reject; + this.leavePromiseHandle.resolve = resolve; + if (timeout) setTimeout(() => resolve(false), timeout); + }); } - // reset scheduled actions so we will not do any new actions. - this.scheduler.resetActions([{ type: DirectMemberhsipManagerActions.Leave, ts: Date.now() }]); - return new Promise((resolve, reject) => { - this.leavePromiseHandle.reject = reject; - this.leavePromiseHandle.resolve = resolve; - if (timeout) setTimeout(() => resolve(false), timeout); - }); + return this.leavePromise; } private leavePromise?: Promise; private leavePromiseHandle: { @@ -586,7 +581,7 @@ export class MembershipManager implements IMembershipManager { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); }, ); - if (!(notFoundHandled || rateLimitHandled)) { + if (!notFoundHandled && !rateLimitHandled) { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } } From 4e60a35d54d540217eb1f406b0bcd332da64e3ee Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 11:45:28 +0100 Subject: [PATCH 036/124] add more leave test cases --- spec/unit/matrixrtc/MembershipManager.spec.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 0019b24fd71..ee26eea188f 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -321,6 +321,29 @@ describe.each([ }); describe("leave()", () => { + // TODO add rate limit cases. + it("resolves delayed leave event when leave is called", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + await flushPromises(); + await manager.leave(); + expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", "send"); + expect(client.sendStateEvent).toHaveBeenCalled(); + }); + it("send leave event when leave is called and resolving delayed leave fails", async () => { + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive); + await flushPromises(); + (client._unstable_updateDelayedEvent as Mock).mockRejectedValue("unknown"); + await manager.leave(); + // We send a normal leave event since we failed using updateDelayedEvent with the "send" action. + expect(client.sendStateEvent).toHaveBeenLastCalledWith( + room.roomId, + "org.matrix.msc3401.call.member", + {}, + "_@alice:example.org_AAAAAAA", + ); + }); // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed it("does nothing if not joined !FailsForLegacy", () => { const manager = new TestMembershipManager({}, room, client, () => undefined); From 4502083c62e202bf748e064fc37f5b23edec9ddf Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 12:26:40 +0100 Subject: [PATCH 037/124] use defer --- spec/unit/matrixrtc/MembershipManager.spec.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index ee26eea188f..a453977cc67 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -24,6 +24,8 @@ import { type Focus, type LivekitFocusActive, type SessionMembershipData } from import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; import { flushPromises } from "../../test-utils/flushPromises"; +import { defer } from "../../../src/utils"; + // import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; function waitForMockCall(method: MockedFunction, returnVal?: any) { @@ -36,14 +38,9 @@ function waitForMockCall(method: MockedFunction, returnVal?: any) { } function createAsyncHandle(method: MockedFunction) { - const handle: { resolve?: (...args: unknown[]) => void; reject?: (...args: any[]) => void } = {}; - method.mockImplementation(() => { - return new Promise((resolve, reject) => { - handle.reject = reject; - handle.resolve = resolve; - }); - }); - return handle; + const { reject, resolve, promise } = defer(); + method.mockImplementation(() => promise); + return { reject, resolve }; } /** @@ -477,7 +474,7 @@ describe.each([ } }); - // The expires logic was removed for the legacy call manager. + // !FailsForLegacy because the expires logic was removed for the legacy call manager. // Delayed events should replace it entirely but before they have wide adoption // the expiration logic still makes sense. // TODO: add git commit when we removed it. From d745605a0fd15991e1ca28ff29cde74c6f63033c Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 12:43:19 +0100 Subject: [PATCH 038/124] review ("Some minor tidying things for now.") --- src/matrixrtc/LegacyMembershipManager.ts | 18 +++++++++++++- src/matrixrtc/MatrixRTCSession.ts | 8 +++---- src/matrixrtc/NewMembershipManager.ts | 30 ++++++++++++++++++------ 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/matrixrtc/LegacyMembershipManager.ts b/src/matrixrtc/LegacyMembershipManager.ts index 01e0b5d58b6..dc86a2ebe75 100644 --- a/src/matrixrtc/LegacyMembershipManager.ts +++ b/src/matrixrtc/LegacyMembershipManager.ts @@ -1,3 +1,19 @@ +/* +Copyright 2025 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + import { EventType } from "../@types/event.ts"; import { UpdateDelayedEventAction } from "../@types/requests.ts"; import type { MatrixClient } from "../client.ts"; @@ -28,7 +44,7 @@ import { type IMembershipManager } from "./NewMembershipManager.ts"; * It is recommended to only use this interface for testing to allow replacing this class. * * @internal - * @deprecated + * @deprecated Use {@link MembershipManager} instead */ export class LegacyMembershipManager implements IMembershipManager { private relativeExpiry: number | undefined; diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 1adf3987163..8c6635dc190 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -71,11 +71,11 @@ export interface MembershipConfig { membershipExpiryTimeout?: number; /** - * The slack in (in milliseconds) which the manager will leave of the meberhsip `expires` time to make sure it + * The slack in (in milliseconds) which the manager will allow before the membership `expires` time to make sure it * sends the updated state event early enough. * - * A slack of 1000ms and a `membershipExpiryTimeout` of 10000ms would result in a memberhsip event update every 9s and - * a memberhsip event that would be considered expired after 10s. + * A slack of 1000ms and a `membershipExpiryTimeout` of 10000ms would result in a membership event update every 9s and + * a membership event that would be considered expired after 10s. */ membershipExpiryTimeoutSlack?: number; @@ -107,7 +107,7 @@ export interface MembershipConfig { */ callMemberEventRetryJitter?: number; /** - * The maximum rate limit retries the manager will do for delayed event sending/updating and state event sending. + * The maximum number of retries that the manager will do for delayed event sending/updating and state event sending when a server rate limit has been hit. */ maximumRateLimitRetryCount?: number; } diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index a0232653493..e7d492073d5 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -1,3 +1,19 @@ +/* +Copyright 2025 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + import { EventType } from "../@types/event.ts"; import { UpdateDelayedEventAction } from "../@types/requests.ts"; import type { MatrixClient } from "../client.ts"; @@ -53,8 +69,8 @@ export interface IMembershipManager { // SCHEDULER TYPES: enum MembershipActionType { SendJoinEvent = "SendJoinEvent", - // -> MembershipActionType.SendJoinEvent if we run in a rate limit and need to retry - // -> MembershipActionType.Update if we successfully send it to schedule the expire event update + // -> MembershipActionType.SendJoinEvent if we run into a rate limit and need to retry + // -> MembershipActionType.Update if we successfully send the join event then schedule the expire event update // -> DelayedLeaveActionType.RestartDelayedEvent to recheck the delayed event Update = "Update", // -> MembershipActionType.Update if the timeout has passed so the next update is required. @@ -87,7 +103,7 @@ function isDelayedLeaveActionType(val: any): val is DelayedLeaveActionType { /** * Actions that are supposed to be used from outside the main handle methods. */ -enum DirectMemberhsipManagerActions { +enum DirectMembershipManagerActions { Join = DelayedLeaveActionType.SendFirstDelayedEvent, Leave = DelayedLeaveActionType.SendScheduledDelayedLeaveEvent, } @@ -119,7 +135,7 @@ interface Action { * The state of the different loops * can also be thought of as the type of the action */ - type: DelayedLeaveActionType | MembershipActionType | DirectMemberhsipManagerActions; + type: DelayedLeaveActionType | MembershipActionType | DirectMembershipManagerActions; } /** @@ -233,7 +249,7 @@ export class MembershipManager implements IMembershipManager { this.focusActive = focusActive; if (!this.scheduler.state.running) { this.scheduler.state.running = true; - return this.scheduler.startWithActions([{ ts: Date.now(), type: DirectMemberhsipManagerActions.Join }]); + return this.scheduler.startWithActions([{ ts: Date.now(), type: DirectMembershipManagerActions.Join }]); } return Promise.resolve(); } @@ -243,7 +259,7 @@ export class MembershipManager implements IMembershipManager { if (!this.leavePromise) { // reset scheduled actions so we will not do any new actions. - this.scheduler.resetActions([{ type: DirectMemberhsipManagerActions.Leave, ts: Date.now() }]); + this.scheduler.resetActions([{ type: DirectMembershipManagerActions.Leave, ts: Date.now() }]); this.leavePromise = new Promise((resolve, reject) => { this.leavePromiseHandle.reject = reject; this.leavePromiseHandle.resolve = resolve; @@ -266,7 +282,7 @@ export class MembershipManager implements IMembershipManager { if (this.isJoined() && !memberships.some(isMyMembership)) { logger.warn("Missing own membership: force re-join"); this.scheduler.state.hasMemberStateEvent = false; - this.scheduler.addAction({ ts: Date.now(), type: DirectMemberhsipManagerActions.Join }); + this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerActions.Join }); } } From 5b35b73face74b7919884a136bb19fe25949b069 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 17:42:00 +0100 Subject: [PATCH 039/124] add onError for join method and cleanup --- src/matrixrtc/LegacyMembershipManager.ts | 2 +- src/matrixrtc/NewMembershipManager.ts | 66 +++++++++++++----------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/matrixrtc/LegacyMembershipManager.ts b/src/matrixrtc/LegacyMembershipManager.ts index dc86a2ebe75..1145810eebb 100644 --- a/src/matrixrtc/LegacyMembershipManager.ts +++ b/src/matrixrtc/LegacyMembershipManager.ts @@ -107,7 +107,7 @@ export class LegacyMembershipManager implements IMembershipManager { return this.relativeExpiry !== undefined; } - public async join(fociPreferred: Focus[], fociActive?: Focus): Promise { + public join(fociPreferred: Focus[], fociActive?: Focus): void { this.ownFocusActive = fociActive; this.ownFociPreferred = fociPreferred; this.relativeExpiry = this.membershipExpiryTimeout; diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index e7d492073d5..bd9cb136ae0 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -47,7 +47,7 @@ export interface IMembershipManager { * @param fociActive the active focus to use in the joined RTC membership event. * @throws can throw if it exceeds a configured maximum retry. */ - join(fociPreferred: Focus[], fociActive?: Focus): Promise; + join(fociPreferred: Focus[], fociActive?: Focus, onError?: (error: unknown) => void): void; /** * Send all necessary events to make this user leave the RTC session. * @param timeout the maximum duration in ms until the promise is forced to resolve. @@ -244,14 +244,22 @@ export class MembershipManager implements IMembershipManager { * @param fociPreferred * @param focusActive */ - public join(fociPreferred: Focus[], focusActive?: Focus): Promise { + public join(fociPreferred: Focus[], focusActive?: Focus, onError?: (error: unknown) => void): void { this.fociPreferred = fociPreferred; this.focusActive = focusActive; if (!this.scheduler.state.running) { this.scheduler.state.running = true; - return this.scheduler.startWithActions([{ ts: Date.now(), type: DirectMembershipManagerActions.Join }]); + const f = async (): Promise => { + try { + await this.scheduler.startWithActions([ + { ts: Date.now(), type: DirectMembershipManagerActions.Join }, + ]); + } catch (e) { + onError?.(e); + } + }; + f(); } - return Promise.resolve(); } public leave(timeout?: number): Promise { @@ -610,7 +618,7 @@ export class MembershipManager implements IMembershipManager { public async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { switch (type) { - case MembershipActionType.Update: + case MembershipActionType.SendJoinEvent: try { await this.client.sendStateEvent( this.room.roomId, @@ -619,40 +627,33 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ); state.nextRelativeExpiry += this.membershipEventExpiryTimeout; - // Success, we reset retries and schedule update. + state.hasMemberStateEvent = true; state.sendMembershipRetries = 0; - + this.scheduler.addAction({ ts: Date.now(), type: DelayedLeaveActionType.RestartDelayedEvent }); this.scheduler.addAction({ ts: Date.now() + this.membershipTimerExpiryTimeout, type: MembershipActionType.Update, }); } catch (e) { - const rateLimitHandled = this.handleRateLimitError( + this.handleRateLimitError( e, state.sendMembershipRetries, - (retryIn) => { - logger.warn("Retry sending membership state event due to rate limit:", e); + (resendDelay) => { state.sendMembershipRetries++; this.scheduler.addAction({ - ts: Date.now() + Math.max(retryIn, this.callMemberEventRetryDelayMinimum), - type: MembershipActionType.Update, + ts: Date.now() + resendDelay, + type: MembershipActionType.SendJoinEvent, }); }, () => { throw Error( - "Exceeded maximum own Membership state update attempts (client.sendStateEvent): " + e, + "Exceeded maximum Own Membership state update attempts (client.sendStateEvent): " + e, ); }, ); - if (!rateLimitHandled) { - this.scheduler.addAction({ - ts: Date.now() + this.callMemberEventRetryDelayMinimum, - type: MembershipActionType.Update, - }); - } } break; - case MembershipActionType.SendJoinEvent: + case MembershipActionType.Update: try { await this.client.sendStateEvent( this.room.roomId, @@ -661,30 +662,37 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ); state.nextRelativeExpiry += this.membershipEventExpiryTimeout; - state.hasMemberStateEvent = true; + // Success, we reset retries and schedule update. state.sendMembershipRetries = 0; - this.scheduler.addAction({ ts: Date.now(), type: DelayedLeaveActionType.RestartDelayedEvent }); + this.scheduler.addAction({ ts: Date.now() + this.membershipTimerExpiryTimeout, type: MembershipActionType.Update, }); } catch (e) { - this.handleRateLimitError( + const rateLimitHandled = this.handleRateLimitError( e, state.sendMembershipRetries, - (resendDelay) => { + (retryIn) => { + logger.warn("Retry sending membership state event due to rate limit:", e); state.sendMembershipRetries++; this.scheduler.addAction({ - ts: Date.now() + resendDelay, - type: MembershipActionType.SendJoinEvent, + ts: Date.now() + Math.max(retryIn, this.callMemberEventRetryDelayMinimum), + type: MembershipActionType.Update, }); }, () => { throw Error( - "Exceeded maximum Own Membership state update attempts (client.sendStateEvent): " + e, + "Exceeded maximum own Membership state update attempts (client.sendStateEvent): " + e, ); }, ); + if (!rateLimitHandled) { + this.scheduler.addAction({ + ts: Date.now() + this.callMemberEventRetryDelayMinimum, + type: MembershipActionType.Update, + }); + } } break; case MembershipActionType.SendLeaveEvent: @@ -694,11 +702,11 @@ export class MembershipManager implements IMembershipManager { // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event try { - this.client.sendStateEvent( + await this.client.sendStateEvent( this.room.roomId, EventType.GroupCallMemberPrefix, {}, - this.makeMembershipStateKey(this.userId, this.deviceId), + this.stateKey, ); state.updateDelayedEventRetries = 0; this.scheduler.resetActions([]); From 7f75daa47f30e3b2d77b638358c59aaf6a90905d Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 17:44:45 +0100 Subject: [PATCH 040/124] use pop instead of filter --- src/matrixrtc/NewMembershipManager.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index bd9cb136ae0..eb474ce184f 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -168,9 +168,9 @@ class ActionScheduler { this.actions = initialActions; while (this.actions.length > 0) { - this.actions.sort((a, b) => a.ts - b.ts); + this.actions.sort((a, b) => b.ts - a.ts); logger.debug("Current MembershipManager action queue: ", this.actions, "\nDate.now: ", +Date.now()); - const nextAction = this.actions[0]; + const nextAction = this.actions.pop()!; this.wakeupPromise = new Promise((resolve) => { this.wakeup = resolve; @@ -197,7 +197,6 @@ class ActionScheduler { this.resetWith = undefined; } - this.actions = this.actions.filter((a) => a !== nextAction); this.actions.push(...this.insertions); this.insertions = []; } From 23c6eecd93d1134c0ae9ce06b8876fbfd1bcb130 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 18:05:36 +0100 Subject: [PATCH 041/124] fixes --- spec/unit/matrixrtc/MembershipManager.spec.ts | 4 ++-- src/matrixrtc/NewMembershipManager.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 8ae9bcb360d..89ed3104356 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -632,7 +632,7 @@ describe.each([ ), ); const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive).catch(delayEventSendError); + manager.join([focus], focusActive, delayEventSendError); await flushPromises(); for (let i = 0; i < 5; i++) { jest.advanceTimersByTime(2000); @@ -653,7 +653,7 @@ describe.each([ ), ); const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.join([focus], focusActive).catch(delayEventRestartError); + manager.join([focus], focusActive, delayEventRestartError); await flushPromises(); for (let i = 0; i < 5; i++) { jest.advanceTimersByTime(1000); diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index eb474ce184f..0817d0cb02e 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -168,9 +168,9 @@ class ActionScheduler { this.actions = initialActions; while (this.actions.length > 0) { - this.actions.sort((a, b) => b.ts - a.ts); + this.actions.sort((a, b) => a.ts - b.ts); logger.debug("Current MembershipManager action queue: ", this.actions, "\nDate.now: ", +Date.now()); - const nextAction = this.actions.pop()!; + const nextAction = this.actions[0]; this.wakeupPromise = new Promise((resolve) => { this.wakeup = resolve; @@ -196,6 +196,7 @@ class ActionScheduler { this.actions = this.resetWith; this.resetWith = undefined; } + this.actions = this.actions.filter((a) => a !== nextAction); this.actions.push(...this.insertions); this.insertions = []; From 724f6908fcbc53047871bdde049aad3891452f45 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 21:17:41 +0100 Subject: [PATCH 042/124] simplify error handling and MembershipAction Only use one membership action enum --- src/matrixrtc/NewMembershipManager.ts | 277 +++++++------------------- 1 file changed, 68 insertions(+), 209 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 0817d0cb02e..1ae29d6d70c 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -68,44 +68,35 @@ export interface IMembershipManager { // SCHEDULER TYPES: enum MembershipActionType { + SendFirstDelayedEvent = "SendFirstDelayedEvent", + // -> MembershipActionType.SendJoinEvent if successful + // -> DelayedLeaveActionType.SendFirstDelayedEvent on error, retry sending the first delayed event. SendJoinEvent = "SendJoinEvent", // -> MembershipActionType.SendJoinEvent if we run into a rate limit and need to retry // -> MembershipActionType.Update if we successfully send the join event then schedule the expire event update // -> DelayedLeaveActionType.RestartDelayedEvent to recheck the delayed event - Update = "Update", - // -> MembershipActionType.Update if the timeout has passed so the next update is required. - SendLeaveEvent = "SendLeaveEvent", // -> MembershipActionType.SendLeaveEvent -} -function isMembershipActionType(val: any): val is MembershipActionType { - return val in MembershipActionType; -} - -enum DelayedLeaveActionType { - SendFirstDelayedEvent = "SendFirstDelayedEvent", - // -> MembershipActionType.SendJoinEvent if successful - // -> DelayedLeaveActionType.SendFirstDelayedEvent on error, retry sending the first delayed event. - SendMainDelayedEvent = "SendMainDelayedEvent", - // -> DelayedLeaveActionType.RestartDelayedEvent on success start updating the delayed event - // -> DelayedLeaveActionType.SendMainDelayedEvent on error try again RestartDelayedEvent = "RestartDelayedEvent", // -> DelayedLeaveActionType.SendMainDelayedEvent on missing delay id but there is a rtc state event // -> DelayedLeaveActionType.SendFirstDelayedEvent on missing delay id and there is no state event // -> DelayedLeaveActionType.RestartDelayedEvent on success we schedule the next restart + UpdateExpiry = "UpdateExpiry", + // -> MembershipActionType.Update if the timeout has passed so the next update is required. + SendMainDelayedEvent = "SendMainDelayedEvent", + // -> DelayedLeaveActionType.RestartDelayedEvent on success start updating the delayed event + // -> DelayedLeaveActionType.SendMainDelayedEvent on error try again SendScheduledDelayedLeaveEvent = "SendScheduledDelayedLeaveEvent", // -> MembershipActionType.SendLeaveEvent on failiour (not found) we need to send the leave manually and cannot use the scheduled delayed event // -> DelayedLeaveActionType.SendScheduledDelayedLeaveEvent on error we try again. -} - -function isDelayedLeaveActionType(val: any): val is DelayedLeaveActionType { - return val in DelayedLeaveActionType; + SendLeaveEvent = "SendLeaveEvent", + // -> MembershipActionType.SendLeaveEvent } /** * Actions that are supposed to be used from outside the main handle methods. */ -enum DirectMembershipManagerActions { - Join = DelayedLeaveActionType.SendFirstDelayedEvent, - Leave = DelayedLeaveActionType.SendScheduledDelayedLeaveEvent, +enum DirectMembershipManagerAction { + Join = MembershipActionType.SendFirstDelayedEvent, + Leave = MembershipActionType.SendScheduledDelayedLeaveEvent, } interface ActionSchedulerState { /** The delayId we got when successfully sending the delayed leave event. @@ -124,6 +115,7 @@ interface ActionSchedulerState { sendMembershipRetries: number; sendDelayedEventRetries: number; updateDelayedEventRetries: number; + retries: number; } interface Action { @@ -135,7 +127,7 @@ interface Action { * The state of the different loops * can also be thought of as the type of the action */ - type: DelayedLeaveActionType | MembershipActionType | DirectMembershipManagerActions; + type: MembershipActionType | DirectMembershipManagerAction; } /** @@ -146,7 +138,7 @@ class ActionScheduler { public constructor( state: ActionSchedulerState, - private manager: Pick, + private manager: Pick, ) { this.state = state; } @@ -182,11 +174,7 @@ class ActionScheduler { this.didWakeUp = false; } else { try { - if (isDelayedLeaveActionType(nextAction.type)) { - await this.manager.delayedLeaveLoopHandler(this.state, nextAction.type); - } else if (isMembershipActionType(nextAction.type)) { - await this.manager.membershipLoopHandler(this.state, nextAction.type); - } + await this.manager.membershipLoopHandler(this.state, nextAction.type as MembershipActionType); } catch (e) { throw Error("The MemberhsipManager has to shut down because of the end condition: " + e); } @@ -252,7 +240,7 @@ export class MembershipManager implements IMembershipManager { const f = async (): Promise => { try { await this.scheduler.startWithActions([ - { ts: Date.now(), type: DirectMembershipManagerActions.Join }, + { ts: Date.now(), type: DirectMembershipManagerAction.Join }, ]); } catch (e) { onError?.(e); @@ -267,7 +255,7 @@ export class MembershipManager implements IMembershipManager { if (!this.leavePromise) { // reset scheduled actions so we will not do any new actions. - this.scheduler.resetActions([{ type: DirectMembershipManagerActions.Leave, ts: Date.now() }]); + this.scheduler.resetActions([{ type: DirectMembershipManagerAction.Leave, ts: Date.now() }]); this.leavePromise = new Promise((resolve, reject) => { this.leavePromiseHandle.reject = reject; this.leavePromiseHandle.resolve = resolve; @@ -290,7 +278,7 @@ export class MembershipManager implements IMembershipManager { if (this.isJoined() && !memberships.some(isMyMembership)) { logger.warn("Missing own membership: force re-join"); this.scheduler.state.hasMemberStateEvent = false; - this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerActions.Join }); + this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerAction.Join }); } } @@ -337,12 +325,10 @@ export class MembershipManager implements IMembershipManager { if (userId === null) throw Error("Missing userId in client"); if (deviceId === null) throw Error("Missing deviceId in client"); this.deviceId = deviceId; - this.userId = userId; this.stateKey = this.makeMembershipStateKey(userId, deviceId); } // Membership Event parameters: - private userId: string; private deviceId: string; private stateKey: string; private fociPreferred?: Focus[]; @@ -401,14 +387,15 @@ export class MembershipManager implements IMembershipManager { sendMembershipRetries: 0, sendDelayedEventRetries: 0, updateDelayedEventRetries: 0, + retries: 0, }, this, ); - // Loop Handlers: - public async delayedLeaveLoopHandler(state: ActionSchedulerState, type: DelayedLeaveActionType): Promise { + // Loop Handler: + public async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { switch (type) { - case DelayedLeaveActionType.SendFirstDelayedEvent: + case MembershipActionType.SendFirstDelayedEvent: // Before we start we check if we come from a state where we have a delay id. if (state.delayId) { // Remove all running updates and restarts @@ -419,33 +406,17 @@ export class MembershipManager implements IMembershipManager { state.updateDelayedEventRetries = 0; this.scheduler.addAction({ ts: Date.now(), - type: DelayedLeaveActionType.SendFirstDelayedEvent, + type: MembershipActionType.SendFirstDelayedEvent, }); } catch (e) { - this.handleRateLimitError( - e, - state.updateDelayedEventRetries, - (retryIn) => { - state.updateDelayedEventRetries++; - this.scheduler.addAction({ - ts: Date.now() + retryIn, - type: DelayedLeaveActionType.SendFirstDelayedEvent, - }); - }, - () => { - throw Error( - "Exceeded maximum delayed event update (cancel) attempts (client._unstable_updateDelayedEvent): " + - e, - ); - }, - ); + if (this.handleRateLimitErr(e, "updateDelayedEvent", type)) break; this.handleNotFoundError(e, () => { // If we get a M_NOT_FOUND we know that the delayed event got already removed. // This means we are good and can set it to undefined and run this again. state.delayId = undefined; this.scheduler.addAction({ ts: Date.now(), - type: DelayedLeaveActionType.SendFirstDelayedEvent, + type: MembershipActionType.SendFirstDelayedEvent, }); }); } @@ -468,38 +439,21 @@ export class MembershipManager implements IMembershipManager { this.handleMaxDelayeExceededError(e, () => { this.scheduler.addAction({ ts: Date.now(), - type: DelayedLeaveActionType.SendFirstDelayedEvent, + type: MembershipActionType.SendFirstDelayedEvent, }); }); - this.handleRateLimitError( - e, - state.sendDelayedEventRetries, - (retryIn) => { - logger.warn("Retry sending delayed disconnection due to rate limit:", e); - state.sendDelayedEventRetries++; - this.scheduler.addAction({ - ts: Date.now() + retryIn, - type: DelayedLeaveActionType.SendFirstDelayedEvent, - }); - }, - () => { - throw Error( - "Exceeded maximum delayed event send attempts (client._unstable_sendDelayedStateEvent): " + - e, - ); - }, - ); + if (this.handleRateLimitErr(e, "updateDelayedEvent", type)) break; } } break; - case DelayedLeaveActionType.RestartDelayedEvent: + case MembershipActionType.RestartDelayedEvent: if (!state.delayId) { // Delay id got reset. This action was used to check if the hs canceled the delayed event when the join state got sent. this.scheduler.addAction({ ts: Date.now(), type: state.hasMemberStateEvent - ? DelayedLeaveActionType.SendMainDelayedEvent - : DelayedLeaveActionType.SendFirstDelayedEvent, + ? MembershipActionType.SendMainDelayedEvent + : MembershipActionType.SendFirstDelayedEvent, }); break; } @@ -508,34 +462,18 @@ export class MembershipManager implements IMembershipManager { state.updateDelayedEventRetries = 0; this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, - type: DelayedLeaveActionType.RestartDelayedEvent, + type: MembershipActionType.RestartDelayedEvent, }); } catch (e) { // TODO this also needs a test: get rate limit while checking id delayed event is scheduled this.handleNotFoundError(e, () => { state.delayId = undefined; - this.scheduler.addAction({ ts: Date.now(), type: DelayedLeaveActionType.SendMainDelayedEvent }); + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendMainDelayedEvent }); }); - this.handleRateLimitError( - e, - state.updateDelayedEventRetries, - (retryIn) => { - state.updateDelayedEventRetries++; - this.scheduler.addAction({ - ts: Date.now() + retryIn, - type: DelayedLeaveActionType.RestartDelayedEvent, - }); - }, - () => { - throw Error( - "Exceeded maximum restart delayed event update attempts (client._unstable_updateDelayedEvent): " + - e, - ); - }, - ); + if (this.handleRateLimitErr(e, "updateDelayedEvent", type)) break; } break; - case DelayedLeaveActionType.SendMainDelayedEvent: + case MembershipActionType.SendMainDelayedEvent: try { const response = await this.client._unstable_sendDelayedStateEvent( this.room.roomId, @@ -549,36 +487,19 @@ export class MembershipManager implements IMembershipManager { state.delayId = response.delay_id; this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, - type: DelayedLeaveActionType.RestartDelayedEvent, + type: MembershipActionType.RestartDelayedEvent, }); } catch (e) { this.handleMaxDelayeExceededError(e, () => { this.scheduler.addAction({ ts: Date.now(), - type: DelayedLeaveActionType.SendMainDelayedEvent, + type: MembershipActionType.SendMainDelayedEvent, }); }); - this.handleRateLimitError( - e, - state.sendMembershipRetries, - (retryIn) => { - logger.warn("Retry sending delayed disconnection due to rate limit:", e); - state.sendMembershipRetries++; - this.scheduler.addAction({ - ts: Date.now() + Math.max(retryIn, this.callMemberEventRetryDelayMinimum), - type: DelayedLeaveActionType.SendMainDelayedEvent, - }); - }, - () => { - throw Error( - "Exceeded maximum send delayed event attempts (client._unstable_sendDelayedStateEvent): " + - e, - ); - }, - ); + if (this.handleRateLimitErr(e, "updateDelayedEvent", type)) break; } break; - case DelayedLeaveActionType.SendScheduledDelayedLeaveEvent: + case MembershipActionType.SendScheduledDelayedLeaveEvent: if (state.delayId) { try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send); @@ -591,20 +512,8 @@ export class MembershipManager implements IMembershipManager { state.delayId = undefined; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); }); - const rateLimitHandled = this.handleRateLimitError( - e, - state.updateDelayedEventRetries, - (retryIn) => { - state.updateDelayedEventRetries++; - this.scheduler.addAction({ - ts: Date.now() + retryIn, - type: DelayedLeaveActionType.SendScheduledDelayedLeaveEvent, - }); - }, - () => { - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); - }, - ); + const rateLimitHandled = this.handleRateLimitErr(e, "updateDelayedEvent", type); + if (!notFoundHandled && !rateLimitHandled) { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } @@ -613,11 +522,6 @@ export class MembershipManager implements IMembershipManager { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } break; - } - } - - public async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { - switch (type) { case MembershipActionType.SendJoinEvent: try { await this.client.sendStateEvent( @@ -629,31 +533,16 @@ export class MembershipManager implements IMembershipManager { state.nextRelativeExpiry += this.membershipEventExpiryTimeout; state.hasMemberStateEvent = true; state.sendMembershipRetries = 0; - this.scheduler.addAction({ ts: Date.now(), type: DelayedLeaveActionType.RestartDelayedEvent }); + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }); this.scheduler.addAction({ ts: Date.now() + this.membershipTimerExpiryTimeout, - type: MembershipActionType.Update, + type: MembershipActionType.UpdateExpiry, }); } catch (e) { - this.handleRateLimitError( - e, - state.sendMembershipRetries, - (resendDelay) => { - state.sendMembershipRetries++; - this.scheduler.addAction({ - ts: Date.now() + resendDelay, - type: MembershipActionType.SendJoinEvent, - }); - }, - () => { - throw Error( - "Exceeded maximum Own Membership state update attempts (client.sendStateEvent): " + e, - ); - }, - ); + if (this.handleRateLimitErr(e, "sendStateEvent", type)) break; } break; - case MembershipActionType.Update: + case MembershipActionType.UpdateExpiry: try { await this.client.sendStateEvent( this.room.roomId, @@ -667,32 +556,15 @@ export class MembershipManager implements IMembershipManager { this.scheduler.addAction({ ts: Date.now() + this.membershipTimerExpiryTimeout, - type: MembershipActionType.Update, + type: MembershipActionType.UpdateExpiry, }); } catch (e) { - const rateLimitHandled = this.handleRateLimitError( - e, - state.sendMembershipRetries, - (retryIn) => { - logger.warn("Retry sending membership state event due to rate limit:", e); - state.sendMembershipRetries++; - this.scheduler.addAction({ - ts: Date.now() + Math.max(retryIn, this.callMemberEventRetryDelayMinimum), - type: MembershipActionType.Update, - }); - }, - () => { - throw Error( - "Exceeded maximum own Membership state update attempts (client.sendStateEvent): " + e, - ); - }, - ); - if (!rateLimitHandled) { - this.scheduler.addAction({ - ts: Date.now() + this.callMemberEventRetryDelayMinimum, - type: MembershipActionType.Update, - }); - } + if (this.handleRateLimitErr(e, "sendStateEvent", type)) break; + + this.scheduler.addAction({ + ts: Date.now() + this.callMemberEventRetryDelayMinimum, + type: MembershipActionType.UpdateExpiry, + }); } break; case MembershipActionType.SendLeaveEvent: @@ -713,19 +585,7 @@ export class MembershipManager implements IMembershipManager { this.leavePromiseHandle.resolve?.(true); state.hasMemberStateEvent = false; } catch (e) { - this.handleRateLimitError( - e, - state.sendMembershipRetries, - (retryIn) => { - this.scheduler.addAction({ - ts: Date.now() + retryIn, - type: MembershipActionType.SendLeaveEvent, - }); - }, - () => { - throw Error("could not send final leave event due to max rate limit retries" + e); - }, - ); + if (this.handleRateLimitErr(e, "sendStateEvent", type)) break; } } } @@ -780,19 +640,16 @@ export class MembershipManager implements IMembershipManager { /** * * @param e - * @param allowedRetries - * @param currentRetries - * @param onRetry - * @param onAbort - * @returns Returns true if it did anything. + * @param method + * @param type + * @returns Returns true if handled the error and rescheduled the correct next action did anything. */ - private handleRateLimitError( - e: unknown, - currentRetries: number, - onRetry: (retryIn: number) => void, - onAbort: () => void, - ): boolean { - if (currentRetries < this.maximumRateLimitRetryCount && e instanceof HTTPError && e.isRateLimitError()) { + private handleRateLimitErr(e: unknown, method: string, type: MembershipActionType): boolean { + if ( + this.scheduler.state.retries < this.maximumRateLimitRetryCount && + e instanceof HTTPError && + e.isRateLimitError() + ) { let resendDelay: number; const defaultMs = 5000; try { @@ -805,11 +662,13 @@ export class MembershipManager implements IMembershipManager { ); resendDelay = defaultMs; } - onRetry(resendDelay); + + this.scheduler.state.retries++; + this.scheduler.addAction({ ts: Date.now() + resendDelay, type }); + return true; } else if (e instanceof HTTPError && e.isRateLimitError()) { - onAbort(); - return true; + throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + e.message); } return false; } From 493f0db7a5bf80fe2891de0e90a6129dea7376d7 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 21:17:49 +0100 Subject: [PATCH 043/124] Add diagram --- src/matrixrtc/NewMembershipManager.ts | 38 ++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 1ae29d6d70c..8ea15fb7c81 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -66,7 +66,43 @@ export interface IMembershipManager { getActiveFocus(): Focus | undefined; } -// SCHEDULER TYPES: +/* SCHEDULER TYPES: + + DirectMembershipManagerAction.Join + ▼ + ┌─────────────────────┐ + │SendFirstDelayedEvent│ + └─────────────────────┘ + │ + ▼ + ┌─────────────┐ + ┌────────────│SendJoinEvent│────────────┐ + │ └─────────────┘ │ + │ ┌─────┐ ┌──────┐ │ ┌──────┐ + ▼ ▼ │ │ ▼ ▼ ▼ │ +┌────────────┐ │ │ ┌───────────────────┐ │ +│UpdateExpiry│ │ │ │RestartDelayedEvent│ │ +└────────────┘ │ │ └───────────────────┘ │ + │ │ │ │ │ │ + └─────┘ └──────┘ │ │ + │ │ + ┌────────────────────┐ │ │ + │SendMainDelayedEvent│◄───────┘ │ + └───────────────────┬┘ │ + │ │ + └─────────────────────┘ + STOP ALL ABOVE + DirectMembershipManagerAction.Leave + ▼ + ┌───────────────────────────────┐ + │ SendScheduledDelayedLeaveEvent│ + └───────────────────────────────┘ + │ + ▼ + ┌──────────────┐ + │SendLeaveEvent│ + └──────────────┘ +*/ enum MembershipActionType { SendFirstDelayedEvent = "SendFirstDelayedEvent", // -> MembershipActionType.SendJoinEvent if successful From 33e203f7079e34675b23f9813da6f6dcbceba030 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 21:23:27 +0100 Subject: [PATCH 044/124] fix new error api in rtc session --- src/matrixrtc/MatrixRTCSession.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 8c6635dc190..2093267483f 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -344,10 +344,10 @@ export class MatrixRTCSession extends TypedEventEmitter + this.membershipManager!.join(fociPreferred, fociActive, (e) => { // TODO: Consider exposing this as a signal from the RTCSession so it can be used in the UI. - logger.error("MembershipManager encountered an unrecoverable error: ", e), - ); + logger.error("MembershipManager encountered an unrecoverable error: ", e); + }); this.encryptionManager!.join(joinConfig); this.emit(MatrixRTCSessionEvent.JoinStateChanged, true); From 9fa9bd6d6aca6553c55283413c93dbbba076b426 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 21:28:05 +0100 Subject: [PATCH 045/124] fix up retry counter --- src/matrixrtc/NewMembershipManager.ts | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 8ea15fb7c81..e46cf09c9e0 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -146,11 +146,7 @@ interface ActionSchedulerState { running: boolean; /** The manager is in the state where its actually connected to the session. */ hasMemberStateEvent: boolean; - // Retry counters that get used to limit the maximum rate limit retires we want to do. - // They get reused for each rate limit loop we run into and reset to 0 on unrecoverable failiour or success. - sendMembershipRetries: number; - sendDelayedEventRetries: number; - updateDelayedEventRetries: number; + // Retry counter retries: number; } @@ -411,7 +407,7 @@ export class MembershipManager implements IMembershipManager { return this.joinConfig?.membershipKeepAlivePeriod ?? 5_000; } private get maximumRateLimitRetryCount(): number { - return this.joinConfig?.maximumRateLimitRetryCount ?? 5; + return this.joinConfig?.maximumRateLimitRetryCount ?? 10; } // Scheduler: private scheduler = new ActionScheduler( @@ -420,9 +416,6 @@ export class MembershipManager implements IMembershipManager { running: false, nextRelativeExpiry: this.membershipEventExpiryTimeout, delayId: undefined, - sendMembershipRetries: 0, - sendDelayedEventRetries: 0, - updateDelayedEventRetries: 0, retries: 0, }, this, @@ -439,7 +432,7 @@ export class MembershipManager implements IMembershipManager { try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel); state.delayId = undefined; - state.updateDelayedEventRetries = 0; + state.retries = 0; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent, @@ -468,7 +461,7 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ); // Success we reset retires and set delayId. - state.sendDelayedEventRetries = 0; + state.retries = 0; state.delayId = response.delay_id; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); } catch (e) { @@ -495,7 +488,7 @@ export class MembershipManager implements IMembershipManager { } try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart); - state.updateDelayedEventRetries = 0; + state.retries = 0; this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, type: MembershipActionType.RestartDelayedEvent, @@ -521,6 +514,7 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ); state.delayId = response.delay_id; + state.retries = 0; this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, type: MembershipActionType.RestartDelayedEvent, @@ -540,7 +534,7 @@ export class MembershipManager implements IMembershipManager { try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send); state.hasMemberStateEvent = false; - state.updateDelayedEventRetries = 0; + state.retries = 0; this.scheduler.resetActions([]); this.leavePromiseHandle.resolve?.(true); } catch (e) { @@ -568,7 +562,7 @@ export class MembershipManager implements IMembershipManager { ); state.nextRelativeExpiry += this.membershipEventExpiryTimeout; state.hasMemberStateEvent = true; - state.sendMembershipRetries = 0; + state.retries = 0; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }); this.scheduler.addAction({ ts: Date.now() + this.membershipTimerExpiryTimeout, @@ -588,7 +582,7 @@ export class MembershipManager implements IMembershipManager { ); state.nextRelativeExpiry += this.membershipEventExpiryTimeout; // Success, we reset retries and schedule update. - state.sendMembershipRetries = 0; + state.retries = 0; this.scheduler.addAction({ ts: Date.now() + this.membershipTimerExpiryTimeout, @@ -616,7 +610,7 @@ export class MembershipManager implements IMembershipManager { {}, this.stateKey, ); - state.updateDelayedEventRetries = 0; + state.retries = 0; this.scheduler.resetActions([]); this.leavePromiseHandle.resolve?.(true); state.hasMemberStateEvent = false; From e835561cda4444875969fb3683c03ff02d0bb8ba Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 24 Feb 2025 21:33:33 +0100 Subject: [PATCH 046/124] fix lints --- src/matrixrtc/NewMembershipManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index e46cf09c9e0..0a946eeadbf 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -269,7 +269,7 @@ export class MembershipManager implements IMembershipManager { this.focusActive = focusActive; if (!this.scheduler.state.running) { this.scheduler.state.running = true; - const f = async (): Promise => { + void (async (): Promise => { try { await this.scheduler.startWithActions([ { ts: Date.now(), type: DirectMembershipManagerAction.Join }, @@ -277,8 +277,7 @@ export class MembershipManager implements IMembershipManager { } catch (e) { onError?.(e); } - }; - f(); + }); } } @@ -312,6 +311,7 @@ export class MembershipManager implements IMembershipManager { this.scheduler.state.hasMemberStateEvent = false; this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerAction.Join }); } + return Promise.resolve(); } public getActiveFocus(): Focus | undefined { From 4c3d1f4c62a684217682e2424e5b3681b85b73b8 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 25 Feb 2025 00:23:41 +0100 Subject: [PATCH 047/124] make unrecoverable errors more explicit --- src/client.ts | 12 ++- src/errors.ts | 10 ++ src/matrixrtc/NewMembershipManager.ts | 144 ++++++++++++++++---------- 3 files changed, 105 insertions(+), 61 deletions(-) diff --git a/src/client.ts b/src/client.ts index 89ec7bcec91..edcd4e1c347 100644 --- a/src/client.ts +++ b/src/client.ts @@ -240,6 +240,7 @@ import { validateAuthMetadataAndKeys, } from "./oidc/index.ts"; import { type EmptyObject } from "./@types/common.ts"; +import { UnsupportedEndpointError } from "./errors.ts"; export type Store = IStore; @@ -3351,7 +3352,7 @@ export class MatrixClient extends TypedEventEmitter { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw Error("Server does not support the delayed events API"); + throw new UnsupportedEndpointError("Server does not support the delayed events API", "sendDelayedEvent"); } this.addThreadRelationIfNeeded(content, threadId, roomId); @@ -3374,7 +3375,10 @@ export class MatrixClient extends TypedEventEmitter { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw Error("Server does not support the delayed events API"); + throw new UnsupportedEndpointError( + "Server does not support the delayed events API", + "sendDelayedStateEvent", + ); } const pathParams = { @@ -3398,7 +3402,7 @@ export class MatrixClient extends TypedEventEmitter { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw Error("Server does not support the delayed events API"); + throw new UnsupportedEndpointError("Server does not support the delayed events API", "getDelayedEvents"); } const queryDict = fromToken ? { from: fromToken } : undefined; @@ -3420,7 +3424,7 @@ export class MatrixClient extends TypedEventEmitter { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw Error("Server does not support the delayed events API"); + throw new UnsupportedEndpointError("Server does not support the delayed events API", "updateDelayedEvent"); } const path = utils.encodeUri("/delayed_events/$delayId", { diff --git a/src/errors.ts b/src/errors.ts index 8345293be53..a88c89188dc 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -52,3 +52,13 @@ export class ClientStoppedError extends Error { super("MatrixClient has been stopped"); } } + +export class UnsupportedEndpointError extends Error { + public constructor( + message: string, + public clientEndpoint: "sendDelayedEvent" | "updateDelayedEvent" | "sendDelayedStateEvent" | "getDelayedEvents", + ) { + super(message); + this.name = "UnsupportedEndpointError"; + } +} diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 0a946eeadbf..64a109cb17a 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -17,6 +17,7 @@ limitations under the License. import { EventType } from "../@types/event.ts"; import { UpdateDelayedEventAction } from "../@types/requests.ts"; import type { MatrixClient } from "../client.ts"; +import { UnsupportedEndpointError } from "../errors.ts"; import { HTTPError, MatrixError } from "../http-api/errors.ts"; import { logger } from "../logger.ts"; import { type Room } from "../models/room.ts"; @@ -241,8 +242,9 @@ class ActionScheduler { } } } + /** - * This Class takes care of the membership management. + * This class takes care of the membership management. * It has the following tasks: * - Send the users leave delayed event before sending the memberhsip * - Sent the users membership if the state machine is started @@ -254,7 +256,6 @@ class ActionScheduler { * - Stop the timer for the delay refresh * - Stop the timer for updateint the state event */ - export class MembershipManager implements IMembershipManager { public isJoined(): boolean { return this.scheduler.state.running; @@ -296,6 +297,7 @@ export class MembershipManager implements IMembershipManager { return this.leavePromise; } + private leavePromise?: Promise; private leavePromiseHandle: { reject?: (reason: any) => void; @@ -426,7 +428,42 @@ export class MembershipManager implements IMembershipManager { switch (type) { case MembershipActionType.SendFirstDelayedEvent: // Before we start we check if we come from a state where we have a delay id. - if (state.delayId) { + if (!state.delayId) { + // Normal case without any previous delayed id. + try { + const response = await this.client._unstable_sendDelayedStateEvent( + this.room.roomId, + { + delay: this.membershipServerSideExpiryTimeout, + }, + EventType.GroupCallMemberPrefix, + {}, // leave event + this.stateKey, + ); + // Success we reset retires and set delayId. + state.retries = 0; + state.delayId = response.delay_id; + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); + } catch (e) { + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; + if (this.maxDelayeExceededErrorHandler(e)) { + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendFirstDelayedEvent, + }); + break; + } + if (this.unsupportedDelayedEndpoint(e)) { + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendJoinEvent, + }); + break; + } + // On any other error we fall back to not using delayed events and send the state event immediately + } + } else { + // Restart case with delayed id. // Remove all running updates and restarts this.scheduler.resetActions([]); try { @@ -438,8 +475,8 @@ export class MembershipManager implements IMembershipManager { type: MembershipActionType.SendFirstDelayedEvent, }); } catch (e) { - if (this.handleRateLimitErr(e, "updateDelayedEvent", type)) break; - this.handleNotFoundError(e, () => { + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; + if (this.notFoundError(e)) { // If we get a M_NOT_FOUND we know that the delayed event got already removed. // This means we are good and can set it to undefined and run this again. state.delayId = undefined; @@ -447,31 +484,15 @@ export class MembershipManager implements IMembershipManager { ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent, }); - }); - } - } else { - try { - const response = await this.client._unstable_sendDelayedStateEvent( - this.room.roomId, - { - delay: this.membershipServerSideExpiryTimeout, - }, - EventType.GroupCallMemberPrefix, - {}, // leave event - this.stateKey, - ); - // Success we reset retires and set delayId. - state.retries = 0; - state.delayId = response.delay_id; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); - } catch (e) { - this.handleMaxDelayeExceededError(e, () => { + break; + } + if (this.unsupportedDelayedEndpoint(e)) { this.scheduler.addAction({ ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, + type: MembershipActionType.SendJoinEvent, }); - }); - if (this.handleRateLimitErr(e, "updateDelayedEvent", type)) break; + break; + } } } break; @@ -494,12 +515,17 @@ export class MembershipManager implements IMembershipManager { type: MembershipActionType.RestartDelayedEvent, }); } catch (e) { - // TODO this also needs a test: get rate limit while checking id delayed event is scheduled - this.handleNotFoundError(e, () => { + if (this.notFoundError(e)) { state.delayId = undefined; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendMainDelayedEvent }); - }); - if (this.handleRateLimitErr(e, "updateDelayedEvent", type)) break; + break; + } + // TODO this also needs a test: get rate limit while checking id delayed event is scheduled + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; + // If the HS does not support delayed events we wont reschedule. + if (this.unsupportedDelayedEndpoint(e)) break; + // In other error cases we have no idea what is happening + throw Error("Could not restart delayed event, even though delayed events are supported. " + e); } break; case MembershipActionType.SendMainDelayedEvent: @@ -520,13 +546,17 @@ export class MembershipManager implements IMembershipManager { type: MembershipActionType.RestartDelayedEvent, }); } catch (e) { - this.handleMaxDelayeExceededError(e, () => { + if (this.maxDelayeExceededErrorHandler(e)) { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendMainDelayedEvent, }); - }); - if (this.handleRateLimitErr(e, "updateDelayedEvent", type)) break; + break; + } + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; + // Don't do any other delayed event work if its not supported. + if (this.unsupportedDelayedEndpoint(e)) break; + throw Error("Could not send delayed event, even though delayed events are supported. " + e); } break; case MembershipActionType.SendScheduledDelayedLeaveEvent: @@ -538,15 +568,16 @@ export class MembershipManager implements IMembershipManager { this.scheduler.resetActions([]); this.leavePromiseHandle.resolve?.(true); } catch (e) { - const notFoundHandled = this.handleNotFoundError(e, () => { + if (this.notFoundError(e)) { state.delayId = undefined; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); - }); - const rateLimitHandled = this.handleRateLimitErr(e, "updateDelayedEvent", type); - - if (!notFoundHandled && !rateLimitHandled) { - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); + break; } + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; + if (this.unsupportedDelayedEndpoint(e)) break; + + // On any other error we fall back to SendLeaveEvent + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } } else { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); @@ -569,7 +600,8 @@ export class MembershipManager implements IMembershipManager { type: MembershipActionType.UpdateExpiry, }); } catch (e) { - if (this.handleRateLimitErr(e, "sendStateEvent", type)) break; + if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; + throw Error("Could not send state event because of unrecoverable error: " + e); } break; case MembershipActionType.UpdateExpiry: @@ -589,12 +621,10 @@ export class MembershipManager implements IMembershipManager { type: MembershipActionType.UpdateExpiry, }); } catch (e) { - if (this.handleRateLimitErr(e, "sendStateEvent", type)) break; + if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; - this.scheduler.addAction({ - ts: Date.now() + this.callMemberEventRetryDelayMinimum, - type: MembershipActionType.UpdateExpiry, - }); + // TODO add timeout/netowrk error + throw Error("Could not update state event with new expiry ts because of unrecoverable error: " + e); } break; case MembershipActionType.SendLeaveEvent: @@ -615,7 +645,8 @@ export class MembershipManager implements IMembershipManager { this.leavePromiseHandle.resolve?.(true); state.hasMemberStateEvent = false; } catch (e) { - if (this.handleRateLimitErr(e, "sendStateEvent", type)) break; + if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; + throw Error("Failed to send Leave event because of: " + e); } } } @@ -643,14 +674,12 @@ export class MembershipManager implements IMembershipManager { foci_preferred: this.fociPreferred ?? [], }; } - private handleNotFoundError(e: unknown, onNotFound: () => void): boolean { - if (e instanceof MatrixError && e.errcode === "M_NOT_FOUND") { - onNotFound(); - return true; - } - return false; + + // Error checks and handlers + private notFoundError(e: unknown): boolean { + return e instanceof MatrixError && e.errcode === "M_NOT_FOUND"; } - private handleMaxDelayeExceededError(e: unknown, didSetupDelayTimeout: () => void): boolean { + private maxDelayeExceededErrorHandler(e: unknown): boolean { if ( e instanceof MatrixError && e.errcode === "M_UNKNOWN" && @@ -660,13 +689,11 @@ export class MembershipManager implements IMembershipManager { if (typeof maxDelayAllowed === "number" && this.membershipServerSideExpiryTimeout > maxDelayAllowed) { this.membershipServerSideExpiryTimeoutOverride = maxDelayAllowed; } - didSetupDelayTimeout(); logger.warn("Retry sending delayed disconnection event due to server timeout limitations:", e); return true; } return false; } - /** * * @param e @@ -674,7 +701,7 @@ export class MembershipManager implements IMembershipManager { * @param type * @returns Returns true if handled the error and rescheduled the correct next action did anything. */ - private handleRateLimitErr(e: unknown, method: string, type: MembershipActionType): boolean { + private rateLimitErrorHandler(e: unknown, method: string, type: MembershipActionType): boolean { if ( this.scheduler.state.retries < this.maximumRateLimitRetryCount && e instanceof HTTPError && @@ -702,4 +729,7 @@ export class MembershipManager implements IMembershipManager { } return false; } + private unsupportedDelayedEndpoint(e: unknown): boolean { + return e instanceof UnsupportedEndpointError; + } } From 3dee61acbad8b73929011bed4a2ad2e2a44e3eba Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 25 Feb 2025 00:36:49 +0100 Subject: [PATCH 048/124] fix tests --- spec/unit/matrixrtc/MembershipManager.spec.ts | 4 ++-- src/matrixrtc/NewMembershipManager.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 8f1030c3c33..551f8d565fc 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -630,7 +630,7 @@ describe.each([ const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive, delayEventSendError); await flushPromises(); - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 10; i++) { jest.advanceTimersByTime(2000); await flushPromises(); } @@ -651,7 +651,7 @@ describe.each([ const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive, delayEventRestartError); await flushPromises(); - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 10; i++) { jest.advanceTimersByTime(1000); await flushPromises(); } diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 64a109cb17a..c23592f107f 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -270,6 +270,7 @@ export class MembershipManager implements IMembershipManager { this.focusActive = focusActive; if (!this.scheduler.state.running) { this.scheduler.state.running = true; + void (async (): Promise => { try { await this.scheduler.startWithActions([ @@ -278,7 +279,7 @@ export class MembershipManager implements IMembershipManager { } catch (e) { onError?.(e); } - }); + })(); } } From 172c3eae171dee2405d646748839a4853ab26f84 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 25 Feb 2025 01:04:31 +0100 Subject: [PATCH 049/124] Allow multiple retries on the rtc state event http requests. --- src/matrixrtc/NewMembershipManager.ts | 78 ++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index c23592f107f..d77d8fee5e2 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -148,6 +148,7 @@ interface ActionSchedulerState { /** The manager is in the state where its actually connected to the session. */ hasMemberStateEvent: boolean; // Retry counter + rateLimitRetries: number; retries: number; } @@ -257,6 +258,8 @@ class ActionScheduler { * - Stop the timer for updateint the state event */ export class MembershipManager implements IMembershipManager { + // PUBLIC: + public isJoined(): boolean { return this.scheduler.state.running; } @@ -363,6 +366,8 @@ export class MembershipManager implements IMembershipManager { this.stateKey = this.makeMembershipStateKey(userId, deviceId); } + // PRIVATE: + // Membership Event parameters: private deviceId: string; private stateKey: string; @@ -412,6 +417,10 @@ export class MembershipManager implements IMembershipManager { private get maximumRateLimitRetryCount(): number { return this.joinConfig?.maximumRateLimitRetryCount ?? 10; } + private get maximumRetryCount(): number { + // TODO allow configuring this via `MembershipConfig`. + return 10; + } // Scheduler: private scheduler = new ActionScheduler( { @@ -419,6 +428,7 @@ export class MembershipManager implements IMembershipManager { running: false, nextRelativeExpiry: this.membershipEventExpiryTimeout, delayId: undefined, + rateLimitRetries: 0, retries: 0, }, this, @@ -442,6 +452,7 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ); // Success we reset retires and set delayId. + state.rateLimitRetries = 0; state.retries = 0; state.delayId = response.delay_id; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); @@ -470,6 +481,7 @@ export class MembershipManager implements IMembershipManager { try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel); state.delayId = undefined; + state.rateLimitRetries = 0; state.retries = 0; this.scheduler.addAction({ ts: Date.now(), @@ -510,6 +522,7 @@ export class MembershipManager implements IMembershipManager { } try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart); + state.rateLimitRetries = 0; state.retries = 0; this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, @@ -541,6 +554,7 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ); state.delayId = response.delay_id; + state.rateLimitRetries = 0; state.retries = 0; this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, @@ -565,6 +579,7 @@ export class MembershipManager implements IMembershipManager { try { await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send); state.hasMemberStateEvent = false; + state.rateLimitRetries = 0; state.retries = 0; this.scheduler.resetActions([]); this.leavePromiseHandle.resolve?.(true); @@ -594,6 +609,7 @@ export class MembershipManager implements IMembershipManager { ); state.nextRelativeExpiry += this.membershipEventExpiryTimeout; state.hasMemberStateEvent = true; + state.rateLimitRetries = 0; state.retries = 0; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }); this.scheduler.addAction({ @@ -602,6 +618,10 @@ export class MembershipManager implements IMembershipManager { }); } catch (e) { if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; + + // Event sending retry (different to rate limit retries) + if (this.retryOnAnyErrorHandler(e, type)) break; + throw Error("Could not send state event because of unrecoverable error: " + e); } break; @@ -615,6 +635,7 @@ export class MembershipManager implements IMembershipManager { ); state.nextRelativeExpiry += this.membershipEventExpiryTimeout; // Success, we reset retries and schedule update. + state.rateLimitRetries = 0; state.retries = 0; this.scheduler.addAction({ @@ -623,8 +644,11 @@ export class MembershipManager implements IMembershipManager { }); } catch (e) { if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; + // TODO add timeout/netowrk error (or just the below) + + // Event sending retry (different to rate limit retries) + if (this.retryOnAnyErrorHandler(e, type)) break; - // TODO add timeout/netowrk error throw Error("Could not update state event with new expiry ts because of unrecoverable error: " + e); } break; @@ -641,12 +665,17 @@ export class MembershipManager implements IMembershipManager { {}, this.stateKey, ); + state.rateLimitRetries = 0; state.retries = 0; this.scheduler.resetActions([]); this.leavePromiseHandle.resolve?.(true); state.hasMemberStateEvent = false; } catch (e) { if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; + + // Event sending retry (different to rate limit retries) + if (this.retryOnAnyErrorHandler(e, type)) break; + throw Error("Failed to send Leave event because of: " + e); } } @@ -677,9 +706,21 @@ export class MembershipManager implements IMembershipManager { } // Error checks and handlers + + /** + * Check if its a NOT_FOUND error + * @param e the error causing this handler check/execution + * @returns true if its a not found error + */ private notFoundError(e: unknown): boolean { return e instanceof MatrixError && e.errcode === "M_NOT_FOUND"; } + + /** + * Check if this is a DelayExceeded timeout and update the TimeoutOverride for the next try + * @param e the error causing this handler check/execution + * @returns true if its a delay exceeded error and we updated the local TimeoutOverride + */ private maxDelayeExceededErrorHandler(e: unknown): boolean { if ( e instanceof MatrixError && @@ -696,15 +737,15 @@ export class MembershipManager implements IMembershipManager { return false; } /** - * - * @param e - * @param method - * @param type + * Check if we have a rate limit error and schedule the same action again if we dont exceed the rate limit retry count yet. + * @param e the error causing this handler check/execution + * @param method the method used for the throw message + * @param type which MembershipActionType we reschedule because of a rate limit. * @returns Returns true if handled the error and rescheduled the correct next action did anything. */ private rateLimitErrorHandler(e: unknown, method: string, type: MembershipActionType): boolean { if ( - this.scheduler.state.retries < this.maximumRateLimitRetryCount && + this.scheduler.state.rateLimitRetries < this.maximumRateLimitRetryCount && e instanceof HTTPError && e.isRateLimitError() ) { @@ -721,7 +762,7 @@ export class MembershipManager implements IMembershipManager { resendDelay = defaultMs; } - this.scheduler.state.retries++; + this.scheduler.state.rateLimitRetries++; this.scheduler.addAction({ ts: Date.now() + resendDelay, type }); return true; @@ -730,6 +771,29 @@ export class MembershipManager implements IMembershipManager { } return false; } + + /** + * Don't Check the error and retry the same MembershipAction again in the configured time and for the configured retry count. + * @param e the error causing this handler check/execution + * @param type the action type that we need to repeat because of the error + * @returns Returns true if we handled the error by rescheduling the correct next action. + */ + private retryOnAnyErrorHandler(e: unknown, type: MembershipActionType): boolean { + if (this.scheduler.state.retries < this.maximumRetryCount) { + this.scheduler.state.retries++; + this.scheduler.addAction({ ts: Date.now() + this.callMemberEventRetryDelayMinimum, type }); + + return true; + } else { + throw Error("Reached maximum (" + this.maximumRetryCount + ") retries cause by: " + e); + } + } + + /** + * Check if its a UnsupportedEndpointError and which implies that we cannot do any delayed event logic + * @param e The error to check + * @returns true it its a UnsupportedEndpointError + */ private unsupportedDelayedEndpoint(e: unknown): boolean { return e instanceof UnsupportedEndpointError; } From 34888639be2b60792f101e4671e82dc943c4b3ad Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 25 Feb 2025 02:25:42 +0100 Subject: [PATCH 050/124] use then catch for startup --- src/matrixrtc/NewMembershipManager.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index d77d8fee5e2..ffda8169acd 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -273,16 +273,9 @@ export class MembershipManager implements IMembershipManager { this.focusActive = focusActive; if (!this.scheduler.state.running) { this.scheduler.state.running = true; - - void (async (): Promise => { - try { - await this.scheduler.startWithActions([ - { ts: Date.now(), type: DirectMembershipManagerAction.Join }, - ]); - } catch (e) { - onError?.(e); - } - })(); + this.scheduler + .startWithActions([{ ts: Date.now(), type: DirectMembershipManagerAction.Join }]) + .catch((e) => onError?.(e)); } } From 8f7744231e673e88674072702a32b42b173baf81 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 25 Feb 2025 09:28:43 +0100 Subject: [PATCH 051/124] no try catch 1 --- .eslintrc.cjs | 2 +- spec/unit/matrixrtc/MembershipManager.spec.ts | 44 ++++---- src/matrixrtc/NewMembershipManager.ts | 104 +++++++++--------- 3 files changed, 81 insertions(+), 69 deletions(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 55c326d3678..b5c61bd9faa 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -156,7 +156,7 @@ module.exports = { }, { // Enable stricter promise rules for the MatrixRTC codebase - files: ["src/matrixrtc/**/*.ts"], + files: ["src/matrixrtc/**/*.ts","spec/unit/matrixrtc/*.ts"], rules: { // Encourage proper usage of Promises: "@typescript-eslint/no-floating-promises": "error", diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 551f8d565fc..36ca642a285 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -27,7 +27,7 @@ import { flushPromises } from "../../test-utils/flushPromises"; import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; import { defer } from "../../../src/utils"; -function waitForMockCall(method: MockedFunction, returnVal?: any) { +function waitForMockCall(method: MockedFunction, returnVal?: Promise) { return new Promise((resolve) => { method.mockImplementation(() => { resolve(); @@ -69,7 +69,8 @@ describe.each([ client = makeMockClient("@alice:example.org", "AAAAAAA"); room = makeMockRoom(membershipTemplate); // Provide a default mock that is like the default "non error" server behaviour. - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + (client._unstable_sendDelayedStateEvent as Mock).mockResolvedValue({ delay_id: "id" }); + (client._unstable_updateDelayedEvent as Mock).mockResolvedValue(undefined); }); afterEach(() => { @@ -83,7 +84,7 @@ describe.each([ expect(manager.isJoined()).toEqual(false); }); - it("returns true after join()", async () => { + it("returns true after join()", () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([]); expect(manager.isJoined()).toEqual(true); @@ -135,11 +136,13 @@ describe.each([ if (useOwnedStateEvents) { room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); } - const updatedDelayedEvent = waitForMockCall(client._unstable_updateDelayedEvent); - const sentDelayedState = waitForMockCall(client._unstable_sendDelayedStateEvent, { - delay_id: "id", - }); + const sentDelayedState = waitForMockCall( + client._unstable_sendDelayedStateEvent, + Promise.resolve({ + delay_id: "id", + }), + ); // preparing the delayed disconnect should handle the delay being too long const sendDelayedStateExceedAttempt = new Promise((resolve) => { @@ -190,6 +193,7 @@ describe.each([ await sendDelayedStateExceedAttempt.then(); // needed to resolve after the send attempt catches await sendDelayedStateAttempt; + await flushPromises(); const callProps = (d: number) => { return [room!.roomId, { delay: d }, "org.matrix.msc3401.call.member", {}, userStateKey]; }; @@ -257,7 +261,7 @@ describe.each([ await flushPromises(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); - it("uses membershipServerSideExpiryTimeout from config", async () => { + it("uses membershipServerSideExpiryTimeout from config", () => { const manager = new TestMembershipManager( { membershipServerSideExpiryTimeout: 123456 }, room, @@ -341,7 +345,9 @@ describe.each([ // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed it("does nothing if not joined !FailsForLegacy", () => { const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.leave(); + const errorFn = jest.fn(); + manager.leave().catch(errorFn); + expect(errorFn).not.toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); expect(client.sendStateEvent).not.toHaveBeenCalled(); }); @@ -394,7 +400,7 @@ describe.each([ describe("onRTCSessionMemberUpdate()", () => { it("does nothing if not joined", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); await flushPromises(); expect(client.sendStateEvent).not.toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); @@ -408,11 +414,11 @@ describe.each([ await flushPromises(); const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` - (client.sendStateEvent as Mock).mockReset(); - (client._unstable_updateDelayedEvent as Mock).mockReset(); - (client._unstable_sendDelayedStateEvent as Mock).mockReset(); + (client.sendStateEvent as Mock).mockClear(); + (client._unstable_updateDelayedEvent as Mock).mockClear(); + (client._unstable_sendDelayedStateEvent as Mock).mockClear(); - manager.onRTCSessionMemberUpdate([ + await manager.onRTCSessionMemberUpdate([ mockCallMembership(membershipTemplate, room.roomId), mockCallMembership(myMembership as SessionMembershipData, room.roomId, client.getUserId() ?? undefined), ]); @@ -425,13 +431,13 @@ describe.each([ const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); await flushPromises(); - // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` + // clearing all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` (client.sendStateEvent as Mock).mockClear(); (client._unstable_updateDelayedEvent as Mock).mockClear(); (client._unstable_sendDelayedStateEvent as Mock).mockClear(); // Our own membership is removed: - manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); await flushPromises(); expect(client.sendStateEvent).toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalled(); @@ -541,10 +547,10 @@ describe.each([ await flushPromises(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + (client._unstable_sendDelayedStateEvent as Mock).mockResolvedValue({ delay_id: "id" }); // Remove our own membership so that there is no reason the send the delayed leave anymore. // the membership is no longer present on the homeserver - manager.onRTCSessionMemberUpdate([]); + await manager.onRTCSessionMemberUpdate([]); // Wait for all timers to be setup await flushPromises(); jest.advanceTimersByTime(1000); @@ -571,7 +577,7 @@ describe.each([ expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); // the user terminated the call locally - manager.leave(); + void manager.leave(); // Wait for all timers to be setup await flushPromises(); diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index ffda8169acd..2c2a3470a3e 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -434,8 +434,8 @@ export class MembershipManager implements IMembershipManager { // Before we start we check if we come from a state where we have a delay id. if (!state.delayId) { // Normal case without any previous delayed id. - try { - const response = await this.client._unstable_sendDelayedStateEvent( + await this.client + ._unstable_sendDelayedStateEvent( this.room.roomId, { delay: this.membershipServerSideExpiryTimeout, @@ -443,63 +443,69 @@ export class MembershipManager implements IMembershipManager { EventType.GroupCallMemberPrefix, {}, // leave event this.stateKey, + ) + .then( + (response) => { + // Success we reset retires and set delayId. + state.rateLimitRetries = 0; + state.retries = 0; + state.delayId = response.delay_id; + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); + }, + (e) => { + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + if (this.maxDelayeExceededErrorHandler(e)) { + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendFirstDelayedEvent, + }); + return; + } + if (this.unsupportedDelayedEndpoint(e)) { + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendJoinEvent, + }); + return; + } + }, ); - // Success we reset retires and set delayId. - state.rateLimitRetries = 0; - state.retries = 0; - state.delayId = response.delay_id; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); - } catch (e) { - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; - if (this.maxDelayeExceededErrorHandler(e)) { - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, - }); - break; - } - if (this.unsupportedDelayedEndpoint(e)) { - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendJoinEvent, - }); - break; - } - // On any other error we fall back to not using delayed events and send the state event immediately - } + // On any other error we fall back to not using delayed events and send the state event immediately } else { // Restart case with delayed id. // Remove all running updates and restarts this.scheduler.resetActions([]); - try { - await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel); - state.delayId = undefined; - state.rateLimitRetries = 0; - state.retries = 0; - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, - }); - } catch (e) { - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; - if (this.notFoundError(e)) { - // If we get a M_NOT_FOUND we know that the delayed event got already removed. - // This means we are good and can set it to undefined and run this again. + await this.client + ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel) + .then(() => { state.delayId = undefined; + state.rateLimitRetries = 0; + state.retries = 0; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent, }); - break; - } - if (this.unsupportedDelayedEndpoint(e)) { - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendJoinEvent, - }); - break; - } - } + }) + .catch((e) => { + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + if (this.notFoundError(e)) { + // If we get a M_NOT_FOUND we know that the delayed event got already removed. + // This means we are good and can set it to undefined and run this again. + state.delayId = undefined; + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendFirstDelayedEvent, + }); + return; + } + if (this.unsupportedDelayedEndpoint(e)) { + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendJoinEvent, + }); + return; + } + }); } break; case MembershipActionType.RestartDelayedEvent: From a1e4f2528726f7a0c83385e7e2af8af2f0165247 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 09:53:13 +0100 Subject: [PATCH 052/124] update expire headroom logic transition from try catch to .then .catch --- spec/unit/matrixrtc/MembershipManager.spec.ts | 33 +- src/matrixrtc/MatrixRTCSession.ts | 8 +- src/matrixrtc/NewMembershipManager.ts | 312 ++++++++++-------- 3 files changed, 190 insertions(+), 163 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 36ca642a285..998286f0d08 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -31,7 +31,7 @@ function waitForMockCall(method: MockedFunction, returnVal?: Promise) return new Promise((resolve) => { method.mockImplementation(() => { resolve(); - return returnVal; + return returnVal ?? Promise.resolve(); }); }); } @@ -71,6 +71,7 @@ describe.each([ // Provide a default mock that is like the default "non error" server behaviour. (client._unstable_sendDelayedStateEvent as Mock).mockResolvedValue({ delay_id: "id" }); (client._unstable_updateDelayedEvent as Mock).mockResolvedValue(undefined); + (client.sendStateEvent as Mock).mockResolvedValue(undefined); }); afterEach(() => { @@ -193,17 +194,17 @@ describe.each([ await sendDelayedStateExceedAttempt.then(); // needed to resolve after the send attempt catches await sendDelayedStateAttempt; - await flushPromises(); const callProps = (d: number) => { return [room!.roomId, { delay: d }, "org.matrix.msc3401.call.member", {}, userStateKey]; }; expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(1, ...callProps(9000)); expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(2, ...callProps(7500)); - jest.advanceTimersByTime(5000); + await jest.advanceTimersByTimeAsync(5000); await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches - jest.advanceTimersByTime(1000); + + await jest.advanceTimersByTimeAsync(1000); expect(client.sendStateEvent).toHaveBeenCalledWith( room!.roomId, @@ -226,9 +227,7 @@ describe.each([ expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. - await flushPromises(); - jest.advanceTimersByTime(5000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(5000); // should update delayed disconnect expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); } @@ -481,27 +480,33 @@ describe.each([ // Delayed events should replace it entirely but before they have wide adoption // the expiration logic still makes sense. // TODO: Add git commit when we removed it. - it("extends `expires` when call still active !FailsForLegacy", async () => { + async function testExpires(expire: number, headroom?: number) { const manager = new TestMembershipManager( - { membershipExpiryTimeout: 10_000 }, + { membershipExpiryTimeout: expire, membershipExpiryTimeoutHeadroom: headroom }, room, client, () => undefined, ); manager.join([focus], focusActive); await waitForMockCall(client.sendStateEvent); + await flushPromises(); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; - expect(sentMembership.expires).toBe(10_000); + expect(sentMembership.expires).toBe(expire); for (let i = 2; i <= 12; i++) { // flush promises before advancing the timers to make sure schedulers are setup - await flushPromises(); - jest.advanceTimersByTime(10_000); + jest.advanceTimersByTime(expire); await flushPromises(); expect(client.sendStateEvent).toHaveBeenCalledTimes(i); const sentMembership = (client.sendStateEvent as Mock).mock.lastCall![2] as SessionMembershipData; - expect(sentMembership.expires).toBe(10_000 * i); + expect(sentMembership.expires).toBe(expire * i); } + } + it("extends `expires` when call still active !FailsForLegacy", async () => { + await testExpires(10_000); + }); + it("extends `expires` using headroom configuration !FailsForLegacy", async () => { + await testExpires(10_000, 1_000); }); }); @@ -612,7 +617,7 @@ describe.each([ expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); // Setup resolve - (client._unstable_updateDelayedEvent as Mock).mockImplementation(() => {}); + (client._unstable_updateDelayedEvent as Mock).mockResolvedValue(undefined); jest.advanceTimersByTime(1000); await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 16868e87e83..826eaca5248 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -71,13 +71,15 @@ export interface MembershipConfig { membershipExpiryTimeout?: number; /** - * The slack in (in milliseconds) which the manager will allow before the membership `expires` time to make sure it + * The time in (in milliseconds) which the manager will prematurely send the updated state event before the membership `expires` time to make sure it * sends the updated state event early enough. * - * A slack of 1000ms and a `membershipExpiryTimeout` of 10000ms would result in a membership event update every 9s and + * A headroom of 1000ms and a `membershipExpiryTimeout` of 10000ms would result in the first membership event update after 9s and * a membership event that would be considered expired after 10s. + * + * This value does not have an effect on the value of `SessionMembershipData.expires`. */ - membershipExpiryTimeoutSlack?: number; + membershipExpiryTimeoutHeadroom?: number; /** * The period (in milliseconds) with which we check that our membership event still exists on the diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 2c2a3470a3e..8bc7283d621 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -139,8 +139,12 @@ interface ActionSchedulerState { /** The delayId we got when successfully sending the delayed leave event. * Gets set to undefined if the server claims it cannot find the delayed event anymore. */ delayId?: string; - /** Stores the value we want to use for the `expires` field in the next own membership update. */ - nextRelativeExpiry: number; + /** Stores how often we have update the `expires` field. + * `expireUpdateIterations` * `membershipEventExpiryTimeout` resolves to the value the expires field should contain next */ + expireUpdateIterations: number; + /** The time at which we send the first state event. The time the call started from the DAG point of view. + * This is used to compute the local sleep timestamps when to next update the member event with a new expires value. */ + startTime: number; /** Flag that gets set once join is called. * The manager tries its best to get the user into the call. * Does not imply the user is actually joined via room state. */ @@ -376,26 +380,15 @@ export class MembershipManager implements IMembershipManager { private get membershipEventExpiryTimeout(): number { return this.joinConfig?.membershipExpiryTimeout ?? DEFAULT_EXPIRE_DURATION; } - private get membershipTimerExpiryTimeout(): number { - let expiryTimeout = this.membershipEventExpiryTimeout; - const expiryTimeoutSlack = this.joinConfig?.membershipExpiryTimeoutSlack; - if (expiryTimeoutSlack) { - if (expiryTimeout > expiryTimeoutSlack) { - expiryTimeout = expiryTimeout - expiryTimeoutSlack; - } else { - logger.warn( - "The membershipExpiryTimeoutSlack is misconfigured. It cannot be less than the membershipExpiryTimeout", - "membershipExpiryTimeout:", - expiryTimeout, - "membershipExpiryTimeoutSlack:", - expiryTimeoutSlack, - ); - } - } else { - // Default Slack - expiryTimeout -= 5_000; - } - return Math.max(expiryTimeout, 1000); + private get membershipEventExpiryTimeoutHeadroom(): number { + return this.joinConfig?.membershipExpiryTimeoutHeadroom ?? 5_000; + } + private computeNextExpiryTs(iteration: number): number { + return ( + this.scheduler.state.startTime + + this.membershipEventExpiryTimeout * iteration - + this.membershipEventExpiryTimeoutHeadroom + ); } private get membershipServerSideExpiryTimeout(): number { return ( @@ -419,10 +412,11 @@ export class MembershipManager implements IMembershipManager { { hasMemberStateEvent: false, running: false, - nextRelativeExpiry: this.membershipEventExpiryTimeout, + startTime: 0, delayId: undefined, rateLimitRetries: 0, retries: 0, + expireUpdateIterations: 0, }, this, ); @@ -430,7 +424,7 @@ export class MembershipManager implements IMembershipManager { // Loop Handler: public async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { switch (type) { - case MembershipActionType.SendFirstDelayedEvent: + case MembershipActionType.SendFirstDelayedEvent: { // Before we start we check if we come from a state where we have a delay id. if (!state.delayId) { // Normal case without any previous delayed id. @@ -508,7 +502,8 @@ export class MembershipManager implements IMembershipManager { }); } break; - case MembershipActionType.RestartDelayedEvent: + } + case MembershipActionType.RestartDelayedEvent: { if (!state.delayId) { // Delay id got reset. This action was used to check if the hs canceled the delayed event when the join state got sent. this.scheduler.addAction({ @@ -519,31 +514,38 @@ export class MembershipManager implements IMembershipManager { }); break; } - try { - await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart); - state.rateLimitRetries = 0; - state.retries = 0; - this.scheduler.addAction({ - ts: Date.now() + this.membershipKeepAlivePeriod, - type: MembershipActionType.RestartDelayedEvent, + const error = await this.client + ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart) + .then(() => { + state.rateLimitRetries = 0; + state.retries = 0; + this.scheduler.addAction({ + ts: Date.now() + this.membershipKeepAlivePeriod, + type: MembershipActionType.RestartDelayedEvent, + }); + }) + .catch((e) => { + if (this.notFoundError(e)) { + state.delayId = undefined; + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendMainDelayedEvent, + }); + return; + } + // TODO this also needs a test: get rate limit while checking id delayed event is scheduled + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + // If the HS does not support delayed events we wont reschedule. + if (this.unsupportedDelayedEndpoint(e)) return; + // In other error cases we have no idea what is happening + return Error("Could not restart delayed event, even though delayed events are supported. " + e); }); - } catch (e) { - if (this.notFoundError(e)) { - state.delayId = undefined; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendMainDelayedEvent }); - break; - } - // TODO this also needs a test: get rate limit while checking id delayed event is scheduled - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; - // If the HS does not support delayed events we wont reschedule. - if (this.unsupportedDelayedEndpoint(e)) break; - // In other error cases we have no idea what is happening - throw Error("Could not restart delayed event, even though delayed events are supported. " + e); - } + if (error instanceof Error) throw error; break; - case MembershipActionType.SendMainDelayedEvent: - try { - const response = await this.client._unstable_sendDelayedStateEvent( + } + case MembershipActionType.SendMainDelayedEvent: { + const error = await this.client + ._unstable_sendDelayedStateEvent( this.room.roomId, { delay: this.membershipServerSideExpiryTimeout, @@ -551,132 +553,150 @@ export class MembershipManager implements IMembershipManager { EventType.GroupCallMemberPrefix, {}, // leave event this.stateKey, - ); - state.delayId = response.delay_id; - state.rateLimitRetries = 0; - state.retries = 0; - this.scheduler.addAction({ - ts: Date.now() + this.membershipKeepAlivePeriod, - type: MembershipActionType.RestartDelayedEvent, - }); - } catch (e) { - if (this.maxDelayeExceededErrorHandler(e)) { + ) + .then((response) => { + state.delayId = response.delay_id; + state.rateLimitRetries = 0; + state.retries = 0; this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendMainDelayedEvent, + ts: Date.now() + this.membershipKeepAlivePeriod, + type: MembershipActionType.RestartDelayedEvent, }); - break; - } - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; - // Don't do any other delayed event work if its not supported. - if (this.unsupportedDelayedEndpoint(e)) break; - throw Error("Could not send delayed event, even though delayed events are supported. " + e); - } + }) + .catch((e) => { + if (this.maxDelayeExceededErrorHandler(e)) { + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendMainDelayedEvent, + }); + return; + } + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + // Don't do any other delayed event work if its not supported. + if (this.unsupportedDelayedEndpoint(e)) return; + return Error("Could not send delayed event, even though delayed events are supported. " + e); + }); + if (error instanceof Error) throw error; break; - case MembershipActionType.SendScheduledDelayedLeaveEvent: + } + case MembershipActionType.SendScheduledDelayedLeaveEvent: { if (state.delayId) { - try { - await this.client._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send); - state.hasMemberStateEvent = false; - state.rateLimitRetries = 0; - state.retries = 0; - this.scheduler.resetActions([]); - this.leavePromiseHandle.resolve?.(true); - } catch (e) { - if (this.notFoundError(e)) { - state.delayId = undefined; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); - break; - } - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) break; - if (this.unsupportedDelayedEndpoint(e)) break; + await this.client + ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send) + .then(() => { + state.hasMemberStateEvent = false; + state.rateLimitRetries = 0; + state.retries = 0; + this.scheduler.resetActions([]); + this.leavePromiseHandle.resolve?.(true); + }) + .catch((e) => { + if (this.notFoundError(e)) { + state.delayId = undefined; + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); + return; + } + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + if (this.unsupportedDelayedEndpoint(e)) return; - // On any other error we fall back to SendLeaveEvent - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); - } + // On any other error we fall back to SendLeaveEvent + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); + }); } else { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } break; - case MembershipActionType.SendJoinEvent: - try { - await this.client.sendStateEvent( + } + case MembershipActionType.SendJoinEvent: { + const error = await this.client + .sendStateEvent( this.room.roomId, EventType.GroupCallMemberPrefix, - this.makeMyMembership(state.nextRelativeExpiry), + this.makeMyMembership(this.membershipEventExpiryTimeout), this.stateKey, - ); - state.nextRelativeExpiry += this.membershipEventExpiryTimeout; - state.hasMemberStateEvent = true; - state.rateLimitRetries = 0; - state.retries = 0; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }); - this.scheduler.addAction({ - ts: Date.now() + this.membershipTimerExpiryTimeout, - type: MembershipActionType.UpdateExpiry, - }); - } catch (e) { - if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; + ) + .then(() => { + state.startTime = Date.now(); + // The next update should already use twice the membershipEventExpiryTimeout + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }); + this.scheduler.addAction({ + ts: this.computeNextExpiryTs(1), + type: MembershipActionType.UpdateExpiry, + }); + state.expireUpdateIterations = 2; + state.hasMemberStateEvent = true; + state.rateLimitRetries = 0; + state.retries = 0; + }) + .catch((e) => { + if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; - // Event sending retry (different to rate limit retries) - if (this.retryOnAnyErrorHandler(e, type)) break; + // Event sending retry (different to rate limit retries) + if (this.retryOnAnyErrorHandler(e, type)) return; - throw Error("Could not send state event because of unrecoverable error: " + e); - } + return Error("Could not send state event because of unrecoverable error: " + e); + }); + if (error instanceof Error) throw error; break; - case MembershipActionType.UpdateExpiry: - try { - await this.client.sendStateEvent( + } + case MembershipActionType.UpdateExpiry: { + const error = await this.client + .sendStateEvent( this.room.roomId, EventType.GroupCallMemberPrefix, - this.makeMyMembership(state.nextRelativeExpiry), + this.makeMyMembership(this.membershipEventExpiryTimeout * state.expireUpdateIterations), this.stateKey, - ); - state.nextRelativeExpiry += this.membershipEventExpiryTimeout; - // Success, we reset retries and schedule update. - state.rateLimitRetries = 0; - state.retries = 0; - - this.scheduler.addAction({ - ts: Date.now() + this.membershipTimerExpiryTimeout, - type: MembershipActionType.UpdateExpiry, - }); - } catch (e) { - if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; - // TODO add timeout/netowrk error (or just the below) + ) + .then(() => { + // Success, we reset retries and schedule update. + this.scheduler.addAction({ + ts: this.computeNextExpiryTs(state.expireUpdateIterations), + type: MembershipActionType.UpdateExpiry, + }); + state.expireUpdateIterations++; + state.rateLimitRetries = 0; + state.retries = 0; + }) + .catch((e) => { + if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; + // TODO add timeout/netowrk error (or just the below) - // Event sending retry (different to rate limit retries) - if (this.retryOnAnyErrorHandler(e, type)) break; + // Event sending retry (different to rate limit retries) + if (this.retryOnAnyErrorHandler(e, type)) return; - throw Error("Could not update state event with new expiry ts because of unrecoverable error: " + e); - } + return Error( + "Could not update state event with new expiry ts because of unrecoverable error: " + e, + ); + }); + if (error instanceof Error) throw error; break; - case MembershipActionType.SendLeaveEvent: + } + case MembershipActionType.SendLeaveEvent: { // We are good already if (!state.hasMemberStateEvent) return; // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event - try { - await this.client.sendStateEvent( - this.room.roomId, - EventType.GroupCallMemberPrefix, - {}, - this.stateKey, - ); - state.rateLimitRetries = 0; - state.retries = 0; - this.scheduler.resetActions([]); - this.leavePromiseHandle.resolve?.(true); - state.hasMemberStateEvent = false; - } catch (e) { - if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) break; + const error = await this.client + .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) + .then(() => { + state.rateLimitRetries = 0; + state.retries = 0; + this.scheduler.resetActions([]); + this.leavePromiseHandle.resolve?.(true); + state.hasMemberStateEvent = false; + }) + .catch((e) => { + if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; - // Event sending retry (different to rate limit retries) - if (this.retryOnAnyErrorHandler(e, type)) break; + // Event sending retry (different to rate limit retries) + if (this.retryOnAnyErrorHandler(e, type)) return; - throw Error("Failed to send Leave event because of: " + e); - } + return Error("Failed to send Leave event because of: " + e); + }); + if (error instanceof Error) throw error; + break; + } } } From 6334faf5ea2e8dd49063de6aa96859a65ae5a7cb Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 10:02:59 +0100 Subject: [PATCH 053/124] replace flushPromise with advanceTimersByTimeAsync --- spec/unit/matrixrtc/MembershipManager.spec.ts | 70 ++++++++----------- 1 file changed, 28 insertions(+), 42 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 998286f0d08..0f46b395d86 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -23,7 +23,6 @@ import { EventType, HTTPError, MatrixError, type Room } from "../../../src"; import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; import { LegacyMembershipManager } from "../../../src/matrixrtc/LegacyMembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; -import { flushPromises } from "../../test-utils/flushPromises"; import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; import { defer } from "../../../src/utils"; @@ -255,9 +254,7 @@ describe.each([ const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); - await flushPromises(); - jest.advanceTimersByTime(5000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(5000); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); it("uses membershipServerSideExpiryTimeout from config", () => { @@ -322,7 +319,7 @@ describe.each([ it("resolves delayed leave event when leave is called", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - await flushPromises(); + await jest.advanceTimersToNextTimerAsync(); await manager.leave(); expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", "send"); expect(client.sendStateEvent).toHaveBeenCalled(); @@ -330,7 +327,7 @@ describe.each([ it("send leave event when leave is called and resolving delayed leave fails", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - await flushPromises(); + await jest.advanceTimersByTimeAsync(100); (client._unstable_updateDelayedEvent as Mock).mockRejectedValue("unknown"); await manager.leave(); // We send a normal leave event since we failed using updateDelayedEvent with the "send" action. @@ -400,7 +397,7 @@ describe.each([ it("does nothing if not joined", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); await manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); - await flushPromises(); + await jest.advanceTimersToNextTimerAsync(); expect(client.sendStateEvent).not.toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); @@ -408,9 +405,7 @@ describe.each([ it("does nothing if own membership still present", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - - await waitForMockCall(client.sendStateEvent); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` (client.sendStateEvent as Mock).mockClear(); @@ -421,7 +416,9 @@ describe.each([ mockCallMembership(membershipTemplate, room.roomId), mockCallMembership(myMembership as SessionMembershipData, room.roomId, client.getUserId() ?? undefined), ]); - await flushPromises(); + + await jest.advanceTimersByTimeAsync(1); + expect(client.sendStateEvent).not.toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); @@ -429,7 +426,7 @@ describe.each([ it("recreates membership if it is missing", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); // clearing all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` (client.sendStateEvent as Mock).mockClear(); (client._unstable_updateDelayedEvent as Mock).mockClear(); @@ -437,7 +434,7 @@ describe.each([ // Our own membership is removed: await manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); expect(client.sendStateEvent).toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalled(); @@ -455,20 +452,18 @@ describe.each([ () => undefined, ); manager.join([focus], focusActive); + await jest.advanceTimersByTimeAsync(1); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); // The first call is from checking id the server deleted the delayed event // so it does not need a `advanceTimersByTime` - await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); // TODO: Check that update delayed event is called with the correct HTTP request timeout // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); for (let i = 2; i <= 12; i++) { // flush promises before advancing the timers to make sure schedulers are setup - await flushPromises(); - jest.advanceTimersByTime(10_000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(10_000); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); // TODO: Check that update delayed event is called with the correct HTTP request timeout @@ -489,14 +484,11 @@ describe.each([ ); manager.join([focus], focusActive); await waitForMockCall(client.sendStateEvent); - await flushPromises(); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; expect(sentMembership.expires).toBe(expire); for (let i = 2; i <= 12; i++) { - // flush promises before advancing the timers to make sure schedulers are setup - jest.advanceTimersByTime(expire); - await flushPromises(); + await jest.advanceTimersByTimeAsync(expire); expect(client.sendStateEvent).toHaveBeenCalledTimes(i); const sentMembership = (client.sendStateEvent as Mock).mock.lastCall![2] as SessionMembershipData; expect(sentMembership.expires).toBe(expire * i); @@ -529,9 +521,8 @@ describe.each([ new Headers({ "Retry-After": "1" }), ), ); - await flushPromises(); - jest.advanceTimersByTime(1000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1000); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); // FailsForLegacy as implementation does not re-check membership before retrying. @@ -549,16 +540,15 @@ describe.each([ // Should call _unstable_sendDelayedStateEvent but not sendStateEvent because of the // RateLimit error. manager.join([focus], focusActive); + await jest.advanceTimersByTimeAsync(1); - await flushPromises(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); (client._unstable_sendDelayedStateEvent as Mock).mockResolvedValue({ delay_id: "id" }); // Remove our own membership so that there is no reason the send the delayed leave anymore. // the membership is no longer present on the homeserver await manager.onRTCSessionMemberUpdate([]); // Wait for all timers to be setup - await flushPromises(); - jest.advanceTimersByTime(1000); + await jest.advanceTimersByTimeAsync(1000); // We should send the first own membership and a new delayed event after the rate limit timeout. expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); @@ -578,16 +568,15 @@ describe.each([ new Headers({ "Retry-After": "1" }), ), ); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); // the user terminated the call locally void manager.leave(); // Wait for all timers to be setup - await flushPromises(); - jest.advanceTimersByTime(1000); - await flushPromises(); + // await flushPromises(); + await jest.advanceTimersByTimeAsync(1000); // No new events should have been sent: expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); @@ -608,18 +597,17 @@ describe.each([ manager.join([focus], focusActive); // Hit rate limit - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); // Hit second rate limit. - jest.advanceTimersByTime(1000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1000); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); // Setup resolve (client._unstable_updateDelayedEvent as Mock).mockResolvedValue(undefined); - jest.advanceTimersByTime(1000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1000); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); @@ -640,10 +628,9 @@ describe.each([ ); const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive, delayEventSendError); - await flushPromises(); + for (let i = 0; i < 10; i++) { - jest.advanceTimersByTime(2000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(2000); } expect(delayEventSendError).toHaveBeenCalled(); }); @@ -661,10 +648,9 @@ describe.each([ ); const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive, delayEventRestartError); - await flushPromises(); + for (let i = 0; i < 10; i++) { - jest.advanceTimersByTime(1000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1000); } expect(delayEventRestartError).toHaveBeenCalled(); }, 5000); From 495bf8ba2ebb62fce1bc7864e5dbce14674d8b95 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 10:28:41 +0100 Subject: [PATCH 054/124] fix leaving special cases --- src/matrixrtc/NewMembershipManager.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 8bc7283d621..bcf5f87ef42 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -283,7 +283,13 @@ export class MembershipManager implements IMembershipManager { } } + /** + * Leave from the call (Send an rtc session event with content: `{}`) + * @param timeout the maximum duration this promise will take to resolve + * @returns true if it managed to leave and false if the timeout condition happened. + */ public leave(timeout?: number): Promise { + if (!this.scheduler.state.running) return Promise.resolve(true); this.scheduler.state.running = false; if (!this.leavePromise) { @@ -673,8 +679,10 @@ export class MembershipManager implements IMembershipManager { } case MembershipActionType.SendLeaveEvent: { // We are good already - if (!state.hasMemberStateEvent) return; - + if (!state.hasMemberStateEvent) { + this.leavePromiseHandle.resolve?.(true); + return; + } // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event const error = await this.client From 5a680b0e9fa43024d93fbad9f675d44b6c0585ac Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 10:29:02 +0100 Subject: [PATCH 055/124] more unrecoverable errors special cases --- spec/unit/matrixrtc/MembershipManager.spec.ts | 32 +++++++-- src/matrixrtc/NewMembershipManager.ts | 72 +++++++++++-------- 2 files changed, 71 insertions(+), 33 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 0f46b395d86..8a7d3725039 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -19,7 +19,7 @@ limitations under the License. import { type MockedFunction, type Mock } from "jest-mock"; -import { EventType, HTTPError, MatrixError, type Room } from "../../../src"; +import { EventType, HTTPError, MatrixError, UnsupportedEndpointError, type Room } from "../../../src"; import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; import { LegacyMembershipManager } from "../../../src/matrixrtc/LegacyMembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; @@ -339,10 +339,10 @@ describe.each([ ); }); // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed - it("does nothing if not joined !FailsForLegacy", () => { + it("does nothing if not joined !FailsForLegacy", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); const errorFn = jest.fn(); - manager.leave().catch(errorFn); + await manager.leave().catch(errorFn); expect(errorFn).not.toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); expect(client.sendStateEvent).not.toHaveBeenCalled(); @@ -572,7 +572,7 @@ describe.each([ await jest.advanceTimersByTimeAsync(1); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); // the user terminated the call locally - void manager.leave(); + await manager.leave(); // Wait for all timers to be setup // await flushPromises(); @@ -653,6 +653,28 @@ describe.each([ await jest.advanceTimersByTimeAsync(1000); } expect(delayEventRestartError).toHaveBeenCalled(); - }, 5000); + }); + it("Errors while sending delayed events dont result in an unrecoverable error. We use the manager without delayed events !FailsForLegacy", async () => { + const unrecoverableError = jest.fn(); + (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue(new HTTPError("unknown", 501)); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive, unrecoverableError); + await jest.advanceTimersByTimeAsync(1); + + expect(unrecoverableError).not.toHaveBeenCalledWith(); + expect(client.sendStateEvent).toHaveBeenCalled(); + }); + it("UnsupportedEndpointError does not in an unrecoverable error. We use the manager without delayed events !FailsForLegacy", async () => { + const unrecoverableError = jest.fn(); + (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue( + new UnsupportedEndpointError("not supported", "sendDelayedStateEvent"), + ); + const manager = new TestMembershipManager({}, room, client, () => undefined); + manager.join([focus], focusActive, unrecoverableError); + await jest.advanceTimersByTimeAsync(1); + + expect(unrecoverableError).not.toHaveBeenCalledWith(); + expect(client.sendStateEvent).toHaveBeenCalled(); + }); }); }); diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index bcf5f87ef42..1b64049cb18 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -169,6 +169,12 @@ interface Action { } /** + * This state machine tracks the state of the current membership participation + * and runs one central timer that wakes up a handler callback with the correct action + * whenever necessary. + * + * It can also be awakened whenever a new action is added which is + * earlier then the current "next awake". * @internal */ class ActionScheduler { @@ -268,9 +274,12 @@ export class MembershipManager implements IMembershipManager { return this.scheduler.state.running; } /** - * @throws can throw if it exceeds a configured maximum retry. + * Puts the MembershipManager in a state where it tries to be joined. + * It will send delayed events and membership events * @param fociPreferred * @param focusActive + * @param onError This will be called once the membership menager encounters an unrecoverable error. + * This should bubble up the the frontend to communicate that the call does not work in the current environment. */ public join(fociPreferred: Focus[], focusActive?: Focus, onError?: (error: unknown) => void): void { this.fociPreferred = fociPreferred; @@ -434,7 +443,7 @@ export class MembershipManager implements IMembershipManager { // Before we start we check if we come from a state where we have a delay id. if (!state.delayId) { // Normal case without any previous delayed id. - await this.client + const error = await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, { @@ -444,32 +453,39 @@ export class MembershipManager implements IMembershipManager { {}, // leave event this.stateKey, ) - .then( - (response) => { - // Success we reset retires and set delayId. - state.rateLimitRetries = 0; - state.retries = 0; - state.delayId = response.delay_id; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); - }, - (e) => { - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; - if (this.maxDelayeExceededErrorHandler(e)) { - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, - }); - return; - } - if (this.unsupportedDelayedEndpoint(e)) { - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendJoinEvent, - }); - return; - } - }, - ); + .then((response) => { + // Success we reset retires and set delayId. + state.rateLimitRetries = 0; + state.retries = 0; + state.delayId = response.delay_id; + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); + }) + .catch((e) => { + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + if (this.maxDelayeExceededErrorHandler(e)) { + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendFirstDelayedEvent, + }); + return; + } + if (this.unsupportedDelayedEndpoint(e)) { + logger.info("Not using deleayed event because the endpoint is not supported"); + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendJoinEvent, + }); + return; + } + return e; + }); + if (error) { + logger.info("Not using deleayed event because: " + error); + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendJoinEvent, + }); + } // On any other error we fall back to not using delayed events and send the state event immediately } else { // Restart case with delayed id. From bb5b0de6579808feb334c5cb7116b83aa8ade4d7 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 10:55:57 +0100 Subject: [PATCH 056/124] move to MatrixRTCSessionManager logger --- src/matrixrtc/NewMembershipManager.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 1b64049cb18..2249d899a26 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -19,7 +19,7 @@ import { UpdateDelayedEventAction } from "../@types/requests.ts"; import type { MatrixClient } from "../client.ts"; import { UnsupportedEndpointError } from "../errors.ts"; import { HTTPError, MatrixError } from "../http-api/errors.ts"; -import { logger } from "../logger.ts"; +import { logger as rootLogger } from "../logger.ts"; import { type Room } from "../models/room.ts"; import { sleep } from "../utils.ts"; import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts"; @@ -27,6 +27,8 @@ import { type Focus } from "./focus.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { type MembershipConfig } from "./MatrixRTCSession.ts"; +const logger = rootLogger.getChild("MatrixRTCSessionManager"); + /** * This interface defines what a MembershipManager uses and exposes. * This interface is what we use to write tests and allows to change the actual implementation From 8005dc84910c0834db93011495961196292034b6 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 10:58:18 +0100 Subject: [PATCH 057/124] add state reset and add another unhandleable error The error occurs if we want to cancel the delayed event we still have an id for but get a non expected error. --- src/matrixrtc/NewMembershipManager.ts | 60 ++++++++++++++++----------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 2249d899a26..be8a9db9060 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -181,7 +181,15 @@ interface Action { */ class ActionScheduler { public state: ActionSchedulerState; - + public static defaultState: ActionSchedulerState = { + hasMemberStateEvent: false, + running: false, + startTime: 0, + delayId: undefined, + rateLimitRetries: 0, + retries: 0, + expireUpdateIterations: 0, + }; public constructor( state: ActionSchedulerState, private manager: Pick, @@ -254,6 +262,9 @@ class ActionScheduler { this.wakeup?.(); } } + public resetState(): void { + this.state = ActionScheduler.defaultState; + } } /** @@ -287,6 +298,7 @@ export class MembershipManager implements IMembershipManager { this.fociPreferred = fociPreferred; this.focusActive = focusActive; if (!this.scheduler.state.running) { + this.scheduler.resetState(); this.scheduler.state.running = true; this.scheduler .startWithActions([{ ts: Date.now(), type: DirectMembershipManagerAction.Join }]) @@ -425,18 +437,7 @@ export class MembershipManager implements IMembershipManager { return 10; } // Scheduler: - private scheduler = new ActionScheduler( - { - hasMemberStateEvent: false, - running: false, - startTime: 0, - delayId: undefined, - rateLimitRetries: 0, - retries: 0, - expireUpdateIterations: 0, - }, - this, - ); + private scheduler = new ActionScheduler(ActionScheduler.defaultState, this); // Loop Handler: public async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { @@ -463,7 +464,7 @@ export class MembershipManager implements IMembershipManager { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); }) .catch((e) => { - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + if (this.rateLimitErrorHandler(e, "sendDelayedStateEvent", type)) return; if (this.maxDelayeExceededErrorHandler(e)) { this.scheduler.addAction({ ts: Date.now(), @@ -481,6 +482,7 @@ export class MembershipManager implements IMembershipManager { } return e; }); + // On any other error we fall back to not using delayed events and send the join state event immediately if (error) { logger.info("Not using deleayed event because: " + error); this.scheduler.addAction({ @@ -488,12 +490,11 @@ export class MembershipManager implements IMembershipManager { type: MembershipActionType.SendJoinEvent, }); } - // On any other error we fall back to not using delayed events and send the state event immediately } else { // Restart case with delayed id. // Remove all running updates and restarts this.scheduler.resetActions([]); - await this.client + const error = await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel) .then(() => { state.delayId = undefined; @@ -523,7 +524,16 @@ export class MembershipManager implements IMembershipManager { }); return; } + return e; }); + if (error) { + // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases + // when state.delayId !== undefined + throw Error( + "We failed to cancel a delayed event where we already had a delay id with an error we cannot automatically handle" + + error, + ); + } } break; } @@ -564,7 +574,7 @@ export class MembershipManager implements IMembershipManager { // In other error cases we have no idea what is happening return Error("Could not restart delayed event, even though delayed events are supported. " + e); }); - if (error instanceof Error) throw error; + if (error) throw error; break; } case MembershipActionType.SendMainDelayedEvent: { @@ -600,12 +610,12 @@ export class MembershipManager implements IMembershipManager { if (this.unsupportedDelayedEndpoint(e)) return; return Error("Could not send delayed event, even though delayed events are supported. " + e); }); - if (error instanceof Error) throw error; + if (error) throw error; break; } case MembershipActionType.SendScheduledDelayedLeaveEvent: { if (state.delayId) { - await this.client + const error = await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send) .then(() => { state.hasMemberStateEvent = false; @@ -622,10 +632,10 @@ export class MembershipManager implements IMembershipManager { } if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; if (this.unsupportedDelayedEndpoint(e)) return; - - // On any other error we fall back to SendLeaveEvent - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); + return e; }); + // On any other error we fall back to SendLeaveEvent + if (error) this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } else { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } @@ -660,7 +670,7 @@ export class MembershipManager implements IMembershipManager { return Error("Could not send state event because of unrecoverable error: " + e); }); - if (error instanceof Error) throw error; + if (error) throw error; break; } case MembershipActionType.UpdateExpiry: { @@ -692,7 +702,7 @@ export class MembershipManager implements IMembershipManager { "Could not update state event with new expiry ts because of unrecoverable error: " + e, ); }); - if (error instanceof Error) throw error; + if (error) throw error; break; } case MembershipActionType.SendLeaveEvent: { @@ -720,7 +730,7 @@ export class MembershipManager implements IMembershipManager { return Error("Failed to send Leave event because of: " + e); }); - if (error instanceof Error) throw error; + if (error) throw error; break; } } From b8aa7fc862834ef07020fa3545ffb961eba740dd Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 11:03:13 +0100 Subject: [PATCH 058/124] missed review fixes --- src/matrixrtc/NewMembershipManager.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index be8a9db9060..6f4eb9af972 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -465,7 +465,7 @@ export class MembershipManager implements IMembershipManager { }) .catch((e) => { if (this.rateLimitErrorHandler(e, "sendDelayedStateEvent", type)) return; - if (this.maxDelayeExceededErrorHandler(e)) { + if (this.maxDelayExceededErrorHandler(e)) { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent, @@ -529,6 +529,8 @@ export class MembershipManager implements IMembershipManager { if (error) { // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases // when state.delayId !== undefined + // We do not use ignore and log this error since we would also need to reset the delayId. + // It is cleaner if we the frontend rejoines instead of resetting the delayId here and behaving like in the success case. throw Error( "We failed to cancel a delayed event where we already had a delay id with an error we cannot automatically handle" + error, @@ -598,14 +600,14 @@ export class MembershipManager implements IMembershipManager { }); }) .catch((e) => { - if (this.maxDelayeExceededErrorHandler(e)) { + if (this.maxDelayExceededErrorHandler(e)) { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendMainDelayedEvent, }); return; } - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + if (this.rateLimitErrorHandler(e, "sendDelayedStateEvent", type)) return; // Don't do any other delayed event work if its not supported. if (this.unsupportedDelayedEndpoint(e)) return; return Error("Could not send delayed event, even though delayed events are supported. " + e); @@ -776,7 +778,7 @@ export class MembershipManager implements IMembershipManager { * @param e the error causing this handler check/execution * @returns true if its a delay exceeded error and we updated the local TimeoutOverride */ - private maxDelayeExceededErrorHandler(e: unknown): boolean { + private maxDelayExceededErrorHandler(e: unknown): boolean { if ( e instanceof MatrixError && e.errcode === "M_UNKNOWN" && From 998e16b6bf52b50586d905469c5e60403c8a7497 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 11:06:43 +0100 Subject: [PATCH 059/124] remove @jest/environment dependency --- package.json | 1 - spec/unit/matrixrtc/MembershipManager.spec.ts | 2 +- spec/unit/matrixrtc/memberManagerTestEnvironment.ts | 5 ----- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/package.json b/package.json index 210bdb6294e..d640fdaadb0 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,6 @@ "@babel/preset-env": "^7.12.11", "@babel/preset-typescript": "^7.12.7", "@casualbot/jest-sonar-reporter": "2.2.7", - "@jest/environment": "^29.7.0", "@peculiar/webcrypto": "^1.4.5", "@stylistic/eslint-plugin": "^3.0.0", "@types/content-type": "^1.1.5", diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index a453977cc67..50f87c039d0 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -327,7 +327,7 @@ describe.each([ expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", "send"); expect(client.sendStateEvent).toHaveBeenCalled(); }); - it("send leave event when leave is called and resolving delayed leave fails", async () => { + it("sends leave event when leave is called and resolving delayed leave fails", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); await flushPromises(); diff --git a/spec/unit/matrixrtc/memberManagerTestEnvironment.ts b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts index 9c6f83400dc..414796ffb81 100644 --- a/spec/unit/matrixrtc/memberManagerTestEnvironment.ts +++ b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts @@ -27,15 +27,10 @@ It is very specific to the MembershipManager.spec.ts file and introduces the fol */ import { TestEnvironment } from "jest-environment-jsdom"; -import { type JestEnvironmentConfig, type EnvironmentContext } from "@jest/environment"; import { logger } from "../../../src/logger"; class CustomEnvironment extends TestEnvironment { - constructor(config: JestEnvironmentConfig, context: EnvironmentContext) { - super(config, context); - } - async handleTestEvent(event: any) { if (event.name === "test_start" && event.test.name.includes("!FailsForLegacy")) { let parent = event.test.parent; From 72994f2bbfdcec82e10f9f52155a9adf691cc349 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 11:16:54 +0100 Subject: [PATCH 060/124] Cleanup awaits and Make mock types more correct. Make every mock return a Promise if the real implementation does return a pormise. --- spec/unit/matrixrtc/MembershipManager.spec.ts | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 50f87c039d0..8283e7fa7b9 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -26,13 +26,11 @@ import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, t import { flushPromises } from "../../test-utils/flushPromises"; import { defer } from "../../../src/utils"; -// import { MembershipManager } from "../../../src/matrixrtc/NewMembershipManager"; - -function waitForMockCall(method: MockedFunction, returnVal?: any) { +function waitForMockCall(method: MockedFunction, returnVal?: Promise) { return new Promise((resolve) => { method.mockImplementation(() => { resolve(); - return returnVal; + return returnVal ?? Promise.resolve(); }); }); } @@ -70,13 +68,15 @@ describe.each([ jest.useFakeTimers(); client = makeMockClient("@alice:example.org", "AAAAAAA"); room = makeMockRoom(membershipTemplate); - // Provide a default mock. Representing the default "non error" server behaviour. - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + // Provide a default mock that is like the default "non error" server behaviour. + (client._unstable_sendDelayedStateEvent as Mock).mockResolvedValue({ delay_id: "id" }); + (client._unstable_updateDelayedEvent as Mock).mockResolvedValue(undefined); + (client.sendStateEvent as Mock).mockResolvedValue(undefined); }); afterEach(() => { jest.useRealTimers(); - // no need to clean up mocks since we will recreate the client + // There is no need to clean up mocks since we will recreate the client. }); describe("isJoined()", () => { @@ -85,7 +85,7 @@ describe.each([ expect(manager.isJoined()).toEqual(false); }); - it("returns true after join()", async () => { + it("returns true after join()", () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([]); expect(manager.isJoined()).toEqual(true); @@ -102,7 +102,6 @@ describe.each([ // Test const memberManager = new TestMembershipManager(undefined, room, client, () => undefined); memberManager.join([focus], focusActive); - // expects await waitForMockCall(client.sendStateEvent); expect(client.sendStateEvent).toHaveBeenCalledWith( @@ -138,11 +137,13 @@ describe.each([ if (useOwnedStateEvents) { room.getVersion = jest.fn().mockReturnValue("org.matrix.msc3757.default"); } - const updatedDelayedEvent = waitForMockCall(client._unstable_updateDelayedEvent); - const sentDelayedState = waitForMockCall(client._unstable_sendDelayedStateEvent, { - delay_id: "id", - }); + const sentDelayedState = waitForMockCall( + client._unstable_sendDelayedStateEvent, + Promise.resolve({ + delay_id: "id", + }), + ); // preparing the delayed disconnect should handle the delay being too long const sendDelayedStateExceedAttempt = new Promise((resolve) => { @@ -260,7 +261,7 @@ describe.each([ await flushPromises(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); - it("uses membershipServerSideExpiryTimeout from config", async () => { + it("uses membershipServerSideExpiryTimeout from config", () => { const manager = new TestMembershipManager( { membershipServerSideExpiryTimeout: 123456 }, room, @@ -411,11 +412,11 @@ describe.each([ await flushPromises(); const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` - (client.sendStateEvent as Mock).mockReset(); - (client._unstable_updateDelayedEvent as Mock).mockReset(); - (client._unstable_sendDelayedStateEvent as Mock).mockReset(); + (client.sendStateEvent as Mock).mockClear(); + (client._unstable_updateDelayedEvent as Mock).mockClear(); + (client._unstable_sendDelayedStateEvent as Mock).mockClear(); - manager.onRTCSessionMemberUpdate([ + await manager.onRTCSessionMemberUpdate([ mockCallMembership(membershipTemplate, room.roomId), mockCallMembership(myMembership as SessionMembershipData, room.roomId, client.getUserId() ?? undefined), ]); @@ -547,7 +548,7 @@ describe.each([ (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); // Remove our own membership so that there is no reason the send the delayed leave anymore. // the membership is no longer present on the homeserver - manager.onRTCSessionMemberUpdate([]); + await manager.onRTCSessionMemberUpdate([]); // Wait for all timers to be setup await flushPromises(); jest.advanceTimersByTime(1000); @@ -574,7 +575,7 @@ describe.each([ expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); // the user terminated the call locally - manager.leave(); + await manager.leave(); // Wait for all timers to be setup await flushPromises(); From ce24845c588a733de69eec296bbc68a4d605e3d6 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 11:17:06 +0100 Subject: [PATCH 061/124] remove flush promise dependency --- spec/unit/matrixrtc/MembershipManager.spec.ts | 108 +++++++----------- 1 file changed, 44 insertions(+), 64 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 8283e7fa7b9..6bb4810d4f4 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -23,7 +23,6 @@ import { EventType, HTTPError, MatrixError, type Room } from "../../../src"; import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; import { LegacyMembershipManager } from "../../../src/matrixrtc/MembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; -import { flushPromises } from "../../test-utils/flushPromises"; import { defer } from "../../../src/utils"; function waitForMockCall(method: MockedFunction, returnVal?: Promise) { @@ -47,8 +46,6 @@ function createAsyncHandle(method: MockedFunction) { */ describe.each([ { TestMembershipManager: LegacyMembershipManager, description: "LegacyMembershipManager" }, - // Here we will add the new implementation of the MembershipManager. - // It is not yet tested since it would currently fail all tests. Adding the MembershipManger looks like this: // { TestMembershipManager: MembershipManager, description: "MembershipManager" }, ])("$description", ({ TestMembershipManager }) => { let client: MockClient; @@ -64,7 +61,7 @@ describe.each([ }; beforeEach(() => { - // Default to fake timers + // Default to fake timers. jest.useFakeTimers(); client = makeMockClient("@alice:example.org", "AAAAAAA"); room = makeMockRoom(membershipTemplate); @@ -200,10 +197,11 @@ describe.each([ expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(1, ...callProps(9000)); expect(client._unstable_sendDelayedStateEvent).toHaveBeenNthCalledWith(2, ...callProps(7500)); - jest.advanceTimersByTime(5000); + await jest.advanceTimersByTimeAsync(5000); await sendStateEventAttempt.then(); // needed to resolve after resendIfRateLimited catches - jest.advanceTimersByTime(1000); + + await jest.advanceTimersByTimeAsync(1000); expect(client.sendStateEvent).toHaveBeenCalledWith( room!.roomId, @@ -226,9 +224,7 @@ describe.each([ expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); // ensures that we reach the code that schedules the timeout for the next delay update before we advance the timers. - await flushPromises(); - jest.advanceTimersByTime(5000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(5000); // should update delayed disconnect expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); } @@ -256,9 +252,7 @@ describe.each([ const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); delayedHandle.reject?.(new HTTPError("rate limited", 429, undefined)); - await flushPromises(); - jest.advanceTimersByTime(5000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(5000); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); it("uses membershipServerSideExpiryTimeout from config", () => { @@ -323,15 +317,15 @@ describe.each([ it("resolves delayed leave event when leave is called", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); await manager.leave(); expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", "send"); expect(client.sendStateEvent).toHaveBeenCalled(); }); - it("sends leave event when leave is called and resolving delayed leave fails", async () => { + it("send leave event when leave is called and resolving delayed leave fails", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); (client._unstable_updateDelayedEvent as Mock).mockRejectedValue("unknown"); await manager.leave(); // We send a normal leave event since we failed using updateDelayedEvent with the "send" action. @@ -343,9 +337,9 @@ describe.each([ ); }); // FailsForLegacy because legacy implementation always sends the empty state event even though it isn't needed - it("does nothing if not joined !FailsForLegacy", () => { + it("does nothing if not joined !FailsForLegacy", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.leave(); + await manager.leave(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); expect(client.sendStateEvent).not.toHaveBeenCalled(); }); @@ -398,8 +392,8 @@ describe.each([ describe("onRTCSessionMemberUpdate()", () => { it("does nothing if not joined", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); - manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); - await flushPromises(); + await manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await jest.advanceTimersToNextTimerAsync(); expect(client.sendStateEvent).not.toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); @@ -407,9 +401,7 @@ describe.each([ it("does nothing if own membership still present", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - - await waitForMockCall(client.sendStateEvent); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); const myMembership = (client.sendStateEvent as Mock).mock.calls[0][2]; // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` (client.sendStateEvent as Mock).mockClear(); @@ -420,7 +412,9 @@ describe.each([ mockCallMembership(membershipTemplate, room.roomId), mockCallMembership(myMembership as SessionMembershipData, room.roomId, client.getUserId() ?? undefined), ]); - await flushPromises(); + + await jest.advanceTimersByTimeAsync(1); + expect(client.sendStateEvent).not.toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); expect(client._unstable_updateDelayedEvent).not.toHaveBeenCalled(); @@ -428,15 +422,15 @@ describe.each([ it("recreates membership if it is missing", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - await flushPromises(); - // reset all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` + await jest.advanceTimersByTimeAsync(1); + // clearing all mocks before checking what happens when calling: `onRTCSessionMemberUpdate` (client.sendStateEvent as Mock).mockClear(); (client._unstable_updateDelayedEvent as Mock).mockClear(); (client._unstable_sendDelayedStateEvent as Mock).mockClear(); - // our own membership is removed: - manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); - await flushPromises(); + // Our own membership is removed: + await manager.onRTCSessionMemberUpdate([mockCallMembership(membershipTemplate, room.roomId)]); + await jest.advanceTimersByTimeAsync(1); expect(client.sendStateEvent).toHaveBeenCalled(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalled(); @@ -444,7 +438,7 @@ describe.each([ }); }); - // TODO: not sure about this name + // TODO: Not sure about this name describe("background timers", () => { it("sends only one keep-alive for delayed leave event per `membershipKeepAlivePeriod`", async () => { const manager = new TestMembershipManager( @@ -454,23 +448,21 @@ describe.each([ () => undefined, ); manager.join([focus], focusActive); + await jest.advanceTimersByTimeAsync(1); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); // The first call is from checking id the server deleted the delayed event // so it does not need a `advanceTimersByTime` - await flushPromises(); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); - // TODO: check that update delayed event is called with the correct HTTP request timeout + // TODO: Check that update delayed event is called with the correct HTTP request timeout // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); for (let i = 2; i <= 12; i++) { // flush promises before advancing the timers to make sure schedulers are setup - await flushPromises(); - jest.advanceTimersByTime(10_000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(10_000); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(i); - // TODO: check that update delayed event is called with the correct HTTP request timeout + // TODO: Check that update delayed event is called with the correct HTTP request timeout // expect(client._unstable_updateDelayedEvent).toHaveBeenLastCalledWith("id", 10_000, { localTimeoutMs: 20_000 }); } }); @@ -492,10 +484,7 @@ describe.each([ const sentMembership = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; expect(sentMembership.expires).toBe(10_000); for (let i = 2; i <= 12; i++) { - // flush promises before advancing the timers to make sure schedulers are setup - await flushPromises(); - jest.advanceTimersByTime(10_000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(10_000); expect(client.sendStateEvent).toHaveBeenCalledTimes(i); const sentMembership = (client.sendStateEvent as Mock).mock.lastCall![2] as SessionMembershipData; expect(sentMembership.expires).toBe(10_000 * i); @@ -522,12 +511,11 @@ describe.each([ new Headers({ "Retry-After": "1" }), ), ); - await flushPromises(); - jest.advanceTimersByTime(1000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1000); + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); }); - // FailsForLegacy as implementation does not re-check membership before retrying + // FailsForLegacy as implementation does not re-check membership before retrying. it("abandons retry loop and sends new own membership if not present anymore !FailsForLegacy", async () => { (client._unstable_sendDelayedStateEvent as any).mockRejectedValue( new MatrixError( @@ -542,21 +530,20 @@ describe.each([ // Should call _unstable_sendDelayedStateEvent but not sendStateEvent because of the // RateLimit error. manager.join([focus], focusActive); + await jest.advanceTimersByTimeAsync(1); - await flushPromises(); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); - (client._unstable_sendDelayedStateEvent as Mock).mockReturnValue({ delay_id: "id" }); + (client._unstable_sendDelayedStateEvent as Mock).mockResolvedValue({ delay_id: "id" }); // Remove our own membership so that there is no reason the send the delayed leave anymore. // the membership is no longer present on the homeserver await manager.onRTCSessionMemberUpdate([]); // Wait for all timers to be setup - await flushPromises(); - jest.advanceTimersByTime(1000); + await jest.advanceTimersByTimeAsync(1000); // We should send the first own membership and a new delayed event after the rate limit timeout. expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(2); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); - // FailsForLegacy as implementation does not re-check membership before retrying + // FailsForLegacy as implementation does not re-check membership before retrying. it("abandons retry loop if leave() was called !FailsForLegacy", async () => { const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); @@ -571,16 +558,15 @@ describe.each([ new Headers({ "Retry-After": "1" }), ), ); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); // the user terminated the call locally await manager.leave(); // Wait for all timers to be setup - await flushPromises(); - jest.advanceTimersByTime(1000); - await flushPromises(); + // await flushPromises(); + await jest.advanceTimersByTimeAsync(1000); // No new events should have been sent: expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); @@ -588,7 +574,7 @@ describe.each([ }); describe("retries sending update delayed leave event restart", () => { it("resends the initial check delayed update event !FailsForLegacy", async () => { - (client._unstable_updateDelayedEvent as any).mockRejectedValue( + (client._unstable_updateDelayedEvent as Mock).mockRejectedValue( new MatrixError( { errcode: "M_LIMIT_EXCEEDED" }, 429, @@ -601,26 +587,20 @@ describe.each([ manager.join([focus], focusActive); // Hit rate limit - await flushPromises(); + await jest.advanceTimersByTimeAsync(1); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(1); // Hit second rate limit. - jest.advanceTimersByTime(1000); - await flushPromises(); + await jest.advanceTimersByTimeAsync(1000); expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(2); // Setup resolve - (client._unstable_updateDelayedEvent as Mock).mockImplementation(() => {}); - jest.advanceTimersByTime(1000); - await flushPromises(); + (client._unstable_updateDelayedEvent as Mock).mockResolvedValue(undefined); + await jest.advanceTimersByTimeAsync(1000); + expect(client._unstable_updateDelayedEvent).toHaveBeenCalledTimes(3); expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); }); - - // describe: "retries sending membership event" - // it: "sends it if still joined at time of retry" - // // TODO: see what this is doing and how its different to: "recreates membership if it is missing" - // it: "abandons it if call no longer joined at time of retry" }); }); From 88f40e44bad6a2b11b68aa20401211380a71330b Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 12:11:29 +0100 Subject: [PATCH 062/124] fix not recreating default state on reset This broke all tests since we only created the state once and than passed by ref --- src/matrixrtc/NewMembershipManager.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 6f4eb9af972..cac136e2b4a 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -181,15 +181,17 @@ interface Action { */ class ActionScheduler { public state: ActionSchedulerState; - public static defaultState: ActionSchedulerState = { - hasMemberStateEvent: false, - running: false, - startTime: 0, - delayId: undefined, - rateLimitRetries: 0, - retries: 0, - expireUpdateIterations: 0, - }; + public static get defaultState(): ActionSchedulerState { + return { + hasMemberStateEvent: false, + running: false, + startTime: 0, + delayId: undefined, + rateLimitRetries: 0, + retries: 0, + expireUpdateIterations: 0, + }; + } public constructor( state: ActionSchedulerState, private manager: Pick, From fe3cc263765d7c9ca2cfd6bf53b6885c54c76244 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 12:53:50 +0100 Subject: [PATCH 063/124] Use per action rate limit and retry counter There can be multiple retries at once so we need to store counters per action e.g. the send update membership and the restart delayed could be rate limited at the same time. --- src/matrixrtc/NewMembershipManager.ts | 72 +++++++++++++-------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index cac136e2b4a..aa13884402c 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -153,9 +153,12 @@ interface ActionSchedulerState { running: boolean; /** The manager is in the state where its actually connected to the session. */ hasMemberStateEvent: boolean; - // Retry counter - rateLimitRetries: number; - retries: number; + // There can be multiple retries at once so we need to store counters per action + // e.g. the send update membership and the restart delayed could be rate limited at the same time. + /** Retry counter for rate limits */ + rateLimitRetries: Map; + /** Retry counter for other errors */ + retries: Map; } interface Action { @@ -187,8 +190,8 @@ class ActionScheduler { running: false, startTime: 0, delayId: undefined, - rateLimitRetries: 0, - retries: 0, + rateLimitRetries: new Map(), + retries: new Map(), expireUpdateIterations: 0, }; } @@ -267,6 +270,10 @@ class ActionScheduler { public resetState(): void { this.state = ActionScheduler.defaultState; } + public resetRateLimitCounter(type: MembershipActionType): void { + this.state.rateLimitRetries.set(type, 0); + this.state.retries.set(type, 0); + } } /** @@ -414,7 +421,7 @@ export class MembershipManager implements IMembershipManager { private get membershipEventExpiryTimeoutHeadroom(): number { return this.joinConfig?.membershipExpiryTimeoutHeadroom ?? 5_000; } - private computeNextExpiryTs(iteration: number): number { + private computeNextExpiryActionTs(iteration: number): number { return ( this.scheduler.state.startTime + this.membershipEventExpiryTimeout * iteration - @@ -460,8 +467,8 @@ export class MembershipManager implements IMembershipManager { ) .then((response) => { // Success we reset retires and set delayId. - state.rateLimitRetries = 0; - state.retries = 0; + state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); + state.retries.set(MembershipActionType.SendFirstDelayedEvent, 0); state.delayId = response.delay_id; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); }) @@ -500,8 +507,7 @@ export class MembershipManager implements IMembershipManager { ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel) .then(() => { state.delayId = undefined; - state.rateLimitRetries = 0; - state.retries = 0; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendFirstDelayedEvent); this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent, @@ -555,8 +561,7 @@ export class MembershipManager implements IMembershipManager { const error = await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart) .then(() => { - state.rateLimitRetries = 0; - state.retries = 0; + this.scheduler.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent); this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, type: MembershipActionType.RestartDelayedEvent, @@ -594,8 +599,7 @@ export class MembershipManager implements IMembershipManager { ) .then((response) => { state.delayId = response.delay_id; - state.rateLimitRetries = 0; - state.retries = 0; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendMainDelayedEvent); this.scheduler.addAction({ ts: Date.now() + this.membershipKeepAlivePeriod, type: MembershipActionType.RestartDelayedEvent, @@ -612,6 +616,8 @@ export class MembershipManager implements IMembershipManager { if (this.rateLimitErrorHandler(e, "sendDelayedStateEvent", type)) return; // Don't do any other delayed event work if its not supported. if (this.unsupportedDelayedEndpoint(e)) return; + // after that + if (this.retryOnAnyErrorHandler(e, type)) return; return Error("Could not send delayed event, even though delayed events are supported. " + e); }); if (error) throw error; @@ -623,8 +629,7 @@ export class MembershipManager implements IMembershipManager { ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send) .then(() => { state.hasMemberStateEvent = false; - state.rateLimitRetries = 0; - state.retries = 0; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendScheduledDelayedLeaveEvent); this.scheduler.resetActions([]); this.leavePromiseHandle.resolve?.(true); }) @@ -638,7 +643,7 @@ export class MembershipManager implements IMembershipManager { if (this.unsupportedDelayedEndpoint(e)) return; return e; }); - // On any other error we fall back to SendLeaveEvent + // On any other error we fall back to SendLeaveEvent (this includes hard errors from rate limiting) if (error) this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } else { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); @@ -658,13 +663,12 @@ export class MembershipManager implements IMembershipManager { // The next update should already use twice the membershipEventExpiryTimeout this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }); this.scheduler.addAction({ - ts: this.computeNextExpiryTs(1), + ts: this.computeNextExpiryActionTs(1), type: MembershipActionType.UpdateExpiry, }); state.expireUpdateIterations = 2; state.hasMemberStateEvent = true; - state.rateLimitRetries = 0; - state.retries = 0; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendJoinEvent); }) .catch((e) => { if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; @@ -688,12 +692,11 @@ export class MembershipManager implements IMembershipManager { .then(() => { // Success, we reset retries and schedule update. this.scheduler.addAction({ - ts: this.computeNextExpiryTs(state.expireUpdateIterations), + ts: this.computeNextExpiryActionTs(state.expireUpdateIterations), type: MembershipActionType.UpdateExpiry, }); state.expireUpdateIterations++; - state.rateLimitRetries = 0; - state.retries = 0; + this.scheduler.resetRateLimitCounter(MembershipActionType.UpdateExpiry); }) .catch((e) => { if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; @@ -720,8 +723,7 @@ export class MembershipManager implements IMembershipManager { const error = await this.client .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) .then(() => { - state.rateLimitRetries = 0; - state.retries = 0; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); this.scheduler.resetActions([]); this.leavePromiseHandle.resolve?.(true); state.hasMemberStateEvent = false; @@ -802,12 +804,9 @@ export class MembershipManager implements IMembershipManager { * @param type which MembershipActionType we reschedule because of a rate limit. * @returns Returns true if handled the error and rescheduled the correct next action did anything. */ - private rateLimitErrorHandler(e: unknown, method: string, type: MembershipActionType): boolean { - if ( - this.scheduler.state.rateLimitRetries < this.maximumRateLimitRetryCount && - e instanceof HTTPError && - e.isRateLimitError() - ) { + private rateLimitErrorHandler(e: unknown, method: string, type: MembershipActionType): boolean | Error { + const rateLimitRetries = this.scheduler.state.rateLimitRetries.get(type) ?? 0; + if (rateLimitRetries < this.maximumRateLimitRetryCount && e instanceof HTTPError && e.isRateLimitError()) { let resendDelay: number; const defaultMs = 5000; try { @@ -820,8 +819,7 @@ export class MembershipManager implements IMembershipManager { ); resendDelay = defaultMs; } - - this.scheduler.state.rateLimitRetries++; + this.scheduler.state.rateLimitRetries.set(type, rateLimitRetries + 1); this.scheduler.addAction({ ts: Date.now() + resendDelay, type }); return true; @@ -838,8 +836,10 @@ export class MembershipManager implements IMembershipManager { * @returns Returns true if we handled the error by rescheduling the correct next action. */ private retryOnAnyErrorHandler(e: unknown, type: MembershipActionType): boolean { - if (this.scheduler.state.retries < this.maximumRetryCount) { - this.scheduler.state.retries++; + const retries = this.scheduler.state.retries.get(type) ?? 0; + + if (retries < this.maximumRetryCount) { + this.scheduler.state.retries.set(type, retries + 1); this.scheduler.addAction({ ts: Date.now() + this.callMemberEventRetryDelayMinimum, type }); return true; @@ -849,7 +849,7 @@ export class MembershipManager implements IMembershipManager { } /** - * Check if its a UnsupportedEndpointError and which implies that we cannot do any delayed event logic + * Check if its an UnsupportedEndpointError and which implies that we cannot do any delayed event logic * @param e The error to check * @returns true it its a UnsupportedEndpointError */ From c50fa32ae47cc5703ede9500261dca0ec48945c3 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 14:18:06 +0100 Subject: [PATCH 064/124] add linting to matrixrtc tests --- .eslintrc.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 55c326d3678..b5c61bd9faa 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -156,7 +156,7 @@ module.exports = { }, { // Enable stricter promise rules for the MatrixRTC codebase - files: ["src/matrixrtc/**/*.ts"], + files: ["src/matrixrtc/**/*.ts","spec/unit/matrixrtc/*.ts"], rules: { // Encourage proper usage of Promises: "@typescript-eslint/no-floating-promises": "error", From 94bb78f894c43ad2888513abfddc23dc6d203a0a Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 14:27:33 +0100 Subject: [PATCH 065/124] Add fix async lints and use matrix rtc logger for test environment. --- spec/unit/matrixrtc/MatrixRTCSession.spec.ts | 10 +++++----- spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts | 2 +- spec/unit/matrixrtc/memberManagerTestEnvironment.ts | 11 ++++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts index 85af425e39b..8d7bf41f548 100644 --- a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts +++ b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts @@ -36,10 +36,10 @@ describe("MatrixRTCSession", () => { client.getDeviceId = jest.fn().mockReturnValue("AAAAAAA"); }); - afterEach(() => { + afterEach(async () => { client.stopClient(); client.matrixRTC.stop(); - if (sess) sess.stop(); + if (sess) await sess.stop(); sess = undefined; }); @@ -496,9 +496,9 @@ describe("MatrixRTCSession", () => { sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); }); - afterEach(() => { + afterEach(async () => { // stop the timers - sess!.leaveRoomSession(); + await sess!.leaveRoomSession(); }); it("creates a key when joining", () => { @@ -595,7 +595,7 @@ describe("MatrixRTCSession", () => { } }); - it("cancels key send event that fail", async () => { + it("cancels key send event that fail", () => { const eventSentinel = {} as unknown as MatrixEvent; client.cancelPendingEvent = jest.fn(); diff --git a/spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts b/spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts index 578c62ac0ac..a2912b82c33 100644 --- a/spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts +++ b/spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts @@ -32,7 +32,7 @@ import { makeMockRoom, makeMockRoomState, membershipTemplate } from "./mocks"; describe("MatrixRTCSessionManager", () => { let client: MatrixClient; - beforeEach(async () => { + beforeEach(() => { client = new MatrixClient({ baseUrl: "base_url" }); client.matrixRTC.start(); }); diff --git a/spec/unit/matrixrtc/memberManagerTestEnvironment.ts b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts index 414796ffb81..04a7381b29b 100644 --- a/spec/unit/matrixrtc/memberManagerTestEnvironment.ts +++ b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts @@ -28,10 +28,11 @@ It is very specific to the MembershipManager.spec.ts file and introduces the fol import { TestEnvironment } from "jest-environment-jsdom"; -import { logger } from "../../../src/logger"; +import { logger as rootLogger } from "../../../src/logger"; +const logger = rootLogger.getChild("MatrixRTCSessionManager"); -class CustomEnvironment extends TestEnvironment { - async handleTestEvent(event: any) { +class MemberManagerTestEnvironment extends TestEnvironment { + handleTestEvent(event: any) { if (event.name === "test_start" && event.test.name.includes("!FailsForLegacy")) { let parent = event.test.parent; let isLegacy = false; @@ -44,10 +45,10 @@ class CustomEnvironment extends TestEnvironment { } } if (isLegacy) { - logger.log("skip test: ", event.test.name); + logger.info("skip test: ", event.test.name); event.test.mode = "skip"; } } } } -module.exports = CustomEnvironment; +module.exports = MemberManagerTestEnvironment; From 1016b05713022ae51db135716f76e2575f87fa32 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 14:35:33 +0100 Subject: [PATCH 066/124] prettier --- .eslintrc.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index b5c61bd9faa..bc66a7408bd 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -156,7 +156,7 @@ module.exports = { }, { // Enable stricter promise rules for the MatrixRTC codebase - files: ["src/matrixrtc/**/*.ts","spec/unit/matrixrtc/*.ts"], + files: ["src/matrixrtc/**/*.ts", "spec/unit/matrixrtc/*.ts"], rules: { // Encourage proper usage of Promises: "@typescript-eslint/no-floating-promises": "error", From a2c8d93bf50975aa88b38c3184664f84787142d7 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 14:42:24 +0100 Subject: [PATCH 067/124] review step 1 --- spec/unit/matrixrtc/MembershipManager.spec.ts | 5 +++-- src/matrixrtc/NewMembershipManager.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 1390f37305e..84275f4ebb3 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -342,6 +342,7 @@ describe.each([ it("does nothing if not joined !FailsForLegacy", async () => { const manager = new TestMembershipManager({}, room, client, () => undefined); await manager.leave(); + expect(async () => await manager.leave()).not.toThrow(); expect(client._unstable_sendDelayedStateEvent).not.toHaveBeenCalled(); expect(client.sendStateEvent).not.toHaveBeenCalled(); }); @@ -652,7 +653,7 @@ describe.each([ } expect(delayEventRestartError).toHaveBeenCalled(); }); - it("Errors while sending delayed events dont result in an unrecoverable error. We use the manager without delayed events !FailsForLegacy", async () => { + it("falls back to using pure state events when some error occurs while sending delayed events !FailsForLegacy", async () => { const unrecoverableError = jest.fn(); (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue(new HTTPError("unknown", 501)); const manager = new TestMembershipManager({}, room, client, () => undefined); @@ -662,7 +663,7 @@ describe.each([ expect(unrecoverableError).not.toHaveBeenCalledWith(); expect(client.sendStateEvent).toHaveBeenCalled(); }); - it("UnsupportedEndpointError does not in an unrecoverable error. We use the manager without delayed events !FailsForLegacy", async () => { + it("falls back to using pure state events when UnsupportedEndpointError encountered for delayed events !FailsForLegacy", async () => { const unrecoverableError = jest.fn(); (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue( new UnsupportedEndpointError("not supported", "sendDelayedStateEvent"), diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index aa13884402c..deff2419aff 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -27,7 +27,7 @@ import { type Focus } from "./focus.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { type MembershipConfig } from "./MatrixRTCSession.ts"; -const logger = rootLogger.getChild("MatrixRTCSessionManager"); +const logger = rootLogger.getChild("MatrixRTCSession"); /** * This interface defines what a MembershipManager uses and exposes. From 218c6d986b7c1961d6c26f20e377f8b341ad5a10 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 14:43:19 +0100 Subject: [PATCH 068/124] change to MatrixRTCSession logger --- spec/unit/matrixrtc/memberManagerTestEnvironment.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/matrixrtc/memberManagerTestEnvironment.ts b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts index 04a7381b29b..f6c6e18dd6d 100644 --- a/spec/unit/matrixrtc/memberManagerTestEnvironment.ts +++ b/spec/unit/matrixrtc/memberManagerTestEnvironment.ts @@ -29,7 +29,7 @@ It is very specific to the MembershipManager.spec.ts file and introduces the fol import { TestEnvironment } from "jest-environment-jsdom"; import { logger as rootLogger } from "../../../src/logger"; -const logger = rootLogger.getChild("MatrixRTCSessionManager"); +const logger = rootLogger.getChild("MatrixRTCSession"); class MemberManagerTestEnvironment extends TestEnvironment { handleTestEvent(event: any) { From 5c43bf2a5b30bbe5f4660698c54b18e6364f680d Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 14:54:46 +0100 Subject: [PATCH 069/124] review step 2 --- src/matrixrtc/NewMembershipManager.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index deff2419aff..90a70b1c24b 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -210,9 +210,10 @@ class ActionScheduler { private insertions: Action[] = []; private resetWith?: Action[]; /** - * This starts the main loop of the memberhsip manager that handles event sending, delayed event sending and delayed event restarting. + * This starts the main loop of the membership manager that handles event sending, delayed event sending and delayed event restarting. * @param initialActions The initial actions the manager will start with. It should be enough to pass: DelayedLeaveActionType.Initial - * @throws This throws an error only if the memberhsip cannot run anymore. For example it reached the maximum retires. + * @returns Promise that resolves once all actions have run and no more are scheduled. + * @throws This throws an error if one of the actions throws. * In most other error cases the manager will try to handle any server errors by itself. */ public async startWithActions(initialActions: Action[]): Promise { @@ -235,7 +236,7 @@ class ActionScheduler { try { await this.manager.membershipLoopHandler(this.state, nextAction.type as MembershipActionType); } catch (e) { - throw Error("The MemberhsipManager has to shut down because of the end condition: " + e); + throw Error("The MembershipManager has to shut down because of the end condition: " + e); } } @@ -279,7 +280,7 @@ class ActionScheduler { /** * This class takes care of the membership management. * It has the following tasks: - * - Send the users leave delayed event before sending the memberhsip + * - Send the users leave delayed event before sending the membership * - Sent the users membership if the state machine is started * - Check if the delayed event was canceled due to sending the membership * - update the delayed event (`restart`) @@ -287,7 +288,7 @@ class ActionScheduler { * - When the state machine is stopped: * - Disconnect the member * - Stop the timer for the delay refresh - * - Stop the timer for updateint the state event + * - Stop the timer for updating the state event */ export class MembershipManager implements IMembershipManager { // PUBLIC: @@ -300,7 +301,7 @@ export class MembershipManager implements IMembershipManager { * It will send delayed events and membership events * @param fociPreferred * @param focusActive - * @param onError This will be called once the membership menager encounters an unrecoverable error. + * @param onError This will be called once the membership manager encounters an unrecoverable error. * This should bubble up the the frontend to communicate that the call does not work in the current environment. */ public join(fociPreferred: Focus[], focusActive?: Focus, onError?: (error: unknown) => void): void { @@ -466,7 +467,7 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ) .then((response) => { - // Success we reset retires and set delayId. + // Success we reset retries and set delayId. state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); state.retries.set(MembershipActionType.SendFirstDelayedEvent, 0); state.delayId = response.delay_id; @@ -482,7 +483,7 @@ export class MembershipManager implements IMembershipManager { return; } if (this.unsupportedDelayedEndpoint(e)) { - logger.info("Not using deleayed event because the endpoint is not supported"); + logger.info("Not using delayed event because the endpoint is not supported"); this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent, @@ -493,7 +494,7 @@ export class MembershipManager implements IMembershipManager { }); // On any other error we fall back to not using delayed events and send the join state event immediately if (error) { - logger.info("Not using deleayed event because: " + error); + logger.info("Not using delayed event because: " + error); this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent, From 453b0cbd38764c6adfa874dc0438b0f3e3158259 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 15:01:01 +0100 Subject: [PATCH 070/124] make LoopHandler Private --- src/matrixrtc/NewMembershipManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 90a70b1c24b..4c76fb06aff 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -197,7 +197,7 @@ class ActionScheduler { } public constructor( state: ActionSchedulerState, - private manager: Pick, + private membershipLoopHandler: (state: ActionSchedulerState, type: MembershipActionType) => Promise, ) { this.state = state; } @@ -234,7 +234,7 @@ class ActionScheduler { this.didWakeUp = false; } else { try { - await this.manager.membershipLoopHandler(this.state, nextAction.type as MembershipActionType); + await this.membershipLoopHandler(this.state, nextAction.type as MembershipActionType); } catch (e) { throw Error("The MembershipManager has to shut down because of the end condition: " + e); } @@ -447,10 +447,10 @@ export class MembershipManager implements IMembershipManager { return 10; } // Scheduler: - private scheduler = new ActionScheduler(ActionScheduler.defaultState, this); + private scheduler = new ActionScheduler(ActionScheduler.defaultState, this.membershipLoopHandler.bind(this)); // Loop Handler: - public async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { + private async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { switch (type) { case MembershipActionType.SendFirstDelayedEvent: { // Before we start we check if we come from a state where we have a delay id. From 86b5a304cac89babb867850c6d57bb426e5cb1a6 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 15:34:34 +0100 Subject: [PATCH 071/124] update config to use NewManager wording --- src/matrixrtc/MatrixRTCSession.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 826eaca5248..3d01f1b2cc2 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -57,10 +57,9 @@ export type MatrixRTCSessionEventHandlerMap = { export interface MembershipConfig { /** - * Use Legacy Manager - * @deprecated + * Use the new Manager */ - useLegacyMembershipManager?: boolean; + useNewMembershipManager?: boolean; /** * The timeout (in milliseconds) after we joined the call, that our membership should expire @@ -334,12 +333,12 @@ export class MatrixRTCSession extends TypedEventEmitter + if (joinConfig?.useNewMembershipManager ?? false) { + this.membershipManager = new MembershipManager(joinConfig, this.room, this.client, () => this.getOldestMembership(), ); } else { - this.membershipManager = new MembershipManager(joinConfig, this.room, this.client, () => + this.membershipManager = new LegacyMembershipManager(joinConfig, this.room, this.client, () => this.getOldestMembership(), ); } From 66c1f8e95816ec77f1b2b7a991bfa98f0d426c91 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 15:45:40 +0100 Subject: [PATCH 072/124] emit error on rtc session if the membership manager encounters one --- src/matrixrtc/MatrixRTCSession.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 3d01f1b2cc2..8bb43e11cf5 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -40,6 +40,8 @@ export enum MatrixRTCSessionEvent { JoinStateChanged = "join_state_changed", // The key used to encrypt media has changed EncryptionKeyChanged = "encryption_key_changed", + /** The membership manager had to shut down caused by an unrecoverable error */ + MembershipManagerError = "membership_manager_error", } export type MatrixRTCSessionEventHandlerMap = { @@ -53,6 +55,7 @@ export type MatrixRTCSessionEventHandlerMap = { encryptionKeyIndex: number, participantId: string, ) => void; + [MatrixRTCSessionEvent.MembershipManagerError]: (error: unknown) => void; }; export interface MembershipConfig { @@ -346,8 +349,8 @@ export class MatrixRTCSession extends TypedEventEmitter { - // TODO: Consider exposing this as a signal from the RTCSession so it can be used in the UI. logger.error("MembershipManager encountered an unrecoverable error: ", e); + this.emit(MatrixRTCSessionEvent.MembershipManagerError, e); }); this.encryptionManager!.join(joinConfig); From fbb2a7067a99cd153a313c735f64ec6e36d32702 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 18:47:13 +0100 Subject: [PATCH 073/124] network error and throw refactor --- src/matrixrtc/NewMembershipManager.ts | 167 ++++++++++++++------------ 1 file changed, 90 insertions(+), 77 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 4c76fb06aff..a9b60ab008d 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -71,16 +71,16 @@ export interface IMembershipManager { /* SCHEDULER TYPES: - DirectMembershipManagerAction.Join - ▼ - ┌─────────────────────┐ - │SendFirstDelayedEvent│ - └─────────────────────┘ - │ - ▼ - ┌─────────────┐ - ┌────────────│SendJoinEvent│────────────┐ - │ └─────────────┘ │ + DirectMembershipManagerAction.Join + ▼ + ┌─────────────────────┐ + │SendFirstDelayedEvent│ + └─────────────────────┘ + │ + ▼ + ┌─────────────┐ + ┌────────────│SendJoinEvent│────────────┐ + │ └─────────────┘ │ │ ┌─────┐ ┌──────┐ │ ┌──────┐ ▼ ▼ │ │ ▼ ▼ ▼ │ ┌────────────┐ │ │ ┌───────────────────┐ │ @@ -94,17 +94,17 @@ export interface IMembershipManager { └───────────────────┬┘ │ │ │ └─────────────────────┘ - STOP ALL ABOVE - DirectMembershipManagerAction.Leave - ▼ - ┌───────────────────────────────┐ - │ SendScheduledDelayedLeaveEvent│ - └───────────────────────────────┘ - │ - ▼ - ┌──────────────┐ - │SendLeaveEvent│ - └──────────────┘ + STOP ALL ABOVE + DirectMembershipManagerAction.Leave + ▼ + ┌───────────────────────────────┐ + │ SendScheduledDelayedLeaveEvent│ + └───────────────────────────────┘ + │ + ▼ + ┌──────────────┐ + │SendLeaveEvent│ + └──────────────┘ */ enum MembershipActionType { SendFirstDelayedEvent = "SendFirstDelayedEvent", @@ -456,7 +456,7 @@ export class MembershipManager implements IMembershipManager { // Before we start we check if we come from a state where we have a delay id. if (!state.delayId) { // Normal case without any previous delayed id. - const error = await this.client + await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, { @@ -490,21 +490,21 @@ export class MembershipManager implements IMembershipManager { }); return; } - return e; - }); - // On any other error we fall back to not using delayed events and send the join state event immediately - if (error) { - logger.info("Not using delayed event because: " + error); - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendJoinEvent, + + if (this.retryOnNetworkError(e, type)) return; + + logger.info("Not using delayed event because: " + e.getMessage()); + // On any other error we fall back to not using delayed events and send the join state event immediately + this.scheduler.addAction({ + ts: Date.now(), + type: MembershipActionType.SendJoinEvent, + }); }); - } } else { // Restart case with delayed id. // Remove all running updates and restarts this.scheduler.resetActions([]); - const error = await this.client + await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel) .then(() => { state.delayId = undefined; @@ -533,18 +533,17 @@ export class MembershipManager implements IMembershipManager { }); return; } - return e; + + if (this.retryOnNetworkError(e, type)) return; + + // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases + // when state.delayId !== undefined + // We do not use ignore and log this error since we would also need to reset the delayId. + // It is cleaner if we the frontend rejoines instead of resetting the delayId here and behaving like in the success case. + throw Error( + "We failed to cancel a delayed event where we already had a delay id with an error we cannot automatically handle", + ); }); - if (error) { - // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases - // when state.delayId !== undefined - // We do not use ignore and log this error since we would also need to reset the delayId. - // It is cleaner if we the frontend rejoines instead of resetting the delayId here and behaving like in the success case. - throw Error( - "We failed to cancel a delayed event where we already had a delay id with an error we cannot automatically handle" + - error, - ); - } } break; } @@ -559,7 +558,7 @@ export class MembershipManager implements IMembershipManager { }); break; } - const error = await this.client + await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart) .then(() => { this.scheduler.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent); @@ -577,18 +576,21 @@ export class MembershipManager implements IMembershipManager { }); return; } - // TODO this also needs a test: get rate limit while checking id delayed event is scheduled - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; // If the HS does not support delayed events we wont reschedule. if (this.unsupportedDelayedEndpoint(e)) return; + + // TODO this also needs a test: get rate limit while checking id delayed event is scheduled + if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + + if (this.retryOnNetworkError(e, type)) return; + // In other error cases we have no idea what is happening - return Error("Could not restart delayed event, even though delayed events are supported. " + e); + throw Error("Could not restart delayed event, even though delayed events are supported. " + e); }); - if (error) throw error; break; } case MembershipActionType.SendMainDelayedEvent: { - const error = await this.client + await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, { @@ -618,15 +620,14 @@ export class MembershipManager implements IMembershipManager { // Don't do any other delayed event work if its not supported. if (this.unsupportedDelayedEndpoint(e)) return; // after that - if (this.retryOnAnyErrorHandler(e, type)) return; - return Error("Could not send delayed event, even though delayed events are supported. " + e); + if (this.retryOnNetworkError(e, type)) return; + throw Error("Could not send delayed event, even though delayed events are supported. " + e); }); - if (error) throw error; break; } case MembershipActionType.SendScheduledDelayedLeaveEvent: { if (state.delayId) { - const error = await this.client + await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send) .then(() => { state.hasMemberStateEvent = false; @@ -642,17 +643,18 @@ export class MembershipManager implements IMembershipManager { } if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; if (this.unsupportedDelayedEndpoint(e)) return; - return e; + if (this.retryOnNetworkError(e, type)) return; + + // On any other error we fall back to SendLeaveEvent (this includes hard errors from rate limiting) + this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); }); - // On any other error we fall back to SendLeaveEvent (this includes hard errors from rate limiting) - if (error) this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } else { this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); } break; } case MembershipActionType.SendJoinEvent: { - const error = await this.client + await this.client .sendStateEvent( this.room.roomId, EventType.GroupCallMemberPrefix, @@ -675,15 +677,14 @@ export class MembershipManager implements IMembershipManager { if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; // Event sending retry (different to rate limit retries) - if (this.retryOnAnyErrorHandler(e, type)) return; + if (this.retryOnNetworkError(e, type)) return; - return Error("Could not send state event because of unrecoverable error: " + e); + throw e; }); - if (error) throw error; break; } case MembershipActionType.UpdateExpiry: { - const error = await this.client + await this.client .sendStateEvent( this.room.roomId, EventType.GroupCallMemberPrefix, @@ -704,13 +705,10 @@ export class MembershipManager implements IMembershipManager { // TODO add timeout/netowrk error (or just the below) // Event sending retry (different to rate limit retries) - if (this.retryOnAnyErrorHandler(e, type)) return; + if (this.retryOnNetworkError(e, type)) return; - return Error( - "Could not update state event with new expiry ts because of unrecoverable error: " + e, - ); + throw e; }); - if (error) throw error; break; } case MembershipActionType.SendLeaveEvent: { @@ -721,7 +719,7 @@ export class MembershipManager implements IMembershipManager { } // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event - const error = await this.client + await this.client .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) .then(() => { this.scheduler.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); @@ -733,11 +731,10 @@ export class MembershipManager implements IMembershipManager { if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; // Event sending retry (different to rate limit retries) - if (this.retryOnAnyErrorHandler(e, type)) return; + if (this.retryOnNetworkError(e, type)) return; - return Error("Failed to send Leave event because of: " + e); + throw e; }); - if (error) throw error; break; } } @@ -806,8 +803,12 @@ export class MembershipManager implements IMembershipManager { * @returns Returns true if handled the error and rescheduled the correct next action did anything. */ private rateLimitErrorHandler(e: unknown, method: string, type: MembershipActionType): boolean | Error { + if (!(e instanceof HTTPError && e.isRateLimitError())) { + return false; + } + const rateLimitRetries = this.scheduler.state.rateLimitRetries.get(type) ?? 0; - if (rateLimitRetries < this.maximumRateLimitRetryCount && e instanceof HTTPError && e.isRateLimitError()) { + if (rateLimitRetries < this.maximumRateLimitRetryCount) { let resendDelay: number; const defaultMs = 5000; try { @@ -824,29 +825,41 @@ export class MembershipManager implements IMembershipManager { this.scheduler.addAction({ ts: Date.now() + resendDelay, type }); return true; - } else if (e instanceof HTTPError && e.isRateLimitError()) { - throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + e.message); } - return false; + throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + e.message); } /** - * Don't Check the error and retry the same MembershipAction again in the configured time and for the configured retry count. + * FIXME Don't Check the error and retry the same MembershipAction again in the configured time and for the configured retry count. * @param e the error causing this handler check/execution * @param type the action type that we need to repeat because of the error * @returns Returns true if we handled the error by rescheduling the correct next action. */ - private retryOnAnyErrorHandler(e: unknown, type: MembershipActionType): boolean { + private retryOnNetworkError(e: unknown, type: MembershipActionType): boolean { + return false; + /* + handle HTTPError with status = 5xx || ConnectionError || AbortError; const retries = this.scheduler.state.retries.get(type) ?? 0; - if (retries < this.maximumRetryCount) { + + if (e instanceof HTTPError && typeof e.httpStatus === "number" && e.httpStatus >= 500 && e.httpStatus < 600) { + logger.warn("Network error while sending state event, retrying in 5s", e); this.scheduler.state.retries.set(type, retries + 1); this.scheduler.addAction({ ts: Date.now() + this.callMemberEventRetryDelayMinimum, type }); + return true; + } + + // timeout + + if (retries < this.maximumRetryCount) { + this.scheduler.state.retries.set(type, retries + 1); + return true; } else { throw Error("Reached maximum (" + this.maximumRetryCount + ") retries cause by: " + e); } + */ } /** From 177e2f4f8c313e13efabe3b31dfa6afa0a901714 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 18:48:13 +0100 Subject: [PATCH 074/124] make accessing the full room deprecated --- src/matrixrtc/MatrixRTCSession.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index cd88d3f2de2..64d19eafd9e 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -230,6 +230,15 @@ export class MatrixRTCSession extends TypedEventEmitter, - public readonly room: Pick, + private roomSubset: Pick, public memberships: CallMembership[], ) { super(); From 9e60a3669edd6577b34b14bea39979b91a8a22ff Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 26 Feb 2025 18:49:33 +0100 Subject: [PATCH 075/124] remove deprecated usage of full room --- src/matrixrtc/MatrixRTCSession.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 64d19eafd9e..8e8377c8959 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -266,13 +266,13 @@ export class MatrixRTCSession extends TypedEventEmitter this.memberships, (keyBin: Uint8Array, encryptionKeyIndex: number, participantId: string) => { this.emit(MatrixRTCSessionEvent.EncryptionKeyChanged, keyBin, encryptionKeyIndex, participantId); @@ -297,7 +297,7 @@ export class MatrixRTCSession extends TypedEventEmitter + this.membershipManager = new LegacyMembershipManager(joinConfig, this.roomSubset, this.client, () => this.getOldestMembership(), ); } @@ -345,11 +345,11 @@ export class MatrixRTCSession extends TypedEventEmitter { if (!this.isJoined()) { - logger.info(`Not joined to session in room ${this.room.roomId}: ignoring leave call`); + logger.info(`Not joined to session in room ${this.roomSubset.roomId}: ignoring leave call`); return false; } - logger.info(`Leaving call session in room ${this.room.roomId}`); + logger.info(`Leaving call session in room ${this.roomSubset.roomId}`); this.encryptionManager.leave(); @@ -489,7 +489,7 @@ export class MatrixRTCSession extends TypedEventEmitter !CallMembership.equal(m, this.memberships[i])); if (changed) { - logger.info(`Memberships for call in room ${this.room.roomId} have changed: emitting`); + logger.info(`Memberships for call in room ${this.roomSubset.roomId} have changed: emitting`); this.emit(MatrixRTCSessionEvent.MembershipsChanged, oldMemberships, this.memberships); void this.membershipManager?.onRTCSessionMemberUpdate(this.memberships); From 74f76b44b522ba8fb141e45f2a0d281445d87f07 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 27 Feb 2025 08:41:10 +0000 Subject: [PATCH 076/124] Clean up the deprecation --- src/matrixrtc/MatrixRTCSession.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 8e8377c8959..d9879f6f820 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -233,12 +233,14 @@ export class MatrixRTCSession extends TypedEventEmitter Date: Thu, 27 Feb 2025 11:00:01 +0100 Subject: [PATCH 077/124] add network error handler and cleanup --- src/matrixrtc/NewMembershipManager.ts | 104 ++++++++++++++------------ 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index a9b60ab008d..aee3921e9b7 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -158,7 +158,7 @@ interface ActionSchedulerState { /** Retry counter for rate limits */ rateLimitRetries: Map; /** Retry counter for other errors */ - retries: Map; + networkErrorRetries: Map; } interface Action { @@ -191,7 +191,7 @@ class ActionScheduler { startTime: 0, delayId: undefined, rateLimitRetries: new Map(), - retries: new Map(), + networkErrorRetries: new Map(), expireUpdateIterations: 0, }; } @@ -273,7 +273,7 @@ class ActionScheduler { } public resetRateLimitCounter(type: MembershipActionType): void { this.state.rateLimitRetries.set(type, 0); - this.state.retries.set(type, 0); + this.state.networkErrorRetries.set(type, 0); } } @@ -469,7 +469,7 @@ export class MembershipManager implements IMembershipManager { .then((response) => { // Success we reset retries and set delayId. state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); - state.retries.set(MembershipActionType.SendFirstDelayedEvent, 0); + state.networkErrorRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); state.delayId = response.delay_id; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); }) @@ -768,51 +768,56 @@ export class MembershipManager implements IMembershipManager { /** * Check if its a NOT_FOUND error - * @param e the error causing this handler check/execution + * @param error the error causing this handler check/execution * @returns true if its a not found error */ - private notFoundError(e: unknown): boolean { - return e instanceof MatrixError && e.errcode === "M_NOT_FOUND"; + private notFoundError(error: unknown): boolean { + return error instanceof MatrixError && error.errcode === "M_NOT_FOUND"; } /** * Check if this is a DelayExceeded timeout and update the TimeoutOverride for the next try - * @param e the error causing this handler check/execution + * @param error the error causing this handler check/execution * @returns true if its a delay exceeded error and we updated the local TimeoutOverride */ - private maxDelayExceededErrorHandler(e: unknown): boolean { + private maxDelayExceededErrorHandler(error: unknown): boolean { if ( - e instanceof MatrixError && - e.errcode === "M_UNKNOWN" && - e.data["org.matrix.msc4140.errcode"] === "M_MAX_DELAY_EXCEEDED" + error instanceof MatrixError && + error.errcode === "M_UNKNOWN" && + error.data["org.matrix.msc4140.errcode"] === "M_MAX_DELAY_EXCEEDED" ) { - const maxDelayAllowed = e.data["org.matrix.msc4140.max_delay"]; + const maxDelayAllowed = error.data["org.matrix.msc4140.max_delay"]; if (typeof maxDelayAllowed === "number" && this.membershipServerSideExpiryTimeout > maxDelayAllowed) { this.membershipServerSideExpiryTimeoutOverride = maxDelayAllowed; } - logger.warn("Retry sending delayed disconnection event due to server timeout limitations:", e); + logger.warn("Retry sending delayed disconnection event due to server timeout limitations:", error); return true; } return false; } + /** * Check if we have a rate limit error and schedule the same action again if we dont exceed the rate limit retry count yet. - * @param e the error causing this handler check/execution + * @param error the error causing this handler check/execution * @param method the method used for the throw message * @param type which MembershipActionType we reschedule because of a rate limit. - * @returns Returns true if handled the error and rescheduled the correct next action did anything. + * @throws If it is a rate limit error and the retry count got exceeded + * @returns Returns true if we handled the error by rescheduling the correct next action. + * Returns false if it is not a network error. */ - private rateLimitErrorHandler(e: unknown, method: string, type: MembershipActionType): boolean | Error { - if (!(e instanceof HTTPError && e.isRateLimitError())) { + private rateLimitErrorHandler(error: unknown, method: string, type: MembershipActionType): boolean { + // "Is rate limit"-boundary + if (!(error instanceof HTTPError && error.isRateLimitError())) { return false; } + // retry boundary const rateLimitRetries = this.scheduler.state.rateLimitRetries.get(type) ?? 0; if (rateLimitRetries < this.maximumRateLimitRetryCount) { let resendDelay: number; const defaultMs = 5000; try { - resendDelay = e.getRetryAfterMs() ?? defaultMs; + resendDelay = error.getRetryAfterMs() ?? defaultMs; logger.info(`Rate limited by server, retrying in ${resendDelay}ms`); } catch (e) { logger.warn( @@ -823,51 +828,56 @@ export class MembershipManager implements IMembershipManager { } this.scheduler.state.rateLimitRetries.set(type, rateLimitRetries + 1); this.scheduler.addAction({ ts: Date.now() + resendDelay, type }); - return true; } - throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + e.message); + + // Failiour + throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + error.message); } /** * FIXME Don't Check the error and retry the same MembershipAction again in the configured time and for the configured retry count. - * @param e the error causing this handler check/execution + * @param error the error causing this handler check/execution * @param type the action type that we need to repeat because of the error - * @returns Returns true if we handled the error by rescheduling the correct next action. + * @throws If it is a network error and the retry count got exceeded + * @returns + * Returns true if we handled the error by rescheduling the correct next action. + * Returns false if it is not a network error. */ - private retryOnNetworkError(e: unknown, type: MembershipActionType): boolean { - return false; - /* - handle HTTPError with status = 5xx || ConnectionError || AbortError; - const retries = this.scheduler.state.retries.get(type) ?? 0; - - - if (e instanceof HTTPError && typeof e.httpStatus === "number" && e.httpStatus >= 500 && e.httpStatus < 600) { - logger.warn("Network error while sending state event, retrying in 5s", e); - this.scheduler.state.retries.set(type, retries + 1); - this.scheduler.addAction({ ts: Date.now() + this.callMemberEventRetryDelayMinimum, type }); - - return true; + private retryOnNetworkError(error: unknown, type: MembershipActionType): boolean { + // "Is a network error"-boundary + const retryDurationString = this.callMemberEventRetryDelayMinimum / 1000 + "s"; + if (error instanceof Error && error.name === "AbortError") { + logger.warn("Network local timeout error while sending event, retrying in " + retryDurationString, error); + } else if ( + error instanceof HTTPError && + typeof error.httpStatus === "number" && + error.httpStatus >= 500 && + error.httpStatus < 600 + ) { + logger.warn("Network error while sending event, retrying in " + retryDurationString, error); + } else { + return false; } - // timeout - + // retry boundary + const retries = this.scheduler.state.networkErrorRetries.get(type) ?? 0; if (retries < this.maximumRetryCount) { - this.scheduler.state.retries.set(type, retries + 1); - + this.scheduler.state.networkErrorRetries.set(type, retries + 1); + this.scheduler.addAction({ ts: Date.now() + this.callMemberEventRetryDelayMinimum, type }); return true; - } else { - throw Error("Reached maximum (" + this.maximumRetryCount + ") retries cause by: " + e); } - */ + + // Failiour + throw Error("Reached maximum (" + this.maximumRetryCount + ") retries cause by: " + error); } /** * Check if its an UnsupportedEndpointError and which implies that we cannot do any delayed event logic - * @param e The error to check - * @returns true it its a UnsupportedEndpointError + * @param error The error to check + * @returns true it its an UnsupportedEndpointError */ - private unsupportedDelayedEndpoint(e: unknown): boolean { - return e instanceof UnsupportedEndpointError; + private unsupportedDelayedEndpoint(error: unknown): boolean { + return error instanceof UnsupportedEndpointError; } } From 631de6ef3020988c1606c9a76166999b77d766e8 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 27 Feb 2025 12:07:20 +0100 Subject: [PATCH 078/124] better logging, another test, make maximumNetworkErrorRetryCount configurable --- spec/unit/matrixrtc/MembershipManager.spec.ts | 25 ++++++++- src/matrixrtc/MatrixRTCSession.ts | 6 ++ src/matrixrtc/NewMembershipManager.ts | 55 ++++++++++++------- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 84275f4ebb3..91f9563de95 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -655,14 +655,33 @@ describe.each([ }); it("falls back to using pure state events when some error occurs while sending delayed events !FailsForLegacy", async () => { const unrecoverableError = jest.fn(); - (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue(new HTTPError("unknown", 501)); + (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue(new HTTPError("unknown", 601)); const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive, unrecoverableError); - await jest.advanceTimersByTimeAsync(1); - + await waitForMockCall(client.sendStateEvent); expect(unrecoverableError).not.toHaveBeenCalledWith(); expect(client.sendStateEvent).toHaveBeenCalled(); }); + it("retries before failing in case its a network error !FailsForLegacy", async () => { + const unrecoverableError = jest.fn(); + (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue(new HTTPError("unknown", 501)); + const manager = new TestMembershipManager( + { callMemberEventRetryDelayMinimum: 1000, maximumNetworkErrorRetryCount: 7 }, + room, + client, + () => undefined, + ); + manager.join([focus], focusActive, unrecoverableError); + for (let retries = 0; retries < 7; retries++) { + expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(retries + 1); + await jest.advanceTimersByTimeAsync(1000); + } + expect(unrecoverableError).toHaveBeenCalled(); + expect(unrecoverableError.mock.lastCall![0].message).toMatch( + "The MembershipManager has to shut down because of the end condition: Error: Reached maximum", + ); + expect(client.sendStateEvent).not.toHaveBeenCalled(); + }); it("falls back to using pure state events when UnsupportedEndpointError encountered for delayed events !FailsForLegacy", async () => { const unrecoverableError = jest.fn(); (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue( diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 1cd36116b3c..d6bb4b9f8ac 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -110,10 +110,16 @@ export interface MembershipConfig { * @deprecated It should be possible to make it stable without this. */ callMemberEventRetryJitter?: number; + /** * The maximum number of retries that the manager will do for delayed event sending/updating and state event sending when a server rate limit has been hit. */ maximumRateLimitRetryCount?: number; + + /** + * The maximum number of retries that the manager will do for delayed event sending/updating and state event sending when a network error occurs. + */ + maximumNetworkErrorRetryCount?: number; } export interface EncryptionConfig { diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index aee3921e9b7..621a6da8fe6 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -312,7 +312,9 @@ export class MembershipManager implements IMembershipManager { this.scheduler.state.running = true; this.scheduler .startWithActions([{ ts: Date.now(), type: DirectMembershipManagerAction.Join }]) - .catch((e) => onError?.(e)); + .catch((e) => { + onError?.(e); + }); } } @@ -442,10 +444,10 @@ export class MembershipManager implements IMembershipManager { private get maximumRateLimitRetryCount(): number { return this.joinConfig?.maximumRateLimitRetryCount ?? 10; } - private get maximumRetryCount(): number { - // TODO allow configuring this via `MembershipConfig`. - return 10; + private get maximumNetworkErrorRetryCount(): number { + return this.joinConfig?.maximumNetworkErrorRetryCount ?? 10; } + // Scheduler: private scheduler = new ActionScheduler(ActionScheduler.defaultState, this.membershipLoopHandler.bind(this)); @@ -482,18 +484,14 @@ export class MembershipManager implements IMembershipManager { }); return; } + if (this.retryOnNetworkError(e, type)) return; + + // log and fall through if (this.unsupportedDelayedEndpoint(e)) { logger.info("Not using delayed event because the endpoint is not supported"); - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendJoinEvent, - }); - return; + } else { + logger.info("Not using delayed event because: " + e); } - - if (this.retryOnNetworkError(e, type)) return; - - logger.info("Not using delayed event because: " + e.getMessage()); // On any other error we fall back to not using delayed events and send the join state event immediately this.scheduler.addAction({ ts: Date.now(), @@ -501,7 +499,13 @@ export class MembershipManager implements IMembershipManager { }); }); } else { - // Restart case with delayed id. + // This can happen if someone else (or another client) removes our own membership event. + // It will trigger `onRTCSessionMemberUpdate` queue `MembershipActionType.SendFirstDelayedEvent`. + // We might still have our delyed event from the previous participation and dependent on the serve this might not + // get automatically removed if the state changes. Hence It would remove our membership unexpectedly shortly after the rejoin. + // + // In this block we will try to cancel this delayed event before setting up a new one. + // Remove all running updates and restarts this.scheduler.resetActions([]); await this.client @@ -702,7 +706,6 @@ export class MembershipManager implements IMembershipManager { }) .catch((e) => { if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; - // TODO add timeout/netowrk error (or just the below) // Event sending retry (different to rate limit retries) if (this.retryOnNetworkError(e, type)) return; @@ -832,7 +835,7 @@ export class MembershipManager implements IMembershipManager { } // Failiour - throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + error.message); + throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + (error as Error)); } /** @@ -846,30 +849,40 @@ export class MembershipManager implements IMembershipManager { */ private retryOnNetworkError(error: unknown, type: MembershipActionType): boolean { // "Is a network error"-boundary + const retries = this.scheduler.state.networkErrorRetries.get(type) ?? 0; const retryDurationString = this.callMemberEventRetryDelayMinimum / 1000 + "s"; + const retryCounterString = "(" + retries + "/" + this.maximumNetworkErrorRetryCount + ")"; if (error instanceof Error && error.name === "AbortError") { - logger.warn("Network local timeout error while sending event, retrying in " + retryDurationString, error); + logger.warn( + "Network local timeout error while sending event, retrying in " + + retryDurationString + + " " + + retryCounterString, + error, + ); } else if ( error instanceof HTTPError && typeof error.httpStatus === "number" && error.httpStatus >= 500 && error.httpStatus < 600 ) { - logger.warn("Network error while sending event, retrying in " + retryDurationString, error); + logger.warn( + "Network error while sending event, retrying in " + retryDurationString + " " + retryCounterString, + error, + ); } else { return false; } // retry boundary - const retries = this.scheduler.state.networkErrorRetries.get(type) ?? 0; - if (retries < this.maximumRetryCount) { + if (retries < this.maximumNetworkErrorRetryCount) { this.scheduler.state.networkErrorRetries.set(type, retries + 1); this.scheduler.addAction({ ts: Date.now() + this.callMemberEventRetryDelayMinimum, type }); return true; } // Failiour - throw Error("Reached maximum (" + this.maximumRetryCount + ") retries cause by: " + error); + throw Error("Reached maximum (" + this.maximumNetworkErrorRetryCount + ") retries cause by: " + error); } /** From 4b459319b6ca8cb47acb2226ac8b32cddfd69a86 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 27 Feb 2025 12:50:44 +0100 Subject: [PATCH 079/124] more logging & refactor leave promise --- src/matrixrtc/NewMembershipManager.ts | 49 ++++++++++++++------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 621a6da8fe6..9e44e67aaf3 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -21,31 +21,30 @@ import { UnsupportedEndpointError } from "../errors.ts"; import { HTTPError, MatrixError } from "../http-api/errors.ts"; import { logger as rootLogger } from "../logger.ts"; import { type Room } from "../models/room.ts"; -import { sleep } from "../utils.ts"; +import { defer, type IDeferred, sleep } from "../utils.ts"; import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts"; import { type Focus } from "./focus.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { type MembershipConfig } from "./MatrixRTCSession.ts"; const logger = rootLogger.getChild("MatrixRTCSession"); - /** * This interface defines what a MembershipManager uses and exposes. - * This interface is what we use to write tests and allows to change the actual implementation - * Without breaking tests because of some internal method renaming. + * This interface is what we use to write tests and allows changing the actual implementation + * without breaking tests because of some internal method renaming. * * @internal */ export interface IMembershipManager { /** * If we are trying to join the session. - * It does not reflect if the room state is already configures to represent us being joined. + * It does not reflect if the room state is already configured to represent us being joined. * It only means that the Manager is running. * @returns true if we intend to be participating in the MatrixRTC session */ isJoined(): boolean; /** - * Start sending all necessary events to make this user participant in the RTC session. + * Start sending all necessary events to make this user participate in the RTC session. * @param fociPreferred the list of preferred foci to use in the joined RTC membership event. * @param fociActive the active focus to use in the joined RTC membership event. * @throws can throw if it exceeds a configured maximum retry. @@ -221,7 +220,6 @@ class ActionScheduler { while (this.actions.length > 0) { this.actions.sort((a, b) => a.ts - b.ts); - logger.debug("Current MembershipManager action queue: ", this.actions, "\nDate.now: ", +Date.now()); const nextAction = this.actions[0]; this.wakeupPromise = new Promise((resolve) => { @@ -233,6 +231,12 @@ class ActionScheduler { // Instead we recompute the actions array and do another iteration. this.didWakeUp = false; } else { + logger.debug( + "Current MembershipManager processing: " + nextAction.type, + "\nQueue:", + this.actions, + "\nDate.now: " + Date.now(), + ); try { await this.membershipLoopHandler(this.state, nextAction.type as MembershipActionType); } catch (e) { @@ -249,6 +253,7 @@ class ActionScheduler { this.actions.push(...this.insertions); this.insertions = []; } + logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); } public addAction(action: Action): void { @@ -307,12 +312,14 @@ export class MembershipManager implements IMembershipManager { public join(fociPreferred: Focus[], focusActive?: Focus, onError?: (error: unknown) => void): void { this.fociPreferred = fociPreferred; this.focusActive = focusActive; + this.leavePromiseDefer = undefined; if (!this.scheduler.state.running) { this.scheduler.resetState(); this.scheduler.state.running = true; this.scheduler .startWithActions([{ ts: Date.now(), type: DirectMembershipManagerAction.Join }]) .catch((e) => { + logger.error("MembershipManager stopped because: ", e); onError?.(e); }); } @@ -327,24 +334,15 @@ export class MembershipManager implements IMembershipManager { if (!this.scheduler.state.running) return Promise.resolve(true); this.scheduler.state.running = false; - if (!this.leavePromise) { + if (!this.leavePromiseDefer) { // reset scheduled actions so we will not do any new actions. this.scheduler.resetActions([{ type: DirectMembershipManagerAction.Leave, ts: Date.now() }]); - this.leavePromise = new Promise((resolve, reject) => { - this.leavePromiseHandle.reject = reject; - this.leavePromiseHandle.resolve = resolve; - if (timeout) setTimeout(() => resolve(false), timeout); - }); + this.leavePromiseDefer = defer(); + if (timeout) setTimeout(() => this.leavePromiseDefer?.resolve(false), timeout); } - - return this.leavePromise; + return this.leavePromiseDefer.promise; } - - private leavePromise?: Promise; - private leavePromiseHandle: { - reject?: (reason: any) => void; - resolve?: (didSendLeaveEvent: boolean) => void; - } = {}; + private leavePromiseDefer?: IDeferred; public async onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { const isMyMembership = (m: CallMembership): boolean => @@ -637,7 +635,8 @@ export class MembershipManager implements IMembershipManager { state.hasMemberStateEvent = false; this.scheduler.resetRateLimitCounter(MembershipActionType.SendScheduledDelayedLeaveEvent); this.scheduler.resetActions([]); - this.leavePromiseHandle.resolve?.(true); + this.leavePromiseDefer?.resolve(true); + this.leavePromiseDefer = undefined; }) .catch((e) => { if (this.notFoundError(e)) { @@ -717,7 +716,8 @@ export class MembershipManager implements IMembershipManager { case MembershipActionType.SendLeaveEvent: { // We are good already if (!state.hasMemberStateEvent) { - this.leavePromiseHandle.resolve?.(true); + this.leavePromiseDefer?.resolve(true); + this.leavePromiseDefer = undefined; return; } // This is only a fallback in case we do not have working delayed events support. @@ -727,7 +727,8 @@ export class MembershipManager implements IMembershipManager { .then(() => { this.scheduler.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); this.scheduler.resetActions([]); - this.leavePromiseHandle.resolve?.(true); + this.leavePromiseDefer?.resolve(true); + this.leavePromiseDefer = undefined; state.hasMemberStateEvent = false; }) .catch((e) => { From 7e20f1164dc4b532df6e71c135be07e7cb561ce8 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 27 Feb 2025 13:14:12 +0100 Subject: [PATCH 080/124] add ConnectionError as possible retry cause --- src/matrixrtc/NewMembershipManager.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 9e44e67aaf3..74a5d2a037a 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -18,7 +18,7 @@ import { EventType } from "../@types/event.ts"; import { UpdateDelayedEventAction } from "../@types/requests.ts"; import type { MatrixClient } from "../client.ts"; import { UnsupportedEndpointError } from "../errors.ts"; -import { HTTPError, MatrixError } from "../http-api/errors.ts"; +import { ConnectionError, HTTPError, MatrixError } from "../http-api/errors.ts"; import { logger as rootLogger } from "../logger.ts"; import { type Room } from "../models/room.ts"; import { defer, type IDeferred, sleep } from "../utils.ts"; @@ -861,6 +861,14 @@ export class MembershipManager implements IMembershipManager { retryCounterString, error, ); + } else if (error instanceof ConnectionError) { + logger.warn( + "Network connection error while sending event, retrying in " + + retryDurationString + + " " + + retryCounterString, + error, + ); } else if ( error instanceof HTTPError && typeof error.httpStatus === "number" && @@ -883,7 +891,9 @@ export class MembershipManager implements IMembershipManager { } // Failiour - throw Error("Reached maximum (" + this.maximumNetworkErrorRetryCount + ") retries cause by: " + error); + throw Error( + "Reached maximum (" + this.maximumNetworkErrorRetryCount + ") retries cause by: " + (error as Error), + ); } /** From ec6a2586efad1a59d225d5e8f5588f9430fa4ed0 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 27 Feb 2025 18:11:22 +0100 Subject: [PATCH 081/124] Make it work in embedded mode with a server that does not support delayed events --- spec/unit/matrixrtc/MembershipManager.spec.ts | 7 ++++++- src/embedded.ts | 9 ++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 741dcbf4a77..9bebfdd6f95 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -246,7 +246,12 @@ describe.each([ const delayedHandle = createAsyncHandle(client._unstable_sendDelayedStateEvent as Mock); const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); - delayedHandle.reject?.(Error("Server does not support the delayed events API")); + delayedHandle.reject?.( + new UnsupportedEndpointError( + "Server does not support the delayed events API", + "sendDelayedStateEvent", + ), + ); expect(client._unstable_sendDelayedStateEvent).toHaveBeenCalledTimes(1); }); it("does try to schedule a delayed leave event again if rate limited", async () => { diff --git a/src/embedded.ts b/src/embedded.ts index 7dc617af119..e80548b9087 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -55,7 +55,7 @@ import { User } from "./models/user.ts"; import { type Room } from "./models/room.ts"; import { type ToDeviceBatch, type ToDevicePayload } from "./models/ToDeviceMessage.ts"; import { MapWithDefault, recursiveMapToObject } from "./utils.ts"; -import { type EmptyObject, TypedEventEmitter } from "./matrix.ts"; +import { type EmptyObject, TypedEventEmitter, UnsupportedEndpointError } from "./matrix.ts"; interface IStateEventRequest { eventType: string; @@ -416,7 +416,10 @@ export class RoomWidgetClient extends MatrixClient { stateKey = "", ): Promise { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw Error("Server does not support the delayed events API"); + throw new UnsupportedEndpointError( + "Server does not support the delayed events API", + "sendDelayedStateEvent", + ); } const response = await this.widgetApi.sendStateEvent( @@ -443,7 +446,7 @@ export class RoomWidgetClient extends MatrixClient { // eslint-disable-next-line public async _unstable_updateDelayedEvent(delayId: string, action: UpdateDelayedEventAction): Promise { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw Error("Server does not support the delayed events API"); + throw new UnsupportedEndpointError("Server does not support the delayed events API", "updateDelayedEvent"); } await this.widgetApi.updateDelayedEvent(delayId, action); From 5d69555b5e03d5cff7b57e20a788dba3c191b973 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 27 Feb 2025 19:43:21 +0100 Subject: [PATCH 082/124] review iteration 1 --- src/matrixrtc/NewMembershipManager.ts | 51 ++++++++++++++------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 74a5d2a037a..db1084069e5 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -37,7 +37,7 @@ const logger = rootLogger.getChild("MatrixRTCSession"); */ export interface IMembershipManager { /** - * If we are trying to join the session. + * If we are trying to join, or have successfully joined the session. * It does not reflect if the room state is already configured to represent us being joined. * It only means that the Manager is running. * @returns true if we intend to be participating in the MatrixRTC session @@ -196,14 +196,13 @@ class ActionScheduler { } public constructor( state: ActionSchedulerState, + /** This is the callback called for each scheduled action (`this.addAction()`) */ private membershipLoopHandler: (state: ActionSchedulerState, type: MembershipActionType) => Promise, ) { this.state = state; } - // state variables for a wakeup mechanism (in case we add some action externally and need to leave the current sleep) - private wakeupPromise?: Promise; + // function for the wakeup mechanism (in case we add an action externally and need to leave the current sleep) private wakeup?: (value: void | PromiseLike) => void; - private didWakeUp = false; private actions: Action[] = []; private insertions: Action[] = []; @@ -221,16 +220,15 @@ class ActionScheduler { while (this.actions.length > 0) { this.actions.sort((a, b) => a.ts - b.ts); const nextAction = this.actions[0]; - - this.wakeupPromise = new Promise((resolve) => { - this.wakeup = resolve; + let didWakeUp = false; + const wakeupPromise = new Promise((resolve) => { + this.wakeup = (): void => { + didWakeUp = true; + resolve(); + }; }); - if (nextAction.ts > Date.now()) await Promise.race([this.wakeupPromise, sleep(nextAction.ts - Date.now())]); - if (this.didWakeUp) { - // In case of a wakeup we do not want to run the next action because the next action now might be sth different. - // Instead we recompute the actions array and do another iteration. - this.didWakeUp = false; - } else { + if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); + if (!didWakeUp) { logger.debug( "Current MembershipManager processing: " + nextAction.type, "\nQueue:", @@ -260,16 +258,14 @@ class ActionScheduler { this.insertions.push(action); const nextTs = this.actions[0]?.ts; if (!nextTs || nextTs > action.ts) { - this.didWakeUp = true; this.wakeup?.(); } } public resetActions(actions: Action[]): void { this.resetWith = actions; const nextTs = this.actions[0]?.ts; - const newestTs = actions.map((a) => a.ts).sort((a, b) => a - b)[0]; + const newestTs = this.resetWith.map((a) => a.ts).sort((a, b) => a - b)[0]; if (nextTs && newestTs && nextTs > newestTs) { - this.didWakeUp = true; this.wakeup?.(); } } @@ -283,10 +279,10 @@ class ActionScheduler { } /** - * This class takes care of the membership management. + * This class is responsible for sending all events relating to the own membership of a matrixRTC call. * It has the following tasks: * - Send the users leave delayed event before sending the membership - * - Sent the users membership if the state machine is started + * - Send the users membership if the state machine is started * - Check if the delayed event was canceled due to sending the membership * - update the delayed event (`restart`) * - Update the state event every ~5h = `DEFAULT_EXPIRE_DURATION` (so it does not get treated as expired) @@ -447,7 +443,9 @@ export class MembershipManager implements IMembershipManager { } // Scheduler: - private scheduler = new ActionScheduler(ActionScheduler.defaultState, this.membershipLoopHandler.bind(this)); + private scheduler = new ActionScheduler(ActionScheduler.defaultState, (state, type) => + this.membershipLoopHandler(state, type), + ); // Loop Handler: private async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { @@ -518,7 +516,7 @@ export class MembershipManager implements IMembershipManager { }) .catch((e) => { if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; - if (this.notFoundError(e)) { + if (this.isNotFoundError(e)) { // If we get a M_NOT_FOUND we know that the delayed event got already removed. // This means we are good and can set it to undefined and run this again. state.delayId = undefined; @@ -570,7 +568,7 @@ export class MembershipManager implements IMembershipManager { }); }) .catch((e) => { - if (this.notFoundError(e)) { + if (this.isNotFoundError(e)) { state.delayId = undefined; this.scheduler.addAction({ ts: Date.now(), @@ -639,7 +637,7 @@ export class MembershipManager implements IMembershipManager { this.leavePromiseDefer = undefined; }) .catch((e) => { - if (this.notFoundError(e)) { + if (this.isNotFoundError(e)) { state.delayId = undefined; this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); return; @@ -649,6 +647,10 @@ export class MembershipManager implements IMembershipManager { if (this.retryOnNetworkError(e, type)) return; // On any other error we fall back to SendLeaveEvent (this includes hard errors from rate limiting) + logger.warn( + "Encountered unexpected error during SendScheduledDelayedLeaveEvent. Falling back to SendLeaveEvent", + e, + ); this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); }); } else { @@ -775,7 +777,7 @@ export class MembershipManager implements IMembershipManager { * @param error the error causing this handler check/execution * @returns true if its a not found error */ - private notFoundError(error: unknown): boolean { + private isNotFoundError(error: unknown): boolean { return error instanceof MatrixError && error.errcode === "M_NOT_FOUND"; } @@ -835,7 +837,6 @@ export class MembershipManager implements IMembershipManager { return true; } - // Failiour throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + (error as Error)); } @@ -876,7 +877,7 @@ export class MembershipManager implements IMembershipManager { error.httpStatus < 600 ) { logger.warn( - "Network error while sending event, retrying in " + retryDurationString + " " + retryCounterString, + "Server error while sending event, retrying in " + retryDurationString + " " + retryCounterString, error, ); } else { From e2d131e2b82a33ba28be9b7700a5e61aa15ab238 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 28 Feb 2025 11:12:56 +0100 Subject: [PATCH 083/124] review iteration 2 --- src/matrixrtc/NewMembershipManager.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index db1084069e5..14872d35984 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -207,6 +207,7 @@ class ActionScheduler { private actions: Action[] = []; private insertions: Action[] = []; private resetWith?: Action[]; + /** * This starts the main loop of the membership manager that handles event sending, delayed event sending and delayed event restarting. * @param initialActions The initial actions the manager will start with. It should be enough to pass: DelayedLeaveActionType.Initial @@ -230,15 +231,14 @@ class ActionScheduler { if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); if (!didWakeUp) { logger.debug( - "Current MembershipManager processing: " + nextAction.type, - "\nQueue:", + `Current MembershipManager processing: ${nextAction.type}\nQueue:`, this.actions, - "\nDate.now: " + Date.now(), + `\nDate.now: "${Date.now()}`, ); try { await this.membershipLoopHandler(this.state, nextAction.type as MembershipActionType); } catch (e) { - throw Error("The MembershipManager has to shut down because of the end condition: " + e); + throw Error(`The MembershipManager shut down because of the end condition: ${e}`); } } @@ -257,21 +257,31 @@ class ActionScheduler { public addAction(action: Action): void { this.insertions.push(action); const nextTs = this.actions[0]?.ts; + const actionString = `${action.type} (ts: ${action.ts})`; if (!nextTs || nextTs > action.ts) { + logger.info(`added action (with wake up): ${actionString}\nAddQueue:`, this.insertions); this.wakeup?.(); + } else { + logger.info(`added action: ${actionString}\nAddQueue:`, this.insertions); } } + public resetActions(actions: Action[]): void { this.resetWith = actions; const nextTs = this.actions[0]?.ts; const newestTs = this.resetWith.map((a) => a.ts).sort((a, b) => a - b)[0]; if (nextTs && newestTs && nextTs > newestTs) { + logger.info("reset actions (with wake up)"); this.wakeup?.(); + } else { + logger.info("reset actions"); } } + public resetState(): void { this.state = ActionScheduler.defaultState; } + public resetRateLimitCounter(type: MembershipActionType): void { this.state.rateLimitRetries.set(type, 0); this.state.networkErrorRetries.set(type, 0); From 79abd60ef8f5eb8628bcfb1adaee38e358cac3a6 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 28 Feb 2025 13:39:32 +0100 Subject: [PATCH 084/124] first step in improving widget error handling --- src/matrixrtc/NewMembershipManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 14872d35984..be88c28cc6f 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -16,7 +16,7 @@ limitations under the License. import { EventType } from "../@types/event.ts"; import { UpdateDelayedEventAction } from "../@types/requests.ts"; -import type { MatrixClient } from "../client.ts"; +import { type MatrixClient } from "../client.ts"; import { UnsupportedEndpointError } from "../errors.ts"; import { ConnectionError, HTTPError, MatrixError } from "../http-api/errors.ts"; import { logger as rootLogger } from "../logger.ts"; @@ -823,7 +823,7 @@ export class MembershipManager implements IMembershipManager { */ private rateLimitErrorHandler(error: unknown, method: string, type: MembershipActionType): boolean { // "Is rate limit"-boundary - if (!(error instanceof HTTPError && error.isRateLimitError())) { + if (!((error instanceof HTTPError || error instanceof MatrixError) && error.isRateLimitError())) { return false; } @@ -881,7 +881,7 @@ export class MembershipManager implements IMembershipManager { error, ); } else if ( - error instanceof HTTPError && + (error instanceof HTTPError || error instanceof MatrixError) && typeof error.httpStatus === "number" && error.httpStatus >= 500 && error.httpStatus < 600 From b9c20cce459bd22fd2d9344a97c1c49952f76565 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 28 Feb 2025 13:52:37 +0100 Subject: [PATCH 085/124] make the embedded client throw ConnectionErrors where desired. --- src/embedded.ts | 73 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/src/embedded.ts b/src/embedded.ts index e80548b9087..d70e4cdb9b5 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -50,7 +50,7 @@ import { } from "./client.ts"; import { SyncApi, SyncState } from "./sync.ts"; import { SlidingSyncSdk } from "./sliding-sync-sdk.ts"; -import { MatrixError } from "./http-api/errors.ts"; +import { ConnectionError, MatrixError } from "./http-api/errors.ts"; import { User } from "./models/user.ts"; import { type Room } from "./models/room.ts"; import { type ToDeviceBatch, type ToDevicePayload } from "./models/ToDeviceMessage.ts"; @@ -358,13 +358,15 @@ export class RoomWidgetClient extends MatrixClient { // Delayed event special case. if (delayOpts) { // TODO: updatePendingEvent for delayed events? - const response = await this.widgetApi.sendRoomEvent( - event.getType(), - content, - room.roomId, - "delay" in delayOpts ? delayOpts.delay : undefined, - "parent_delay_id" in delayOpts ? delayOpts.parent_delay_id : undefined, - ); + const response = await this.widgetApi + .sendRoomEvent( + event.getType(), + content, + room.roomId, + "delay" in delayOpts ? delayOpts.delay : undefined, + "parent_delay_id" in delayOpts ? delayOpts.parent_delay_id : undefined, + ) + .catch(timeoutToConnectionError); return this.validateSendDelayedEventResponse(response); } @@ -374,7 +376,9 @@ export class RoomWidgetClient extends MatrixClient { let response: ISendEventFromWidgetResponseData; try { - response = await this.widgetApi.sendRoomEvent(event.getType(), content, room.roomId); + response = await this.widgetApi + .sendRoomEvent(event.getType(), content, room.roomId) + .catch(timeoutToConnectionError); } catch (e) { this.updatePendingEventStatus(room, event, EventStatus.NOT_SENT); throw e; @@ -397,7 +401,9 @@ export class RoomWidgetClient extends MatrixClient { content: any, stateKey = "", ): Promise { - const response = await this.widgetApi.sendStateEvent(eventType, stateKey, content, roomId); + const response = await this.widgetApi + .sendStateEvent(eventType, stateKey, content, roomId) + .catch(timeoutToConnectionError); if (response.event_id === undefined) { throw new Error("'event_id' absent from response to an event request"); } @@ -422,14 +428,16 @@ export class RoomWidgetClient extends MatrixClient { ); } - const response = await this.widgetApi.sendStateEvent( - eventType, - stateKey, - content, - roomId, - "delay" in delayOpts ? delayOpts.delay : undefined, - "parent_delay_id" in delayOpts ? delayOpts.parent_delay_id : undefined, - ); + const response = await this.widgetApi + .sendStateEvent( + eventType, + stateKey, + content, + roomId, + "delay" in delayOpts ? delayOpts.delay : undefined, + "parent_delay_id" in delayOpts ? delayOpts.parent_delay_id : undefined, + ) + .catch(timeoutToConnectionError); return this.validateSendDelayedEventResponse(response); } @@ -449,17 +457,19 @@ export class RoomWidgetClient extends MatrixClient { throw new UnsupportedEndpointError("Server does not support the delayed events API", "updateDelayedEvent"); } - await this.widgetApi.updateDelayedEvent(delayId, action); + await this.widgetApi.updateDelayedEvent(delayId, action).catch(timeoutToConnectionError); return {}; } public async sendToDevice(eventType: string, contentMap: SendToDeviceContentMap): Promise { - await this.widgetApi.sendToDevice(eventType, false, recursiveMapToObject(contentMap)); + await this.widgetApi + .sendToDevice(eventType, false, recursiveMapToObject(contentMap)) + .catch(timeoutToConnectionError); return {}; } public async getOpenIdToken(): Promise { - const token = await this.widgetApi.requestOpenIDConnectToken(); + const token = await this.widgetApi.requestOpenIDConnectToken().catch(timeoutToConnectionError); // the IOpenIDCredentials from the widget-api and IOpenIDToken form the matrix-js-sdk are compatible. // we still recreate the token to make this transparent and catch'able by the linter in case the types change in the future. return { @@ -477,7 +487,9 @@ export class RoomWidgetClient extends MatrixClient { contentMap.getOrCreate(userId).set(deviceId, payload); } - await this.widgetApi.sendToDevice(eventType, false, recursiveMapToObject(contentMap)); + await this.widgetApi + .sendToDevice(eventType, false, recursiveMapToObject(contentMap)) + .catch(timeoutToConnectionError); } public async encryptAndSendToDevices(userDeviceInfoArr: OlmDevice[], payload: object): Promise { @@ -487,7 +499,9 @@ export class RoomWidgetClient extends MatrixClient { contentMap.getOrCreate(userId).set(deviceId, payload); } - await this.widgetApi.sendToDevice((payload as { type: string }).type, true, recursiveMapToObject(contentMap)); + await this.widgetApi + .sendToDevice((payload as { type: string }).type, true, recursiveMapToObject(contentMap)) + .catch(timeoutToConnectionError); } /** @@ -508,7 +522,9 @@ export class RoomWidgetClient extends MatrixClient { encrypted: boolean, contentMap: SendToDeviceContentMap, ): Promise { - await this.widgetApi.sendToDevice(eventType, encrypted, recursiveMapToObject(contentMap)); + await this.widgetApi + .sendToDevice(eventType, encrypted, recursiveMapToObject(contentMap)) + .catch(timeoutToConnectionError); } // Overridden since we get TURN servers automatically over the widget API, @@ -673,3 +689,12 @@ function processAndThrow(error: unknown): never { throw error; } } + +function timeoutToConnectionError(error: unknown): never { + // TODO: this should not check on error.message but instead it should be a specific type + // error instanceof WidgetTimeoutError + if (error instanceof Error && error.message === "Request timed out") { + throw new ConnectionError("widget api timeout"); + } + throw error; +} From 8a711c3a279296248621741a8456942dda8b74a0 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 28 Feb 2025 14:04:06 +0100 Subject: [PATCH 086/124] fix tests --- spec/unit/embedded.spec.ts | 40 +++++++++---------- spec/unit/matrixrtc/MembershipManager.spec.ts | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts index fc430678de6..0b27541db31 100644 --- a/spec/unit/embedded.spec.ts +++ b/spec/unit/embedded.spec.ts @@ -49,26 +49,26 @@ const testOIDCToken = { token_type: "Bearer", }; class MockWidgetApi extends EventEmitter { - public start = jest.fn(); - public requestCapability = jest.fn(); - public requestCapabilities = jest.fn(); - public requestCapabilityForRoomTimeline = jest.fn(); - public requestCapabilityToSendEvent = jest.fn(); - public requestCapabilityToReceiveEvent = jest.fn(); - public requestCapabilityToSendMessage = jest.fn(); - public requestCapabilityToReceiveMessage = jest.fn(); - public requestCapabilityToSendState = jest.fn(); - public requestCapabilityToReceiveState = jest.fn(); - public requestCapabilityToSendToDevice = jest.fn(); - public requestCapabilityToReceiveToDevice = jest.fn(); + public start = jest.fn().mockResolvedValue(undefined); + public requestCapability = jest.fn().mockResolvedValue(undefined); + public requestCapabilities = jest.fn().mockResolvedValue(undefined); + public requestCapabilityForRoomTimeline = jest.fn().mockResolvedValue(undefined); + public requestCapabilityToSendEvent = jest.fn().mockResolvedValue(undefined); + public requestCapabilityToReceiveEvent = jest.fn().mockResolvedValue(undefined); + public requestCapabilityToSendMessage = jest.fn().mockResolvedValue(undefined); + public requestCapabilityToReceiveMessage = jest.fn().mockResolvedValue(undefined); + public requestCapabilityToSendState = jest.fn().mockResolvedValue(undefined); + public requestCapabilityToReceiveState = jest.fn().mockResolvedValue(undefined); + public requestCapabilityToSendToDevice = jest.fn().mockResolvedValue(undefined); + public requestCapabilityToReceiveToDevice = jest.fn().mockResolvedValue(undefined); public sendRoomEvent = jest.fn( - (eventType: string, content: unknown, roomId?: string, delay?: number, parentDelayId?: string) => + async (eventType: string, content: unknown, roomId?: string, delay?: number, parentDelayId?: string) => delay === undefined && parentDelayId === undefined ? { event_id: `$${Math.random()}` } : { delay_id: `id-${Math.random()}` }, ); public sendStateEvent = jest.fn( - ( + async ( eventType: string, stateKey: string, content: unknown, @@ -80,17 +80,17 @@ class MockWidgetApi extends EventEmitter { ? { event_id: `$${Math.random()}` } : { delay_id: `id-${Math.random()}` }, ); - public updateDelayedEvent = jest.fn(); - public sendToDevice = jest.fn(); - public requestOpenIDConnectToken = jest.fn(() => { + public updateDelayedEvent = jest.fn().mockResolvedValue(undefined); + public sendToDevice = jest.fn().mockResolvedValue(undefined); + public requestOpenIDConnectToken = jest.fn(async () => { return testOIDCToken; return new Promise(() => { return testOIDCToken; }); }); - public readStateEvents = jest.fn(() => []); - public getTurnServers = jest.fn(() => []); - public sendContentLoaded = jest.fn(); + public readStateEvents = jest.fn(async () => []); + public getTurnServers = jest.fn(async () => []); + public sendContentLoaded = jest.fn().mockResolvedValue(undefined); public transport = { reply: jest.fn(), diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 9bebfdd6f95..3bdf0ab0062 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -682,7 +682,7 @@ describe.each([ } expect(unrecoverableError).toHaveBeenCalled(); expect(unrecoverableError.mock.lastCall![0].message).toMatch( - "The MembershipManager has to shut down because of the end condition: Error: Reached maximum", + "The MembershipManager shut down because of the end condition", ); expect(client.sendStateEvent).not.toHaveBeenCalled(); }); From f4c4b7d5fe6aa0f224564b088173be60f387a3b9 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 28 Feb 2025 16:21:42 +0100 Subject: [PATCH 087/124] delayed event sending widget mode stop gap fix. --- src/matrixrtc/NewMembershipManager.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index be88c28cc6f..53029791812 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -872,6 +872,22 @@ export class MembershipManager implements IMembershipManager { retryCounterString, error, ); + } else if (error instanceof Error && error.message.includes("updating delayed event")) { + // TODO: We do not want a error message matching here but instead sth more sophiticated: + // The error originates because of https://github.com/matrix-org/matrix-widget-api/blob/5d81d4a26ff69e4bd3ddc79a884c9527999fb2f4/src/ClientWidgetApi.ts#L698-L701 + // uses `e` instance of HttpError (and not MatrixError) + // The element web widget driver (only checks for MatrixError) is then failing to process (`processError`) it as a typed error: https://github.com/element-hq/element-web/blob/471712cbf06a067e5499bd5d2d7a75f693d9a12d/src/stores/widgets/StopGapWidgetDriver.ts#L711-L715 + // So it will not call: `error.asWidgetApiErrorData()` which is also missing for `HttpError` + // + // A proper fix would be to either find a place to convert the `HttpError` into a `MatrixError` and the `processError` + // method to handle it as expected or to adjust `processError` to also process `HttpError`'s. + logger.warn( + "delayed event update timeout error, retrying in " + + retryDurationString + + " " + + retryCounterString, + error, + ); } else if (error instanceof ConnectionError) { logger.warn( "Network connection error while sending event, retrying in " + From dabb8aab22083eb30b928ed53d6493a84acf6593 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 28 Feb 2025 16:23:19 +0100 Subject: [PATCH 088/124] improve comment --- src/matrixrtc/NewMembershipManager.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 53029791812..9cbf08349b8 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -873,7 +873,9 @@ export class MembershipManager implements IMembershipManager { error, ); } else if (error instanceof Error && error.message.includes("updating delayed event")) { - // TODO: We do not want a error message matching here but instead sth more sophiticated: + // TODO: We do not want error message matching here but instead the error should be a typed HTTPError + // and be handled below automatically (the same as in the SPA case). + // // The error originates because of https://github.com/matrix-org/matrix-widget-api/blob/5d81d4a26ff69e4bd3ddc79a884c9527999fb2f4/src/ClientWidgetApi.ts#L698-L701 // uses `e` instance of HttpError (and not MatrixError) // The element web widget driver (only checks for MatrixError) is then failing to process (`processError`) it as a typed error: https://github.com/element-hq/element-web/blob/471712cbf06a067e5499bd5d2d7a75f693d9a12d/src/stores/widgets/StopGapWidgetDriver.ts#L711-L715 @@ -882,10 +884,7 @@ export class MembershipManager implements IMembershipManager { // A proper fix would be to either find a place to convert the `HttpError` into a `MatrixError` and the `processError` // method to handle it as expected or to adjust `processError` to also process `HttpError`'s. logger.warn( - "delayed event update timeout error, retrying in " + - retryDurationString + - " " + - retryCounterString, + "delayed event update timeout error, retrying in " + retryDurationString + " " + retryCounterString, error, ); } else if (error instanceof ConnectionError) { From a2c4295c151b26919ad2fb7c4c06faa8badc4cea Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 3 Mar 2025 10:51:01 +0100 Subject: [PATCH 089/124] fix unrecoverable error joinState (and add JoinStateChanged) emission. --- src/matrixrtc/MatrixRTCSession.ts | 1 + src/matrixrtc/NewMembershipManager.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index d6bb4b9f8ac..3e64ed7dbfa 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -368,6 +368,7 @@ export class MatrixRTCSession extends TypedEventEmitter { logger.error("MembershipManager encountered an unrecoverable error: ", e); this.emit(MatrixRTCSessionEvent.MembershipManagerError, e); + this.emit(MatrixRTCSessionEvent.JoinStateChanged, this.isJoined()); }); this.encryptionManager!.join(joinConfig); diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 9cbf08349b8..e058e117e82 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -325,6 +325,9 @@ export class MembershipManager implements IMembershipManager { this.scheduler .startWithActions([{ ts: Date.now(), type: DirectMembershipManagerAction.Join }]) .catch((e) => { + // Set the rtc session to left state since we cannot recover from here and the consumer user of the + // MatrixRTCSession class needs to manually rejoin. + this.scheduler.state.running = false; logger.error("MembershipManager stopped because: ", e); onError?.(e); }); From 06545e810c3f5c9ccced58f966b0e49e9fd25d36 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 3 Mar 2025 11:24:32 +0100 Subject: [PATCH 090/124] check that we do not add multipe sendFirstDelayed Events --- src/matrixrtc/NewMembershipManager.ts | 37 +++++++++++++++++---------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index e058e117e82..85b28377636 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -204,7 +204,10 @@ class ActionScheduler { // function for the wakeup mechanism (in case we add an action externally and need to leave the current sleep) private wakeup?: (value: void | PromiseLike) => void; - private actions: Action[] = []; + private _actions: Action[] = []; + public get actions(): Action[] { + return this._actions; + } private insertions: Action[] = []; private resetWith?: Action[]; @@ -216,11 +219,11 @@ class ActionScheduler { * In most other error cases the manager will try to handle any server errors by itself. */ public async startWithActions(initialActions: Action[]): Promise { - this.actions = initialActions; + this._actions = initialActions; - while (this.actions.length > 0) { - this.actions.sort((a, b) => a.ts - b.ts); - const nextAction = this.actions[0]; + while (this._actions.length > 0) { + this._actions.sort((a, b) => a.ts - b.ts); + const nextAction = this._actions[0]; let didWakeUp = false; const wakeupPromise = new Promise((resolve) => { this.wakeup = (): void => { @@ -232,7 +235,7 @@ class ActionScheduler { if (!didWakeUp) { logger.debug( `Current MembershipManager processing: ${nextAction.type}\nQueue:`, - this.actions, + this._actions, `\nDate.now: "${Date.now()}`, ); try { @@ -243,12 +246,12 @@ class ActionScheduler { } if (this.resetWith) { - this.actions = this.resetWith; + this._actions = this.resetWith; this.resetWith = undefined; } - this.actions = this.actions.filter((a) => a !== nextAction); + this._actions = this._actions.filter((a) => a !== nextAction); - this.actions.push(...this.insertions); + this._actions.push(...this.insertions); this.insertions = []; } logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); @@ -256,7 +259,7 @@ class ActionScheduler { public addAction(action: Action): void { this.insertions.push(action); - const nextTs = this.actions[0]?.ts; + const nextTs = this._actions[0]?.ts; const actionString = `${action.type} (ts: ${action.ts})`; if (!nextTs || nextTs > action.ts) { logger.info(`added action (with wake up): ${actionString}\nAddQueue:`, this.insertions); @@ -268,7 +271,7 @@ class ActionScheduler { public resetActions(actions: Action[]): void { this.resetWith = actions; - const nextTs = this.actions[0]?.ts; + const nextTs = this._actions[0]?.ts; const newestTs = this.resetWith.map((a) => a.ts).sort((a, b) => a - b)[0]; if (nextTs && newestTs && nextTs > newestTs) { logger.info("reset actions (with wake up)"); @@ -358,9 +361,15 @@ export class MembershipManager implements IMembershipManager { m.sender === this.client.getUserId() && m.deviceId === this.client.getDeviceId(); if (this.isJoined() && !memberships.some(isMyMembership)) { - logger.warn("Missing own membership: force re-join"); - this.scheduler.state.hasMemberStateEvent = false; - this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerAction.Join }); + if (this.scheduler.actions.find((a) => a.type === DirectMembershipManagerAction.Join)) { + logger.error( + "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already have one", + ); + } else { + logger.warn("Missing own membership: force re-join"); + this.scheduler.state.hasMemberStateEvent = false; + this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerAction.Join }); + } } return Promise.resolve(); } From 9c2eab3889af614bb9409967b4ecba097de8ce80 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 3 Mar 2025 16:08:56 +0100 Subject: [PATCH 091/124] also check insertions queue --- src/matrixrtc/NewMembershipManager.ts | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 85b28377636..aa5012ad3fd 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -208,7 +208,10 @@ class ActionScheduler { public get actions(): Action[] { return this._actions; } - private insertions: Action[] = []; + private _insertions: Action[] = []; + public get insertions(): Action[] { + return this._insertions; + } private resetWith?: Action[]; /** @@ -251,21 +254,21 @@ class ActionScheduler { } this._actions = this._actions.filter((a) => a !== nextAction); - this._actions.push(...this.insertions); - this.insertions = []; + this._actions.push(...this._insertions); + this._insertions = []; } logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); } public addAction(action: Action): void { - this.insertions.push(action); + this._insertions.push(action); const nextTs = this._actions[0]?.ts; const actionString = `${action.type} (ts: ${action.ts})`; if (!nextTs || nextTs > action.ts) { - logger.info(`added action (with wake up): ${actionString}\nAddQueue:`, this.insertions); + logger.info(`added action (with wake up): ${actionString}\nAddQueue:`, this._insertions); this.wakeup?.(); } else { - logger.info(`added action: ${actionString}\nAddQueue:`, this.insertions); + logger.info(`added action: ${actionString}\nAddQueue:`, this._insertions); } } @@ -363,7 +366,13 @@ export class MembershipManager implements IMembershipManager { if (this.isJoined() && !memberships.some(isMyMembership)) { if (this.scheduler.actions.find((a) => a.type === DirectMembershipManagerAction.Join)) { logger.error( - "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already have one", + "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already has on int the Queue\nActionQueueOnMemberUpdate:", + this.scheduler.actions, + ); + } else if (this.scheduler.insertions.find((a) => a.type === DirectMembershipManagerAction.Join)) { + logger.error( + "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already have one in the Insertion queue\nInsertionQueueOnMemberUpdate: ", + this.scheduler.insertions, ); } else { logger.warn("Missing own membership: force re-join"); From f46f09932739c66cbc186bf8f7357bd996b39d03 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 4 Mar 2025 15:21:46 +0100 Subject: [PATCH 092/124] always log "Missing own membership: force re-join" --- src/matrixrtc/NewMembershipManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index aa5012ad3fd..9831f8d2516 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -364,6 +364,7 @@ export class MembershipManager implements IMembershipManager { m.sender === this.client.getUserId() && m.deviceId === this.client.getDeviceId(); if (this.isJoined() && !memberships.some(isMyMembership)) { + logger.warn("Missing own membership: force re-join"); if (this.scheduler.actions.find((a) => a.type === DirectMembershipManagerAction.Join)) { logger.error( "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already has on int the Queue\nActionQueueOnMemberUpdate:", @@ -375,7 +376,6 @@ export class MembershipManager implements IMembershipManager { this.scheduler.insertions, ); } else { - logger.warn("Missing own membership: force re-join"); this.scheduler.state.hasMemberStateEvent = false; this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerAction.Join }); } From 0603181cd11c8e696d79635263ea3258c369f684 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 4 Mar 2025 19:41:07 +0100 Subject: [PATCH 093/124] Do not update the membership if we are in any (a later) state of sending our own state. The scheduled states MembershipActionType.SendFirstDelayedEvent and MembershipActionType.SendJoinEvent both imply that we are already trying to send our own membership state event. --- src/matrixrtc/NewMembershipManager.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 9831f8d2516..aafc0e2c69f 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -349,6 +349,8 @@ export class MembershipManager implements IMembershipManager { if (!this.scheduler.state.running) return Promise.resolve(true); this.scheduler.state.running = false; + // We use the promise to track if we already scheduled a leave event + // So we do not check scheduler.actions/scheduler.insertions if (!this.leavePromiseDefer) { // reset scheduled actions so we will not do any new actions. this.scheduler.resetActions([{ type: DirectMembershipManagerAction.Leave, ts: Date.now() }]); @@ -364,18 +366,27 @@ export class MembershipManager implements IMembershipManager { m.sender === this.client.getUserId() && m.deviceId === this.client.getDeviceId(); if (this.isJoined() && !memberships.some(isMyMembership)) { + // If one of these actions are scheduled or are getting inserted in the next iteration, we should already + // take care of our missing membership. + const sendingMembershipActions = [ + MembershipActionType.SendFirstDelayedEvent, + MembershipActionType.SendJoinEvent, + ]; logger.warn("Missing own membership: force re-join"); - if (this.scheduler.actions.find((a) => a.type === DirectMembershipManagerAction.Join)) { + if (this.scheduler.actions.find((a) => sendingMembershipActions.includes(a.type as MembershipActionType))) { logger.error( - "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already has on int the Queue\nActionQueueOnMemberUpdate:", + "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already have one in the Queue\nActionQueueOnMemberUpdate:", this.scheduler.actions, ); - } else if (this.scheduler.insertions.find((a) => a.type === DirectMembershipManagerAction.Join)) { + } else if ( + this.scheduler.insertions.find((a) => sendingMembershipActions.includes(a.type as MembershipActionType)) + ) { logger.error( - "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already have one in the Insertion queue\nInsertionQueueOnMemberUpdate: ", + "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already have one in the Insertion Queue\nInsertionQueueOnMemberUpdate: ", this.scheduler.insertions, ); } else { + // Only react to our own membership missing if we have not already scheduled sending a new membership DirectMembershipManagerAction.Join this.scheduler.state.hasMemberStateEvent = false; this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerAction.Join }); } From 987a0a87beca032991aa1202866e494c09fb44e9 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 4 Mar 2025 22:01:34 +0100 Subject: [PATCH 094/124] make leave reset actually stop the manager. The reset case was not covered properly. There are cases where it is not allowed to add additional events after a reset and cases where we want to add more events after the reset. We need to allow this as a reset property. --- src/matrixrtc/NewMembershipManager.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index aafc0e2c69f..79fadd309ee 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -213,6 +213,7 @@ class ActionScheduler { return this._insertions; } private resetWith?: Action[]; + private resetWithoutInsertions = false; /** * This starts the main loop of the membership manager that handles event sending, delayed event sending and delayed event restarting. @@ -253,8 +254,10 @@ class ActionScheduler { this.resetWith = undefined; } this._actions = this._actions.filter((a) => a !== nextAction); - - this._actions.push(...this._insertions); + if (!this.resetWithoutInsertions) { + this._actions.push(...this._insertions); + this.resetWithoutInsertions = false; + } this._insertions = []; } logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); @@ -272,8 +275,9 @@ class ActionScheduler { } } - public resetActions(actions: Action[]): void { + public resetActions(actions: Action[], withoutInsertions = true): void { this.resetWith = actions; + this.resetWithoutInsertions = withoutInsertions; const nextTs = this._actions[0]?.ts; const newestTs = this.resetWith.map((a) => a.ts).sort((a, b) => a - b)[0]; if (nextTs && newestTs && nextTs > newestTs) { @@ -353,7 +357,11 @@ export class MembershipManager implements IMembershipManager { // So we do not check scheduler.actions/scheduler.insertions if (!this.leavePromiseDefer) { // reset scheduled actions so we will not do any new actions. - this.scheduler.resetActions([{ type: DirectMembershipManagerAction.Leave, ts: Date.now() }]); + if (this.scheduler.state.hasMemberStateEvent) { + this.scheduler.resetActions([{ type: DirectMembershipManagerAction.Leave, ts: Date.now() }]); + } else { + this.scheduler.resetActions([]); + } this.leavePromiseDefer = defer(); if (timeout) setTimeout(() => this.leavePromiseDefer?.resolve(false), timeout); } @@ -545,7 +553,7 @@ export class MembershipManager implements IMembershipManager { // In this block we will try to cancel this delayed event before setting up a new one. // Remove all running updates and restarts - this.scheduler.resetActions([]); + this.scheduler.resetActions([], false); await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel) .then(() => { From 27b689ab9e586645c8c03e7a4ed6bbd74b12aba3 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 5 Mar 2025 14:40:16 +0100 Subject: [PATCH 095/124] fix tests (and implementation) --- spec/unit/matrixrtc/MembershipManager.spec.ts | 4 ++-- src/matrixrtc/NewMembershipManager.ts | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 3bdf0ab0062..5f3783a17c6 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -335,6 +335,7 @@ describe.each([ await jest.advanceTimersByTimeAsync(1); (client._unstable_updateDelayedEvent as Mock).mockRejectedValue("unknown"); await manager.leave(); + // We send a normal leave event since we failed using updateDelayedEvent with the "send" action. expect(client.sendStateEvent).toHaveBeenLastCalledWith( room.roomId, @@ -557,7 +558,7 @@ describe.each([ expect(client.sendStateEvent).toHaveBeenCalledTimes(1); }); // FailsForLegacy as implementation does not re-check membership before retrying. - it("abandons retry loop if leave() was called !FailsForLegacy", async () => { + it("abandons retry loop if leave() was called before sending state event !FailsForLegacy", async () => { const handle = createAsyncHandle(client._unstable_sendDelayedStateEvent); const manager = new TestMembershipManager({}, room, client, () => undefined); @@ -578,7 +579,6 @@ describe.each([ await manager.leave(); // Wait for all timers to be setup - // await flushPromises(); await jest.advanceTimersByTimeAsync(1000); // No new events should have been sent: diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 79fadd309ee..86c574709c3 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -254,9 +254,10 @@ class ActionScheduler { this.resetWith = undefined; } this._actions = this._actions.filter((a) => a !== nextAction); - if (!this.resetWithoutInsertions) { - this._actions.push(...this._insertions); + if (this.resetWithoutInsertions) { this.resetWithoutInsertions = false; + } else { + this._actions.push(...this._insertions); } this._insertions = []; } @@ -280,7 +281,7 @@ class ActionScheduler { this.resetWithoutInsertions = withoutInsertions; const nextTs = this._actions[0]?.ts; const newestTs = this.resetWith.map((a) => a.ts).sort((a, b) => a - b)[0]; - if (nextTs && newestTs && nextTs > newestTs) { + if (actions.length === 0 || (nextTs && newestTs && nextTs > newestTs)) { logger.info("reset actions (with wake up)"); this.wakeup?.(); } else { @@ -357,12 +358,15 @@ export class MembershipManager implements IMembershipManager { // So we do not check scheduler.actions/scheduler.insertions if (!this.leavePromiseDefer) { // reset scheduled actions so we will not do any new actions. + this.leavePromiseDefer = defer(); if (this.scheduler.state.hasMemberStateEvent) { this.scheduler.resetActions([{ type: DirectMembershipManagerAction.Leave, ts: Date.now() }]); } else { this.scheduler.resetActions([]); + // We don't do anything else here since we are already in the left state. + // We do not care about canceling a pending delayed leave event it will set it to {} again = no-op. + this.leavePromiseDefer?.resolve(true); } - this.leavePromiseDefer = defer(); if (timeout) setTimeout(() => this.leavePromiseDefer?.resolve(false), timeout); } return this.leavePromiseDefer.promise; From 6d674073a16e7dae91d0da41ace62d47a21bc492 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 5 Mar 2025 18:46:00 +0000 Subject: [PATCH 096/124] Allow MembershipManger to be set at runtime via JoinConfig.membershipManagerFactory --- src/matrixrtc/MatrixRTCSession.ts | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 3e64ed7dbfa..36207a6cb08 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -64,6 +64,20 @@ export interface MembershipConfig { */ useNewMembershipManager?: boolean; + membershipManagerFactory?: ( + joinConfig: MembershipConfig | undefined, + room: Pick, + client: Pick< + MatrixClient, + | "getUserId" + | "getDeviceId" + | "sendStateEvent" + | "_unstable_sendDelayedStateEvent" + | "_unstable_updateDelayedEvent" + >, + getOldestMembership: () => CallMembership | undefined, + ) => IMembershipManager; + /** * The timeout (in milliseconds) after we joined the call, that our membership should expire * unless we have explicitly updated it. @@ -353,7 +367,14 @@ export class MatrixRTCSession extends TypedEventEmitter this.getOldestMembership(), + ); + } else if (joinConfig?.useNewMembershipManager ?? false) { this.membershipManager = new MembershipManager(joinConfig, this.roomSubset, this.client, () => this.getOldestMembership(), ); From c5435a5a7d46c4826af9e9e3e3f5b3f11c586f3e Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 5 Mar 2025 19:28:03 +0000 Subject: [PATCH 097/124] Map actions into status as a sanity check --- src/matrixrtc/NewMembershipManager.ts | 92 +++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 86c574709c3..c7086a70135 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -68,6 +68,17 @@ export interface IMembershipManager { getActiveFocus(): Focus | undefined; } +enum Status { + Disconnected = "Disconnected", + Connecting = "Connecting", + ConnectingFailed = "ConnectingFailed", + Connected = "Connected", + Reconnecting = "Reconnecting", + Disconnecting = "Disconnecting", + Stuck = "Stuck", + Unknown = "Unknown", +} + /* SCHEDULER TYPES: DirectMembershipManagerAction.Join @@ -236,6 +247,9 @@ class ActionScheduler { }; }); if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); + + logger.info("MembershipManager ActionScheduler awakened. status=" + this.status); + if (!didWakeUp) { logger.debug( `Current MembershipManager processing: ${nextAction.type}\nQueue:`, @@ -297,6 +311,45 @@ class ActionScheduler { this.state.rateLimitRetries.set(type, 0); this.state.networkErrorRetries.set(type, 0); } + + public get status(): Status { + const actions = [...this.actions, ...this.insertions]; + + logger.info(`foo ${actions.map((a) => a.type)}`); + if (actions.length === 1) { + const { type } = actions[0]; + switch (type) { + case MembershipActionType.SendFirstDelayedEvent: + case MembershipActionType.SendJoinEvent: + case MembershipActionType.SendMainDelayedEvent: + return Status.Connecting; + case MembershipActionType.UpdateExpiry: // where no delayed events + return Status.Connected; + case MembershipActionType.SendScheduledDelayedLeaveEvent: + case MembershipActionType.SendLeaveEvent: + return Status.Disconnecting; + default: + // pass through as not expected + } + } else if (actions.length === 2) { + const types = actions.map((a) => a.type); + // normal state for connected with delayed events + if ( + (types.includes(MembershipActionType.RestartDelayedEvent) || + types.includes(MembershipActionType.SendMainDelayedEvent)) && + types.includes(MembershipActionType.UpdateExpiry) + ) { + return Status.Connected; + } + } + + if (!this.state.running) { + return Status.Disconnected; + } + + logger.error("MembershipManager has an unknown state. Actions: ", actions); + return Status.Unknown; + } } /** @@ -318,6 +371,45 @@ export class MembershipManager implements IMembershipManager { public isJoined(): boolean { return this.scheduler.state.running; } + + public status(): Status { + if (!this.scheduler.state.running) { + return Status.Disconnected; + } + const actions = [...this.scheduler.actions, ...this.scheduler.insertions]; + + if (actions.length === 1) { + const { type } = actions[0]; + switch (type) { + case DirectMembershipManagerAction.Join: + case MembershipActionType.SendFirstDelayedEvent: + case MembershipActionType.SendJoinEvent: + case MembershipActionType.SendMainDelayedEvent: + return Status.Connecting; + case MembershipActionType.UpdateExpiry: // where no delayed events + return Status.Connected; + case DirectMembershipManagerAction.Leave: + case MembershipActionType.SendScheduledDelayedLeaveEvent: + case MembershipActionType.SendLeaveEvent: + return Status.Disconnecting; + default: + // pass through as not expected + } + } else if (actions.length === 2) { + const types = actions.map((a) => a.type); + // normal state for connected with delayed events + if ( + types.includes(MembershipActionType.RestartDelayedEvent) && + types.includes(MembershipActionType.UpdateExpiry) + ) { + return Status.Connected; + } + } + + logger.error("MembershipManager has an unknown state. Actions: ", actions); + return Status.Unknown; + } + /** * Puts the MembershipManager in a state where it tries to be joined. * It will send delayed events and membership events From c4eaf5efc96b0faa9d1f9c1d69bba4d2f6f1189a Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 6 Mar 2025 11:12:24 +0000 Subject: [PATCH 098/124] Log status change after applying actions --- src/matrixrtc/NewMembershipManager.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index c7086a70135..1a479ff690f 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -263,6 +263,7 @@ class ActionScheduler { } } + const oldStatus = this.status; if (this.resetWith) { this._actions = this.resetWith; this.resetWith = undefined; @@ -274,6 +275,9 @@ class ActionScheduler { this._actions.push(...this._insertions); } this._insertions = []; + logger.info( + `MembershipManager ActionScheduler applied action changes. Status: ${oldStatus} -> ${this.status}`, + ); } logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); } From 40dd4f6575b34e4ca1c9b9475229fcdd18b4c6b1 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 6 Mar 2025 11:23:26 +0000 Subject: [PATCH 099/124] Add todo --- src/matrixrtc/MatrixRTCSession.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 36207a6cb08..fa2545148a6 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -64,6 +64,11 @@ export interface MembershipConfig { */ useNewMembershipManager?: boolean; + /** + * This is just for testing + * + * TODO: remove as part of PR cleanup + */ membershipManagerFactory?: ( joinConfig: MembershipConfig | undefined, room: Pick, From d59f9d9fa6f3486209c7f0017d5d850c97804a35 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 6 Mar 2025 11:32:03 +0000 Subject: [PATCH 100/124] Cleanup --- src/matrixrtc/NewMembershipManager.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 1a479ff690f..d3c78abf278 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -319,7 +319,6 @@ class ActionScheduler { public get status(): Status { const actions = [...this.actions, ...this.insertions]; - logger.info(`foo ${actions.map((a) => a.type)}`); if (actions.length === 1) { const { type } = actions[0]; switch (type) { From 5942f051afd3bd2c050fb0ccb409d22d8d294037 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 6 Mar 2025 11:36:25 +0000 Subject: [PATCH 101/124] Log transition from earlier status --- src/matrixrtc/NewMembershipManager.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index d3c78abf278..b5968925792 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -248,7 +248,8 @@ class ActionScheduler { }); if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); - logger.info("MembershipManager ActionScheduler awakened. status=" + this.status); + const oldStatus = this.status; + logger.info(`MembershipManager ActionScheduler awakened. status=${oldStatus}`); if (!didWakeUp) { logger.debug( @@ -263,7 +264,7 @@ class ActionScheduler { } } - const oldStatus = this.status; + logger.info(`MembershipManager ActionScheduler intermediate status=${this.status}`); if (this.resetWith) { this._actions = this.resetWith; this.resetWith = undefined; From 31b6e45d5dedaf864256def73d4b1df137bdc2f4 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 7 Mar 2025 11:53:52 +0100 Subject: [PATCH 102/124] remove redundant status implementation also add TODO comment to not forget about this. --- src/matrixrtc/NewMembershipManager.ts | 74 ++++++++++++++------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index b5968925792..ca4a4d0266f 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -318,7 +318,7 @@ class ActionScheduler { } public get status(): Status { - const actions = [...this.actions, ...this.insertions]; + const actions = [...this.actions]; if (actions.length === 1) { const { type } = actions[0]; @@ -377,41 +377,43 @@ export class MembershipManager implements IMembershipManager { } public status(): Status { - if (!this.scheduler.state.running) { - return Status.Disconnected; - } - const actions = [...this.scheduler.actions, ...this.scheduler.insertions]; - - if (actions.length === 1) { - const { type } = actions[0]; - switch (type) { - case DirectMembershipManagerAction.Join: - case MembershipActionType.SendFirstDelayedEvent: - case MembershipActionType.SendJoinEvent: - case MembershipActionType.SendMainDelayedEvent: - return Status.Connecting; - case MembershipActionType.UpdateExpiry: // where no delayed events - return Status.Connected; - case DirectMembershipManagerAction.Leave: - case MembershipActionType.SendScheduledDelayedLeaveEvent: - case MembershipActionType.SendLeaveEvent: - return Status.Disconnecting; - default: - // pass through as not expected - } - } else if (actions.length === 2) { - const types = actions.map((a) => a.type); - // normal state for connected with delayed events - if ( - types.includes(MembershipActionType.RestartDelayedEvent) && - types.includes(MembershipActionType.UpdateExpiry) - ) { - return Status.Connected; - } - } - - logger.error("MembershipManager has an unknown state. Actions: ", actions); - return Status.Unknown; + return this.scheduler.status; + // TODO figure out if this is still needed and different to the scheduler status + // if (!this.scheduler.state.running) { + // return Status.Disconnected; + // } + // const actions = [...this.scheduler.actions]; + + // if (actions.length === 1) { + // const { type } = actions[0]; + // switch (type) { + // case DirectMembershipManagerAction.Join: + // case MembershipActionType.SendFirstDelayedEvent: + // case MembershipActionType.SendJoinEvent: + // case MembershipActionType.SendMainDelayedEvent: + // return Status.Connecting; + // case MembershipActionType.UpdateExpiry: // where no delayed events + // return Status.Connected; + // case DirectMembershipManagerAction.Leave: + // case MembershipActionType.SendScheduledDelayedLeaveEvent: + // case MembershipActionType.SendLeaveEvent: + // return Status.Disconnecting; + // default: + // // pass through as not expected + // } + // } else if (actions.length === 2) { + // const types = actions.map((a) => a.type); + // // normal state for connected with delayed events + // if ( + // types.includes(MembershipActionType.RestartDelayedEvent) && + // types.includes(MembershipActionType.UpdateExpiry) + // ) { + // return Status.Connected; + // } + // } + + // logger.error("MembershipManager has an unknown state. Actions: ", actions); + // return Status.Unknown; } /** From 2b464c94285da8062436e55ec27d26f247d291e8 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 7 Mar 2025 11:09:34 +0000 Subject: [PATCH 103/124] More cleanup --- src/matrixrtc/NewMembershipManager.ts | 41 --------------------------- 1 file changed, 41 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index ca4a4d0266f..a6223f088fa 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -264,7 +264,6 @@ class ActionScheduler { } } - logger.info(`MembershipManager ActionScheduler intermediate status=${this.status}`); if (this.resetWith) { this._actions = this.resetWith; this.resetWith = undefined; @@ -376,46 +375,6 @@ export class MembershipManager implements IMembershipManager { return this.scheduler.state.running; } - public status(): Status { - return this.scheduler.status; - // TODO figure out if this is still needed and different to the scheduler status - // if (!this.scheduler.state.running) { - // return Status.Disconnected; - // } - // const actions = [...this.scheduler.actions]; - - // if (actions.length === 1) { - // const { type } = actions[0]; - // switch (type) { - // case DirectMembershipManagerAction.Join: - // case MembershipActionType.SendFirstDelayedEvent: - // case MembershipActionType.SendJoinEvent: - // case MembershipActionType.SendMainDelayedEvent: - // return Status.Connecting; - // case MembershipActionType.UpdateExpiry: // where no delayed events - // return Status.Connected; - // case DirectMembershipManagerAction.Leave: - // case MembershipActionType.SendScheduledDelayedLeaveEvent: - // case MembershipActionType.SendLeaveEvent: - // return Status.Disconnecting; - // default: - // // pass through as not expected - // } - // } else if (actions.length === 2) { - // const types = actions.map((a) => a.type); - // // normal state for connected with delayed events - // if ( - // types.includes(MembershipActionType.RestartDelayedEvent) && - // types.includes(MembershipActionType.UpdateExpiry) - // ) { - // return Status.Connected; - // } - // } - - // logger.error("MembershipManager has an unknown state. Actions: ", actions); - // return Status.Unknown; - } - /** * Puts the MembershipManager in a state where it tries to be joined. * It will send delayed events and membership events From b501057fb47f05bf2d4630789a8c5169548bd774 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 7 Mar 2025 11:39:37 +0000 Subject: [PATCH 104/124] Consider insertions in status() --- src/matrixrtc/NewMembershipManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index a6223f088fa..c9aca3b9c11 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -317,7 +317,7 @@ class ActionScheduler { } public get status(): Status { - const actions = [...this.actions]; + const actions = [...this.actions, ...this.insertions]; if (actions.length === 1) { const { type } = actions[0]; From 075e186caf7254dda14900ab2b822eaaa041a704 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 7 Mar 2025 12:03:04 +0000 Subject: [PATCH 105/124] Log duration for emitting MatrixRTCSessionEvent.MembershipsChanged --- src/matrixrtc/MatrixRTCSession.ts | 5 ++++- src/utils.ts | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index fa2545148a6..c2db7582956 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -28,6 +28,7 @@ import { type MatrixEvent } from "../models/event.ts"; import { MembershipManager, type IMembershipManager } from "./NewMembershipManager.ts"; import { EncryptionManager, type IEncryptionManager, type Statistics } from "./EncryptionManager.ts"; import { LegacyMembershipManager } from "./LegacyMembershipManager.ts"; +import { logDurationSync } from "../utils.ts"; const logger = rootLogger.getChild("MatrixRTCSession"); @@ -558,7 +559,9 @@ export class MatrixRTCSession extends TypedEventEmitter { + this.emit(MatrixRTCSessionEvent.MembershipsChanged, oldMemberships, this.memberships); + }); void this.membershipManager?.onRTCSessionMemberUpdate(this.memberships); } diff --git a/src/utils.ts b/src/utils.ts index 93ef2bf73a9..9a822f92f8a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -405,6 +405,23 @@ export async function logDuration(logger: BaseLogger, name: string, block: () } } +/** + * Utility to log the duration of a synchronous block. + * + * @param logger - The logger to log to. + * @param name - The name of the operation. + * @param block - The block to execute. + */ +export function logDurationSync(logger: BaseLogger, name: string, block: () => T): T { + const start = Date.now(); + try { + return block(); + } finally { + const end = Date.now(); + logger.debug(`[Perf]: ${name} took ${end - start}ms`); + } +} + /** * Promise/async version of {@link setImmediate}. * From 306518009b192544a46309ddeae241d75c035d3c Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 7 Mar 2025 14:20:17 +0100 Subject: [PATCH 106/124] add another valid condition for connected --- src/matrixrtc/NewMembershipManager.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index c9aca3b9c11..c784ba1a587 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -344,6 +344,16 @@ class ActionScheduler { ) { return Status.Connected; } + } else if (actions.length === 3) { + const types = actions.map((a) => a.type); + // It is a correct connected state if we already schedule the next Restart but have not yet cleaned up + // the current restart. + if ( + types.filter((t) => t === MembershipActionType.RestartDelayedEvent).length === 2 && + types.includes(MembershipActionType.UpdateExpiry) + ) { + return Status.Connected; + } } if (!this.state.running) { From 72cc607d945d997e201e4ebd6c4f78623bae4786 Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 7 Mar 2025 14:35:50 +0100 Subject: [PATCH 107/124] some TODO cleanup --- src/embedded.ts | 4 ++++ src/matrixrtc/MatrixRTCSession.ts | 32 ++++--------------------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/src/embedded.ts b/src/embedded.ts index d70e4cdb9b5..fde54740f04 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -690,6 +690,10 @@ function processAndThrow(error: unknown): never { } } +/** + * This converts an "Request timed out" error from the PostmessageTransport into a ConnectionError. + * It either throws the original error or a new ConnectionError. + **/ function timeoutToConnectionError(error: unknown): never { // TODO: this should not check on error.message but instead it should be a specific type // error instanceof WidgetTimeoutError diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index c2db7582956..4006c124fb1 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -61,28 +61,11 @@ export type MatrixRTCSessionEventHandlerMap = { export interface MembershipConfig { /** - * Use the new Manager - */ - useNewMembershipManager?: boolean; - - /** - * This is just for testing + * Use the new Manager. * - * TODO: remove as part of PR cleanup + * Default: `false`. */ - membershipManagerFactory?: ( - joinConfig: MembershipConfig | undefined, - room: Pick, - client: Pick< - MatrixClient, - | "getUserId" - | "getDeviceId" - | "sendStateEvent" - | "_unstable_sendDelayedStateEvent" - | "_unstable_updateDelayedEvent" - >, - getOldestMembership: () => CallMembership | undefined, - ) => IMembershipManager; + useNewMembershipManager?: boolean; /** * The timeout (in milliseconds) after we joined the call, that our membership should expire @@ -373,14 +356,7 @@ export class MatrixRTCSession extends TypedEventEmitter this.getOldestMembership(), - ); - } else if (joinConfig?.useNewMembershipManager ?? false) { + if (joinConfig?.useNewMembershipManager ?? false) { this.membershipManager = new MembershipManager(joinConfig, this.roomSubset, this.client, () => this.getOldestMembership(), ); From e8d588da17b3e4404478fe86edc9dc1efba28d5e Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 7 Mar 2025 14:48:46 +0100 Subject: [PATCH 108/124] review add warning when using addAction while the scheduler is not running. --- src/matrixrtc/NewMembershipManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index c784ba1a587..aa8022cfcfe 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -292,6 +292,8 @@ class ActionScheduler { } else { logger.info(`added action: ${actionString}\nAddQueue:`, this._insertions); } + if (!this.state.running) + logger.warn(`MembershipManager is not running but we try to add Action: ${actionString}`); } public resetActions(actions: Action[], withoutInsertions = true): void { From 47a7e4cff044e83fbb6aafd6700a3af707ef39cc Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 7 Mar 2025 14:52:53 +0100 Subject: [PATCH 109/124] es lint --- src/matrixrtc/NewMembershipManager.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index aa8022cfcfe..acd286032a7 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -292,8 +292,9 @@ class ActionScheduler { } else { logger.info(`added action: ${actionString}\nAddQueue:`, this._insertions); } - if (!this.state.running) + if (!this.state.running) { logger.warn(`MembershipManager is not running but we try to add Action: ${actionString}`); + } } public resetActions(actions: Action[], withoutInsertions = true): void { From fce95bfe5e514441344de4734095165f39f6f4eb Mon Sep 17 00:00:00 2001 From: Timo Date: Sat, 8 Mar 2025 12:02:36 +0100 Subject: [PATCH 110/124] refactor to return based handler approach (remove insertions array) --- src/matrixrtc/NewMembershipManager.ts | 434 +++++++++++++------------- 1 file changed, 210 insertions(+), 224 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index acd286032a7..1d4c844c33a 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -20,7 +20,7 @@ import { type MatrixClient } from "../client.ts"; import { UnsupportedEndpointError } from "../errors.ts"; import { ConnectionError, HTTPError, MatrixError } from "../http-api/errors.ts"; import { logger as rootLogger } from "../logger.ts"; -import { type Room } from "../models/room.ts"; +import { Room } from "../models/room.ts"; import { defer, type IDeferred, sleep } from "../utils.ts"; import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts"; import { type Focus } from "./focus.ts"; @@ -140,13 +140,6 @@ enum MembershipActionType { // -> MembershipActionType.SendLeaveEvent } -/** - * Actions that are supposed to be used from outside the main handle methods. - */ -enum DirectMembershipManagerAction { - Join = MembershipActionType.SendFirstDelayedEvent, - Leave = MembershipActionType.SendScheduledDelayedLeaveEvent, -} interface ActionSchedulerState { /** The delayId we got when successfully sending the delayed leave event. * Gets set to undefined if the server claims it cannot find the delayed event anymore. */ @@ -180,9 +173,12 @@ interface Action { * The state of the different loops * can also be thought of as the type of the action */ - type: MembershipActionType | DirectMembershipManagerAction; + type: MembershipActionType; +} +interface ActionUpdate { + setActions?: Action[]; + addActions?: Action[]; } - /** * This state machine tracks the state of the current membership participation * and runs one central timer that wakes up a handler callback with the correct action @@ -198,33 +194,32 @@ class ActionScheduler { return { hasMemberStateEvent: false, running: false, - startTime: 0, delayId: undefined, + + startTime: 0, rateLimitRetries: new Map(), networkErrorRetries: new Map(), - expireUpdateIterations: 0, + expireUpdateIterations: 1, }; } public constructor( state: ActionSchedulerState, /** This is the callback called for each scheduled action (`this.addAction()`) */ - private membershipLoopHandler: (state: ActionSchedulerState, type: MembershipActionType) => Promise, + private membershipLoopHandler: ( + state: ActionSchedulerState, + type: MembershipActionType, + ) => Promise, ) { this.state = state; } // function for the wakeup mechanism (in case we add an action externally and need to leave the current sleep) - private wakeup?: (value: void | PromiseLike) => void; - + private wakeup: (update: ActionUpdate) => void = (update: ActionUpdate): void => { + logger.error("Cannot call wakeup before calling `startWithJoin()`"); + }; private _actions: Action[] = []; public get actions(): Action[] { return this._actions; } - private _insertions: Action[] = []; - public get insertions(): Action[] { - return this._insertions; - } - private resetWith?: Action[]; - private resetWithoutInsertions = false; /** * This starts the main loop of the membership manager that handles event sending, delayed event sending and delayed event restarting. @@ -233,16 +228,19 @@ class ActionScheduler { * @throws This throws an error if one of the actions throws. * In most other error cases the manager will try to handle any server errors by itself. */ - public async startWithActions(initialActions: Action[]): Promise { - this._actions = initialActions; + public async startWithJoin(): Promise { + this._actions = [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }]; while (this._actions.length > 0) { + // Sort so next (smallest ts) action is at the beginning this._actions.sort((a, b) => a.ts - b.ts); const nextAction = this._actions[0]; - let didWakeUp = false; + let wakeupUpdate: ActionUpdate | undefined = undefined; + + // while we await for the next action, wakeup has to resolve the wakeupPromise const wakeupPromise = new Promise((resolve) => { - this.wakeup = (): void => { - didWakeUp = true; + this.wakeup = (update: ActionUpdate): void => { + wakeupUpdate = update; resolve(); }; }); @@ -251,30 +249,35 @@ class ActionScheduler { const oldStatus = this.status; logger.info(`MembershipManager ActionScheduler awakened. status=${oldStatus}`); - if (!didWakeUp) { + let handlerResult: ActionUpdate = {}; + if (!wakeupUpdate) { logger.debug( `Current MembershipManager processing: ${nextAction.type}\nQueue:`, this._actions, `\nDate.now: "${Date.now()}`, ); try { - await this.membershipLoopHandler(this.state, nextAction.type as MembershipActionType); + // `this.wakeup` can also be called and sets the `wakupUpdate` object while we are in the handler. + handlerResult = await this.membershipLoopHandler( + this.state, + nextAction.type as MembershipActionType, + ); } catch (e) { throw Error(`The MembershipManager shut down because of the end condition: ${e}`); } } + // remove the processed action only after we are done processing + this._actions.splice(0, 1); + // The wakeupUpdate always wins since that is a direct external update. + let { addActions, setActions } = wakeupUpdate ?? handlerResult; - if (this.resetWith) { - this._actions = this.resetWith; - this.resetWith = undefined; + if (setActions) { + this._actions = setActions; } - this._actions = this._actions.filter((a) => a !== nextAction); - if (this.resetWithoutInsertions) { - this.resetWithoutInsertions = false; - } else { - this._actions.push(...this._insertions); + if (addActions) { + this._actions.push(...addActions); } - this._insertions = []; + logger.info( `MembershipManager ActionScheduler applied action changes. Status: ${oldStatus} -> ${this.status}`, ); @@ -282,32 +285,11 @@ class ActionScheduler { logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); } - public addAction(action: Action): void { - this._insertions.push(action); - const nextTs = this._actions[0]?.ts; - const actionString = `${action.type} (ts: ${action.ts})`; - if (!nextTs || nextTs > action.ts) { - logger.info(`added action (with wake up): ${actionString}\nAddQueue:`, this._insertions); - this.wakeup?.(); - } else { - logger.info(`added action: ${actionString}\nAddQueue:`, this._insertions); - } - if (!this.state.running) { - logger.warn(`MembershipManager is not running but we try to add Action: ${actionString}`); - } + public initiateJoin(): void { + this.wakeup?.({ setActions: [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }] }); } - - public resetActions(actions: Action[], withoutInsertions = true): void { - this.resetWith = actions; - this.resetWithoutInsertions = withoutInsertions; - const nextTs = this._actions[0]?.ts; - const newestTs = this.resetWith.map((a) => a.ts).sort((a, b) => a - b)[0]; - if (actions.length === 0 || (nextTs && newestTs && nextTs > newestTs)) { - logger.info("reset actions (with wake up)"); - this.wakeup?.(); - } else { - logger.info("reset actions"); - } + public initiateLeave(): void { + this.wakeup?.({ setActions: [{ ts: Date.now(), type: MembershipActionType.SendScheduledDelayedLeaveEvent }] }); } public resetState(): void { @@ -320,10 +302,8 @@ class ActionScheduler { } public get status(): Status { - const actions = [...this.actions, ...this.insertions]; - - if (actions.length === 1) { - const { type } = actions[0]; + if (this.actions.length === 1) { + const { type } = this.actions[0]; switch (type) { case MembershipActionType.SendFirstDelayedEvent: case MembershipActionType.SendJoinEvent: @@ -337,8 +317,8 @@ class ActionScheduler { default: // pass through as not expected } - } else if (actions.length === 2) { - const types = actions.map((a) => a.type); + } else if (this.actions.length === 2) { + const types = this.actions.map((a) => a.type); // normal state for connected with delayed events if ( (types.includes(MembershipActionType.RestartDelayedEvent) || @@ -347,8 +327,8 @@ class ActionScheduler { ) { return Status.Connected; } - } else if (actions.length === 3) { - const types = actions.map((a) => a.type); + } else if (this.actions.length === 3) { + const types = this.actions.map((a) => a.type); // It is a correct connected state if we already schedule the next Restart but have not yet cleaned up // the current restart. if ( @@ -363,7 +343,7 @@ class ActionScheduler { return Status.Disconnected; } - logger.error("MembershipManager has an unknown state. Actions: ", actions); + logger.error("MembershipManager has an unknown state. Actions: ", this.actions); return Status.Unknown; } } @@ -403,15 +383,13 @@ export class MembershipManager implements IMembershipManager { if (!this.scheduler.state.running) { this.scheduler.resetState(); this.scheduler.state.running = true; - this.scheduler - .startWithActions([{ ts: Date.now(), type: DirectMembershipManagerAction.Join }]) - .catch((e) => { - // Set the rtc session to left state since we cannot recover from here and the consumer user of the - // MatrixRTCSession class needs to manually rejoin. - this.scheduler.state.running = false; - logger.error("MembershipManager stopped because: ", e); - onError?.(e); - }); + this.scheduler.startWithJoin().catch((e) => { + // Set the rtc session to left state since we cannot recover from here and the consumer user of the + // MatrixRTCSession class needs to manually rejoin. + this.scheduler.state.running = false; + logger.error("MembershipManager stopped because: ", e); + onError?.(e); + }); } } @@ -429,14 +407,7 @@ export class MembershipManager implements IMembershipManager { if (!this.leavePromiseDefer) { // reset scheduled actions so we will not do any new actions. this.leavePromiseDefer = defer(); - if (this.scheduler.state.hasMemberStateEvent) { - this.scheduler.resetActions([{ type: DirectMembershipManagerAction.Leave, ts: Date.now() }]); - } else { - this.scheduler.resetActions([]); - // We don't do anything else here since we are already in the left state. - // We do not care about canceling a pending delayed leave event it will set it to {} again = no-op. - this.leavePromiseDefer?.resolve(true); - } + this.scheduler.initiateLeave(); if (timeout) setTimeout(() => this.leavePromiseDefer?.resolve(false), timeout); } return this.leavePromiseDefer.promise; @@ -460,17 +431,10 @@ export class MembershipManager implements IMembershipManager { "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already have one in the Queue\nActionQueueOnMemberUpdate:", this.scheduler.actions, ); - } else if ( - this.scheduler.insertions.find((a) => sendingMembershipActions.includes(a.type as MembershipActionType)) - ) { - logger.error( - "NewMembershipManger tried adding another `SendFirstDelayedEvent` actions even though we already have one in the Insertion Queue\nInsertionQueueOnMemberUpdate: ", - this.scheduler.insertions, - ); } else { // Only react to our own membership missing if we have not already scheduled sending a new membership DirectMembershipManagerAction.Join this.scheduler.state.hasMemberStateEvent = false; - this.scheduler.addAction({ ts: Date.now(), type: DirectMembershipManagerAction.Join }); + this.scheduler.initiateJoin(); } } return Promise.resolve(); @@ -572,13 +536,16 @@ export class MembershipManager implements IMembershipManager { ); // Loop Handler: - private async membershipLoopHandler(state: ActionSchedulerState, type: MembershipActionType): Promise { + private async membershipLoopHandler( + state: ActionSchedulerState, + type: MembershipActionType, + ): Promise { switch (type) { case MembershipActionType.SendFirstDelayedEvent: { // Before we start we check if we come from a state where we have a delay id. if (!state.delayId) { - // Normal case without any previous delayed id. - await this.client + // this.sendFirstDelayedLeaveEvent(state, type);// Normal case without any previous delayed id. + return await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, { @@ -593,30 +560,32 @@ export class MembershipManager implements IMembershipManager { state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); state.networkErrorRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); state.delayId = response.delay_id; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendJoinEvent }); + return createAddActionUpdate(MembershipActionType.SendJoinEvent); }) .catch((e) => { - if (this.rateLimitErrorHandler(e, "sendDelayedStateEvent", type)) return; - if (this.maxDelayExceededErrorHandler(e)) { - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, - }); - return; + if (this.manageMaxDelayExceededSituation(e)) { + return { + addActions: [ + { + ts: Date.now(), + type: MembershipActionType.SendFirstDelayedEvent, + }, + ], + }; } - if (this.retryOnNetworkError(e, type)) return; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); + if (updateLimit) return updateLimit; // log and fall through - if (this.unsupportedDelayedEndpoint(e)) { + if (this.isUnsupportedDelayedEndpoint(e)) { logger.info("Not using delayed event because the endpoint is not supported"); } else { logger.info("Not using delayed event because: " + e); } // On any other error we fall back to not using delayed events and send the join state event immediately - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendJoinEvent, - }); + return createAddActionUpdate(MembershipActionType.SendJoinEvent); }); } else { // This can happen if someone else (or another client) removes our own membership event. @@ -627,39 +596,45 @@ export class MembershipManager implements IMembershipManager { // In this block we will try to cancel this delayed event before setting up a new one. // Remove all running updates and restarts - this.scheduler.resetActions([], false); - await this.client + // this.scheduler.resetActions([], false); + let resetActionUpdate = { setActions: [] }; + return await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel) .then(() => { state.delayId = undefined; this.scheduler.resetRateLimitCounter(MembershipActionType.SendFirstDelayedEvent); - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, - }); + return { + ...resetActionUpdate, + ...createAddActionUpdate(MembershipActionType.SendFirstDelayedEvent), + }; }) .catch((e) => { - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; + const rateLimitActionUpdate = this.actionUpdateFromRateLimitError( + e, + "updateDelayedEvent", + type, + ); + if (rateLimitActionUpdate) return { ...resetActionUpdate, ...rateLimitActionUpdate }; + + const networkErrorActionupdate = this.actionUpdateFromNetworkErrorRetry(e, type); + if (networkErrorActionupdate) return { ...resetActionUpdate, ...networkErrorActionupdate }; + if (this.isNotFoundError(e)) { // If we get a M_NOT_FOUND we know that the delayed event got already removed. // This means we are good and can set it to undefined and run this again. state.delayId = undefined; - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, - }); - return; + return { + ...resetActionUpdate, + ...createAddActionUpdate(MembershipActionType.SendFirstDelayedEvent), + }; } - if (this.unsupportedDelayedEndpoint(e)) { - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendJoinEvent, - }); - return; + if (this.isUnsupportedDelayedEndpoint(e)) { + return { + ...resetActionUpdate, + ...createAddActionUpdate(MembershipActionType.SendJoinEvent), + }; } - if (this.retryOnNetworkError(e, type)) return; - // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases // when state.delayId !== undefined // We do not use ignore and log this error since we would also need to reset the delayId. @@ -674,39 +649,34 @@ export class MembershipManager implements IMembershipManager { case MembershipActionType.RestartDelayedEvent: { if (!state.delayId) { // Delay id got reset. This action was used to check if the hs canceled the delayed event when the join state got sent. - this.scheduler.addAction({ - ts: Date.now(), - type: state.hasMemberStateEvent + return createAddActionUpdate( + state.hasMemberStateEvent ? MembershipActionType.SendMainDelayedEvent : MembershipActionType.SendFirstDelayedEvent, - }); - break; + ); } - await this.client + return await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart) .then(() => { this.scheduler.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent); - this.scheduler.addAction({ - ts: Date.now() + this.membershipKeepAlivePeriod, - type: MembershipActionType.RestartDelayedEvent, - }); + return createAddActionUpdate( + MembershipActionType.RestartDelayedEvent, + this.membershipKeepAlivePeriod, + ); }) .catch((e) => { if (this.isNotFoundError(e)) { state.delayId = undefined; - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendMainDelayedEvent, - }); - return; + return createAddActionUpdate(MembershipActionType.SendMainDelayedEvent); } // If the HS does not support delayed events we wont reschedule. - if (this.unsupportedDelayedEndpoint(e)) return; + if (this.isUnsupportedDelayedEndpoint(e)) return {}; // TODO this also needs a test: get rate limit while checking id delayed event is scheduled - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; - - if (this.retryOnNetworkError(e, type)) return; + const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; // In other error cases we have no idea what is happening throw Error("Could not restart delayed event, even though delayed events are supported. " + e); @@ -714,7 +684,7 @@ export class MembershipManager implements IMembershipManager { break; } case MembershipActionType.SendMainDelayedEvent: { - await this.client + return await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, { @@ -727,63 +697,68 @@ export class MembershipManager implements IMembershipManager { .then((response) => { state.delayId = response.delay_id; this.scheduler.resetRateLimitCounter(MembershipActionType.SendMainDelayedEvent); - this.scheduler.addAction({ - ts: Date.now() + this.membershipKeepAlivePeriod, - type: MembershipActionType.RestartDelayedEvent, - }); + return createAddActionUpdate( + MembershipActionType.RestartDelayedEvent, + this.membershipKeepAlivePeriod, + ); }) .catch((e) => { - if (this.maxDelayExceededErrorHandler(e)) { - this.scheduler.addAction({ - ts: Date.now(), - type: MembershipActionType.SendMainDelayedEvent, - }); - return; - } - if (this.rateLimitErrorHandler(e, "sendDelayedStateEvent", type)) return; // Don't do any other delayed event work if its not supported. - if (this.unsupportedDelayedEndpoint(e)) return; - // after that - if (this.retryOnNetworkError(e, type)) return; + if (this.isUnsupportedDelayedEndpoint(e)) return {}; + + if (this.manageMaxDelayExceededSituation(e)) { + return createAddActionUpdate(MembershipActionType.SendMainDelayedEvent); + } + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + throw Error("Could not send delayed event, even though delayed events are supported. " + e); }); - break; } case MembershipActionType.SendScheduledDelayedLeaveEvent: { + // We are already good + if (!state.hasMemberStateEvent) { + this.leavePromiseDefer?.resolve(true); + this.leavePromiseDefer = undefined; + return { setActions: [] }; + } if (state.delayId) { - await this.client + return await this.client ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send) .then(() => { state.hasMemberStateEvent = false; this.scheduler.resetRateLimitCounter(MembershipActionType.SendScheduledDelayedLeaveEvent); - this.scheduler.resetActions([]); + this.leavePromiseDefer?.resolve(true); this.leavePromiseDefer = undefined; + return { setActions: [] }; }) .catch((e) => { + if (this.isUnsupportedDelayedEndpoint(e)) return {}; if (this.isNotFoundError(e)) { state.delayId = undefined; - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); - return; + return createAddActionUpdate(MembershipActionType.SendLeaveEvent); } - if (this.rateLimitErrorHandler(e, "updateDelayedEvent", type)) return; - if (this.unsupportedDelayedEndpoint(e)) return; - if (this.retryOnNetworkError(e, type)) return; + const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; // On any other error we fall back to SendLeaveEvent (this includes hard errors from rate limiting) logger.warn( "Encountered unexpected error during SendScheduledDelayedLeaveEvent. Falling back to SendLeaveEvent", e, ); - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); + return createAddActionUpdate(MembershipActionType.SendLeaveEvent); }); } else { - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.SendLeaveEvent }); + return createAddActionUpdate(MembershipActionType.SendLeaveEvent); } - break; } case MembershipActionType.SendJoinEvent: { - await this.client + return await this.client .sendStateEvent( this.room.roomId, EventType.GroupCallMemberPrefix, @@ -793,79 +768,83 @@ export class MembershipManager implements IMembershipManager { .then(() => { state.startTime = Date.now(); // The next update should already use twice the membershipEventExpiryTimeout - this.scheduler.addAction({ ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }); - this.scheduler.addAction({ - ts: this.computeNextExpiryActionTs(1), - type: MembershipActionType.UpdateExpiry, - }); - state.expireUpdateIterations = 2; + + state.expireUpdateIterations = 1; state.hasMemberStateEvent = true; this.scheduler.resetRateLimitCounter(MembershipActionType.SendJoinEvent); + return { + addActions: [ + { ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }, + { + ts: this.computeNextExpiryActionTs(state.expireUpdateIterations), + type: MembershipActionType.UpdateExpiry, + }, + ], + }; }) .catch((e) => { - if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; - - // Event sending retry (different to rate limit retries) - if (this.retryOnNetworkError(e, type)) return; - + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; throw e; }); - break; } case MembershipActionType.UpdateExpiry: { - await this.client + const nextExpireUpdateIteration = state.expireUpdateIterations + 1; + return await this.client .sendStateEvent( this.room.roomId, EventType.GroupCallMemberPrefix, - this.makeMyMembership(this.membershipEventExpiryTimeout * state.expireUpdateIterations), + this.makeMyMembership(this.membershipEventExpiryTimeout * nextExpireUpdateIteration), this.stateKey, ) .then(() => { // Success, we reset retries and schedule update. - this.scheduler.addAction({ - ts: this.computeNextExpiryActionTs(state.expireUpdateIterations), - type: MembershipActionType.UpdateExpiry, - }); - state.expireUpdateIterations++; this.scheduler.resetRateLimitCounter(MembershipActionType.UpdateExpiry); + state.expireUpdateIterations = nextExpireUpdateIteration; + return { + addActions: [ + { + ts: this.computeNextExpiryActionTs(nextExpireUpdateIteration), + type: MembershipActionType.UpdateExpiry, + }, + ], + }; }) .catch((e) => { - if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; - - // Event sending retry (different to rate limit retries) - if (this.retryOnNetworkError(e, type)) return; - + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; throw e; }); - break; } case MembershipActionType.SendLeaveEvent: { // We are good already if (!state.hasMemberStateEvent) { this.leavePromiseDefer?.resolve(true); this.leavePromiseDefer = undefined; - return; + return { setActions: [] }; } // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event - await this.client + return await this.client .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) .then(() => { this.scheduler.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); - this.scheduler.resetActions([]); this.leavePromiseDefer?.resolve(true); this.leavePromiseDefer = undefined; state.hasMemberStateEvent = false; + return { setActions: [] }; }) .catch((e) => { - if (this.rateLimitErrorHandler(e, "sendStateEvent", type)) return; - - // Event sending retry (different to rate limit retries) - if (this.retryOnNetworkError(e, type)) return; - + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; throw e; }); - break; } } } @@ -910,7 +889,7 @@ export class MembershipManager implements IMembershipManager { * @param error the error causing this handler check/execution * @returns true if its a delay exceeded error and we updated the local TimeoutOverride */ - private maxDelayExceededErrorHandler(error: unknown): boolean { + private manageMaxDelayExceededSituation(error: unknown): boolean { if ( error instanceof MatrixError && error.errcode === "M_UNKNOWN" && @@ -935,10 +914,14 @@ export class MembershipManager implements IMembershipManager { * @returns Returns true if we handled the error by rescheduling the correct next action. * Returns false if it is not a network error. */ - private rateLimitErrorHandler(error: unknown, method: string, type: MembershipActionType): boolean { + private actionUpdateFromRateLimitError( + error: unknown, + method: string, + type: MembershipActionType, + ): ActionUpdate | undefined { // "Is rate limit"-boundary if (!((error instanceof HTTPError || error instanceof MatrixError) && error.isRateLimitError())) { - return false; + return undefined; } // retry boundary @@ -957,8 +940,7 @@ export class MembershipManager implements IMembershipManager { resendDelay = defaultMs; } this.scheduler.state.rateLimitRetries.set(type, rateLimitRetries + 1); - this.scheduler.addAction({ ts: Date.now() + resendDelay, type }); - return true; + return createAddActionUpdate(type, resendDelay); } throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + (error as Error)); @@ -973,7 +955,7 @@ export class MembershipManager implements IMembershipManager { * Returns true if we handled the error by rescheduling the correct next action. * Returns false if it is not a network error. */ - private retryOnNetworkError(error: unknown, type: MembershipActionType): boolean { + private actionUpdateFromNetworkErrorRetry(error: unknown, type: MembershipActionType): ActionUpdate | undefined { // "Is a network error"-boundary const retries = this.scheduler.state.networkErrorRetries.get(type) ?? 0; const retryDurationString = this.callMemberEventRetryDelayMinimum / 1000 + "s"; @@ -1020,14 +1002,13 @@ export class MembershipManager implements IMembershipManager { error, ); } else { - return false; + return undefined; } // retry boundary if (retries < this.maximumNetworkErrorRetryCount) { this.scheduler.state.networkErrorRetries.set(type, retries + 1); - this.scheduler.addAction({ ts: Date.now() + this.callMemberEventRetryDelayMinimum, type }); - return true; + return createAddActionUpdate(type, this.callMemberEventRetryDelayMinimum); } // Failiour @@ -1041,7 +1022,12 @@ export class MembershipManager implements IMembershipManager { * @param error The error to check * @returns true it its an UnsupportedEndpointError */ - private unsupportedDelayedEndpoint(error: unknown): boolean { + private isUnsupportedDelayedEndpoint(error: unknown): boolean { return error instanceof UnsupportedEndpointError; } } +function createAddActionUpdate(type: MembershipActionType, offset?: number): ActionUpdate { + return { + addActions: [{ ts: Date.now() + (offset ?? 0), type }], + }; +} From 246354a9cd1bff94bb8edd74bda1ec4939982503 Mon Sep 17 00:00:00 2001 From: Timo Date: Sat, 8 Mar 2025 12:35:16 +0100 Subject: [PATCH 111/124] refactor: Move action scheduler --- src/matrixrtc/NewMembershipManager.ts | 235 +----------------- .../NewMembershipManagerActionScheduler.ts | 229 +++++++++++++++++ 2 files changed, 239 insertions(+), 225 deletions(-) create mode 100644 src/matrixrtc/NewMembershipManagerActionScheduler.ts diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 1d4c844c33a..4ccde7eb901 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -21,13 +21,15 @@ import { UnsupportedEndpointError } from "../errors.ts"; import { ConnectionError, HTTPError, MatrixError } from "../http-api/errors.ts"; import { logger as rootLogger } from "../logger.ts"; import { Room } from "../models/room.ts"; -import { defer, type IDeferred, sleep } from "../utils.ts"; +import { defer, type IDeferred } from "../utils.ts"; import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts"; import { type Focus } from "./focus.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { type MembershipConfig } from "./MatrixRTCSession.ts"; +import { ActionScheduler, ActionSchedulerState, ActionUpdate } from "./NewMembershipManagerActionScheduler.ts"; const logger = rootLogger.getChild("MatrixRTCSession"); + /** * This interface defines what a MembershipManager uses and exposes. * This interface is what we use to write tests and allows changing the actual implementation @@ -68,20 +70,7 @@ export interface IMembershipManager { getActiveFocus(): Focus | undefined; } -enum Status { - Disconnected = "Disconnected", - Connecting = "Connecting", - ConnectingFailed = "ConnectingFailed", - Connected = "Connected", - Reconnecting = "Reconnecting", - Disconnecting = "Disconnecting", - Stuck = "Stuck", - Unknown = "Unknown", -} - -/* SCHEDULER TYPES: - - DirectMembershipManagerAction.Join +/* MembershipActionTypes: ▼ ┌─────────────────────┐ │SendFirstDelayedEvent│ @@ -105,7 +94,6 @@ enum Status { │ │ └─────────────────────┘ STOP ALL ABOVE - DirectMembershipManagerAction.Leave ▼ ┌───────────────────────────────┐ │ SendScheduledDelayedLeaveEvent│ @@ -115,8 +103,13 @@ enum Status { ┌──────────────┐ │SendLeaveEvent│ └──────────────┘ + */ -enum MembershipActionType { +/** + * The different types of actions the MembershipManager can take. + * @internal + */ +export enum MembershipActionType { SendFirstDelayedEvent = "SendFirstDelayedEvent", // -> MembershipActionType.SendJoinEvent if successful // -> DelayedLeaveActionType.SendFirstDelayedEvent on error, retry sending the first delayed event. @@ -140,214 +133,6 @@ enum MembershipActionType { // -> MembershipActionType.SendLeaveEvent } -interface ActionSchedulerState { - /** The delayId we got when successfully sending the delayed leave event. - * Gets set to undefined if the server claims it cannot find the delayed event anymore. */ - delayId?: string; - /** Stores how often we have update the `expires` field. - * `expireUpdateIterations` * `membershipEventExpiryTimeout` resolves to the value the expires field should contain next */ - expireUpdateIterations: number; - /** The time at which we send the first state event. The time the call started from the DAG point of view. - * This is used to compute the local sleep timestamps when to next update the member event with a new expires value. */ - startTime: number; - /** Flag that gets set once join is called. - * The manager tries its best to get the user into the call. - * Does not imply the user is actually joined via room state. */ - running: boolean; - /** The manager is in the state where its actually connected to the session. */ - hasMemberStateEvent: boolean; - // There can be multiple retries at once so we need to store counters per action - // e.g. the send update membership and the restart delayed could be rate limited at the same time. - /** Retry counter for rate limits */ - rateLimitRetries: Map; - /** Retry counter for other errors */ - networkErrorRetries: Map; -} - -interface Action { - /** - * When this action should be executed - */ - ts: number; - /** - * The state of the different loops - * can also be thought of as the type of the action - */ - type: MembershipActionType; -} -interface ActionUpdate { - setActions?: Action[]; - addActions?: Action[]; -} -/** - * This state machine tracks the state of the current membership participation - * and runs one central timer that wakes up a handler callback with the correct action - * whenever necessary. - * - * It can also be awakened whenever a new action is added which is - * earlier then the current "next awake". - * @internal - */ -class ActionScheduler { - public state: ActionSchedulerState; - public static get defaultState(): ActionSchedulerState { - return { - hasMemberStateEvent: false, - running: false, - delayId: undefined, - - startTime: 0, - rateLimitRetries: new Map(), - networkErrorRetries: new Map(), - expireUpdateIterations: 1, - }; - } - public constructor( - state: ActionSchedulerState, - /** This is the callback called for each scheduled action (`this.addAction()`) */ - private membershipLoopHandler: ( - state: ActionSchedulerState, - type: MembershipActionType, - ) => Promise, - ) { - this.state = state; - } - // function for the wakeup mechanism (in case we add an action externally and need to leave the current sleep) - private wakeup: (update: ActionUpdate) => void = (update: ActionUpdate): void => { - logger.error("Cannot call wakeup before calling `startWithJoin()`"); - }; - private _actions: Action[] = []; - public get actions(): Action[] { - return this._actions; - } - - /** - * This starts the main loop of the membership manager that handles event sending, delayed event sending and delayed event restarting. - * @param initialActions The initial actions the manager will start with. It should be enough to pass: DelayedLeaveActionType.Initial - * @returns Promise that resolves once all actions have run and no more are scheduled. - * @throws This throws an error if one of the actions throws. - * In most other error cases the manager will try to handle any server errors by itself. - */ - public async startWithJoin(): Promise { - this._actions = [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }]; - - while (this._actions.length > 0) { - // Sort so next (smallest ts) action is at the beginning - this._actions.sort((a, b) => a.ts - b.ts); - const nextAction = this._actions[0]; - let wakeupUpdate: ActionUpdate | undefined = undefined; - - // while we await for the next action, wakeup has to resolve the wakeupPromise - const wakeupPromise = new Promise((resolve) => { - this.wakeup = (update: ActionUpdate): void => { - wakeupUpdate = update; - resolve(); - }; - }); - if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); - - const oldStatus = this.status; - logger.info(`MembershipManager ActionScheduler awakened. status=${oldStatus}`); - - let handlerResult: ActionUpdate = {}; - if (!wakeupUpdate) { - logger.debug( - `Current MembershipManager processing: ${nextAction.type}\nQueue:`, - this._actions, - `\nDate.now: "${Date.now()}`, - ); - try { - // `this.wakeup` can also be called and sets the `wakupUpdate` object while we are in the handler. - handlerResult = await this.membershipLoopHandler( - this.state, - nextAction.type as MembershipActionType, - ); - } catch (e) { - throw Error(`The MembershipManager shut down because of the end condition: ${e}`); - } - } - // remove the processed action only after we are done processing - this._actions.splice(0, 1); - // The wakeupUpdate always wins since that is a direct external update. - let { addActions, setActions } = wakeupUpdate ?? handlerResult; - - if (setActions) { - this._actions = setActions; - } - if (addActions) { - this._actions.push(...addActions); - } - - logger.info( - `MembershipManager ActionScheduler applied action changes. Status: ${oldStatus} -> ${this.status}`, - ); - } - logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); - } - - public initiateJoin(): void { - this.wakeup?.({ setActions: [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }] }); - } - public initiateLeave(): void { - this.wakeup?.({ setActions: [{ ts: Date.now(), type: MembershipActionType.SendScheduledDelayedLeaveEvent }] }); - } - - public resetState(): void { - this.state = ActionScheduler.defaultState; - } - - public resetRateLimitCounter(type: MembershipActionType): void { - this.state.rateLimitRetries.set(type, 0); - this.state.networkErrorRetries.set(type, 0); - } - - public get status(): Status { - if (this.actions.length === 1) { - const { type } = this.actions[0]; - switch (type) { - case MembershipActionType.SendFirstDelayedEvent: - case MembershipActionType.SendJoinEvent: - case MembershipActionType.SendMainDelayedEvent: - return Status.Connecting; - case MembershipActionType.UpdateExpiry: // where no delayed events - return Status.Connected; - case MembershipActionType.SendScheduledDelayedLeaveEvent: - case MembershipActionType.SendLeaveEvent: - return Status.Disconnecting; - default: - // pass through as not expected - } - } else if (this.actions.length === 2) { - const types = this.actions.map((a) => a.type); - // normal state for connected with delayed events - if ( - (types.includes(MembershipActionType.RestartDelayedEvent) || - types.includes(MembershipActionType.SendMainDelayedEvent)) && - types.includes(MembershipActionType.UpdateExpiry) - ) { - return Status.Connected; - } - } else if (this.actions.length === 3) { - const types = this.actions.map((a) => a.type); - // It is a correct connected state if we already schedule the next Restart but have not yet cleaned up - // the current restart. - if ( - types.filter((t) => t === MembershipActionType.RestartDelayedEvent).length === 2 && - types.includes(MembershipActionType.UpdateExpiry) - ) { - return Status.Connected; - } - } - - if (!this.state.running) { - return Status.Disconnected; - } - - logger.error("MembershipManager has an unknown state. Actions: ", this.actions); - return Status.Unknown; - } -} - /** * This class is responsible for sending all events relating to the own membership of a matrixRTC call. * It has the following tasks: diff --git a/src/matrixrtc/NewMembershipManagerActionScheduler.ts b/src/matrixrtc/NewMembershipManagerActionScheduler.ts new file mode 100644 index 00000000000..37874479a6a --- /dev/null +++ b/src/matrixrtc/NewMembershipManagerActionScheduler.ts @@ -0,0 +1,229 @@ +import { logger as rootLogger } from "../logger"; +import { sleep } from "../utils"; +import { MembershipActionType } from "./NewMembershipManager"; + +const logger = rootLogger.getChild("MatrixRTCSession"); + +/** + * @internal + */ +export interface ActionSchedulerState { + /** The delayId we got when successfully sending the delayed leave event. + * Gets set to undefined if the server claims it cannot find the delayed event anymore. */ + delayId?: string; + /** Stores how often we have update the `expires` field. + * `expireUpdateIterations` * `membershipEventExpiryTimeout` resolves to the value the expires field should contain next */ + expireUpdateIterations: number; + /** The time at which we send the first state event. The time the call started from the DAG point of view. + * This is used to compute the local sleep timestamps when to next update the member event with a new expires value. */ + startTime: number; + /** Flag that gets set once join is called. + * The manager tries its best to get the user into the call. + * Does not imply the user is actually joined via room state. */ + running: boolean; + /** The manager is in the state where its actually connected to the session. */ + hasMemberStateEvent: boolean; + // There can be multiple retries at once so we need to store counters per action + // e.g. the send update membership and the restart delayed could be rate limited at the same time. + /** Retry counter for rate limits */ + rateLimitRetries: Map; + /** Retry counter for other errors */ + networkErrorRetries: Map; +} +/** @internal */ +export interface Action { + /** + * When this action should be executed + */ + ts: number; + /** + * The state of the different loops + * can also be thought of as the type of the action + */ + type: MembershipActionType; +} +/** @internal */ +export interface ActionUpdate { + setActions?: Action[]; + addActions?: Action[]; +} + +enum Status { + Disconnected = "Disconnected", + Connecting = "Connecting", + ConnectingFailed = "ConnectingFailed", + Connected = "Connected", + Reconnecting = "Reconnecting", + Disconnecting = "Disconnecting", + Stuck = "Stuck", + Unknown = "Unknown", +} + +/** + * This scheduler tracks the state of the current membership participation + * and runs one central timer that wakes up a handler callback with the correct action + state + * whenever necessary. + * + * It can also be awakened whenever a new action is added which is + * earlier then the current "next awake". + * @internal + */ +export class ActionScheduler { + public state: ActionSchedulerState; + public static get defaultState(): ActionSchedulerState { + return { + hasMemberStateEvent: false, + running: false, + delayId: undefined, + + startTime: 0, + rateLimitRetries: new Map(), + networkErrorRetries: new Map(), + expireUpdateIterations: 1, + }; + } + public constructor( + state: ActionSchedulerState, + /** This is the callback called for each scheduled action (`this.addAction()`) */ + private membershipLoopHandler: ( + state: ActionSchedulerState, + type: MembershipActionType, + ) => Promise, + ) { + this.state = state; + } + // function for the wakeup mechanism (in case we add an action externally and need to leave the current sleep) + private wakeup: (update: ActionUpdate) => void = (update: ActionUpdate): void => { + logger.error("Cannot call wakeup before calling `startWithJoin()`"); + }; + private _actions: Action[] = []; + public get actions(): Action[] { + return this._actions; + } + + /** + * This starts the main loop of the membership manager that handles event sending, delayed event sending and delayed event restarting. + * @param initialActions The initial actions the manager will start with. It should be enough to pass: DelayedLeaveActionType.Initial + * @returns Promise that resolves once all actions have run and no more are scheduled. + * @throws This throws an error if one of the actions throws. + * In most other error cases the manager will try to handle any server errors by itself. + */ + public async startWithJoin(): Promise { + this._actions = [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }]; + + while (this._actions.length > 0) { + // Sort so next (smallest ts) action is at the beginning + this._actions.sort((a, b) => a.ts - b.ts); + const nextAction = this._actions[0]; + let wakeupUpdate: ActionUpdate | undefined = undefined; + + // while we await for the next action, wakeup has to resolve the wakeupPromise + const wakeupPromise = new Promise((resolve) => { + this.wakeup = (update: ActionUpdate): void => { + wakeupUpdate = update; + resolve(); + }; + }); + if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); + + const oldStatus = this.status; + logger.info(`MembershipManager ActionScheduler awakened. status=${oldStatus}`); + + let handlerResult: ActionUpdate = {}; + if (!wakeupUpdate) { + logger.debug( + `Current MembershipManager processing: ${nextAction.type}\nQueue:`, + this._actions, + `\nDate.now: "${Date.now()}`, + ); + try { + // `this.wakeup` can also be called and sets the `wakupUpdate` object while we are in the handler. + handlerResult = await this.membershipLoopHandler( + this.state, + nextAction.type as MembershipActionType, + ); + } catch (e) { + throw Error(`The MembershipManager shut down because of the end condition: ${e}`); + } + } + // remove the processed action only after we are done processing + this._actions.splice(0, 1); + // The wakeupUpdate always wins since that is a direct external update. + let { addActions, setActions } = wakeupUpdate ?? handlerResult; + + if (setActions) { + this._actions = setActions; + } + if (addActions) { + this._actions.push(...addActions); + } + + logger.info( + `MembershipManager ActionScheduler applied action changes. Status: ${oldStatus} -> ${this.status}`, + ); + } + logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); + } + + public initiateJoin(): void { + this.wakeup?.({ setActions: [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }] }); + } + public initiateLeave(): void { + this.wakeup?.({ setActions: [{ ts: Date.now(), type: MembershipActionType.SendScheduledDelayedLeaveEvent }] }); + } + + public resetState(): void { + this.state = ActionScheduler.defaultState; + } + + public resetRateLimitCounter(type: MembershipActionType): void { + this.state.rateLimitRetries.set(type, 0); + this.state.networkErrorRetries.set(type, 0); + } + + public get status(): Status { + if (this.actions.length === 1) { + const { type } = this.actions[0]; + switch (type) { + case MembershipActionType.SendFirstDelayedEvent: + case MembershipActionType.SendJoinEvent: + case MembershipActionType.SendMainDelayedEvent: + return Status.Connecting; + case MembershipActionType.UpdateExpiry: // where no delayed events + return Status.Connected; + case MembershipActionType.SendScheduledDelayedLeaveEvent: + case MembershipActionType.SendLeaveEvent: + return Status.Disconnecting; + default: + // pass through as not expected + } + } else if (this.actions.length === 2) { + const types = this.actions.map((a) => a.type); + // normal state for connected with delayed events + if ( + (types.includes(MembershipActionType.RestartDelayedEvent) || + types.includes(MembershipActionType.SendMainDelayedEvent)) && + types.includes(MembershipActionType.UpdateExpiry) + ) { + return Status.Connected; + } + } else if (this.actions.length === 3) { + const types = this.actions.map((a) => a.type); + // It is a correct connected state if we already schedule the next Restart but have not yet cleaned up + // the current restart. + if ( + types.filter((t) => t === MembershipActionType.RestartDelayedEvent).length === 2 && + types.includes(MembershipActionType.UpdateExpiry) + ) { + return Status.Connected; + } + } + + if (!this.state.running) { + return Status.Disconnected; + } + + logger.error("MembershipManager has an unknown state. Actions: ", this.actions); + return Status.Unknown; + } +} From b00a630d3462dc129bd08adc9dd08d5a51b247d3 Mon Sep 17 00:00:00 2001 From: Timo Date: Sat, 8 Mar 2025 12:35:52 +0100 Subject: [PATCH 112/124] refactor: move different handler cases into separate functions --- src/matrixrtc/NewMembershipManager.ts | 552 ++++++++++++++------------ 1 file changed, 294 insertions(+), 258 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 4ccde7eb901..90765766953 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -147,8 +147,6 @@ export enum MembershipActionType { * - Stop the timer for updating the state event */ export class MembershipManager implements IMembershipManager { - // PUBLIC: - public isJoined(): boolean { return this.scheduler.state.running; } @@ -271,8 +269,6 @@ export class MembershipManager implements IMembershipManager { this.stateKey = this.makeMembershipStateKey(userId, deviceId); } - // PRIVATE: - // Membership Event parameters: private deviceId: string; private stateKey: string; @@ -320,7 +316,7 @@ export class MembershipManager implements IMembershipManager { this.membershipLoopHandler(state, type), ); - // Loop Handler: + // LOOP HANDLER: private async membershipLoopHandler( state: ActionSchedulerState, type: MembershipActionType, @@ -330,48 +326,7 @@ export class MembershipManager implements IMembershipManager { // Before we start we check if we come from a state where we have a delay id. if (!state.delayId) { // this.sendFirstDelayedLeaveEvent(state, type);// Normal case without any previous delayed id. - return await this.client - ._unstable_sendDelayedStateEvent( - this.room.roomId, - { - delay: this.membershipServerSideExpiryTimeout, - }, - EventType.GroupCallMemberPrefix, - {}, // leave event - this.stateKey, - ) - .then((response) => { - // Success we reset retries and set delayId. - state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); - state.networkErrorRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); - state.delayId = response.delay_id; - return createAddActionUpdate(MembershipActionType.SendJoinEvent); - }) - .catch((e) => { - if (this.manageMaxDelayExceededSituation(e)) { - return { - addActions: [ - { - ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, - }, - ], - }; - } - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); - if (updateLimit) return updateLimit; - - // log and fall through - if (this.isUnsupportedDelayedEndpoint(e)) { - logger.info("Not using delayed event because the endpoint is not supported"); - } else { - logger.info("Not using delayed event because: " + e); - } - // On any other error we fall back to not using delayed events and send the join state event immediately - return createAddActionUpdate(MembershipActionType.SendJoinEvent); - }); + return this.sendFirstDelayedLeaveEvent(state, type); } else { // This can happen if someone else (or another client) removes our own membership event. // It will trigger `onRTCSessionMemberUpdate` queue `MembershipActionType.SendFirstDelayedEvent`. @@ -380,56 +335,8 @@ export class MembershipManager implements IMembershipManager { // // In this block we will try to cancel this delayed event before setting up a new one. - // Remove all running updates and restarts - // this.scheduler.resetActions([], false); - let resetActionUpdate = { setActions: [] }; - return await this.client - ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Cancel) - .then(() => { - state.delayId = undefined; - this.scheduler.resetRateLimitCounter(MembershipActionType.SendFirstDelayedEvent); - return { - ...resetActionUpdate, - ...createAddActionUpdate(MembershipActionType.SendFirstDelayedEvent), - }; - }) - .catch((e) => { - const rateLimitActionUpdate = this.actionUpdateFromRateLimitError( - e, - "updateDelayedEvent", - type, - ); - if (rateLimitActionUpdate) return { ...resetActionUpdate, ...rateLimitActionUpdate }; - - const networkErrorActionupdate = this.actionUpdateFromNetworkErrorRetry(e, type); - if (networkErrorActionupdate) return { ...resetActionUpdate, ...networkErrorActionupdate }; - - if (this.isNotFoundError(e)) { - // If we get a M_NOT_FOUND we know that the delayed event got already removed. - // This means we are good and can set it to undefined and run this again. - state.delayId = undefined; - return { - ...resetActionUpdate, - ...createAddActionUpdate(MembershipActionType.SendFirstDelayedEvent), - }; - } - if (this.isUnsupportedDelayedEndpoint(e)) { - return { - ...resetActionUpdate, - ...createAddActionUpdate(MembershipActionType.SendJoinEvent), - }; - } - - // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases - // when state.delayId !== undefined - // We do not use ignore and log this error since we would also need to reset the delayId. - // It is cleaner if we the frontend rejoines instead of resetting the delayId here and behaving like in the success case. - throw Error( - "We failed to cancel a delayed event where we already had a delay id with an error we cannot automatically handle", - ); - }); + return this.cancelKnownDelayIdBeforeSendFirstDelayedEvent(state, type, state.delayId); } - break; } case MembershipActionType.RestartDelayedEvent: { if (!state.delayId) { @@ -440,67 +347,10 @@ export class MembershipManager implements IMembershipManager { : MembershipActionType.SendFirstDelayedEvent, ); } - return await this.client - ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Restart) - .then(() => { - this.scheduler.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent); - return createAddActionUpdate( - MembershipActionType.RestartDelayedEvent, - this.membershipKeepAlivePeriod, - ); - }) - .catch((e) => { - if (this.isNotFoundError(e)) { - state.delayId = undefined; - return createAddActionUpdate(MembershipActionType.SendMainDelayedEvent); - } - // If the HS does not support delayed events we wont reschedule. - if (this.isUnsupportedDelayedEndpoint(e)) return {}; - - // TODO this also needs a test: get rate limit while checking id delayed event is scheduled - const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; - - // In other error cases we have no idea what is happening - throw Error("Could not restart delayed event, even though delayed events are supported. " + e); - }); - break; + return this.restartDelayedEvent(state, type, state.delayId); } case MembershipActionType.SendMainDelayedEvent: { - return await this.client - ._unstable_sendDelayedStateEvent( - this.room.roomId, - { - delay: this.membershipServerSideExpiryTimeout, - }, - EventType.GroupCallMemberPrefix, - {}, // leave event - this.stateKey, - ) - .then((response) => { - state.delayId = response.delay_id; - this.scheduler.resetRateLimitCounter(MembershipActionType.SendMainDelayedEvent); - return createAddActionUpdate( - MembershipActionType.RestartDelayedEvent, - this.membershipKeepAlivePeriod, - ); - }) - .catch((e) => { - // Don't do any other delayed event work if its not supported. - if (this.isUnsupportedDelayedEndpoint(e)) return {}; - - if (this.manageMaxDelayExceededSituation(e)) { - return createAddActionUpdate(MembershipActionType.SendMainDelayedEvent); - } - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; - - throw Error("Could not send delayed event, even though delayed events are supported. " + e); - }); + return this.sendMainDelayedEvent(state, type); } case MembershipActionType.SendScheduledDelayedLeaveEvent: { // We are already good @@ -510,100 +360,16 @@ export class MembershipManager implements IMembershipManager { return { setActions: [] }; } if (state.delayId) { - return await this.client - ._unstable_updateDelayedEvent(state.delayId, UpdateDelayedEventAction.Send) - .then(() => { - state.hasMemberStateEvent = false; - this.scheduler.resetRateLimitCounter(MembershipActionType.SendScheduledDelayedLeaveEvent); - - this.leavePromiseDefer?.resolve(true); - this.leavePromiseDefer = undefined; - return { setActions: [] }; - }) - .catch((e) => { - if (this.isUnsupportedDelayedEndpoint(e)) return {}; - if (this.isNotFoundError(e)) { - state.delayId = undefined; - return createAddActionUpdate(MembershipActionType.SendLeaveEvent); - } - const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; - - // On any other error we fall back to SendLeaveEvent (this includes hard errors from rate limiting) - logger.warn( - "Encountered unexpected error during SendScheduledDelayedLeaveEvent. Falling back to SendLeaveEvent", - e, - ); - return createAddActionUpdate(MembershipActionType.SendLeaveEvent); - }); + return this.sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent(state, type, state.delayId); } else { return createAddActionUpdate(MembershipActionType.SendLeaveEvent); } } case MembershipActionType.SendJoinEvent: { - return await this.client - .sendStateEvent( - this.room.roomId, - EventType.GroupCallMemberPrefix, - this.makeMyMembership(this.membershipEventExpiryTimeout), - this.stateKey, - ) - .then(() => { - state.startTime = Date.now(); - // The next update should already use twice the membershipEventExpiryTimeout - - state.expireUpdateIterations = 1; - state.hasMemberStateEvent = true; - this.scheduler.resetRateLimitCounter(MembershipActionType.SendJoinEvent); - return { - addActions: [ - { ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }, - { - ts: this.computeNextExpiryActionTs(state.expireUpdateIterations), - type: MembershipActionType.UpdateExpiry, - }, - ], - }; - }) - .catch((e) => { - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; - throw e; - }); + return this.sendJoinEvent(state, type); } case MembershipActionType.UpdateExpiry: { - const nextExpireUpdateIteration = state.expireUpdateIterations + 1; - return await this.client - .sendStateEvent( - this.room.roomId, - EventType.GroupCallMemberPrefix, - this.makeMyMembership(this.membershipEventExpiryTimeout * nextExpireUpdateIteration), - this.stateKey, - ) - .then(() => { - // Success, we reset retries and schedule update. - this.scheduler.resetRateLimitCounter(MembershipActionType.UpdateExpiry); - state.expireUpdateIterations = nextExpireUpdateIteration; - return { - addActions: [ - { - ts: this.computeNextExpiryActionTs(nextExpireUpdateIteration), - type: MembershipActionType.UpdateExpiry, - }, - ], - }; - }) - .catch((e) => { - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; - throw e; - }); + return this.updateExpiryOnJoinedEvent(state, type); } case MembershipActionType.SendLeaveEvent: { // We are good already @@ -614,26 +380,296 @@ export class MembershipManager implements IMembershipManager { } // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event - return await this.client - .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) - .then(() => { - this.scheduler.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); - this.leavePromiseDefer?.resolve(true); - this.leavePromiseDefer = undefined; - state.hasMemberStateEvent = false; - return { setActions: [] }; - }) - .catch((e) => { - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; - throw e; - }); + return this.sendFallbackLeaveEvent(state, type); } } } + // HANDLERS (used in the membershipLoopHandler) + + private async sendFirstDelayedLeaveEvent( + state: ActionSchedulerState, + type: MembershipActionType, + ): Promise { + return await this.client + ._unstable_sendDelayedStateEvent( + this.room.roomId, + { + delay: this.membershipServerSideExpiryTimeout, + }, + EventType.GroupCallMemberPrefix, + {}, // leave event + this.stateKey, + ) + .then((response) => { + // Success we reset retries and set delayId. + state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); + state.networkErrorRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); + state.delayId = response.delay_id; + return createAddActionUpdate(MembershipActionType.SendJoinEvent); + }) + .catch((e) => { + if (this.manageMaxDelayExceededSituation(e)) { + return { + addActions: [ + { + ts: Date.now(), + type: MembershipActionType.SendFirstDelayedEvent, + }, + ], + }; + } + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); + if (updateLimit) return updateLimit; + + // log and fall through + if (this.isUnsupportedDelayedEndpoint(e)) { + logger.info("Not using delayed event because the endpoint is not supported"); + } else { + logger.info("Not using delayed event because: " + e); + } + // On any other error we fall back to not using delayed events and send the join state event immediately + return createAddActionUpdate(MembershipActionType.SendJoinEvent); + }); + } + + private async cancelKnownDelayIdBeforeSendFirstDelayedEvent( + state: ActionSchedulerState, + type: MembershipActionType, + delayId: string, + ): Promise { + // Remove all running updates and restarts + const resetActionUpdate = { setActions: [] }; + return await this.client + ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Cancel) + .then(() => { + state.delayId = undefined; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendFirstDelayedEvent); + return { + ...resetActionUpdate, + ...createAddActionUpdate(MembershipActionType.SendFirstDelayedEvent), + }; + }) + .catch((e) => { + const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); + if (updateLimit) return { ...resetActionUpdate, ...updateLimit }; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return { ...resetActionUpdate, ...updateNetwork }; + + if (this.isNotFoundError(e)) { + // If we get a M_NOT_FOUND we know that the delayed event got already removed. + // This means we are good and can set it to undefined and run this again. + state.delayId = undefined; + return { + ...resetActionUpdate, + ...createAddActionUpdate(MembershipActionType.SendFirstDelayedEvent), + }; + } + if (this.isUnsupportedDelayedEndpoint(e)) { + return { + ...resetActionUpdate, + ...createAddActionUpdate(MembershipActionType.SendJoinEvent), + }; + } + + // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases + // when state.delayId !== undefined + // We do not use ignore and log this error since we would also need to reset the delayId. + // It is cleaner if we the frontend rejoines instead of resetting the delayId here and behaving like in the success case. + throw Error( + "We failed to cancel a delayed event where we already had a delay id with an error we cannot automatically handle", + ); + }); + } + + private async restartDelayedEvent( + state: ActionSchedulerState, + type: MembershipActionType, + delayId: string, + ): Promise { + return await this.client + ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart) + .then(() => { + this.scheduler.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent); + return createAddActionUpdate(MembershipActionType.RestartDelayedEvent, this.membershipKeepAlivePeriod); + }) + .catch((e) => { + if (this.isNotFoundError(e)) { + state.delayId = undefined; + return createAddActionUpdate(MembershipActionType.SendMainDelayedEvent); + } + // If the HS does not support delayed events we wont reschedule. + if (this.isUnsupportedDelayedEndpoint(e)) return {}; + + // TODO this also needs a test: get rate limit while checking id delayed event is scheduled + const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + + // In other error cases we have no idea what is happening + throw Error("Could not restart delayed event, even though delayed events are supported. " + e); + }); + } + + private async sendMainDelayedEvent(state: ActionSchedulerState, type: MembershipActionType): Promise { + return await this.client + ._unstable_sendDelayedStateEvent( + this.room.roomId, + { + delay: this.membershipServerSideExpiryTimeout, + }, + EventType.GroupCallMemberPrefix, + {}, // leave event + this.stateKey, + ) + .then((response) => { + state.delayId = response.delay_id; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendMainDelayedEvent); + return createAddActionUpdate(MembershipActionType.RestartDelayedEvent, this.membershipKeepAlivePeriod); + }) + .catch((e) => { + // Don't do any other delayed event work if its not supported. + if (this.isUnsupportedDelayedEndpoint(e)) return {}; + + if (this.manageMaxDelayExceededSituation(e)) { + return createAddActionUpdate(MembershipActionType.SendMainDelayedEvent); + } + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + + throw Error("Could not send delayed event, even though delayed events are supported. " + e); + }); + } + + private async sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent( + state: ActionSchedulerState, + type: MembershipActionType, + delayId: string, + ): Promise { + return await this.client + ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Send) + .then(() => { + state.hasMemberStateEvent = false; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendScheduledDelayedLeaveEvent); + + this.leavePromiseDefer?.resolve(true); + this.leavePromiseDefer = undefined; + return { setActions: [] }; + }) + .catch((e) => { + if (this.isUnsupportedDelayedEndpoint(e)) return {}; + if (this.isNotFoundError(e)) { + state.delayId = undefined; + return createAddActionUpdate(MembershipActionType.SendLeaveEvent); + } + const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + + // On any other error we fall back to SendLeaveEvent (this includes hard errors from rate limiting) + logger.warn( + "Encountered unexpected error during SendScheduledDelayedLeaveEvent. Falling back to SendLeaveEvent", + e, + ); + return createAddActionUpdate(MembershipActionType.SendLeaveEvent); + }); + } + + private async sendJoinEvent(state: ActionSchedulerState, type: MembershipActionType): Promise { + return await this.client + .sendStateEvent( + this.room.roomId, + EventType.GroupCallMemberPrefix, + this.makeMyMembership(this.membershipEventExpiryTimeout), + this.stateKey, + ) + .then(() => { + state.startTime = Date.now(); + // The next update should already use twice the membershipEventExpiryTimeout + + state.expireUpdateIterations = 1; + state.hasMemberStateEvent = true; + this.scheduler.resetRateLimitCounter(MembershipActionType.SendJoinEvent); + return { + addActions: [ + { ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }, + { + ts: this.computeNextExpiryActionTs(state.expireUpdateIterations), + type: MembershipActionType.UpdateExpiry, + }, + ], + }; + }) + .catch((e) => { + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + throw e; + }); + } + + private async updateExpiryOnJoinedEvent( + state: ActionSchedulerState, + type: MembershipActionType, + ): Promise { + const nextExpireUpdateIteration = state.expireUpdateIterations + 1; + return await this.client + .sendStateEvent( + this.room.roomId, + EventType.GroupCallMemberPrefix, + this.makeMyMembership(this.membershipEventExpiryTimeout * nextExpireUpdateIteration), + this.stateKey, + ) + .then(() => { + // Success, we reset retries and schedule update. + this.scheduler.resetRateLimitCounter(MembershipActionType.UpdateExpiry); + state.expireUpdateIterations = nextExpireUpdateIteration; + return { + addActions: [ + { + ts: this.computeNextExpiryActionTs(nextExpireUpdateIteration), + type: MembershipActionType.UpdateExpiry, + }, + ], + }; + }) + .catch((e) => { + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + throw e; + }); + } + private async sendFallbackLeaveEvent( + state: ActionSchedulerState, + type: MembershipActionType, + ): Promise { + return await this.client + .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) + .then(() => { + this.scheduler.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); + this.leavePromiseDefer?.resolve(true); + this.leavePromiseDefer = undefined; + state.hasMemberStateEvent = false; + return { setActions: [] }; + }) + .catch((e) => { + const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); + if (updateNetwork) return updateNetwork; + throw e; + }); + } + // HELPERS private makeMembershipStateKey(localUserId: string, localDeviceId: string): string { const stateKey = `${localUserId}_${localDeviceId}`; From 798c77c40ef498f588cbbca38096ae4e94e1497b Mon Sep 17 00:00:00 2001 From: Timo Date: Sat, 8 Mar 2025 12:39:19 +0100 Subject: [PATCH 113/124] linter --- src/matrixrtc/NewMembershipManager.ts | 8 ++++++-- src/matrixrtc/NewMembershipManagerActionScheduler.ts | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 90765766953..1517fe67adf 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -20,13 +20,17 @@ import { type MatrixClient } from "../client.ts"; import { UnsupportedEndpointError } from "../errors.ts"; import { ConnectionError, HTTPError, MatrixError } from "../http-api/errors.ts"; import { logger as rootLogger } from "../logger.ts"; -import { Room } from "../models/room.ts"; +import { type Room } from "../models/room.ts"; import { defer, type IDeferred } from "../utils.ts"; import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts"; import { type Focus } from "./focus.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { type MembershipConfig } from "./MatrixRTCSession.ts"; -import { ActionScheduler, ActionSchedulerState, ActionUpdate } from "./NewMembershipManagerActionScheduler.ts"; +import { + ActionScheduler, + type ActionSchedulerState, + type ActionUpdate, +} from "./NewMembershipManagerActionScheduler.ts"; const logger = rootLogger.getChild("MatrixRTCSession"); diff --git a/src/matrixrtc/NewMembershipManagerActionScheduler.ts b/src/matrixrtc/NewMembershipManagerActionScheduler.ts index 37874479a6a..1600ca467a6 100644 --- a/src/matrixrtc/NewMembershipManagerActionScheduler.ts +++ b/src/matrixrtc/NewMembershipManagerActionScheduler.ts @@ -1,6 +1,6 @@ -import { logger as rootLogger } from "../logger"; -import { sleep } from "../utils"; -import { MembershipActionType } from "./NewMembershipManager"; +import { logger as rootLogger } from "../logger.ts"; +import { sleep } from "../utils.ts"; +import { MembershipActionType } from "./NewMembershipManager.ts"; const logger = rootLogger.getChild("MatrixRTCSession"); @@ -149,7 +149,7 @@ export class ActionScheduler { // remove the processed action only after we are done processing this._actions.splice(0, 1); // The wakeupUpdate always wins since that is a direct external update. - let { addActions, setActions } = wakeupUpdate ?? handlerResult; + const { addActions, setActions } = wakeupUpdate ?? handlerResult; if (setActions) { this._actions = setActions; From 9cd235898b0171623ae017763642db47706b4f75 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 10 Mar 2025 13:55:40 +0100 Subject: [PATCH 114/124] review: delayed events endpoint error --- spec/unit/matrixrtc/MembershipManager.spec.ts | 8 ++++---- src/client.ts | 19 ++++++++++++++----- src/embedded.ts | 9 ++++++--- src/errors.ts | 7 +++++-- src/matrixrtc/NewMembershipManager.ts | 8 ++++---- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index 5f3783a17c6..db6c33c1223 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -19,7 +19,7 @@ limitations under the License. import { type MockedFunction, type Mock } from "jest-mock"; -import { EventType, HTTPError, MatrixError, UnsupportedEndpointError, type Room } from "../../../src"; +import { EventType, HTTPError, MatrixError, UnsupportedDelayedEventsEndpointError, type Room } from "../../../src"; import { type Focus, type LivekitFocusActive, type SessionMembershipData } from "../../../src/matrixrtc"; import { LegacyMembershipManager } from "../../../src/matrixrtc/LegacyMembershipManager"; import { makeMockClient, makeMockRoom, membershipTemplate, mockCallMembership, type MockClient } from "./mocks"; @@ -247,7 +247,7 @@ describe.each([ const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive); delayedHandle.reject?.( - new UnsupportedEndpointError( + new UnsupportedDelayedEventsEndpointError( "Server does not support the delayed events API", "sendDelayedStateEvent", ), @@ -686,10 +686,10 @@ describe.each([ ); expect(client.sendStateEvent).not.toHaveBeenCalled(); }); - it("falls back to using pure state events when UnsupportedEndpointError encountered for delayed events !FailsForLegacy", async () => { + it("falls back to using pure state events when UnsupportedDelayedEventsEndpointError encountered for delayed events !FailsForLegacy", async () => { const unrecoverableError = jest.fn(); (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue( - new UnsupportedEndpointError("not supported", "sendDelayedStateEvent"), + new UnsupportedDelayedEventsEndpointError("not supported", "sendDelayedStateEvent"), ); const manager = new TestMembershipManager({}, room, client, () => undefined); manager.join([focus], focusActive, unrecoverableError); diff --git a/src/client.ts b/src/client.ts index edcd4e1c347..9638676a839 100644 --- a/src/client.ts +++ b/src/client.ts @@ -240,7 +240,7 @@ import { validateAuthMetadataAndKeys, } from "./oidc/index.ts"; import { type EmptyObject } from "./@types/common.ts"; -import { UnsupportedEndpointError } from "./errors.ts"; +import { UnsupportedDelayedEventsEndpointError } from "./errors.ts"; export type Store = IStore; @@ -3352,7 +3352,10 @@ export class MatrixClient extends TypedEventEmitter { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw new UnsupportedEndpointError("Server does not support the delayed events API", "sendDelayedEvent"); + throw new UnsupportedDelayedEventsEndpointError( + "Server does not support the delayed events API", + "sendDelayedEvent", + ); } this.addThreadRelationIfNeeded(content, threadId, roomId); @@ -3375,7 +3378,7 @@ export class MatrixClient extends TypedEventEmitter { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw new UnsupportedEndpointError( + throw new UnsupportedDelayedEventsEndpointError( "Server does not support the delayed events API", "sendDelayedStateEvent", ); @@ -3402,7 +3405,10 @@ export class MatrixClient extends TypedEventEmitter { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw new UnsupportedEndpointError("Server does not support the delayed events API", "getDelayedEvents"); + throw new UnsupportedDelayedEventsEndpointError( + "Server does not support the delayed events API", + "getDelayedEvents", + ); } const queryDict = fromToken ? { from: fromToken } : undefined; @@ -3424,7 +3430,10 @@ export class MatrixClient extends TypedEventEmitter { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw new UnsupportedEndpointError("Server does not support the delayed events API", "updateDelayedEvent"); + throw new UnsupportedDelayedEventsEndpointError( + "Server does not support the delayed events API", + "updateDelayedEvent", + ); } const path = utils.encodeUri("/delayed_events/$delayId", { diff --git a/src/embedded.ts b/src/embedded.ts index fde54740f04..0882872e5ad 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -55,7 +55,7 @@ import { User } from "./models/user.ts"; import { type Room } from "./models/room.ts"; import { type ToDeviceBatch, type ToDevicePayload } from "./models/ToDeviceMessage.ts"; import { MapWithDefault, recursiveMapToObject } from "./utils.ts"; -import { type EmptyObject, TypedEventEmitter, UnsupportedEndpointError } from "./matrix.ts"; +import { type EmptyObject, TypedEventEmitter, UnsupportedDelayedEventsEndpointError } from "./matrix.ts"; interface IStateEventRequest { eventType: string; @@ -422,7 +422,7 @@ export class RoomWidgetClient extends MatrixClient { stateKey = "", ): Promise { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw new UnsupportedEndpointError( + throw new UnsupportedDelayedEventsEndpointError( "Server does not support the delayed events API", "sendDelayedStateEvent", ); @@ -454,7 +454,10 @@ export class RoomWidgetClient extends MatrixClient { // eslint-disable-next-line public async _unstable_updateDelayedEvent(delayId: string, action: UpdateDelayedEventAction): Promise { if (!(await this.doesServerSupportUnstableFeature(UNSTABLE_MSC4140_DELAYED_EVENTS))) { - throw new UnsupportedEndpointError("Server does not support the delayed events API", "updateDelayedEvent"); + throw new UnsupportedDelayedEventsEndpointError( + "Server does not support the delayed events API", + "updateDelayedEvent", + ); } await this.widgetApi.updateDelayedEvent(delayId, action).catch(timeoutToConnectionError); diff --git a/src/errors.ts b/src/errors.ts index a88c89188dc..8baf7979bc4 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -53,12 +53,15 @@ export class ClientStoppedError extends Error { } } -export class UnsupportedEndpointError extends Error { +/** + * This error is thrown when the Homeserver does not support the delayed events enpdpoints. + */ +export class UnsupportedDelayedEventsEndpointError extends Error { public constructor( message: string, public clientEndpoint: "sendDelayedEvent" | "updateDelayedEvent" | "sendDelayedStateEvent" | "getDelayedEvents", ) { super(message); - this.name = "UnsupportedEndpointError"; + this.name = "UnsupportedDelayedEventsEndpointError"; } } diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 1517fe67adf..4ae84612d82 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -17,7 +17,7 @@ limitations under the License. import { EventType } from "../@types/event.ts"; import { UpdateDelayedEventAction } from "../@types/requests.ts"; import { type MatrixClient } from "../client.ts"; -import { UnsupportedEndpointError } from "../errors.ts"; +import { UnsupportedDelayedEventsEndpointError } from "../errors.ts"; import { ConnectionError, HTTPError, MatrixError } from "../http-api/errors.ts"; import { logger as rootLogger } from "../logger.ts"; import { type Room } from "../models/room.ts"; @@ -843,12 +843,12 @@ export class MembershipManager implements IMembershipManager { } /** - * Check if its an UnsupportedEndpointError and which implies that we cannot do any delayed event logic + * Check if its an UnsupportedDelayedEventsEndpointError and which implies that we cannot do any delayed event logic * @param error The error to check - * @returns true it its an UnsupportedEndpointError + * @returns true it its an UnsupportedDelayedEventsEndpointError */ private isUnsupportedDelayedEndpoint(error: unknown): boolean { - return error instanceof UnsupportedEndpointError; + return error instanceof UnsupportedDelayedEventsEndpointError; } } function createAddActionUpdate(type: MembershipActionType, offset?: number): ActionUpdate { From de492d95cea4593f962bc759507501c36d42a7db Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 10 Mar 2025 13:56:31 +0100 Subject: [PATCH 115/124] review --- spec/unit/matrixrtc/MembershipManager.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index db6c33c1223..a7a5143afbd 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -618,7 +618,7 @@ describe.each([ }); describe("unrecoverable errors", () => { // !FailsForLegacy because legacy does not have a retry limit and no mechanism to communicate unrecoverable errors. - it("throws, when reaching maximum number of retires for initial delayed event creation !FailsForLegacy", async () => { + it("throws, when reaching maximum number of retries for initial delayed event creation !FailsForLegacy", async () => { const delayEventSendError = jest.fn(); (client._unstable_sendDelayedStateEvent as Mock).mockRejectedValue( new MatrixError( @@ -638,7 +638,7 @@ describe.each([ expect(delayEventSendError).toHaveBeenCalled(); }); // !FailsForLegacy because legacy does not have a retry limit and no mechanism to communicate unrecoverable errors. - it("throws, when reaching maximum number of retires !FailsForLegacy", async () => { + it("throws, when reaching maximum number of retries !FailsForLegacy", async () => { const delayEventRestartError = jest.fn(); (client._unstable_updateDelayedEvent as Mock).mockRejectedValue( new MatrixError( @@ -695,7 +695,7 @@ describe.each([ manager.join([focus], focusActive, unrecoverableError); await jest.advanceTimersByTimeAsync(1); - expect(unrecoverableError).not.toHaveBeenCalledWith(); + expect(unrecoverableError).not.toHaveBeenCalled(); expect(client.sendStateEvent).toHaveBeenCalled(); }); }); From 31c5cfdbdf56f0ad0dee9ef845b7e62ec733f43b Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Mon, 10 Mar 2025 16:27:52 +0000 Subject: [PATCH 116/124] Suggestions from pair review --- src/matrixrtc/NewMembershipManager.ts | 58 ++++++++++--------- .../NewMembershipManagerActionScheduler.ts | 30 ++++++---- 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 4ae84612d82..ab1cb1c844f 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -345,7 +345,7 @@ export class MembershipManager implements IMembershipManager { case MembershipActionType.RestartDelayedEvent: { if (!state.delayId) { // Delay id got reset. This action was used to check if the hs canceled the delayed event when the join state got sent. - return createAddActionUpdate( + return createInsertActionUpdate( state.hasMemberStateEvent ? MembershipActionType.SendMainDelayedEvent : MembershipActionType.SendFirstDelayedEvent, @@ -361,12 +361,12 @@ export class MembershipManager implements IMembershipManager { if (!state.hasMemberStateEvent) { this.leavePromiseDefer?.resolve(true); this.leavePromiseDefer = undefined; - return { setActions: [] }; + return { replace: [] }; } if (state.delayId) { return this.sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent(state, type, state.delayId); } else { - return createAddActionUpdate(MembershipActionType.SendLeaveEvent); + return createInsertActionUpdate(MembershipActionType.SendLeaveEvent); } } case MembershipActionType.SendJoinEvent: { @@ -380,7 +380,7 @@ export class MembershipManager implements IMembershipManager { if (!state.hasMemberStateEvent) { this.leavePromiseDefer?.resolve(true); this.leavePromiseDefer = undefined; - return { setActions: [] }; + return { replace: [] }; } // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event @@ -410,12 +410,12 @@ export class MembershipManager implements IMembershipManager { state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); state.networkErrorRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); state.delayId = response.delay_id; - return createAddActionUpdate(MembershipActionType.SendJoinEvent); + return createInsertActionUpdate(MembershipActionType.SendJoinEvent); }) .catch((e) => { if (this.manageMaxDelayExceededSituation(e)) { return { - addActions: [ + insert: [ { ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent, @@ -435,7 +435,7 @@ export class MembershipManager implements IMembershipManager { logger.info("Not using delayed event because: " + e); } // On any other error we fall back to not using delayed events and send the join state event immediately - return createAddActionUpdate(MembershipActionType.SendJoinEvent); + return createInsertActionUpdate(MembershipActionType.SendJoinEvent); }); } @@ -445,7 +445,7 @@ export class MembershipManager implements IMembershipManager { delayId: string, ): Promise { // Remove all running updates and restarts - const resetActionUpdate = { setActions: [] }; + const resetActionUpdate = { replace: [] }; return await this.client ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Cancel) .then(() => { @@ -453,7 +453,7 @@ export class MembershipManager implements IMembershipManager { this.scheduler.resetRateLimitCounter(MembershipActionType.SendFirstDelayedEvent); return { ...resetActionUpdate, - ...createAddActionUpdate(MembershipActionType.SendFirstDelayedEvent), + ...createInsertActionUpdate(MembershipActionType.SendFirstDelayedEvent), }; }) .catch((e) => { @@ -468,13 +468,13 @@ export class MembershipManager implements IMembershipManager { state.delayId = undefined; return { ...resetActionUpdate, - ...createAddActionUpdate(MembershipActionType.SendFirstDelayedEvent), + ...createInsertActionUpdate(MembershipActionType.SendFirstDelayedEvent), }; } if (this.isUnsupportedDelayedEndpoint(e)) { return { ...resetActionUpdate, - ...createAddActionUpdate(MembershipActionType.SendJoinEvent), + ...createInsertActionUpdate(MembershipActionType.SendJoinEvent), }; } @@ -497,12 +497,15 @@ export class MembershipManager implements IMembershipManager { ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart) .then(() => { this.scheduler.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent); - return createAddActionUpdate(MembershipActionType.RestartDelayedEvent, this.membershipKeepAlivePeriod); + return createInsertActionUpdate( + MembershipActionType.RestartDelayedEvent, + this.membershipKeepAlivePeriod, + ); }) .catch((e) => { if (this.isNotFoundError(e)) { state.delayId = undefined; - return createAddActionUpdate(MembershipActionType.SendMainDelayedEvent); + return createInsertActionUpdate(MembershipActionType.SendMainDelayedEvent); } // If the HS does not support delayed events we wont reschedule. if (this.isUnsupportedDelayedEndpoint(e)) return {}; @@ -532,14 +535,17 @@ export class MembershipManager implements IMembershipManager { .then((response) => { state.delayId = response.delay_id; this.scheduler.resetRateLimitCounter(MembershipActionType.SendMainDelayedEvent); - return createAddActionUpdate(MembershipActionType.RestartDelayedEvent, this.membershipKeepAlivePeriod); + return createInsertActionUpdate( + MembershipActionType.RestartDelayedEvent, + this.membershipKeepAlivePeriod, + ); }) .catch((e) => { // Don't do any other delayed event work if its not supported. if (this.isUnsupportedDelayedEndpoint(e)) return {}; if (this.manageMaxDelayExceededSituation(e)) { - return createAddActionUpdate(MembershipActionType.SendMainDelayedEvent); + return createInsertActionUpdate(MembershipActionType.SendMainDelayedEvent); } const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); if (updateLimit) return updateLimit; @@ -563,13 +569,13 @@ export class MembershipManager implements IMembershipManager { this.leavePromiseDefer?.resolve(true); this.leavePromiseDefer = undefined; - return { setActions: [] }; + return { replace: [] }; }) .catch((e) => { if (this.isUnsupportedDelayedEndpoint(e)) return {}; if (this.isNotFoundError(e)) { state.delayId = undefined; - return createAddActionUpdate(MembershipActionType.SendLeaveEvent); + return createInsertActionUpdate(MembershipActionType.SendLeaveEvent); } const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); if (updateLimit) return updateLimit; @@ -581,7 +587,7 @@ export class MembershipManager implements IMembershipManager { "Encountered unexpected error during SendScheduledDelayedLeaveEvent. Falling back to SendLeaveEvent", e, ); - return createAddActionUpdate(MembershipActionType.SendLeaveEvent); + return createInsertActionUpdate(MembershipActionType.SendLeaveEvent); }); } @@ -601,7 +607,7 @@ export class MembershipManager implements IMembershipManager { state.hasMemberStateEvent = true; this.scheduler.resetRateLimitCounter(MembershipActionType.SendJoinEvent); return { - addActions: [ + insert: [ { ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }, { ts: this.computeNextExpiryActionTs(state.expireUpdateIterations), @@ -636,7 +642,7 @@ export class MembershipManager implements IMembershipManager { this.scheduler.resetRateLimitCounter(MembershipActionType.UpdateExpiry); state.expireUpdateIterations = nextExpireUpdateIteration; return { - addActions: [ + insert: [ { ts: this.computeNextExpiryActionTs(nextExpireUpdateIteration), type: MembershipActionType.UpdateExpiry, @@ -663,7 +669,7 @@ export class MembershipManager implements IMembershipManager { this.leavePromiseDefer?.resolve(true); this.leavePromiseDefer = undefined; state.hasMemberStateEvent = false; - return { setActions: [] }; + return { replace: [] }; }) .catch((e) => { const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); @@ -765,7 +771,7 @@ export class MembershipManager implements IMembershipManager { resendDelay = defaultMs; } this.scheduler.state.rateLimitRetries.set(type, rateLimitRetries + 1); - return createAddActionUpdate(type, resendDelay); + return createInsertActionUpdate(type, resendDelay); } throw Error("Exceeded maximum retries for " + type + " attempts (client." + method + "): " + (error as Error)); @@ -833,10 +839,10 @@ export class MembershipManager implements IMembershipManager { // retry boundary if (retries < this.maximumNetworkErrorRetryCount) { this.scheduler.state.networkErrorRetries.set(type, retries + 1); - return createAddActionUpdate(type, this.callMemberEventRetryDelayMinimum); + return createInsertActionUpdate(type, this.callMemberEventRetryDelayMinimum); } - // Failiour + // Failure throw Error( "Reached maximum (" + this.maximumNetworkErrorRetryCount + ") retries cause by: " + (error as Error), ); @@ -851,8 +857,8 @@ export class MembershipManager implements IMembershipManager { return error instanceof UnsupportedDelayedEventsEndpointError; } } -function createAddActionUpdate(type: MembershipActionType, offset?: number): ActionUpdate { +function createInsertActionUpdate(type: MembershipActionType, offset?: number): ActionUpdate { return { - addActions: [{ ts: Date.now() + (offset ?? 0), type }], + insert: [{ ts: Date.now() + (offset ?? 0), type }], }; } diff --git a/src/matrixrtc/NewMembershipManagerActionScheduler.ts b/src/matrixrtc/NewMembershipManagerActionScheduler.ts index 1600ca467a6..267ccfd0919 100644 --- a/src/matrixrtc/NewMembershipManagerActionScheduler.ts +++ b/src/matrixrtc/NewMembershipManagerActionScheduler.ts @@ -1,4 +1,5 @@ import { logger as rootLogger } from "../logger.ts"; +import { type EmptyObject } from "../matrix.ts"; import { sleep } from "../utils.ts"; import { MembershipActionType } from "./NewMembershipManager.ts"; @@ -43,10 +44,16 @@ export interface Action { type: MembershipActionType; } /** @internal */ -export interface ActionUpdate { - setActions?: Action[]; - addActions?: Action[]; -} +export type ActionUpdate = + | { + /** Replace all existing scheduled actions with this new array */ + replace: Action[]; + } + | { + /** Add these actions to the existing scheduled actions */ + insert: Action[]; + } + | EmptyObject; enum Status { Disconnected = "Disconnected", @@ -149,13 +156,12 @@ export class ActionScheduler { // remove the processed action only after we are done processing this._actions.splice(0, 1); // The wakeupUpdate always wins since that is a direct external update. - const { addActions, setActions } = wakeupUpdate ?? handlerResult; + const actionUpdate = wakeupUpdate ?? handlerResult; - if (setActions) { - this._actions = setActions; - } - if (addActions) { - this._actions.push(...addActions); + if ("replace" in actionUpdate) { + this._actions = actionUpdate.replace; + } else if ("insert" in actionUpdate) { + this._actions.push(...actionUpdate.insert); } logger.info( @@ -166,10 +172,10 @@ export class ActionScheduler { } public initiateJoin(): void { - this.wakeup?.({ setActions: [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }] }); + this.wakeup?.({ replace: [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }] }); } public initiateLeave(): void { - this.wakeup?.({ setActions: [{ ts: Date.now(), type: MembershipActionType.SendScheduledDelayedLeaveEvent }] }); + this.wakeup?.({ replace: [{ ts: Date.now(), type: MembershipActionType.SendScheduledDelayedLeaveEvent }] }); } public resetState(): void { From 6af4730919ec07ce9aaad8de35c27ac6b98a3019 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Mon, 10 Mar 2025 16:31:37 +0000 Subject: [PATCH 117/124] resetState is actually only used internally --- src/matrixrtc/NewMembershipManager.ts | 1 - src/matrixrtc/NewMembershipManagerActionScheduler.ts | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index ab1cb1c844f..235f5a88762 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -168,7 +168,6 @@ export class MembershipManager implements IMembershipManager { this.focusActive = focusActive; this.leavePromiseDefer = undefined; if (!this.scheduler.state.running) { - this.scheduler.resetState(); this.scheduler.state.running = true; this.scheduler.startWithJoin().catch((e) => { // Set the rtc session to left state since we cannot recover from here and the consumer user of the diff --git a/src/matrixrtc/NewMembershipManagerActionScheduler.ts b/src/matrixrtc/NewMembershipManagerActionScheduler.ts index 267ccfd0919..1ed71cc71c0 100644 --- a/src/matrixrtc/NewMembershipManagerActionScheduler.ts +++ b/src/matrixrtc/NewMembershipManagerActionScheduler.ts @@ -116,6 +116,7 @@ export class ActionScheduler { * In most other error cases the manager will try to handle any server errors by itself. */ public async startWithJoin(): Promise { + this.state = ActionScheduler.defaultState; this._actions = [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }]; while (this._actions.length > 0) { @@ -178,10 +179,6 @@ export class ActionScheduler { this.wakeup?.({ replace: [{ ts: Date.now(), type: MembershipActionType.SendScheduledDelayedLeaveEvent }] }); } - public resetState(): void { - this.state = ActionScheduler.defaultState; - } - public resetRateLimitCounter(type: MembershipActionType): void { this.state.rateLimitRetries.set(type, 0); this.state.networkErrorRetries.set(type, 0); From 3e4caffd587ae21a9c51cd61a225ae8172032b9f Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Mon, 10 Mar 2025 16:37:28 +0000 Subject: [PATCH 118/124] Revert "resetState is actually only used internally" This reverts commit 6af4730919ec07ce9aaad8de35c27ac6b98a3019. --- src/matrixrtc/NewMembershipManager.ts | 1 + src/matrixrtc/NewMembershipManagerActionScheduler.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 235f5a88762..ab1cb1c844f 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -168,6 +168,7 @@ export class MembershipManager implements IMembershipManager { this.focusActive = focusActive; this.leavePromiseDefer = undefined; if (!this.scheduler.state.running) { + this.scheduler.resetState(); this.scheduler.state.running = true; this.scheduler.startWithJoin().catch((e) => { // Set the rtc session to left state since we cannot recover from here and the consumer user of the diff --git a/src/matrixrtc/NewMembershipManagerActionScheduler.ts b/src/matrixrtc/NewMembershipManagerActionScheduler.ts index 1ed71cc71c0..267ccfd0919 100644 --- a/src/matrixrtc/NewMembershipManagerActionScheduler.ts +++ b/src/matrixrtc/NewMembershipManagerActionScheduler.ts @@ -116,7 +116,6 @@ export class ActionScheduler { * In most other error cases the manager will try to handle any server errors by itself. */ public async startWithJoin(): Promise { - this.state = ActionScheduler.defaultState; this._actions = [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }]; while (this._actions.length > 0) { @@ -179,6 +178,10 @@ export class ActionScheduler { this.wakeup?.({ replace: [{ ts: Date.now(), type: MembershipActionType.SendScheduledDelayedLeaveEvent }] }); } + public resetState(): void { + this.state = ActionScheduler.defaultState; + } + public resetRateLimitCounter(type: MembershipActionType): void { this.state.rateLimitRetries.set(type, 0); this.state.networkErrorRetries.set(type, 0); From 2c6062f8c6da0d05d6c84933db51a697c9fc54fc Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 11 Mar 2025 09:32:26 +0100 Subject: [PATCH 119/124] refactor: running is part of the scheduler (not state) --- src/matrixrtc/NewMembershipManager.ts | 58 ++++----- .../NewMembershipManagerActionScheduler.ts | 117 ++++++++++-------- 2 files changed, 91 insertions(+), 84 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index ab1cb1c844f..0018736ba49 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -152,7 +152,7 @@ export enum MembershipActionType { */ export class MembershipManager implements IMembershipManager { public isJoined(): boolean { - return this.scheduler.state.running; + return this.scheduler.running; } /** @@ -164,20 +164,26 @@ export class MembershipManager implements IMembershipManager { * This should bubble up the the frontend to communicate that the call does not work in the current environment. */ public join(fociPreferred: Focus[], focusActive?: Focus, onError?: (error: unknown) => void): void { + if (this.isJoined()) { + logger.error("MembershipManager is already running. Ignoring join request."); + return; + } this.fociPreferred = fociPreferred; this.focusActive = focusActive; this.leavePromiseDefer = undefined; - if (!this.scheduler.state.running) { - this.scheduler.resetState(); - this.scheduler.state.running = true; - this.scheduler.startWithJoin().catch((e) => { - // Set the rtc session to left state since we cannot recover from here and the consumer user of the - // MatrixRTCSession class needs to manually rejoin. - this.scheduler.state.running = false; + + this.scheduler.resetState(); + + this.scheduler + .startWithJoin() + .catch((e) => { logger.error("MembershipManager stopped because: ", e); onError?.(e); + }) + .then(() => { + this.leavePromiseDefer?.resolve(true); + this.leavePromiseDefer = undefined; }); - } } /** @@ -186,8 +192,7 @@ export class MembershipManager implements IMembershipManager { * @returns true if it managed to leave and false if the timeout condition happened. */ public leave(timeout?: number): Promise { - if (!this.scheduler.state.running) return Promise.resolve(true); - this.scheduler.state.running = false; + if (!this.scheduler.running) return Promise.resolve(true); // We use the promise to track if we already scheduled a leave event // So we do not check scheduler.actions/scheduler.insertions @@ -359,8 +364,6 @@ export class MembershipManager implements IMembershipManager { case MembershipActionType.SendScheduledDelayedLeaveEvent: { // We are already good if (!state.hasMemberStateEvent) { - this.leavePromiseDefer?.resolve(true); - this.leavePromiseDefer = undefined; return { replace: [] }; } if (state.delayId) { @@ -378,8 +381,6 @@ export class MembershipManager implements IMembershipManager { case MembershipActionType.SendLeaveEvent: { // We are good already if (!state.hasMemberStateEvent) { - this.leavePromiseDefer?.resolve(true); - this.leavePromiseDefer = undefined; return { replace: [] }; } // This is only a fallback in case we do not have working delayed events support. @@ -451,10 +452,7 @@ export class MembershipManager implements IMembershipManager { .then(() => { state.delayId = undefined; this.scheduler.resetRateLimitCounter(MembershipActionType.SendFirstDelayedEvent); - return { - ...resetActionUpdate, - ...createInsertActionUpdate(MembershipActionType.SendFirstDelayedEvent), - }; + return createReplaceActionUpdate(MembershipActionType.SendFirstDelayedEvent); }) .catch((e) => { const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); @@ -466,16 +464,10 @@ export class MembershipManager implements IMembershipManager { // If we get a M_NOT_FOUND we know that the delayed event got already removed. // This means we are good and can set it to undefined and run this again. state.delayId = undefined; - return { - ...resetActionUpdate, - ...createInsertActionUpdate(MembershipActionType.SendFirstDelayedEvent), - }; + return createReplaceActionUpdate(MembershipActionType.SendFirstDelayedEvent); } if (this.isUnsupportedDelayedEndpoint(e)) { - return { - ...resetActionUpdate, - ...createInsertActionUpdate(MembershipActionType.SendJoinEvent), - }; + return createReplaceActionUpdate(MembershipActionType.SendJoinEvent); } // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases @@ -567,8 +559,6 @@ export class MembershipManager implements IMembershipManager { state.hasMemberStateEvent = false; this.scheduler.resetRateLimitCounter(MembershipActionType.SendScheduledDelayedLeaveEvent); - this.leavePromiseDefer?.resolve(true); - this.leavePromiseDefer = undefined; return { replace: [] }; }) .catch((e) => { @@ -666,8 +656,6 @@ export class MembershipManager implements IMembershipManager { .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) .then(() => { this.scheduler.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); - this.leavePromiseDefer?.resolve(true); - this.leavePromiseDefer = undefined; state.hasMemberStateEvent = false; return { replace: [] }; }) @@ -689,6 +677,7 @@ export class MembershipManager implements IMembershipManager { return `_${stateKey}`; } } + /** * Constructs our own membership */ @@ -857,8 +846,15 @@ export class MembershipManager implements IMembershipManager { return error instanceof UnsupportedDelayedEventsEndpointError; } } + function createInsertActionUpdate(type: MembershipActionType, offset?: number): ActionUpdate { return { insert: [{ ts: Date.now() + (offset ?? 0), type }], }; } + +function createReplaceActionUpdate(type: MembershipActionType, offset?: number): ActionUpdate { + return { + replace: [{ ts: Date.now() + (offset ?? 0), type }], + }; +} diff --git a/src/matrixrtc/NewMembershipManagerActionScheduler.ts b/src/matrixrtc/NewMembershipManagerActionScheduler.ts index 267ccfd0919..4463ada79ab 100644 --- a/src/matrixrtc/NewMembershipManagerActionScheduler.ts +++ b/src/matrixrtc/NewMembershipManagerActionScheduler.ts @@ -18,10 +18,6 @@ export interface ActionSchedulerState { /** The time at which we send the first state event. The time the call started from the DAG point of view. * This is used to compute the local sleep timestamps when to next update the member event with a new expires value. */ startTime: number; - /** Flag that gets set once join is called. - * The manager tries its best to get the user into the call. - * Does not imply the user is actually joined via room state. */ - running: boolean; /** The manager is in the state where its actually connected to the session. */ hasMemberStateEvent: boolean; // There can be multiple retries at once so we need to store counters per action @@ -31,6 +27,7 @@ export interface ActionSchedulerState { /** Retry counter for other errors */ networkErrorRetries: Map; } + /** @internal */ export interface Action { /** @@ -43,6 +40,7 @@ export interface Action { */ type: MembershipActionType; } + /** @internal */ export type ActionUpdate = | { @@ -76,11 +74,11 @@ enum Status { * @internal */ export class ActionScheduler { + public running = false; public state: ActionSchedulerState; public static get defaultState(): ActionSchedulerState { return { hasMemberStateEvent: false, - running: false, delayId: undefined, startTime: 0, @@ -116,58 +114,71 @@ export class ActionScheduler { * In most other error cases the manager will try to handle any server errors by itself. */ public async startWithJoin(): Promise { + if (this.running) { + logger.error("Cannot call startWithJoin() on NewMembershipActionScheduler while already running"); + return; + } + this.running = true; this._actions = [{ ts: Date.now(), type: MembershipActionType.SendFirstDelayedEvent }]; - - while (this._actions.length > 0) { - // Sort so next (smallest ts) action is at the beginning - this._actions.sort((a, b) => a.ts - b.ts); - const nextAction = this._actions[0]; - let wakeupUpdate: ActionUpdate | undefined = undefined; - - // while we await for the next action, wakeup has to resolve the wakeupPromise - const wakeupPromise = new Promise((resolve) => { - this.wakeup = (update: ActionUpdate): void => { - wakeupUpdate = update; - resolve(); - }; - }); - if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); - - const oldStatus = this.status; - logger.info(`MembershipManager ActionScheduler awakened. status=${oldStatus}`); - - let handlerResult: ActionUpdate = {}; - if (!wakeupUpdate) { - logger.debug( - `Current MembershipManager processing: ${nextAction.type}\nQueue:`, - this._actions, - `\nDate.now: "${Date.now()}`, - ); - try { - // `this.wakeup` can also be called and sets the `wakupUpdate` object while we are in the handler. - handlerResult = await this.membershipLoopHandler( - this.state, - nextAction.type as MembershipActionType, + try { + while (this._actions.length > 0) { + // Sort so next (smallest ts) action is at the beginning + this._actions.sort((a, b) => a.ts - b.ts); + const nextAction = this._actions[0]; + let wakeupUpdate: ActionUpdate | undefined = undefined; + + // while we await for the next action, wakeup has to resolve the wakeupPromise + const wakeupPromise = new Promise((resolve) => { + this.wakeup = (update: ActionUpdate): void => { + wakeupUpdate = update; + resolve(); + }; + }); + if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); + + const oldStatus = this.status; + logger.info(`MembershipManager ActionScheduler awakened. status=${oldStatus}`); + + let handlerResult: ActionUpdate = {}; + if (!wakeupUpdate) { + logger.debug( + `Current MembershipManager processing: ${nextAction.type}\nQueue:`, + this._actions, + `\nDate.now: "${Date.now()}`, ); - } catch (e) { - throw Error(`The MembershipManager shut down because of the end condition: ${e}`); + try { + // `this.wakeup` can also be called and sets the `wakupUpdate` object while we are in the handler. + handlerResult = await this.membershipLoopHandler( + this.state, + nextAction.type as MembershipActionType, + ); + } catch (e) { + throw Error(`The MembershipManager shut down because of the end condition: ${e}`); + } + } + // remove the processed action only after we are done processing + this._actions.splice(0, 1); + // The wakeupUpdate always wins since that is a direct external update. + const actionUpdate = wakeupUpdate ?? handlerResult; + + if ("replace" in actionUpdate) { + this._actions = actionUpdate.replace; + } else if ("insert" in actionUpdate) { + this._actions.push(...actionUpdate.insert); } - } - // remove the processed action only after we are done processing - this._actions.splice(0, 1); - // The wakeupUpdate always wins since that is a direct external update. - const actionUpdate = wakeupUpdate ?? handlerResult; - - if ("replace" in actionUpdate) { - this._actions = actionUpdate.replace; - } else if ("insert" in actionUpdate) { - this._actions.push(...actionUpdate.insert); - } - logger.info( - `MembershipManager ActionScheduler applied action changes. Status: ${oldStatus} -> ${this.status}`, - ); + logger.info( + `MembershipManager ActionScheduler applied action changes. Status: ${oldStatus} -> ${this.status}`, + ); + } + } catch (e) { + // Set the rtc session "not running" state since we cannot recover from here and the consumer user of the + // MatrixRTCSession class needs to manually rejoin. + this.running = false; + throw e; } + this.running = false; + logger.debug("Leave MembershipManager ActionScheduler loop (no more actions)"); } @@ -225,7 +236,7 @@ export class ActionScheduler { } } - if (!this.state.running) { + if (!this.running) { return Status.Disconnected; } From 2f77b7e34849ceebf31b92de4112625ff0c14510 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 11 Mar 2025 10:07:52 +0100 Subject: [PATCH 120/124] refactor: move everything state related from schduler to manager. --- src/matrixrtc/NewMembershipManager.ts | 245 ++++++++++++------ .../NewMembershipManagerActionScheduler.ts | 123 +-------- 2 files changed, 169 insertions(+), 199 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 0018736ba49..516577af130 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -26,11 +26,7 @@ import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipDat import { type Focus } from "./focus.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { type MembershipConfig } from "./MatrixRTCSession.ts"; -import { - ActionScheduler, - type ActionSchedulerState, - type ActionUpdate, -} from "./NewMembershipManagerActionScheduler.ts"; +import { ActionScheduler, type ActionUpdate } from "./NewMembershipManagerActionScheduler.ts"; const logger = rootLogger.getChild("MatrixRTCSession"); @@ -137,6 +133,40 @@ export enum MembershipActionType { // -> MembershipActionType.SendLeaveEvent } +/** + * @internal + */ +export interface ActionSchedulerState { + /** The delayId we got when successfully sending the delayed leave event. + * Gets set to undefined if the server claims it cannot find the delayed event anymore. */ + delayId?: string; + /** Stores how often we have update the `expires` field. + * `expireUpdateIterations` * `membershipEventExpiryTimeout` resolves to the value the expires field should contain next */ + expireUpdateIterations: number; + /** The time at which we send the first state event. The time the call started from the DAG point of view. + * This is used to compute the local sleep timestamps when to next update the member event with a new expires value. */ + startTime: number; + /** The manager is in the state where its actually connected to the session. */ + hasMemberStateEvent: boolean; + // There can be multiple retries at once so we need to store counters per action + // e.g. the send update membership and the restart delayed could be rate limited at the same time. + /** Retry counter for rate limits */ + rateLimitRetries: Map; + /** Retry counter for other errors */ + networkErrorRetries: Map; +} + +enum Status { + Disconnected = "Disconnected", + Connecting = "Connecting", + ConnectingFailed = "ConnectingFailed", + Connected = "Connected", + Reconnecting = "Reconnecting", + Disconnecting = "Disconnecting", + Stuck = "Stuck", + Unknown = "Unknown", +} + /** * This class is responsible for sending all events relating to the own membership of a matrixRTC call. * It has the following tasks: @@ -172,7 +202,7 @@ export class MembershipManager implements IMembershipManager { this.focusActive = focusActive; this.leavePromiseDefer = undefined; - this.scheduler.resetState(); + this.state = MembershipManager.defaultState; this.scheduler .startWithJoin() @@ -225,7 +255,7 @@ export class MembershipManager implements IMembershipManager { ); } else { // Only react to our own membership missing if we have not already scheduled sending a new membership DirectMembershipManagerAction.Join - this.scheduler.state.hasMemberStateEvent = false; + this.state.hasMemberStateEvent = false; this.scheduler.initiateJoin(); } } @@ -276,9 +306,23 @@ export class MembershipManager implements IMembershipManager { if (deviceId === null) throw Error("Missing deviceId in client"); this.deviceId = deviceId; this.stateKey = this.makeMembershipStateKey(userId, deviceId); + this.state = MembershipManager.defaultState; } - // Membership Event parameters: + // MembershipManager mutable state. + public state: ActionSchedulerState; + public static get defaultState(): ActionSchedulerState { + return { + hasMemberStateEvent: false, + delayId: undefined, + + startTime: 0, + rateLimitRetries: new Map(), + networkErrorRetries: new Map(), + expireUpdateIterations: 1, + }; + } + // Membership Event static parameters: private deviceId: string; private stateKey: string; private fociPreferred?: Focus[]; @@ -298,7 +342,7 @@ export class MembershipManager implements IMembershipManager { } private computeNextExpiryActionTs(iteration: number): number { return ( - this.scheduler.state.startTime + + this.state.startTime + this.membershipEventExpiryTimeout * iteration - this.membershipEventExpiryTimeoutHeadroom ); @@ -321,21 +365,26 @@ export class MembershipManager implements IMembershipManager { } // Scheduler: - private scheduler = new ActionScheduler(ActionScheduler.defaultState, (state, type) => - this.membershipLoopHandler(state, type), - ); + private oldStatus?: Status; + private scheduler = new ActionScheduler((type): Promise => { + if (this.oldStatus) { + // we put this at the beginngin of the actsion scheduler loop handle callback since it is a loop this + // is equivalent to running it at the end of the loop. (just after applying the status/action list changes) + logger.debug(`MembershipManager applied action changes. Status: ${this.oldStatus} -> ${this.status}`); + } + this.oldStatus = this.status; + logger.debug(`MembershipManager before processing action. status=${this.oldStatus}`); + return this.membershipLoopHandler(type); + }); // LOOP HANDLER: - private async membershipLoopHandler( - state: ActionSchedulerState, - type: MembershipActionType, - ): Promise { + private async membershipLoopHandler(type: MembershipActionType): Promise { + this.oldStatus = this.status; switch (type) { case MembershipActionType.SendFirstDelayedEvent: { // Before we start we check if we come from a state where we have a delay id. - if (!state.delayId) { - // this.sendFirstDelayedLeaveEvent(state, type);// Normal case without any previous delayed id. - return this.sendFirstDelayedLeaveEvent(state, type); + if (!this.state.delayId) { + return this.sendFirstDelayedLeaveEvent(type); // Normal case without any previous delayed id. } else { // This can happen if someone else (or another client) removes our own membership event. // It will trigger `onRTCSessionMemberUpdate` queue `MembershipActionType.SendFirstDelayedEvent`. @@ -344,58 +393,55 @@ export class MembershipManager implements IMembershipManager { // // In this block we will try to cancel this delayed event before setting up a new one. - return this.cancelKnownDelayIdBeforeSendFirstDelayedEvent(state, type, state.delayId); + return this.cancelKnownDelayIdBeforeSendFirstDelayedEvent(type, this.state.delayId); } } case MembershipActionType.RestartDelayedEvent: { - if (!state.delayId) { + if (!this.state.delayId) { // Delay id got reset. This action was used to check if the hs canceled the delayed event when the join state got sent. return createInsertActionUpdate( - state.hasMemberStateEvent + this.state.hasMemberStateEvent ? MembershipActionType.SendMainDelayedEvent : MembershipActionType.SendFirstDelayedEvent, ); } - return this.restartDelayedEvent(state, type, state.delayId); + return this.restartDelayedEvent(type, this.state.delayId); } case MembershipActionType.SendMainDelayedEvent: { - return this.sendMainDelayedEvent(state, type); + return this.sendMainDelayedEvent(type); } case MembershipActionType.SendScheduledDelayedLeaveEvent: { // We are already good - if (!state.hasMemberStateEvent) { + if (!this.state.hasMemberStateEvent) { return { replace: [] }; } - if (state.delayId) { - return this.sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent(state, type, state.delayId); + if (this.state.delayId) { + return this.sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent(type, this.state.delayId); } else { return createInsertActionUpdate(MembershipActionType.SendLeaveEvent); } } case MembershipActionType.SendJoinEvent: { - return this.sendJoinEvent(state, type); + return this.sendJoinEvent(type); } case MembershipActionType.UpdateExpiry: { - return this.updateExpiryOnJoinedEvent(state, type); + return this.updateExpiryOnJoinedEvent(type); } case MembershipActionType.SendLeaveEvent: { // We are good already - if (!state.hasMemberStateEvent) { + if (!this.state.hasMemberStateEvent) { return { replace: [] }; } // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event - return this.sendFallbackLeaveEvent(state, type); + return this.sendFallbackLeaveEvent(type); } } } // HANDLERS (used in the membershipLoopHandler) - private async sendFirstDelayedLeaveEvent( - state: ActionSchedulerState, - type: MembershipActionType, - ): Promise { + private async sendFirstDelayedLeaveEvent(type: MembershipActionType): Promise { return await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, @@ -408,9 +454,9 @@ export class MembershipManager implements IMembershipManager { ) .then((response) => { // Success we reset retries and set delayId. - state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); - state.networkErrorRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); - state.delayId = response.delay_id; + this.state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); + this.state.networkErrorRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); + this.state.delayId = response.delay_id; return createInsertActionUpdate(MembershipActionType.SendJoinEvent); }) .catch((e) => { @@ -441,7 +487,6 @@ export class MembershipManager implements IMembershipManager { } private async cancelKnownDelayIdBeforeSendFirstDelayedEvent( - state: ActionSchedulerState, type: MembershipActionType, delayId: string, ): Promise { @@ -450,8 +495,8 @@ export class MembershipManager implements IMembershipManager { return await this.client ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Cancel) .then(() => { - state.delayId = undefined; - this.scheduler.resetRateLimitCounter(MembershipActionType.SendFirstDelayedEvent); + this.state.delayId = undefined; + this.resetRateLimitCounter(MembershipActionType.SendFirstDelayedEvent); return createReplaceActionUpdate(MembershipActionType.SendFirstDelayedEvent); }) .catch((e) => { @@ -463,7 +508,7 @@ export class MembershipManager implements IMembershipManager { if (this.isNotFoundError(e)) { // If we get a M_NOT_FOUND we know that the delayed event got already removed. // This means we are good and can set it to undefined and run this again. - state.delayId = undefined; + this.state.delayId = undefined; return createReplaceActionUpdate(MembershipActionType.SendFirstDelayedEvent); } if (this.isUnsupportedDelayedEndpoint(e)) { @@ -480,15 +525,11 @@ export class MembershipManager implements IMembershipManager { }); } - private async restartDelayedEvent( - state: ActionSchedulerState, - type: MembershipActionType, - delayId: string, - ): Promise { + private async restartDelayedEvent(type: MembershipActionType, delayId: string): Promise { return await this.client ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart) .then(() => { - this.scheduler.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent); + this.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent); return createInsertActionUpdate( MembershipActionType.RestartDelayedEvent, this.membershipKeepAlivePeriod, @@ -496,7 +537,7 @@ export class MembershipManager implements IMembershipManager { }) .catch((e) => { if (this.isNotFoundError(e)) { - state.delayId = undefined; + this.state.delayId = undefined; return createInsertActionUpdate(MembershipActionType.SendMainDelayedEvent); } // If the HS does not support delayed events we wont reschedule. @@ -513,7 +554,7 @@ export class MembershipManager implements IMembershipManager { }); } - private async sendMainDelayedEvent(state: ActionSchedulerState, type: MembershipActionType): Promise { + private async sendMainDelayedEvent(type: MembershipActionType): Promise { return await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, @@ -525,8 +566,8 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ) .then((response) => { - state.delayId = response.delay_id; - this.scheduler.resetRateLimitCounter(MembershipActionType.SendMainDelayedEvent); + this.state.delayId = response.delay_id; + this.resetRateLimitCounter(MembershipActionType.SendMainDelayedEvent); return createInsertActionUpdate( MembershipActionType.RestartDelayedEvent, this.membershipKeepAlivePeriod, @@ -549,22 +590,21 @@ export class MembershipManager implements IMembershipManager { } private async sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent( - state: ActionSchedulerState, type: MembershipActionType, delayId: string, ): Promise { return await this.client ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Send) .then(() => { - state.hasMemberStateEvent = false; - this.scheduler.resetRateLimitCounter(MembershipActionType.SendScheduledDelayedLeaveEvent); + this.state.hasMemberStateEvent = false; + this.resetRateLimitCounter(MembershipActionType.SendScheduledDelayedLeaveEvent); return { replace: [] }; }) .catch((e) => { if (this.isUnsupportedDelayedEndpoint(e)) return {}; if (this.isNotFoundError(e)) { - state.delayId = undefined; + this.state.delayId = undefined; return createInsertActionUpdate(MembershipActionType.SendLeaveEvent); } const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); @@ -581,7 +621,7 @@ export class MembershipManager implements IMembershipManager { }); } - private async sendJoinEvent(state: ActionSchedulerState, type: MembershipActionType): Promise { + private async sendJoinEvent(type: MembershipActionType): Promise { return await this.client .sendStateEvent( this.room.roomId, @@ -590,17 +630,16 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ) .then(() => { - state.startTime = Date.now(); + this.state.startTime = Date.now(); // The next update should already use twice the membershipEventExpiryTimeout - - state.expireUpdateIterations = 1; - state.hasMemberStateEvent = true; - this.scheduler.resetRateLimitCounter(MembershipActionType.SendJoinEvent); + this.state.expireUpdateIterations = 1; + this.state.hasMemberStateEvent = true; + this.resetRateLimitCounter(MembershipActionType.SendJoinEvent); return { insert: [ { ts: Date.now(), type: MembershipActionType.RestartDelayedEvent }, { - ts: this.computeNextExpiryActionTs(state.expireUpdateIterations), + ts: this.computeNextExpiryActionTs(this.state.expireUpdateIterations), type: MembershipActionType.UpdateExpiry, }, ], @@ -615,11 +654,8 @@ export class MembershipManager implements IMembershipManager { }); } - private async updateExpiryOnJoinedEvent( - state: ActionSchedulerState, - type: MembershipActionType, - ): Promise { - const nextExpireUpdateIteration = state.expireUpdateIterations + 1; + private async updateExpiryOnJoinedEvent(type: MembershipActionType): Promise { + const nextExpireUpdateIteration = this.state.expireUpdateIterations + 1; return await this.client .sendStateEvent( this.room.roomId, @@ -629,8 +665,8 @@ export class MembershipManager implements IMembershipManager { ) .then(() => { // Success, we reset retries and schedule update. - this.scheduler.resetRateLimitCounter(MembershipActionType.UpdateExpiry); - state.expireUpdateIterations = nextExpireUpdateIteration; + this.resetRateLimitCounter(MembershipActionType.UpdateExpiry); + this.state.expireUpdateIterations = nextExpireUpdateIteration; return { insert: [ { @@ -648,15 +684,12 @@ export class MembershipManager implements IMembershipManager { throw e; }); } - private async sendFallbackLeaveEvent( - state: ActionSchedulerState, - type: MembershipActionType, - ): Promise { + private async sendFallbackLeaveEvent(type: MembershipActionType): Promise { return await this.client .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) .then(() => { - this.scheduler.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); - state.hasMemberStateEvent = false; + this.resetRateLimitCounter(MembershipActionType.SendLeaveEvent); + this.state.hasMemberStateEvent = false; return { replace: [] }; }) .catch((e) => { @@ -745,7 +778,7 @@ export class MembershipManager implements IMembershipManager { } // retry boundary - const rateLimitRetries = this.scheduler.state.rateLimitRetries.get(type) ?? 0; + const rateLimitRetries = this.state.rateLimitRetries.get(type) ?? 0; if (rateLimitRetries < this.maximumRateLimitRetryCount) { let resendDelay: number; const defaultMs = 5000; @@ -759,7 +792,7 @@ export class MembershipManager implements IMembershipManager { ); resendDelay = defaultMs; } - this.scheduler.state.rateLimitRetries.set(type, rateLimitRetries + 1); + this.state.rateLimitRetries.set(type, rateLimitRetries + 1); return createInsertActionUpdate(type, resendDelay); } @@ -777,7 +810,7 @@ export class MembershipManager implements IMembershipManager { */ private actionUpdateFromNetworkErrorRetry(error: unknown, type: MembershipActionType): ActionUpdate | undefined { // "Is a network error"-boundary - const retries = this.scheduler.state.networkErrorRetries.get(type) ?? 0; + const retries = this.state.networkErrorRetries.get(type) ?? 0; const retryDurationString = this.callMemberEventRetryDelayMinimum / 1000 + "s"; const retryCounterString = "(" + retries + "/" + this.maximumNetworkErrorRetryCount + ")"; if (error instanceof Error && error.name === "AbortError") { @@ -827,7 +860,7 @@ export class MembershipManager implements IMembershipManager { // retry boundary if (retries < this.maximumNetworkErrorRetryCount) { - this.scheduler.state.networkErrorRetries.set(type, retries + 1); + this.state.networkErrorRetries.set(type, retries + 1); return createInsertActionUpdate(type, this.callMemberEventRetryDelayMinimum); } @@ -845,6 +878,58 @@ export class MembershipManager implements IMembershipManager { private isUnsupportedDelayedEndpoint(error: unknown): boolean { return error instanceof UnsupportedDelayedEventsEndpointError; } + + private resetRateLimitCounter(type: MembershipActionType): void { + this.state.rateLimitRetries.set(type, 0); + this.state.networkErrorRetries.set(type, 0); + } + + public get status(): Status { + const actions = this.scheduler.actions; + if (actions.length === 1) { + const { type } = actions[0]; + switch (type) { + case MembershipActionType.SendFirstDelayedEvent: + case MembershipActionType.SendJoinEvent: + case MembershipActionType.SendMainDelayedEvent: + return Status.Connecting; + case MembershipActionType.UpdateExpiry: // where no delayed events + return Status.Connected; + case MembershipActionType.SendScheduledDelayedLeaveEvent: + case MembershipActionType.SendLeaveEvent: + return Status.Disconnecting; + default: + // pass through as not expected + } + } else if (actions.length === 2) { + const types = actions.map((a) => a.type); + // normal state for connected with delayed events + if ( + (types.includes(MembershipActionType.RestartDelayedEvent) || + types.includes(MembershipActionType.SendMainDelayedEvent)) && + types.includes(MembershipActionType.UpdateExpiry) + ) { + return Status.Connected; + } + } else if (actions.length === 3) { + const types = actions.map((a) => a.type); + // It is a correct connected state if we already schedule the next Restart but have not yet cleaned up + // the current restart. + if ( + types.filter((t) => t === MembershipActionType.RestartDelayedEvent).length === 2 && + types.includes(MembershipActionType.UpdateExpiry) + ) { + return Status.Connected; + } + } + + if (!this.scheduler.running) { + return Status.Disconnected; + } + + logger.error("MembershipManager has an unknown state. Actions: ", actions); + return Status.Unknown; + } } function createInsertActionUpdate(type: MembershipActionType, offset?: number): ActionUpdate { diff --git a/src/matrixrtc/NewMembershipManagerActionScheduler.ts b/src/matrixrtc/NewMembershipManagerActionScheduler.ts index 4463ada79ab..8bb37bc116d 100644 --- a/src/matrixrtc/NewMembershipManagerActionScheduler.ts +++ b/src/matrixrtc/NewMembershipManagerActionScheduler.ts @@ -5,29 +5,6 @@ import { MembershipActionType } from "./NewMembershipManager.ts"; const logger = rootLogger.getChild("MatrixRTCSession"); -/** - * @internal - */ -export interface ActionSchedulerState { - /** The delayId we got when successfully sending the delayed leave event. - * Gets set to undefined if the server claims it cannot find the delayed event anymore. */ - delayId?: string; - /** Stores how often we have update the `expires` field. - * `expireUpdateIterations` * `membershipEventExpiryTimeout` resolves to the value the expires field should contain next */ - expireUpdateIterations: number; - /** The time at which we send the first state event. The time the call started from the DAG point of view. - * This is used to compute the local sleep timestamps when to next update the member event with a new expires value. */ - startTime: number; - /** The manager is in the state where its actually connected to the session. */ - hasMemberStateEvent: boolean; - // There can be multiple retries at once so we need to store counters per action - // e.g. the send update membership and the restart delayed could be rate limited at the same time. - /** Retry counter for rate limits */ - rateLimitRetries: Map; - /** Retry counter for other errors */ - networkErrorRetries: Map; -} - /** @internal */ export interface Action { /** @@ -53,17 +30,6 @@ export type ActionUpdate = } | EmptyObject; -enum Status { - Disconnected = "Disconnected", - Connecting = "Connecting", - ConnectingFailed = "ConnectingFailed", - Connected = "Connected", - Reconnecting = "Reconnecting", - Disconnecting = "Disconnecting", - Stuck = "Stuck", - Unknown = "Unknown", -} - /** * This scheduler tracks the state of the current membership participation * and runs one central timer that wakes up a handler callback with the correct action + state @@ -75,28 +41,12 @@ enum Status { */ export class ActionScheduler { public running = false; - public state: ActionSchedulerState; - public static get defaultState(): ActionSchedulerState { - return { - hasMemberStateEvent: false, - delayId: undefined, - startTime: 0, - rateLimitRetries: new Map(), - networkErrorRetries: new Map(), - expireUpdateIterations: 1, - }; - } public constructor( - state: ActionSchedulerState, /** This is the callback called for each scheduled action (`this.addAction()`) */ - private membershipLoopHandler: ( - state: ActionSchedulerState, - type: MembershipActionType, - ) => Promise, - ) { - this.state = state; - } + private membershipLoopHandler: (type: MembershipActionType) => Promise, + ) {} + // function for the wakeup mechanism (in case we add an action externally and need to leave the current sleep) private wakeup: (update: ActionUpdate) => void = (update: ActionUpdate): void => { logger.error("Cannot call wakeup before calling `startWithJoin()`"); @@ -136,9 +86,6 @@ export class ActionScheduler { }); if (nextAction.ts > Date.now()) await Promise.race([wakeupPromise, sleep(nextAction.ts - Date.now())]); - const oldStatus = this.status; - logger.info(`MembershipManager ActionScheduler awakened. status=${oldStatus}`); - let handlerResult: ActionUpdate = {}; if (!wakeupUpdate) { logger.debug( @@ -148,10 +95,7 @@ export class ActionScheduler { ); try { // `this.wakeup` can also be called and sets the `wakupUpdate` object while we are in the handler. - handlerResult = await this.membershipLoopHandler( - this.state, - nextAction.type as MembershipActionType, - ); + handlerResult = await this.membershipLoopHandler(nextAction.type as MembershipActionType); } catch (e) { throw Error(`The MembershipManager shut down because of the end condition: ${e}`); } @@ -166,10 +110,6 @@ export class ActionScheduler { } else if ("insert" in actionUpdate) { this._actions.push(...actionUpdate.insert); } - - logger.info( - `MembershipManager ActionScheduler applied action changes. Status: ${oldStatus} -> ${this.status}`, - ); } } catch (e) { // Set the rtc session "not running" state since we cannot recover from here and the consumer user of the @@ -188,59 +128,4 @@ export class ActionScheduler { public initiateLeave(): void { this.wakeup?.({ replace: [{ ts: Date.now(), type: MembershipActionType.SendScheduledDelayedLeaveEvent }] }); } - - public resetState(): void { - this.state = ActionScheduler.defaultState; - } - - public resetRateLimitCounter(type: MembershipActionType): void { - this.state.rateLimitRetries.set(type, 0); - this.state.networkErrorRetries.set(type, 0); - } - - public get status(): Status { - if (this.actions.length === 1) { - const { type } = this.actions[0]; - switch (type) { - case MembershipActionType.SendFirstDelayedEvent: - case MembershipActionType.SendJoinEvent: - case MembershipActionType.SendMainDelayedEvent: - return Status.Connecting; - case MembershipActionType.UpdateExpiry: // where no delayed events - return Status.Connected; - case MembershipActionType.SendScheduledDelayedLeaveEvent: - case MembershipActionType.SendLeaveEvent: - return Status.Disconnecting; - default: - // pass through as not expected - } - } else if (this.actions.length === 2) { - const types = this.actions.map((a) => a.type); - // normal state for connected with delayed events - if ( - (types.includes(MembershipActionType.RestartDelayedEvent) || - types.includes(MembershipActionType.SendMainDelayedEvent)) && - types.includes(MembershipActionType.UpdateExpiry) - ) { - return Status.Connected; - } - } else if (this.actions.length === 3) { - const types = this.actions.map((a) => a.type); - // It is a correct connected state if we already schedule the next Restart but have not yet cleaned up - // the current restart. - if ( - types.filter((t) => t === MembershipActionType.RestartDelayedEvent).length === 2 && - types.includes(MembershipActionType.UpdateExpiry) - ) { - return Status.Connected; - } - } - - if (!this.running) { - return Status.Disconnected; - } - - logger.error("MembershipManager has an unknown state. Actions: ", this.actions); - return Status.Unknown; - } } From 9af3bfeb314c5881fb1c8feb7c8c90260119e9e0 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 11 Mar 2025 10:35:12 +0100 Subject: [PATCH 121/124] review --- src/matrixrtc/NewMembershipManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 516577af130..9fcc5157a93 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -206,13 +206,13 @@ export class MembershipManager implements IMembershipManager { this.scheduler .startWithJoin() - .catch((e) => { - logger.error("MembershipManager stopped because: ", e); - onError?.(e); - }) .then(() => { this.leavePromiseDefer?.resolve(true); this.leavePromiseDefer = undefined; + }) + .catch((e) => { + logger.error("MembershipManager stopped because: ", e); + onError?.(e); }); } From 2729f3a74576e24906c8d9a0b317b8cbceb3ce47 Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Tue, 11 Mar 2025 10:54:04 +0100 Subject: [PATCH 122/124] Update src/matrixrtc/NewMembershipManager.ts Co-authored-by: Hugh Nimmo-Smith --- src/matrixrtc/NewMembershipManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 9fcc5157a93..a05fdc18762 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -368,7 +368,7 @@ export class MembershipManager implements IMembershipManager { private oldStatus?: Status; private scheduler = new ActionScheduler((type): Promise => { if (this.oldStatus) { - // we put this at the beginngin of the actsion scheduler loop handle callback since it is a loop this + // we put this at the beginning of the actions scheduler loop handle callback since it is a loop this // is equivalent to running it at the end of the loop. (just after applying the status/action list changes) logger.debug(`MembershipManager applied action changes. Status: ${this.oldStatus} -> ${this.status}`); } From 288edf5b808e966f616e861f7538e0142cdd63f1 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 11 Mar 2025 11:20:04 +0100 Subject: [PATCH 123/124] review --- src/matrixrtc/NewMembershipManager.ts | 130 ++++++++++++-------------- 1 file changed, 58 insertions(+), 72 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index a05fdc18762..59c0b224f58 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -311,7 +311,7 @@ export class MembershipManager implements IMembershipManager { // MembershipManager mutable state. public state: ActionSchedulerState; - public static get defaultState(): ActionSchedulerState { + private static get defaultState(): ActionSchedulerState { return { hasMemberStateEvent: false, delayId: undefined, @@ -384,16 +384,16 @@ export class MembershipManager implements IMembershipManager { case MembershipActionType.SendFirstDelayedEvent: { // Before we start we check if we come from a state where we have a delay id. if (!this.state.delayId) { - return this.sendFirstDelayedLeaveEvent(type); // Normal case without any previous delayed id. + return this.sendFirstDelayedLeaveEvent(); // Normal case without any previous delayed id. } else { // This can happen if someone else (or another client) removes our own membership event. // It will trigger `onRTCSessionMemberUpdate` queue `MembershipActionType.SendFirstDelayedEvent`. - // We might still have our delyed event from the previous participation and dependent on the serve this might not + // We might still have our delayed event from the previous participation and dependent on the server this might not // get automatically removed if the state changes. Hence It would remove our membership unexpectedly shortly after the rejoin. // // In this block we will try to cancel this delayed event before setting up a new one. - return this.cancelKnownDelayIdBeforeSendFirstDelayedEvent(type, this.state.delayId); + return this.cancelKnownDelayIdBeforeSendFirstDelayedEvent(this.state.delayId); } } case MembershipActionType.RestartDelayedEvent: { @@ -405,10 +405,10 @@ export class MembershipManager implements IMembershipManager { : MembershipActionType.SendFirstDelayedEvent, ); } - return this.restartDelayedEvent(type, this.state.delayId); + return this.restartDelayedEvent(this.state.delayId); } case MembershipActionType.SendMainDelayedEvent: { - return this.sendMainDelayedEvent(type); + return this.sendMainDelayedEvent(); } case MembershipActionType.SendScheduledDelayedLeaveEvent: { // We are already good @@ -416,16 +416,16 @@ export class MembershipManager implements IMembershipManager { return { replace: [] }; } if (this.state.delayId) { - return this.sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent(type, this.state.delayId); + return this.sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent(this.state.delayId); } else { return createInsertActionUpdate(MembershipActionType.SendLeaveEvent); } } case MembershipActionType.SendJoinEvent: { - return this.sendJoinEvent(type); + return this.sendJoinEvent(); } case MembershipActionType.UpdateExpiry: { - return this.updateExpiryOnJoinedEvent(type); + return this.updateExpiryOnJoinedEvent(); } case MembershipActionType.SendLeaveEvent: { // We are good already @@ -434,14 +434,13 @@ export class MembershipManager implements IMembershipManager { } // This is only a fallback in case we do not have working delayed events support. // first we should try to just send the scheduled leave event - return this.sendFallbackLeaveEvent(type); + return this.sendFallbackLeaveEvent(); } } } // HANDLERS (used in the membershipLoopHandler) - - private async sendFirstDelayedLeaveEvent(type: MembershipActionType): Promise { + private async sendFirstDelayedLeaveEvent(): Promise { return await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, @@ -453,27 +452,19 @@ export class MembershipManager implements IMembershipManager { this.stateKey, ) .then((response) => { - // Success we reset retries and set delayId. + // On success we reset retries and set delayId. this.state.rateLimitRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); this.state.networkErrorRetries.set(MembershipActionType.SendFirstDelayedEvent, 0); this.state.delayId = response.delay_id; return createInsertActionUpdate(MembershipActionType.SendJoinEvent); }) .catch((e) => { + const repeatActionType = MembershipActionType.SendFirstDelayedEvent; if (this.manageMaxDelayExceededSituation(e)) { - return { - insert: [ - { - ts: Date.now(), - type: MembershipActionType.SendFirstDelayedEvent, - }, - ], - }; + return createInsertActionUpdate(repeatActionType); } - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); - if (updateLimit) return updateLimit; + const update = this.actionUpdateFromErrors(e, repeatActionType, "sendDelayedStateEvent"); + if (update) return update; // log and fall through if (this.isUnsupportedDelayedEndpoint(e)) { @@ -486,12 +477,8 @@ export class MembershipManager implements IMembershipManager { }); } - private async cancelKnownDelayIdBeforeSendFirstDelayedEvent( - type: MembershipActionType, - delayId: string, - ): Promise { + private async cancelKnownDelayIdBeforeSendFirstDelayedEvent(delayId: string): Promise { // Remove all running updates and restarts - const resetActionUpdate = { replace: [] }; return await this.client ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Cancel) .then(() => { @@ -500,22 +487,22 @@ export class MembershipManager implements IMembershipManager { return createReplaceActionUpdate(MembershipActionType.SendFirstDelayedEvent); }) .catch((e) => { - const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); - if (updateLimit) return { ...resetActionUpdate, ...updateLimit }; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return { ...resetActionUpdate, ...updateNetwork }; + const repeatActionType = MembershipActionType.SendFirstDelayedEvent; + const update = this.actionUpdateFromErrors(e, repeatActionType, "updateDelayedEvent"); + if (update) return update; if (this.isNotFoundError(e)) { // If we get a M_NOT_FOUND we know that the delayed event got already removed. // This means we are good and can set it to undefined and run this again. this.state.delayId = undefined; - return createReplaceActionUpdate(MembershipActionType.SendFirstDelayedEvent); + return createReplaceActionUpdate(repeatActionType); } if (this.isUnsupportedDelayedEndpoint(e)) { return createReplaceActionUpdate(MembershipActionType.SendJoinEvent); } + // We do not just ignore and log this error since we would also need to reset the delayId. - // This becomes an unhandle-able error case since sth is signifciantly off if we dont hit any of the above cases + // This becomes an unrecoverable error case since something is significantly off if we don't hit any of the above cases // when state.delayId !== undefined // We do not use ignore and log this error since we would also need to reset the delayId. // It is cleaner if we the frontend rejoines instead of resetting the delayId here and behaving like in the success case. @@ -525,7 +512,7 @@ export class MembershipManager implements IMembershipManager { }); } - private async restartDelayedEvent(type: MembershipActionType, delayId: string): Promise { + private async restartDelayedEvent(delayId: string): Promise { return await this.client ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart) .then(() => { @@ -536,6 +523,7 @@ export class MembershipManager implements IMembershipManager { ); }) .catch((e) => { + const repeatActionType = MembershipActionType.RestartDelayedEvent; if (this.isNotFoundError(e)) { this.state.delayId = undefined; return createInsertActionUpdate(MembershipActionType.SendMainDelayedEvent); @@ -544,17 +532,15 @@ export class MembershipManager implements IMembershipManager { if (this.isUnsupportedDelayedEndpoint(e)) return {}; // TODO this also needs a test: get rate limit while checking id delayed event is scheduled - const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; + const update = this.actionUpdateFromErrors(e, repeatActionType, "updateDelayedEvent"); + if (update) return update; // In other error cases we have no idea what is happening throw Error("Could not restart delayed event, even though delayed events are supported. " + e); }); } - private async sendMainDelayedEvent(type: MembershipActionType): Promise { + private async sendMainDelayedEvent(): Promise { return await this.client ._unstable_sendDelayedStateEvent( this.room.roomId, @@ -574,25 +560,21 @@ export class MembershipManager implements IMembershipManager { ); }) .catch((e) => { + const repeatActionType = MembershipActionType.SendMainDelayedEvent; // Don't do any other delayed event work if its not supported. if (this.isUnsupportedDelayedEndpoint(e)) return {}; if (this.manageMaxDelayExceededSituation(e)) { - return createInsertActionUpdate(MembershipActionType.SendMainDelayedEvent); + return createInsertActionUpdate(repeatActionType); } - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendDelayedStateEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; + const update = this.actionUpdateFromErrors(e, repeatActionType, "updateDelayedEvent"); + if (update) return update; throw Error("Could not send delayed event, even though delayed events are supported. " + e); }); } - private async sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent( - type: MembershipActionType, - delayId: string, - ): Promise { + private async sendScheduledDelayedLeaveEventOrFallbackToSendLeaveEvent(delayId: string): Promise { return await this.client ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Send) .then(() => { @@ -602,26 +584,25 @@ export class MembershipManager implements IMembershipManager { return { replace: [] }; }) .catch((e) => { + const repeatActionType = MembershipActionType.SendLeaveEvent; if (this.isUnsupportedDelayedEndpoint(e)) return {}; if (this.isNotFoundError(e)) { this.state.delayId = undefined; - return createInsertActionUpdate(MembershipActionType.SendLeaveEvent); + return createInsertActionUpdate(repeatActionType); } - const updateLimit = this.actionUpdateFromRateLimitError(e, "updateDelayedEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; + const update = this.actionUpdateFromErrors(e, repeatActionType, "updateDelayedEvent"); + if (update) return update; // On any other error we fall back to SendLeaveEvent (this includes hard errors from rate limiting) logger.warn( "Encountered unexpected error during SendScheduledDelayedLeaveEvent. Falling back to SendLeaveEvent", e, ); - return createInsertActionUpdate(MembershipActionType.SendLeaveEvent); + return createInsertActionUpdate(repeatActionType); }); } - private async sendJoinEvent(type: MembershipActionType): Promise { + private async sendJoinEvent(): Promise { return await this.client .sendStateEvent( this.room.roomId, @@ -646,15 +627,13 @@ export class MembershipManager implements IMembershipManager { }; }) .catch((e) => { - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; + const update = this.actionUpdateFromErrors(e, MembershipActionType.SendJoinEvent, "sendStateEvent"); + if (update) return update; throw e; }); } - private async updateExpiryOnJoinedEvent(type: MembershipActionType): Promise { + private async updateExpiryOnJoinedEvent(): Promise { const nextExpireUpdateIteration = this.state.expireUpdateIterations + 1; return await this.client .sendStateEvent( @@ -677,14 +656,13 @@ export class MembershipManager implements IMembershipManager { }; }) .catch((e) => { - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; + const update = this.actionUpdateFromErrors(e, MembershipActionType.UpdateExpiry, "sendStateEvent"); + if (update) return update; + throw e; }); } - private async sendFallbackLeaveEvent(type: MembershipActionType): Promise { + private async sendFallbackLeaveEvent(): Promise { return await this.client .sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, {}, this.stateKey) .then(() => { @@ -693,10 +671,8 @@ export class MembershipManager implements IMembershipManager { return { replace: [] }; }) .catch((e) => { - const updateLimit = this.actionUpdateFromRateLimitError(e, "sendStateEvent", type); - if (updateLimit) return updateLimit; - const updateNetwork = this.actionUpdateFromNetworkErrorRetry(e, type); - if (updateNetwork) return updateNetwork; + const update = this.actionUpdateFromErrors(e, MembershipActionType.SendLeaveEvent, "sendStateEvent"); + if (update) return update; throw e; }); } @@ -758,6 +734,16 @@ export class MembershipManager implements IMembershipManager { return false; } + private actionUpdateFromErrors( + error: unknown, + type: MembershipActionType, + method: string, + ): ActionUpdate | undefined { + const updateLimit = this.actionUpdateFromRateLimitError(error, method, type); + if (updateLimit) return updateLimit; + const updateNetwork = this.actionUpdateFromNetworkErrorRetry(error, type); + if (updateNetwork) return updateNetwork; + } /** * Check if we have a rate limit error and schedule the same action again if we dont exceed the rate limit retry count yet. * @param error the error causing this handler check/execution From 08ffababfe7a20761ceec19d29bf65913d153bc4 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 11 Mar 2025 14:33:37 +0100 Subject: [PATCH 124/124] public -> private + missed review fiexes (comment typos) --- src/matrixrtc/NewMembershipManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrixrtc/NewMembershipManager.ts b/src/matrixrtc/NewMembershipManager.ts index 59c0b224f58..9977364a921 100644 --- a/src/matrixrtc/NewMembershipManager.ts +++ b/src/matrixrtc/NewMembershipManager.ts @@ -310,7 +310,7 @@ export class MembershipManager implements IMembershipManager { } // MembershipManager mutable state. - public state: ActionSchedulerState; + private state: ActionSchedulerState; private static get defaultState(): ActionSchedulerState { return { hasMemberStateEvent: false, @@ -368,7 +368,7 @@ export class MembershipManager implements IMembershipManager { private oldStatus?: Status; private scheduler = new ActionScheduler((type): Promise => { if (this.oldStatus) { - // we put this at the beginning of the actions scheduler loop handle callback since it is a loop this + // we put this at the beginning of the actions scheduler loop handle callback since it is a loop this // is equivalent to running it at the end of the loop. (just after applying the status/action list changes) logger.debug(`MembershipManager applied action changes. Status: ${this.oldStatus} -> ${this.status}`); } @@ -504,8 +504,8 @@ export class MembershipManager implements IMembershipManager { // This becomes an unrecoverable error case since something is significantly off if we don't hit any of the above cases // when state.delayId !== undefined - // We do not use ignore and log this error since we would also need to reset the delayId. - // It is cleaner if we the frontend rejoines instead of resetting the delayId here and behaving like in the success case. + // We do not just ignore and log this error since we would also need to reset the delayId. + // It is cleaner if we, the frontend, rejoins instead of resetting the delayId here and behaving like in the success case. throw Error( "We failed to cancel a delayed event where we already had a delay id with an error we cannot automatically handle", );