From 39bcd8d56d1e51e62c4559daae97a867bd887741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 21 May 2020 10:10:15 +0200 Subject: [PATCH 1/5] EventIndex: Add a checkpoint if a room turns into a encrypted one. --- src/indexing/EventIndex.js | 69 +++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index fac7c92b65..3f08574d7a 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -182,6 +182,14 @@ export default class EventIndex extends EventEmitter { return; } + if (ev.getType() === "m.room.encryption") { + console.log("EventIndex: Adding checkpoint for newly encrypted room", + room.roomId); + + this.addRoomCheckpoint(room.roomId, true); + return; + } + // If the event is not yet decrypted mark it for the // Event.decrypted callback. if (ev.isBeingDecrypted()) { @@ -234,26 +242,12 @@ export default class EventIndex extends EventEmitter { */ onTimelineReset = async (room, timelineSet, resetAllTimelines) => { if (room === null) return; - - const indexManager = PlatformPeg.get().getEventIndexingManager(); if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; - const timeline = room.getLiveTimeline(); - const token = timeline.getPaginationToken("b"); - - const backwardsCheckpoint = { - roomId: room.roomId, - token: token, - fullCrawl: false, - direction: "b", - }; - console.log("EventIndex: Added checkpoint because of a limited timeline", backwardsCheckpoint); - await indexManager.addCrawlerCheckpoint(backwardsCheckpoint); - - this.crawlerCheckpoints.push(backwardsCheckpoint); + this.addRoomCheckpoint(room.roomId, false); } /** @@ -319,6 +313,51 @@ export default class EventIndex extends EventEmitter { this.emit("changedCheckpoint", this.currentRoom()); } + async addEventsFromLiveTimeline(timeline) { + let events = timeline.getEvents(); + + for (let i = 0; i < events.length; i++) { + const ev = events[i]; + await this.addLiveEventToIndex(ev); + } + } + + async addRoomCheckpoint(roomId, fullCrawl = false) { + const indexManager = PlatformPeg.get().getEventIndexingManager(); + const client = MatrixClientPeg.get(); + const room = client.getRoom(roomId); + + if (!room) return; + + const timeline = room.getLiveTimeline(); + let token = timeline.getPaginationToken("b"); + + if(!token) { + // The room doesn't contain any tokens, meaning the live timeline + // contains all the events, add those to the index. + await this.addEventsFromLiveTimeline(timeline); + return; + } + + const checkpoint = { + roomId: room.roomId, + token: token, + fullCrawl: fullCrawl, + direction: "b", + }; + + console.log("EventIndex: Adding checkpoint", checkpoint); + + try{ + await indexManager.addCrawlerCheckpoint(checkpoint); + } catch (e) { + console.log("EventIndex: Error adding new checkpoint for room", + room.roomId, checkpoint, e); + } + + this.crawlerCheckpoints.push(checkpoint); + } + /** * The main crawler loop. * From f802668fff31587d541bb3059a4b42db5fca4c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 21 May 2020 10:10:46 +0200 Subject: [PATCH 2/5] EventIndex: Add a missing await. --- src/indexing/EventIndex.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index 3f08574d7a..aefe849370 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -302,7 +302,7 @@ export default class EventIndex extends EventEmitter { avatar_url: ev.sender.getMxcAvatarUrl(), }; - indexManager.addEventToIndex(e, profile); + await indexManager.addEventToIndex(e, profile); } /** From ea35fc2881d8b47261d5f46fe0b3ea9e7f53b594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 21 May 2020 12:35:47 +0200 Subject: [PATCH 3/5] EventIndex: Fix some lint issues. --- src/indexing/EventIndex.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index aefe849370..f67738ca68 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -244,8 +244,8 @@ export default class EventIndex extends EventEmitter { if (room === null) return; if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; - console.log("EventIndex: Added checkpoint because of a limited timeline", - backwardsCheckpoint); + console.log("EventIndex: Adding a checkpoint because of a limited timeline", + room.roomId); this.addRoomCheckpoint(room.roomId, false); } @@ -314,7 +314,7 @@ export default class EventIndex extends EventEmitter { } async addEventsFromLiveTimeline(timeline) { - let events = timeline.getEvents(); + const events = timeline.getEvents(); for (let i = 0; i < events.length; i++) { const ev = events[i]; @@ -330,9 +330,9 @@ export default class EventIndex extends EventEmitter { if (!room) return; const timeline = room.getLiveTimeline(); - let token = timeline.getPaginationToken("b"); + const token = timeline.getPaginationToken("b"); - if(!token) { + if (!token) { // The room doesn't contain any tokens, meaning the live timeline // contains all the events, add those to the index. await this.addEventsFromLiveTimeline(timeline); @@ -348,7 +348,7 @@ export default class EventIndex extends EventEmitter { console.log("EventIndex: Adding checkpoint", checkpoint); - try{ + try { await indexManager.addCrawlerCheckpoint(checkpoint); } catch (e) { console.log("EventIndex: Error adding new checkpoint for room", From 7a2bb4b112daec2392f04b64dfc067aeb35af6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 8 Jun 2020 16:43:20 +0200 Subject: [PATCH 4/5] EventIndex: Check if a newly encrypted room is indexed before adding a checkpoint. --- src/indexing/BaseEventIndexManager.ts | 13 +++++++++++ src/indexing/EventIndex.js | 33 ++++++++++++++++++++------- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/indexing/BaseEventIndexManager.ts b/src/indexing/BaseEventIndexManager.ts index c40d1300ea..32ab3b34fe 100644 --- a/src/indexing/BaseEventIndexManager.ts +++ b/src/indexing/BaseEventIndexManager.ts @@ -134,6 +134,19 @@ export default abstract class BaseEventIndexManager { throw new Error("Unimplemented"); } + /** + * Check if the room with the given id is already indexed. + * + * @param {string} roomId The ID of the room which we want to check if it + * has been already indexed. + * + * @return {Promise} Returns true if the index contains events for + * the given room, false otherwise. + */ + isRoomIndexed(roomId: string): Promise { + throw new Error("Unimplemented"); + } + /** * Get statistical information of the index. * diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index f67738ca68..fe7c71cfa6 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -62,6 +62,7 @@ export default class EventIndex extends EventEmitter { client.on('Event.decrypted', this.onEventDecrypted); client.on('Room.timelineReset', this.onTimelineReset); client.on('Room.redaction', this.onRedaction); + client.on('RoomState.events', this.onRoomStateEvent); } /** @@ -76,6 +77,7 @@ export default class EventIndex extends EventEmitter { client.removeListener('Event.decrypted', this.onEventDecrypted); client.removeListener('Room.timelineReset', this.onTimelineReset); client.removeListener('Room.redaction', this.onRedaction); + client.removeListener('RoomState.events', this.onRoomStateEvent); } /** @@ -182,14 +184,6 @@ export default class EventIndex extends EventEmitter { return; } - if (ev.getType() === "m.room.encryption") { - console.log("EventIndex: Adding checkpoint for newly encrypted room", - room.roomId); - - this.addRoomCheckpoint(room.roomId, true); - return; - } - // If the event is not yet decrypted mark it for the // Event.decrypted callback. if (ev.isBeingDecrypted()) { @@ -202,6 +196,15 @@ export default class EventIndex extends EventEmitter { } } + onRoomStateEvent = async (ev, state) => { + if (!MatrixClientPeg.get().isRoomEncrypted(state.roomId)) return; + + if (ev.getType() === "m.room.encryption" && !await this.isRoomIndexed(state.roomId)) { + console.log("EventIndex: Adding a checkpoint for a newly encrypted room", room.roomId); + this.addRoomCheckpoint(state.roomId, true); + } + } + /* * The Event.decrypted listener. * @@ -847,6 +850,20 @@ export default class EventIndex extends EventEmitter { return indexManager.getStats(); } + /** + * Check if the room with the given id is already indexed. + * + * @param {string} roomId The ID of the room which we want to check if it + * has been already indexed. + * + * @return {Promise} Returns true if the index contains events for + * the given room, false otherwise. + */ + async isRoomIndexed(roomId) { + const indexManager = PlatformPeg.get().getEventIndexingManager(); + return indexManager.isRoomIndexed(roomId); + } + /** * Get the room that we are currently crawling. * From 2c81d3eda84b8da8d1fd259608b9d9a1582c7003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 8 Jun 2020 17:30:26 +0200 Subject: [PATCH 5/5] EventIndex: Use the correct variable to get the room id. --- src/indexing/EventIndex.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index fe7c71cfa6..17c3691bc6 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -200,7 +200,7 @@ export default class EventIndex extends EventEmitter { if (!MatrixClientPeg.get().isRoomEncrypted(state.roomId)) return; if (ev.getType() === "m.room.encryption" && !await this.isRoomIndexed(state.roomId)) { - console.log("EventIndex: Adding a checkpoint for a newly encrypted room", room.roomId); + console.log("EventIndex: Adding a checkpoint for a newly encrypted room", state.roomId); this.addRoomCheckpoint(state.roomId, true); } }