From 0b7f23fa5491a97b598c4d94881108893520b030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 28 Jan 2020 15:06:43 +0100 Subject: [PATCH 1/3] EventIndex: Improve the documentation a bit. --- src/indexing/BaseEventIndexManager.js | 4 +- src/indexing/EventIndex.js | 131 +++++++++++++++++++++----- 2 files changed, 107 insertions(+), 28 deletions(-) diff --git a/src/indexing/BaseEventIndexManager.js b/src/indexing/BaseEventIndexManager.js index 8ebaddc3ab..7a3b583088 100644 --- a/src/indexing/BaseEventIndexManager.js +++ b/src/indexing/BaseEventIndexManager.js @@ -153,8 +153,8 @@ export default class BaseEventIndexManager { /** * Search the event index using the given term for matching events. * - * @param {SearchArgs} searchArgs The search configuration sets what should - * be searched for and what should be contained in the search result. + * @param {SearchArgs} searchArgs The search configuration for the search, + * sets the search term and determines the search result contents. * * @return {Promise<[SearchResult]>} A promise that will resolve to an array * of search results once the search is done. diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index 9628920cd7..5d40267fcd 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -51,6 +51,9 @@ export default class EventIndex extends EventEmitter { this.registerListeners(); } + /** + * Register event listeners that are necessary for the event index to work. + */ registerListeners() { const client = MatrixClientPeg.get(); @@ -60,6 +63,9 @@ export default class EventIndex extends EventEmitter { client.on('Room.timelineReset', this.onTimelineReset); } + /** + * Remove the event index specific event listeners. + */ removeListeners() { const client = MatrixClientPeg.get(); if (client === null) return; @@ -116,6 +122,15 @@ export default class EventIndex extends EventEmitter { })); } + /** + * The sync event listener. + * + * The listener has two cases: + * - First sync after start up, check if the index is empty, add + * initial checkpoints, if so. Start the crawler background task. + * - Every other sync, tell the event index to commit all the queued up + * live events + */ onSync = async (state, prevState, data) => { const indexManager = PlatformPeg.get().getEventIndexingManager(); @@ -139,6 +154,14 @@ export default class EventIndex extends EventEmitter { } } + /** + * The Room.timeline listener. + * + * This listener waits for live events in encrypted rooms, if they are + * decrypted or unencrypted we queue them to be added to the index, + * otherwise we save their event id and wait for them in the Event.decrypted + * listener. + */ onRoomTimeline = async (ev, room, toStartOfTimeline, removed, data) => { // We only index encrypted rooms locally. if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; @@ -162,6 +185,12 @@ export default class EventIndex extends EventEmitter { } } + /** + * The Event.decrypted listener. + * + * Checks if the event was marked for addition in the Room.timeline + * listener, if so queues it up to be added to the index. + */ onEventDecrypted = async (ev, err) => { const eventId = ev.getId(); @@ -171,6 +200,41 @@ export default class EventIndex extends EventEmitter { await this.addLiveEventToIndex(ev); } + /** + * The Room.timelineReset listener. + * + * Listens for timeline resets that are caused by a limited timeline to + * re-add checkpoints for rooms that need to be crawled again. + */ + onTimelineReset = async (room, timelineSet, resetAllTimelines) => { + if (room === null) return; + + const indexManager = PlatformPeg.get().getEventIndexingManager(); + if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; + + const timeline = room.getLiveTimeline(); + const token = timeline.getPaginationToken("b"); + + const backwardsCheckpoint = { + roomId: room.roomId, + token: token, + fullCrawl: false, + direction: "b", + }; + + console.log("EventIndex: Added checkpoint because of a limited timeline", + backwardsCheckpoint); + + await indexManager.addCrawlerCheckpoint(backwardsCheckpoint); + + this.crawlerCheckpoints.push(backwardsCheckpoint); + } + + /** + * Queue up live events to be added to the event index. + * + * @param {MatrixEvent} ev The event that should be added to the index. + */ async addLiveEventToIndex(ev) { const indexManager = PlatformPeg.get().getEventIndexingManager(); @@ -190,10 +254,24 @@ export default class EventIndex extends EventEmitter { indexManager.addEventToIndex(e, profile); } + /** + * Emmit that the crawler has changed the checkpoint that it's currently + * handling. + */ emitNewCheckpoint() { this.emit("changedCheckpoint", this.currentRoom()); } + /** + * The main crawler loop. + * + * Goes through crawlerCheckpoints and fetches events from the server to be + * added to the EventIndex. + * + * If a /room/{roomId}/messages request doesn't contain any events, stop the + * crawl, otherwise create a new checkpoint and push it to the + * crawlerCheckpoints queue so we go through them in a round-robin way. + */ async crawlerFunc() { let cancelled = false; @@ -328,8 +406,6 @@ export default class EventIndex extends EventEmitter { ].indexOf(value.getType()) >= 0 && !value.isRedacted() && !value.isDecryptionFailure() ); - // TODO do we need to check if the event has all the valid - // attributes? }; // TODO if there are no events at this point we're missing a lot @@ -394,40 +470,28 @@ export default class EventIndex extends EventEmitter { console.log("EventIndex: Stopping crawler function"); } - onTimelineReset = async (room, timelineSet, resetAllTimelines) => { - if (room === null) return; - - const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; - - const timeline = room.getLiveTimeline(); - const token = timeline.getPaginationToken("b"); - - const backwardsCheckpoint = { - roomId: room.roomId, - token: token, - fullCrawl: false, - direction: "b", - }; - - console.log("EventIndex: Added checkpoint because of a limited timeline", - backwardsCheckpoint); - - await indexManager.addCrawlerCheckpoint(backwardsCheckpoint); - - this.crawlerCheckpoints.push(backwardsCheckpoint); - } - + /** + * Start the crawler background task. + */ startCrawler() { if (this._crawler !== null) return; this.crawlerFunc(); } + /** + * Stop the crawler background task. + */ stopCrawler() { if (this._crawler === null) return; this._crawler.cancel(); } + /** + * Close the event index. + * + * This removes all the MatrixClient event listeners, stops the crawler + * task, and closes the index. + */ async close() { const indexManager = PlatformPeg.get().getEventIndexingManager(); this.removeListeners(); @@ -435,6 +499,15 @@ export default class EventIndex extends EventEmitter { return indexManager.closeEventIndex(); } + /** + * Search the event index using the given term for matching events. + * + * @param {SearchArgs} searchArgs The search configuration for the search, + * sets the search term and determines the search result contents. + * + * @return {Promise<[SearchResult]>} A promise that will resolve to an array + * of search results once the search is done. + */ async search(searchArgs) { const indexManager = PlatformPeg.get().getEventIndexingManager(); return indexManager.searchEventIndex(searchArgs); @@ -634,6 +707,12 @@ export default class EventIndex extends EventEmitter { return paginationPromise; } + /** + * Get statistical information of the index. + * + * @return {Promise} A promise that will resolve to the index + * statistics. + */ async getStats() { const indexManager = PlatformPeg.get().getEventIndexingManager(); return indexManager.getStats(); From 948ec203c63856863661256f0ea9bfa60455c4c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 28 Jan 2020 15:07:29 +0100 Subject: [PATCH 2/3] BaseEventIndexManager: Fix the type hints for the empty promises. --- src/indexing/BaseEventIndexManager.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/indexing/BaseEventIndexManager.js b/src/indexing/BaseEventIndexManager.js index 7a3b583088..66904f9264 100644 --- a/src/indexing/BaseEventIndexManager.js +++ b/src/indexing/BaseEventIndexManager.js @@ -105,7 +105,7 @@ export default class BaseEventIndexManager { * @return {Promise} A promise that will resolve when the event index is * initialized. */ - async initEventIndex(): Promise<> { + async initEventIndex(): Promise { throw new Error("Unimplemented"); } @@ -146,7 +146,7 @@ export default class BaseEventIndexManager { * @return {Promise} A promise that will resolve once the queued up events * were added to the index. */ - async commitLiveEvents(): Promise<> { + async commitLiveEvents(): Promise { throw new Error("Unimplemented"); } @@ -197,7 +197,7 @@ export default class BaseEventIndexManager { * @return {Promise} A promise that will resolve once the checkpoint has * been stored. */ - async addCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise<> { + async addCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise { throw new Error("Unimplemented"); } @@ -210,7 +210,7 @@ export default class BaseEventIndexManager { * @return {Promise} A promise that will resolve once the checkpoint has * been removed. */ - async removeCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise<> { + async removeCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise { throw new Error("Unimplemented"); } @@ -250,7 +250,7 @@ export default class BaseEventIndexManager { * @return {Promise} A promise that will resolve once the event index has * been closed. */ - async closeEventIndex(): Promise<> { + async closeEventIndex(): Promise { throw new Error("Unimplemented"); } @@ -260,7 +260,7 @@ export default class BaseEventIndexManager { * @return {Promise} A promise that will resolve once the event index has * been deleted. */ - async deleteEventIndex(): Promise<> { + async deleteEventIndex(): Promise { throw new Error("Unimplemented"); } } From 5cabfb63a43a62e3a28c61f926bba32c312a8be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 28 Jan 2020 16:13:56 +0100 Subject: [PATCH 3/3] EventIndex: Don't mark the event listener method docs as jsdoc comments. --- src/indexing/EventIndex.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index 5d40267fcd..e1ec0d1d1c 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -122,7 +122,7 @@ export default class EventIndex extends EventEmitter { })); } - /** + /* * The sync event listener. * * The listener has two cases: @@ -154,7 +154,7 @@ export default class EventIndex extends EventEmitter { } } - /** + /* * The Room.timeline listener. * * This listener waits for live events in encrypted rooms, if they are @@ -185,7 +185,7 @@ export default class EventIndex extends EventEmitter { } } - /** + /* * The Event.decrypted listener. * * Checks if the event was marked for addition in the Room.timeline @@ -200,7 +200,7 @@ export default class EventIndex extends EventEmitter { await this.addLiveEventToIndex(ev); } - /** + /* * The Room.timelineReset listener. * * Listens for timeline resets that are caused by a limited timeline to