From 0e5fec5febcf7c3bae9cc24946a5bf214e0de28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 31 Jan 2020 15:37:06 +0100 Subject: [PATCH 1/3] EventIndex: Catch errors when removing a crawler checkpoint. --- src/indexing/EventIndex.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index e1ec0d1d1c..55f8dc9986 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -356,7 +356,11 @@ export default class EventIndex extends EventEmitter { console.log("EventIndex: Done with the checkpoint", checkpoint); // We got to the start/end of our timeline, lets just // delete our checkpoint and go back to sleep. - await indexManager.removeCrawlerCheckpoint(checkpoint); + try { + await indexManager.removeCrawlerCheckpoint(checkpoint); + } catch (e) { + console.log("EventIndex: Error removing checkpoint", checkpoint, e); + } continue; } From 5eb510387c9aa08a061d5fd17bed3efc104b48eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 31 Jan 2020 16:37:30 +0100 Subject: [PATCH 2/3] EventIndex: Don't process checkpoints that don't have a valid token. Some rooms might have null as a token, don't add those to be crawled nor to the index. Catch any other errors that might occur during the addition of an initial checkpoint and log them. --- src/indexing/EventIndex.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index 55f8dc9986..ba6c1b6509 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -115,10 +115,20 @@ export default class EventIndex extends EventEmitter { direction: "f", }; - await indexManager.addCrawlerCheckpoint(backCheckpoint); - await indexManager.addCrawlerCheckpoint(forwardCheckpoint); - this.crawlerCheckpoints.push(backCheckpoint); - this.crawlerCheckpoints.push(forwardCheckpoint); + try { + if (backCheckpoint.token) { + await indexManager.addCrawlerCheckpoint(backCheckpoint); + this.crawlerCheckpoints.push(backCheckpoint); + } + + if (forwardCheckpoint.token) { + await indexManager.addCrawlerCheckpoint(forwardCheckpoint); + this.crawlerCheckpoints.push(forwardCheckpoint); + } + } catch (e) { + console.log("EventIndex: Error adding initial checkpoints for room", + room.roomId, backCheckpoint, forwardCheckpoint, e); + } })); } From 01c1c2364b69e7e0f9ba7c183e0129a954c4e441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 31 Jan 2020 18:44:52 +0100 Subject: [PATCH 3/3] ManageEventIndexDialog: Add back a control for the crawler sleep time. --- .../dialogs/eventindex/ManageEventIndexDialog.js | 16 ++++++++++++++++ src/i18n/strings/en_EN.json | 2 ++ src/indexing/EventIndex.js | 7 ++----- src/settings/Settings.js | 9 +++++++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/async-components/views/dialogs/eventindex/ManageEventIndexDialog.js b/src/async-components/views/dialogs/eventindex/ManageEventIndexDialog.js index b7ea87b1b2..58605a3263 100644 --- a/src/async-components/views/dialogs/eventindex/ManageEventIndexDialog.js +++ b/src/async-components/views/dialogs/eventindex/ManageEventIndexDialog.js @@ -18,6 +18,7 @@ import React from 'react'; import * as sdk from '../../../../index'; import PropTypes from 'prop-types'; import { _t } from '../../../../languageHandler'; +import SettingsStore, {SettingLevel} from "../../../../settings/SettingsStore"; import Modal from '../../../../Modal'; import {formatBytes, formatCountLong} from "../../../../utils/FormattingUtils"; @@ -39,6 +40,8 @@ export default class ManageEventIndexDialog extends React.Component { eventCount: 0, roomCount: 0, currentRoom: null, + crawlerSleepTime: + SettingsStore.getValueAt(SettingLevel.DEVICE, 'crawlerSleepTime'), }; } @@ -104,6 +107,11 @@ export default class ManageEventIndexDialog extends React.Component { this.props.onFinished(true); } + _onCrawlerSleepTimeChange = (e) => { + this.setState({crawlerSleepTime: e.target.value}); + SettingsStore.setValue("crawlerSleepTime", null, SettingLevel.DEVICE, e.target.value); + } + render() { let crawlerState; @@ -115,6 +123,8 @@ export default class ManageEventIndexDialog extends React.Component { ); } + const Field = sdk.getComponent('views.elements.Field'); + const eventIndexingSettings = (
{ @@ -127,6 +137,12 @@ export default class ManageEventIndexDialog extends React.Component { {_t("Indexed messages:")} {formatCountLong(this.state.eventCount)}
{_t("Number of rooms:")} {formatCountLong(this.state.roomCount)}
{crawlerState}
+
); diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 47032dcddc..78c837986b 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -415,6 +415,7 @@ "Show previews/thumbnails for images": "Show previews/thumbnails for images", "Enable message search in encrypted rooms": "Enable message search in encrypted rooms", "Keep secret storage passphrase in memory for this session": "Keep secret storage passphrase in memory for this session", + "How fast should messages be downloaded.": "How fast should messages be downloaded.", "Collecting app version information": "Collecting app version information", "Collecting logs": "Collecting logs", "Uploading report": "Uploading report", @@ -2094,6 +2095,7 @@ "Space used:": "Space used:", "Indexed messages:": "Indexed messages:", "Number of rooms:": "Number of rooms:", + "Message downloading sleep time(ms)": "Message downloading sleep time(ms)", "Failed to set direct chat tag": "Failed to set direct chat tag", "Failed to remove tag %(tagName)s from room": "Failed to remove tag %(tagName)s from room", "Failed to add tag %(tagName)s to room": "Failed to add tag %(tagName)s to room" diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index ba6c1b6509..cbd739602c 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -18,6 +18,7 @@ import PlatformPeg from "../PlatformPeg"; import {MatrixClientPeg} from "../MatrixClientPeg"; import {EventTimeline, RoomMember} from 'matrix-js-sdk'; import {sleep} from "../utils/promise"; +import SettingsStore, {SettingLevel} from "../settings/SettingsStore"; import {EventEmitter} from "events"; /* @@ -30,7 +31,6 @@ export default class EventIndex extends EventEmitter { // The time in ms that the crawler will wait loop iterations if there // have not been any checkpoints to consume in the last iteration. this._crawlerIdleTime = 5000; - this._crawlerSleepTime = 3000; // The maximum number of events our crawler should fetch in a single // crawl. this._eventsPerCrawl = 100; @@ -299,10 +299,7 @@ export default class EventIndex extends EventEmitter { let idle = false; while (!cancelled) { - // This is a low priority task and we don't want to spam our - // homeserver with /messages requests so we set a hefty timeout - // here. - let sleepTime = this._crawlerSleepTime; + let sleepTime = SettingsStore.getValueAt(SettingLevel.DEVICE, 'crawlerSleepTime'); // Don't let the user configure a lower sleep time than 100 ms. sleepTime = Math.max(sleepTime, 100); diff --git a/src/settings/Settings.js b/src/settings/Settings.js index ecaf9cd2e3..b5a966523e 100644 --- a/src/settings/Settings.js +++ b/src/settings/Settings.js @@ -492,8 +492,13 @@ export const SETTINGS = { default: true, }, "keepSecretStoragePassphraseForSession": { + supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + displayName: _td("Keep secret storage passphrase in memory for this session"), + default: false, + }, + "crawlerSleepTime": { supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, - displayName: _td("Keep secret storage passphrase in memory for this session"), - default: false, + displayName: _td("How fast should messages be downloaded."), + default: 3000, }, };