Merge pull request #4020 from matrix-org/poljar/eventindex-er

Checkpoint addition fixes and return of the crawler sleep time setting.

src/async-components/views/dialogs/eventindex/ManageEventIndexDialog.js

@@ -18,6 +18,7 @@ import React from 'react';
 import * as sdk from '../../../../index';
 import PropTypes from 'prop-types';
 import { _t } from '../../../../languageHandler';
+import SettingsStore, {SettingLevel} from "../../../../settings/SettingsStore";
 
 import Modal from '../../../../Modal';
 import {formatBytes, formatCountLong} from "../../../../utils/FormattingUtils";
@@ -39,6 +40,8 @@ export default class ManageEventIndexDialog extends React.Component {
             eventCount: 0,
             roomCount: 0,
             currentRoom: null,
+            crawlerSleepTime:
+                SettingsStore.getValueAt(SettingLevel.DEVICE, 'crawlerSleepTime'),
         };
     }
 
@@ -104,6 +107,11 @@ export default class ManageEventIndexDialog extends React.Component {
         this.props.onFinished(true);
     }
 
+    _onCrawlerSleepTimeChange = (e) => {
+        this.setState({crawlerSleepTime: e.target.value});
+        SettingsStore.setValue("crawlerSleepTime", null, SettingLevel.DEVICE, e.target.value);
+    }
+
     render() {
         let crawlerState;
 
@@ -115,6 +123,8 @@ export default class ManageEventIndexDialog extends React.Component {
             );
         }
 
+        const Field = sdk.getComponent('views.elements.Field');
+
         const eventIndexingSettings = (
             <div>
                 {
@@ -127,6 +137,12 @@ export default class ManageEventIndexDialog extends React.Component {
                     {_t("Indexed messages:")} {formatCountLong(this.state.eventCount)}<br />
                     {_t("Number of rooms:")} {formatCountLong(this.state.roomCount)}<br />
                     {crawlerState}<br />
+                    <Field
+                        id={"crawlerSleepTimeMs"}
+                        label={_t('Message downloading sleep time(ms)')}
+                        type='number'
+                        value={this.state.crawlerSleepTime}
+                        onChange={this._onCrawlerSleepTimeChange} />
                 </div>
             </div>
         );

src/i18n/strings/en_EN.json

@@ -415,6 +415,7 @@
     "Show previews/thumbnails for images": "Show previews/thumbnails for images",
     "Enable message search in encrypted rooms": "Enable message search in encrypted rooms",
     "Keep secret storage passphrase in memory for this session": "Keep secret storage passphrase in memory for this session",
+    "How fast should messages be downloaded.": "How fast should messages be downloaded.",
     "Collecting app version information": "Collecting app version information",
     "Collecting logs": "Collecting logs",
     "Uploading report": "Uploading report",
@@ -2094,6 +2095,7 @@
     "Space used:": "Space used:",
     "Indexed messages:": "Indexed messages:",
     "Number of rooms:": "Number of rooms:",
+    "Message downloading sleep time(ms)": "Message downloading sleep time(ms)",
     "Failed to set direct chat tag": "Failed to set direct chat tag",
     "Failed to remove tag %(tagName)s from room": "Failed to remove tag %(tagName)s from room",
     "Failed to add tag %(tagName)s to room": "Failed to add tag %(tagName)s to room"

src/indexing/EventIndex.js

@@ -18,6 +18,7 @@ import PlatformPeg from "../PlatformPeg";
 import {MatrixClientPeg} from "../MatrixClientPeg";
 import {EventTimeline, RoomMember} from 'matrix-js-sdk';
 import {sleep} from "../utils/promise";
+import SettingsStore, {SettingLevel} from "../settings/SettingsStore";
 import {EventEmitter} from "events";
 
 /*
@@ -30,7 +31,6 @@ export default class EventIndex extends EventEmitter {
         // The time in ms that the crawler will wait loop iterations if there
         // have not been any checkpoints to consume in the last iteration.
         this._crawlerIdleTime = 5000;
-        this._crawlerSleepTime = 3000;
         // The maximum number of events our crawler should fetch in a single
         // crawl.
         this._eventsPerCrawl = 100;
@@ -115,10 +115,20 @@ export default class EventIndex extends EventEmitter {
                 direction: "f",
             };
 
-            await indexManager.addCrawlerCheckpoint(backCheckpoint);
-            await indexManager.addCrawlerCheckpoint(forwardCheckpoint);
-            this.crawlerCheckpoints.push(backCheckpoint);
-            this.crawlerCheckpoints.push(forwardCheckpoint);
+            try {
+                if (backCheckpoint.token) {
+                    await indexManager.addCrawlerCheckpoint(backCheckpoint);
+                    this.crawlerCheckpoints.push(backCheckpoint);
+                }
+
+                if (forwardCheckpoint.token) {
+                    await indexManager.addCrawlerCheckpoint(forwardCheckpoint);
+                    this.crawlerCheckpoints.push(forwardCheckpoint);
+                }
+            } catch (e) {
+                console.log("EventIndex: Error adding initial checkpoints for room",
+                            room.roomId, backCheckpoint, forwardCheckpoint, e);
+            }
         }));
     }
 
@@ -289,10 +299,7 @@ export default class EventIndex extends EventEmitter {
         let idle = false;
 
         while (!cancelled) {
-            // This is a low priority task and we don't want to spam our
-            // homeserver with /messages requests so we set a hefty timeout
-            // here.
-            let sleepTime = this._crawlerSleepTime;
+            let sleepTime = SettingsStore.getValueAt(SettingLevel.DEVICE, 'crawlerSleepTime');
 
             // Don't let the user configure a lower sleep time than 100 ms.
             sleepTime = Math.max(sleepTime, 100);
@@ -356,7 +363,11 @@ export default class EventIndex extends EventEmitter {
                 console.log("EventIndex: Done with the checkpoint", checkpoint);
                 // We got to the start/end of our timeline, lets just
                 // delete our checkpoint and go back to sleep.
-                await indexManager.removeCrawlerCheckpoint(checkpoint);
+                try {
+                    await indexManager.removeCrawlerCheckpoint(checkpoint);
+                } catch (e) {
+                    console.log("EventIndex: Error removing checkpoint", checkpoint, e);
+                }
                 continue;
             }
 

src/settings/Settings.js

@@ -492,8 +492,13 @@ export const SETTINGS = {
         default: true,
     },
     "keepSecretStoragePassphraseForSession": {
+         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+         displayName: _td("Keep secret storage passphrase in memory for this session"),
+         default: false,
+    },
+    "crawlerSleepTime": {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
-        displayName: _td("Keep secret storage passphrase in memory for this session"),
-        default: false,
+        displayName: _td("How fast should messages be downloaded."),
+        default: 3000,
     },
 };