From e94f3422dfdc2045d76a8481ada485f56cd0a6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 29 May 2020 11:44:08 +0200 Subject: [PATCH 01/19] Searching: Add support to paginate Seshat search results. --- src/Searching.js | 44 ++++++++++++++++++++++++++- src/components/structures/RoomView.js | 5 ++- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 663328fe41..c16906ade7 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -88,6 +88,7 @@ async function localSearch(searchTerm, roomId = undefined) { } const emptyResult = { + seshatQuery: searchArgs, results: [], highlights: [], }; @@ -97,6 +98,7 @@ async function localSearch(searchTerm, roomId = undefined) { const eventIndex = EventIndexPeg.get(); const localResult = await eventIndex.search(searchArgs); + emptyResult.seshatQuery.next_batch = localResult.next_batch; const response = { search_categories: { @@ -104,8 +106,25 @@ async function localSearch(searchTerm, roomId = undefined) { }, }; - const result = MatrixClientPeg.get()._processRoomEventsSearch( + return MatrixClientPeg.get()._processRoomEventsSearch( emptyResult, response); +} + +async function paginatedLocalSearch(searchResult) { + const eventIndex = EventIndexPeg.get(); + + let searchArgs = searchResult.seshatQuery; + + const localResult = await eventIndex.search(searchArgs); + + const response = { + search_categories: { + room_events: localResult, + }, + }; + + const result = MatrixClientPeg.get()._processRoomEventsSearch(searchResult, response); + searchResult.pendingRequest = null; return result; } @@ -132,6 +151,29 @@ function eventIndexSearch(term, roomId = undefined) { return searchPromise; } +function eventIndexSearchPagination(searchResult) { + const client = MatrixClientPeg.get(); + const query = searchResult.seshatQuery; + + if (!query) { + return client.backPaginateRoomEventsSearch(searchResult); + } else { + const promise = paginatedLocalSearch(searchResult); + searchResult.pendingRequest = promise; + return promise; + } +} + +export function searchPagination(searchResult) { + const eventIndex = EventIndexPeg.get(); + const client = MatrixClientPeg.get(); + + if (searchResult.pendingRequest) return searchResult.pendingRequest; + + if (eventIndex === null) return client.backPaginateRoomEventsSearch(searchResult); + else return eventIndexSearchPagination(searchResult); +} + export default function eventSearch(term, roomId = undefined) { const eventIndex = EventIndexPeg.get(); diff --git a/src/components/structures/RoomView.js b/src/components/structures/RoomView.js index 49d7e3c238..6796984037 100644 --- a/src/components/structures/RoomView.js +++ b/src/components/structures/RoomView.js @@ -39,7 +39,7 @@ import Tinter from '../../Tinter'; import rate_limited_func from '../../ratelimitedfunc'; import * as ObjectUtils from '../../ObjectUtils'; import * as Rooms from '../../Rooms'; -import eventSearch from '../../Searching'; +import eventSearch, {searchPagination} from '../../Searching'; import {isOnlyCtrlOrCmdIgnoreShiftKeyEvent, isOnlyCtrlOrCmdKeyEvent, Key} from '../../Keyboard'; @@ -1035,8 +1035,7 @@ export default createReactClass({ if (this.state.searchResults.next_batch) { debuglog("requesting more search results"); - const searchPromise = this.context.backPaginateRoomEventsSearch( - this.state.searchResults); + const searchPromise = searchPagination(this.state.searchResults); return this._handleSearchResult(searchPromise); } else { debuglog("no more search results"); From c1ef193e3a8ba0e221d2655095fabc016127b4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 29 May 2020 16:32:57 +0200 Subject: [PATCH 02/19] Searching: Add initial pagination for combined searches. --- src/Searching.js | 122 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 114 insertions(+), 8 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index c16906ade7..1960f2a2b2 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -71,6 +71,18 @@ async function combinedSearch(searchTerm) { result.highlights = localResult.highlights.concat( serverSideResult.highlights); + result.seshatQuery = localResult.seshatQuery; + result.serverSideNextBatch = serverSideResult.next_batch; + result._query = serverSideResult._query; + + // We need the next batch to be set for the client to know that it can + // paginate further. + if (serverSideResult.next_batch) { + result.next_batch = serverSideResult.next_batch; + } else { + result.next_batch = localResult.next_batch; + } + return result; } @@ -106,16 +118,16 @@ async function localSearch(searchTerm, roomId = undefined) { }, }; - return MatrixClientPeg.get()._processRoomEventsSearch( - emptyResult, response); + return MatrixClientPeg.get()._processRoomEventsSearch(emptyResult, response); } -async function paginatedLocalSearch(searchResult) { +async function localPagination(searchResult) { const eventIndex = EventIndexPeg.get(); - let searchArgs = searchResult.seshatQuery; + const searchArgs = searchResult.seshatQuery; const localResult = await eventIndex.search(searchArgs); + searchResult.seshatQuery.next_batch = localResult.next_batch; const response = { search_categories: { @@ -129,6 +141,92 @@ async function paginatedLocalSearch(searchResult) { return result; } +/** + * Combine the local and server search results + */ +function combineResults(previousSearchResult, localResult = undefined, serverSideResult = undefined) { + // // cachedResults = previousSearchResult.cachedResults; + // if (localResult) { + // previousSearchResult.seshatQuery.next_batch = localResult.next_batch; + // } + const compare = (a, b) => { + const aEvent = a.result; + const bEvent = b.result; + + if (aEvent.origin_server_ts > + bEvent.origin_server_ts) return -1; + if (aEvent.origin_server_ts < + bEvent.origin_server_ts) return 1; + return 0; + }; + + const result = {}; + + result.count = previousSearchResult.count; + + if (localResult && serverSideResult) { + result.results = localResult.results.concat(serverSideResult.results).sort(compare); + result.highlights = localResult.highlights.concat(serverSideResult.highlights); + } else if (localResult) { + result.results = localResult.results; + result.highlights = localResult.highlights; + } else { + result.results = serverSideResult.results; + result.highlights = serverSideResult.highlights; + } + + if (localResult) { + previousSearchResult.seshatQuery.next_batch = localResult.next_batch; + result.next_batch = localResult.next_batch; + } + + if (serverSideResult && serverSideResult.next_batch) { + previousSearchResult.serverSideNextBatch = serverSideResult.next_batch; + result.next_batch = serverSideResult.next_batch; + } + + console.log("HELLOO COMBINING RESULTS", localResult, serverSideResult, result); + + return result +} + +async function combinedPagination(searchResult) { + const eventIndex = EventIndexPeg.get(); + const client = MatrixClientPeg.get(); + + console.log("HELLOOO WORLD"); + + const searchArgs = searchResult.seshatQuery; + + let localResult; + let serverSideResult; + + if (searchArgs.next_batch) { + localResult = await eventIndex.search(searchArgs); + } + + if (searchResult.serverSideNextBatch) { + const body = {body: searchResult._query, next_batch: searchResult.serverSideNextBatch}; + serverSideResult = await client.search(body); + } + + const combinedResult = combineResults(searchResult, localResult, serverSideResult.search_categories.room_events); + + const response = { + search_categories: { + room_events: combinedResult, + }, + }; + + const result = client._processRoomEventsSearch(searchResult, response); + + console.log("HELLO NEW RESULT", searchResult); + + searchResult.pendingRequest = null; + + return result +} + function eventIndexSearch(term, roomId = undefined) { let searchPromise; @@ -153,14 +251,22 @@ function eventIndexSearch(term, roomId = undefined) { function eventIndexSearchPagination(searchResult) { const client = MatrixClientPeg.get(); - const query = searchResult.seshatQuery; - if (!query) { + const seshatQuery = searchResult.seshatQuery; + const serverQuery = searchResult._query; + + if (!seshatQuery) { return client.backPaginateRoomEventsSearch(searchResult); - } else { - const promise = paginatedLocalSearch(searchResult); + } else if (!serverQuery) { + const promise = localPagination(searchResult); searchResult.pendingRequest = promise; + return promise; + } else { + const promise = combinedPagination(searchResult); + searchResult.pendingRequest = promise; + + return promise } } From b6198e0ab996770c7ca8a08ae2cd91e9071a5e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 3 Jun 2020 14:16:02 +0200 Subject: [PATCH 03/19] Searching: Combine the events before letting the client process them. --- src/Searching.js | 194 +++++++++++++++++++++++++++-------------------- 1 file changed, 113 insertions(+), 81 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 1960f2a2b2..5fc5b31544 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -17,7 +17,9 @@ limitations under the License. import EventIndexPeg from "./indexing/EventIndexPeg"; import {MatrixClientPeg} from "./MatrixClientPeg"; -function serverSideSearch(term, roomId = undefined) { +async function serverSideSearch(term, roomId = undefined, processResult = true) { + const client = MatrixClientPeg.get(); + let filter; if (roomId !== undefined) { // XXX: it's unintuitive that the filter for searching doesn't have @@ -27,19 +29,59 @@ function serverSideSearch(term, roomId = undefined) { }; } - const searchPromise = MatrixClientPeg.get().searchRoomEvents({ - filter, - term, - }); + const body = { + search_categories: { + room_events: { + search_term: term, + filter: filter, + order_by: "recent", + event_context: { + before_limit: 1, + after_limit: 1, + include_profile: true, + }, + }, + }, + }; - return searchPromise; + const response = await client.search({body: body}); + + if (processResult) { + const searchResult = { + _query: body, + results: [], + highlights: [], + }; + + return client._processRoomEventsSearch(searchResult, response); + } + + const result = { + response: response, + query: body, + }; + + return result; +} + +function compareEvents(a, b) { + const aEvent = a.result; + const bEvent = b.result; + + if (aEvent.origin_server_ts > + bEvent.origin_server_ts) return -1; + if (aEvent.origin_server_ts < + bEvent.origin_server_ts) return 1; + return 0; } async function combinedSearch(searchTerm) { + const client = MatrixClientPeg.get(); + // Create two promises, one for the local search, one for the // server-side search. - const serverSidePromise = serverSideSearch(searchTerm); - const localPromise = localSearch(searchTerm); + const serverSidePromise = serverSideSearch(searchTerm, undefined, false); + const localPromise = localSearch(searchTerm, undefined, false); // Wait for both promises to resolve. await Promise.all([serverSidePromise, localPromise]); @@ -48,45 +90,34 @@ async function combinedSearch(searchTerm) { const localResult = await localPromise; const serverSideResult = await serverSidePromise; - // Combine the search results into one result. - const result = {}; + const serverQuery = serverSideResult.query; + const serverResponse = serverSideResult.response; - // Our localResult and serverSideResult are both ordered by - // recency separately, when we combine them the order might not - // be the right one so we need to sort them. - const compare = (a, b) => { - const aEvent = a.context.getEvent().event; - const bEvent = b.context.getEvent().event; + const localQuery = localResult.query; + const localResponse = localResult.response; - if (aEvent.origin_server_ts > - bEvent.origin_server_ts) return -1; - if (aEvent.origin_server_ts < - bEvent.origin_server_ts) return 1; - return 0; + const emptyResult = { + seshatQuery: localQuery, + _query: serverQuery, + serverSideNextBatch: serverResponse.next_batch, + results: [], + highlights: [], }; - result.count = localResult.count + serverSideResult.count; - result.results = localResult.results.concat( - serverSideResult.results).sort(compare); - result.highlights = localResult.highlights.concat( - serverSideResult.highlights); + const combinedResult = combineResponses(emptyResult, localResponse, serverResponse.search_categories.room_events); - result.seshatQuery = localResult.seshatQuery; - result.serverSideNextBatch = serverSideResult.next_batch; - result._query = serverSideResult._query; + const response = { + search_categories: { + room_events: combinedResult, + }, + }; - // We need the next batch to be set for the client to know that it can - // paginate further. - if (serverSideResult.next_batch) { - result.next_batch = serverSideResult.next_batch; - } else { - result.next_batch = localResult.next_batch; - } + const result = client._processRoomEventsSearch(emptyResult, response); return result; } -async function localSearch(searchTerm, roomId = undefined) { +async function localSearch(searchTerm, roomId = undefined, processResult = true) { const searchArgs = { search_term: searchTerm, before_limit: 1, @@ -110,15 +141,27 @@ async function localSearch(searchTerm, roomId = undefined) { const eventIndex = EventIndexPeg.get(); const localResult = await eventIndex.search(searchArgs); - emptyResult.seshatQuery.next_batch = localResult.next_batch; - const response = { - search_categories: { - room_events: localResult, - }, - }; + if (processResult) { + emptyResult.seshatQuery.next_batch = localResult.next_batch; - return MatrixClientPeg.get()._processRoomEventsSearch(emptyResult, response); + const response = { + search_categories: { + room_events: localResult, + }, + }; + + return MatrixClientPeg.get()._processRoomEventsSearch(emptyResult, response); + } + + searchArgs.next_batch = localResult.next_batch; + + const result = { + response: localResult, + query: searchArgs, + } + + return result; } async function localPagination(searchResult) { @@ -144,57 +187,46 @@ async function localPagination(searchResult) { /** * Combine the local and server search results */ -function combineResults(previousSearchResult, localResult = undefined, serverSideResult = undefined) { - // // cachedResults = previousSearchResult.cachedResults; - // if (localResult) { - // previousSearchResult.seshatQuery.next_batch = localResult.next_batch; - // } - const compare = (a, b) => { - const aEvent = a.result; - const bEvent = b.result; +function combineResponses(previousSearchResult, localEvents = undefined, serverEvents = undefined) { + const response = {}; - if (aEvent.origin_server_ts > - bEvent.origin_server_ts) return -1; - if (aEvent.origin_server_ts < - bEvent.origin_server_ts) return 1; - return 0; - }; - - const result = {}; - - result.count = previousSearchResult.count; - - if (localResult && serverSideResult) { - result.results = localResult.results.concat(serverSideResult.results).sort(compare); - result.highlights = localResult.highlights.concat(serverSideResult.highlights); - } else if (localResult) { - result.results = localResult.results; - result.highlights = localResult.highlights; + if (previousSearchResult.count) { + response.count = previousSearchResult.count; } else { - result.results = serverSideResult.results; - result.highlights = serverSideResult.highlights; + response.count = localEvents.count + serverEvents.count; } - if (localResult) { - previousSearchResult.seshatQuery.next_batch = localResult.next_batch; - result.next_batch = localResult.next_batch; + if (localEvents && serverEvents) { + response.results = localEvents.results.concat(serverEvents.results).sort(compareEvents); + response.highlights = localEvents.highlights.concat(serverEvents.highlights); + } else if (localEvents) { + response.results = localEvents.results; + response.highlights = localEvents.highlights; + } else { + response.results = serverEvents.results; + response.highlights = serverEvents.highlights; } - if (serverSideResult && serverSideResult.next_batch) { - previousSearchResult.serverSideNextBatch = serverSideResult.next_batch; - result.next_batch = serverSideResult.next_batch; + if (localEvents) { + previousSearchResult.seshatQuery.next_batch = localEvents.next_batch; + response.next_batch = localEvents.next_batch; } - console.log("HELLOO COMBINING RESULTS", localResult, serverSideResult, result); + if (serverEvents && serverEvents.next_batch) { + previousSearchResult.serverSideNextBatch = serverEvents.next_batch; + response.next_batch = serverEvents.next_batch; + } - return result + console.log("HELLOO COMBINING RESULTS", localEvents, serverEvents, response); + + return response } async function combinedPagination(searchResult) { const eventIndex = EventIndexPeg.get(); const client = MatrixClientPeg.get(); - console.log("HELLOOO WORLD"); + console.log("HELLOOO WORLD", searchResult.oldestEventFrom); const searchArgs = searchResult.seshatQuery; @@ -210,7 +242,7 @@ async function combinedPagination(searchResult) { serverSideResult = await client.search(body); } - const combinedResult = combineResults(searchResult, localResult, serverSideResult.search_categories.room_events); + const combinedResult = combineResponses(searchResult, localResult, serverSideResult.search_categories.room_events); const response = { search_categories: { From c2e0e10553f1bd3f08a6ea12aaa60c34f1653ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 3 Jun 2020 14:30:21 +0200 Subject: [PATCH 04/19] Searching: Split out more logic that combines the events. --- src/Searching.js | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 5fc5b31544..1a5c8d7a48 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -184,18 +184,9 @@ async function localPagination(searchResult) { return result; } -/** - * Combine the local and server search results - */ -function combineResponses(previousSearchResult, localEvents = undefined, serverEvents = undefined) { +function combineEvents(localEvents = undefined, serverEvents = undefined, cachedEvents = undefined) { const response = {}; - if (previousSearchResult.count) { - response.count = previousSearchResult.count; - } else { - response.count = localEvents.count + serverEvents.count; - } - if (localEvents && serverEvents) { response.results = localEvents.results.concat(serverEvents.results).sort(compareEvents); response.highlights = localEvents.highlights.concat(serverEvents.highlights); @@ -207,6 +198,21 @@ function combineResponses(previousSearchResult, localEvents = undefined, serverE response.highlights = serverEvents.highlights; } + return response; +} + +/** + * Combine the local and server search responses + */ +function combineResponses(previousSearchResult, localEvents = undefined, serverEvents = undefined) { + const response = combineEvents(localEvents, serverEvents); + + if (previousSearchResult.count) { + response.count = previousSearchResult.count; + } else { + response.count = localEvents.count + serverEvents.count; + } + if (localEvents) { previousSearchResult.seshatQuery.next_batch = localEvents.next_batch; response.next_batch = localEvents.next_batch; From 28f7d23bfc674bfd899695a352f385462e8ad383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 4 Jun 2020 11:18:53 +0200 Subject: [PATCH 05/19] Searching: Correctly order the combined search results. --- src/Searching.js | 108 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 91 insertions(+), 17 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 1a5c8d7a48..42a092edb0 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -184,20 +184,78 @@ async function localPagination(searchResult) { return result; } -function combineEvents(localEvents = undefined, serverEvents = undefined, cachedEvents = undefined) { +function combineEvents(previousSearchResult, localEvents = undefined, serverEvents = undefined) { const response = {}; - if (localEvents && serverEvents) { - response.results = localEvents.results.concat(serverEvents.results).sort(compareEvents); - response.highlights = localEvents.highlights.concat(serverEvents.highlights); - } else if (localEvents) { - response.results = localEvents.results; - response.highlights = localEvents.highlights; - } else { - response.results = serverEvents.results; - response.highlights = serverEvents.highlights; + let oldestEventFrom = "server"; + let cachedEvents; + + if (previousSearchResult.oldestEventFrom) { + oldestEventFrom = previousSearchResult.oldestEventFrom; } + if (previousSearchResult.cachedEvents) { + cachedEvents = previousSearchResult.cachedEvents; + } + + if (localEvents && serverEvents) { + const oldestLocalEvent = localEvents.results[localEvents.results.length - 1].result; + const oldestServerEvent = serverEvents.results[serverEvents.results.length - 1].result; + + if (oldestLocalEvent.origin_server_ts <= oldestServerEvent.origin_server_ts) { + oldestEventFrom = "local"; + } + + const combinedEvents = localEvents.results.concat(serverEvents.results).sort(compareEvents); + response.results = combinedEvents.slice(0, 10); + response.highlights = localEvents.highlights.concat(serverEvents.highlights); + previousSearchResult.cachedEvents = combinedEvents.slice(10); + console.log("HELLOO COMBINED", combinedEvents); + } else if (localEvents) { + if (cachedEvents && cachedEvents.length > 0) { + const oldestLocalEvent = localEvents.results[localEvents.results.length - 1].result; + const oldestCachedEvent = cachedEvents[cachedEvents.length - 1].result; + + if (oldestLocalEvent.origin_server_ts <= oldestCachedEvent.origin_server_ts) { + oldestEventFrom = "local"; + } + + const combinedEvents = localEvents.results.concat(cachedEvents).sort(compareEvents); + response.results = combinedEvents.slice(0, 10); + previousSearchResult.cachedEvents = combinedEvents.slice(10); + } else { + response.results = localEvents.results; + } + + response.highlights = localEvents.highlights; + } else if (serverEvents) { + console.log("HEEEEELOO WHAT'S GOING ON", cachedEvents); + if (cachedEvents && cachedEvents.length > 0) { + const oldestServerEvent = serverEvents.results[serverEvents.results.length - 1].result; + const oldestCachedEvent = cachedEvents[cachedEvents.length - 1].result; + + if (oldestServerEvent.origin_server_ts <= oldestCachedEvent.origin_server_ts) { + oldestEventFrom = "server"; + } + + const combinedEvents = serverEvents.results.concat(cachedEvents).sort(compareEvents); + response.results = combinedEvents.slice(0, 10); + previousSearchResult.cachedEvents = combinedEvents.slice(10); + } else { + response.results = serverEvents.results; + } + response.highlights = serverEvents.highlights; + } else { + if (cachedEvents && cachedEvents.length > 0) { + response.results = cachedEvents; + } + response.highlights = []; + + delete previousSearchResult.cachedEvents; + } + + previousSearchResult.oldestEventFrom = oldestEventFrom; + return response; } @@ -205,7 +263,7 @@ function combineEvents(localEvents = undefined, serverEvents = undefined, cached * Combine the local and server search responses */ function combineResponses(previousSearchResult, localEvents = undefined, serverEvents = undefined) { - const response = combineEvents(localEvents, serverEvents); + const response = combineEvents(previousSearchResult, localEvents, serverEvents); if (previousSearchResult.count) { response.count = previousSearchResult.count; @@ -215,12 +273,20 @@ function combineResponses(previousSearchResult, localEvents = undefined, serverE if (localEvents) { previousSearchResult.seshatQuery.next_batch = localEvents.next_batch; - response.next_batch = localEvents.next_batch; } - if (serverEvents && serverEvents.next_batch) { + if (serverEvents) { previousSearchResult.serverSideNextBatch = serverEvents.next_batch; - response.next_batch = serverEvents.next_batch; + } + + if (previousSearchResult.seshatQuery.next_batch) { + response.next_batch = previousSearchResult.seshatQuery.next_batch; + } else if (previousSearchResult.serverSideNextBatch) { + response.next_batch = previousSearchResult.serverSideNextBatch; + } + + if (!response.next_batch && previousSearchResult.cachedEvents && previousSearchResult.cachedEvents.length > 0) { + response.next_batch = "cached"; } console.log("HELLOO COMBINING RESULTS", localEvents, serverEvents, response); @@ -239,16 +305,24 @@ async function combinedPagination(searchResult) { let localResult; let serverSideResult; - if (searchArgs.next_batch) { + const oldestEventFrom = searchResult.oldestEventFrom; + + if ((searchArgs.next_batch && oldestEventFrom === "server") || (!searchResult.serverSideNextBatch && searchArgs.next_batch)) { localResult = await eventIndex.search(searchArgs); } - if (searchResult.serverSideNextBatch) { + if ((searchResult.serverSideNextBatch && oldestEventFrom === "local") || (!searchArgs.next_batch && searchResult.serverSideNextBatch)) { const body = {body: searchResult._query, next_batch: searchResult.serverSideNextBatch}; serverSideResult = await client.search(body); } - const combinedResult = combineResponses(searchResult, localResult, serverSideResult.search_categories.room_events); + let serverEvents; + + if (serverSideResult) { + serverEvents = serverSideResult.search_categories.room_events; + } + + const combinedResult = combineResponses(searchResult, localResult, serverEvents); const response = { search_categories: { From c6462e11ecd385eb1bb6c0c7f5279aaef1b8584a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 4 Jun 2020 12:12:09 +0200 Subject: [PATCH 06/19] Searching: Fix some lint issues. --- src/Searching.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 42a092edb0..6932836273 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -159,7 +159,7 @@ async function localSearch(searchTerm, roomId = undefined, processResult = true) const result = { response: localResult, query: searchArgs, - } + }; return result; } @@ -291,7 +291,7 @@ function combineResponses(previousSearchResult, localEvents = undefined, serverE console.log("HELLOO COMBINING RESULTS", localEvents, serverEvents, response); - return response + return response; } async function combinedPagination(searchResult) { @@ -307,11 +307,13 @@ async function combinedPagination(searchResult) { const oldestEventFrom = searchResult.oldestEventFrom; - if ((searchArgs.next_batch && oldestEventFrom === "server") || (!searchResult.serverSideNextBatch && searchArgs.next_batch)) { + if ((searchArgs.next_batch && oldestEventFrom === "server") || + (!searchResult.serverSideNextBatch && searchArgs.next_batch)) { localResult = await eventIndex.search(searchArgs); } - if ((searchResult.serverSideNextBatch && oldestEventFrom === "local") || (!searchArgs.next_batch && searchResult.serverSideNextBatch)) { + if ((searchResult.serverSideNextBatch && oldestEventFrom === "local") || + (!searchArgs.next_batch && searchResult.serverSideNextBatch)) { const body = {body: searchResult._query, next_batch: searchResult.serverSideNextBatch}; serverSideResult = await client.search(body); } @@ -336,7 +338,7 @@ async function combinedPagination(searchResult) { searchResult.pendingRequest = null; - return result + return result; } function eventIndexSearch(term, roomId = undefined) { @@ -378,7 +380,7 @@ function eventIndexSearchPagination(searchResult) { const promise = combinedPagination(searchResult); searchResult.pendingRequest = promise; - return promise + return promise; } } From df0659431456d035b323b1a2a6477a84b7cf5af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 4 Jun 2020 15:25:02 +0200 Subject: [PATCH 07/19] Searching: Refactor out some code that combines the search results. --- src/Searching.js | 100 ++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 57 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 6932836273..11652a6019 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -17,6 +17,8 @@ limitations under the License. import EventIndexPeg from "./indexing/EventIndexPeg"; import {MatrixClientPeg} from "./MatrixClientPeg"; +const SEARCH_LIMIT = 10; + async function serverSideSearch(term, roomId = undefined, processResult = true) { const client = MatrixClientPeg.get(); @@ -26,6 +28,7 @@ async function serverSideSearch(term, roomId = undefined, processResult = true) // the same shape as the v2 filter API :( filter = { rooms: [roomId], + limit: SEARCH_LIMIT, }; } @@ -96,16 +99,21 @@ async function combinedSearch(searchTerm) { const localQuery = localResult.query; const localResponse = localResult.response; + // Store our queries for later on so we can support pagination. const emptyResult = { seshatQuery: localQuery, _query: serverQuery, serverSideNextBatch: serverResponse.next_batch, + cachedEvents: [], + oldestEventFrom: "server", results: [], highlights: [], }; + // Combine our results. const combinedResult = combineResponses(emptyResult, localResponse, serverResponse.search_categories.room_events); + // Let the client process the combined result. const response = { search_categories: { room_events: combinedResult, @@ -122,6 +130,7 @@ async function localSearch(searchTerm, roomId = undefined, processResult = true) search_term: searchTerm, before_limit: 1, after_limit: 1, + limit: SEARCH_LIMIT, order_by_recency: true, room_id: undefined, }; @@ -184,73 +193,53 @@ async function localPagination(searchResult) { return result; } +function compareOldestEvents(firstResults, secondResults) { + try { + const oldestFirstEvent = firstResults.results[firstResults.results.length - 1].result; + const oldestSecondEvent = secondResults.results[secondResults.results.length - 1].result; + + if (oldestFirstEvent.origin_server_ts <= oldestSecondEvent.origin_server_ts) { + return -1 + } else { + return 1 + } + } catch { + return 0 + } +} + +function combineEventSources(previousSearchResult, response, a, b) { + const combinedEvents = a.concat(b).sort(compareEvents); + response.results = combinedEvents.slice(0, SEARCH_LIMIT); + previousSearchResult.cachedEvents = combinedEvents.slice(SEARCH_LIMIT); +} + function combineEvents(previousSearchResult, localEvents = undefined, serverEvents = undefined) { const response = {}; - let oldestEventFrom = "server"; - let cachedEvents; - - if (previousSearchResult.oldestEventFrom) { - oldestEventFrom = previousSearchResult.oldestEventFrom; - } - - if (previousSearchResult.cachedEvents) { - cachedEvents = previousSearchResult.cachedEvents; - } + const cachedEvents = previousSearchResult.cachedEvents; + let oldestEventFrom = previousSearchResult.oldestEventFrom; + response.highlights = previousSearchResult.highlights; if (localEvents && serverEvents) { - const oldestLocalEvent = localEvents.results[localEvents.results.length - 1].result; - const oldestServerEvent = serverEvents.results[serverEvents.results.length - 1].result; - - if (oldestLocalEvent.origin_server_ts <= oldestServerEvent.origin_server_ts) { + if (compareOldestEvents(localEvents, serverEvents) < 0) { oldestEventFrom = "local"; } - const combinedEvents = localEvents.results.concat(serverEvents.results).sort(compareEvents); - response.results = combinedEvents.slice(0, 10); + combineEventSources(previousSearchResult, response, localEvents.results, serverEvents.results); response.highlights = localEvents.highlights.concat(serverEvents.highlights); - previousSearchResult.cachedEvents = combinedEvents.slice(10); - console.log("HELLOO COMBINED", combinedEvents); } else if (localEvents) { - if (cachedEvents && cachedEvents.length > 0) { - const oldestLocalEvent = localEvents.results[localEvents.results.length - 1].result; - const oldestCachedEvent = cachedEvents[cachedEvents.length - 1].result; - - if (oldestLocalEvent.origin_server_ts <= oldestCachedEvent.origin_server_ts) { - oldestEventFrom = "local"; - } - - const combinedEvents = localEvents.results.concat(cachedEvents).sort(compareEvents); - response.results = combinedEvents.slice(0, 10); - previousSearchResult.cachedEvents = combinedEvents.slice(10); - } else { - response.results = localEvents.results; + if (compareOldestEvents(localEvents, cachedEvents) < 0) { + oldestEventFrom = "local"; } - - response.highlights = localEvents.highlights; + combineEventSources(previousSearchResult, response, localEvents.results, cachedEvents); } else if (serverEvents) { - console.log("HEEEEELOO WHAT'S GOING ON", cachedEvents); - if (cachedEvents && cachedEvents.length > 0) { - const oldestServerEvent = serverEvents.results[serverEvents.results.length - 1].result; - const oldestCachedEvent = cachedEvents[cachedEvents.length - 1].result; - - if (oldestServerEvent.origin_server_ts <= oldestCachedEvent.origin_server_ts) { - oldestEventFrom = "server"; - } - - const combinedEvents = serverEvents.results.concat(cachedEvents).sort(compareEvents); - response.results = combinedEvents.slice(0, 10); - previousSearchResult.cachedEvents = combinedEvents.slice(10); - } else { - response.results = serverEvents.results; + if (compareOldestEvents(serverEvents, cachedEvents) < 0) { + oldestEventFrom = "server"; } - response.highlights = serverEvents.highlights; + combineEventSources(previousSearchResult, response, serverEvents.results, cachedEvents); } else { - if (cachedEvents && cachedEvents.length > 0) { - response.results = cachedEvents; - } - response.highlights = []; - + response.results = cachedEvents; delete previousSearchResult.cachedEvents; } @@ -298,15 +287,12 @@ async function combinedPagination(searchResult) { const eventIndex = EventIndexPeg.get(); const client = MatrixClientPeg.get(); - console.log("HELLOOO WORLD", searchResult.oldestEventFrom); - const searchArgs = searchResult.seshatQuery; + const oldestEventFrom = searchResult.oldestEventFrom; let localResult; let serverSideResult; - const oldestEventFrom = searchResult.oldestEventFrom; - if ((searchArgs.next_batch && oldestEventFrom === "server") || (!searchResult.serverSideNextBatch && searchArgs.next_batch)) { localResult = await eventIndex.search(searchArgs); From 96ca47381c94ea4219a326bb56d37ad51c210ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 4 Jun 2020 15:25:32 +0200 Subject: [PATCH 08/19] Searching: Add more docs explaining what's going on in the pagination. --- src/Searching.js | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 11652a6019..7ea1463a5e 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -71,10 +71,9 @@ function compareEvents(a, b) { const aEvent = a.result; const bEvent = b.result; - if (aEvent.origin_server_ts > - bEvent.origin_server_ts) return -1; - if (aEvent.origin_server_ts < - bEvent.origin_server_ts) return 1; + if (aEvent.origin_server_ts > bEvent.origin_server_ts) return -1; + if (aEvent.origin_server_ts < bEvent.origin_server_ts) return 1; + return 0; } @@ -254,26 +253,37 @@ function combineEvents(previousSearchResult, localEvents = undefined, serverEven function combineResponses(previousSearchResult, localEvents = undefined, serverEvents = undefined) { const response = combineEvents(previousSearchResult, localEvents, serverEvents); + // Our first search will contain counts from both sources, subsequent + // pagination requests will fetch responses only from one of the sources, so + // reuse the first count when we're paginating. if (previousSearchResult.count) { response.count = previousSearchResult.count; } else { response.count = localEvents.count + serverEvents.count; } + // Update our next batch tokens for the given search sources. if (localEvents) { previousSearchResult.seshatQuery.next_batch = localEvents.next_batch; } - if (serverEvents) { previousSearchResult.serverSideNextBatch = serverEvents.next_batch; } + // Set the response next batch token to one of the tokens from the sources, + // this makes sure that if we exhaust one of the sources we continue with + // the other one. if (previousSearchResult.seshatQuery.next_batch) { response.next_batch = previousSearchResult.seshatQuery.next_batch; } else if (previousSearchResult.serverSideNextBatch) { response.next_batch = previousSearchResult.serverSideNextBatch; } + // We collected all search results from the server as well as from Seshat, + // we still have some events cached that we'll want to display on the next + // pagination request. + // + // Provide a fake next batch token for that case. if (!response.next_batch && previousSearchResult.cachedEvents && previousSearchResult.cachedEvents.length > 0) { response.next_batch = "cached"; } @@ -356,13 +366,18 @@ function eventIndexSearchPagination(searchResult) { const serverQuery = searchResult._query; if (!seshatQuery) { + // This is a search in a non-encrypted room. Do the normal server-side + // pagination. return client.backPaginateRoomEventsSearch(searchResult); } else if (!serverQuery) { + // This is a search in a encrypted room. Do a local pagination. const promise = localPagination(searchResult); searchResult.pendingRequest = promise; return promise; } else { + // We have both queries around, this is a search across all rooms so a + // combined pagination needs to be done. const promise = combinedPagination(searchResult); searchResult.pendingRequest = promise; From 46fd36568affdb948ca3d632027ae56f2301d135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 4 Jun 2020 15:33:51 +0200 Subject: [PATCH 09/19] Searching: Always set the limit when searching. The spec doesn't mention any default limit so different homeservers might use different defaults, since we don't want Riot to behave differently depending on the homeserver bite the bullet of sending an additional 10 or so bytes when searching. --- src/Searching.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 7ea1463a5e..f40e44a84d 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -22,15 +22,11 @@ const SEARCH_LIMIT = 10; async function serverSideSearch(term, roomId = undefined, processResult = true) { const client = MatrixClientPeg.get(); - let filter; - if (roomId !== undefined) { - // XXX: it's unintuitive that the filter for searching doesn't have - // the same shape as the v2 filter API :( - filter = { - rooms: [roomId], - limit: SEARCH_LIMIT, - }; - } + const filter = { + limit: SEARCH_LIMIT, + }; + + if (roomId !== undefined) filter.rooms = [roomId]; const body = { search_categories: { From 94d4aa17ee738bc10c78f138da04c8b23d1d4ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 4 Jun 2020 15:35:18 +0200 Subject: [PATCH 10/19] Searching: Remove some debug logs. --- src/Searching.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index f40e44a84d..cb20176804 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -284,8 +284,6 @@ function combineResponses(previousSearchResult, localEvents = undefined, serverE response.next_batch = "cached"; } - console.log("HELLOO COMBINING RESULTS", localEvents, serverEvents, response); - return response; } @@ -326,8 +324,6 @@ async function combinedPagination(searchResult) { const result = client._processRoomEventsSearch(searchResult, response); - console.log("HELLO NEW RESULT", searchResult); - searchResult.pendingRequest = null; return result; From bf13032d5b9c14ab675db164b09a14ca7cb4e0de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 4 Jun 2020 15:51:06 +0200 Subject: [PATCH 11/19] Searching: Fix a lint issue and expand some docs. --- src/Searching.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index cb20176804..29556f0757 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -194,12 +194,12 @@ function compareOldestEvents(firstResults, secondResults) { const oldestSecondEvent = secondResults.results[secondResults.results.length - 1].result; if (oldestFirstEvent.origin_server_ts <= oldestSecondEvent.origin_server_ts) { - return -1 + return -1; } else { - return 1 + return 1; } } catch { - return 0 + return 0; } } @@ -209,6 +209,19 @@ function combineEventSources(previousSearchResult, response, a, b) { previousSearchResult.cachedEvents = combinedEvents.slice(SEARCH_LIMIT); } +/** + * Combine the events from our event sources into a sorted result + * + * @param {object} previousSearchResult A search result from a previous search + * call. + * @param {object} localEvents An unprocessed search result from the event + * index. + * @param {object} serverEvents An unprocessed search result from the server. + * + * @ return {object} A response object that combines the events from the + * different event sources. + * + */ function combineEvents(previousSearchResult, localEvents = undefined, serverEvents = undefined) { const response = {}; From 5dfe5ac135458710a185d5416e3de944e176abb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 4 Jun 2020 16:57:28 +0200 Subject: [PATCH 12/19] Searching: More docs and a bit of simplification. --- src/Searching.js | 134 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 127 insertions(+), 7 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 29556f0757..d2fd67fa27 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -204,7 +204,9 @@ function compareOldestEvents(firstResults, secondResults) { } function combineEventSources(previousSearchResult, response, a, b) { + // Merge event sources and sort the events. const combinedEvents = a.concat(b).sort(compareEvents); + // Put half of the events in the response, and cache the other half. response.results = combinedEvents.slice(0, SEARCH_LIMIT); previousSearchResult.cachedEvents = combinedEvents.slice(SEARCH_LIMIT); } @@ -212,13 +214,104 @@ function combineEventSources(previousSearchResult, response, a, b) { /** * Combine the events from our event sources into a sorted result * + * This method will first be called from the combinedSearch() method. In this + * case we will fetch SEARCH_LIMIT events from the server and the local index. + * + * The method will put the SEARCH_LIMIT newest events from the server and the + * local index in the results part of the response, the rest will be put in the + * cachedEvents field of the previousSearchResult (in this case an empty search + * result). + * + * Every subsequent call will be made from the combinedPagination() method, in + * this case we will combine the cachedEvents and the next SEARCH_LIMIT events + * from either the server or the local index. + * + * Since we have two event sources and we need to sort the results by date we + * need keep on looking for the oldest event. We are implementing a variation of + * a sliding window. + * + * If we set SEARCH_LIMIT to 3: + * + * Server events [01, 02, 04, 06, 07, 08, 11, 13] + * |01, 02, 04| + * Local events [03, 05, 09, 10, 12, 14, 15, 16] + * |03, 05, 09| + * + * We note that the oldest event is from the local index, and we combine the + * results: + * + * Server window [01, 02, 04] + * Local window [03, 05, 09] + * + * Combined events [01, 02, 03, 04, 05, 09] + * + * We split the combined result in the part that we want to present and a part + * that will be cached. + * + * Presented events [01, 02, 03] + * Cached events [04, 05, 09] + * + * We slide the window for the server since the oldest event is from the local + * index. + * + * Server events [01, 02, 04, 06, 07, 08, 11, 13] + * |06, 07, 08| + * Local events [03, 05, 09, 10, 12, 14, 15, 16] + * |XX, XX, XX| + * Cached events [04, 05, 09] + * + * We note that the oldest event is from the server and we combine the new + * server events with the cached ones. + * + * Cached events [04, 05, 09] + * Server events [06, 07, 08] + * + * Combined events [04, 05, 06, 07, 08, 09] + * + * We split again. + * + * Presented events [04, 05, 06] + * Cached events [07, 08, 09] + * + * We slide the local window, the oldest event is on the server. + * + * Server events [01, 02, 04, 06, 07, 08, 11, 13] + * |XX, XX, XX| + * Local events [03, 05, 09, 10, 12, 14, 15, 16] + * |10, 12, 14| + * + * Cached events [07, 08, 09] + * Local events [10, 12, 14] + * Combined events [07, 08, 09, 10, 12, 14] + * + * Presented events [07, 08, 09] + * Cached events [10, 12, 14] + * + * Next up we slide the server window again. + * + * Server events [01, 02, 04, 06, 07, 08, 11, 13] + * |11, 13| + * Local events [03, 05, 09, 10, 12, 14, 15, 16] + * |XX, XX, XX| + * + * Cached events [10, 12, 14] + * Server events [11, 13] + * Combined events [10, 11, 12, 13, 14] + * + * Presented events [10, 11, 12] + * Cached events [13, 14] + * + * We have one source exhausted, we fetch the rest of our events from the other + * source and combine it with our cached events. + * + * * @param {object} previousSearchResult A search result from a previous search * call. * @param {object} localEvents An unprocessed search result from the event * index. * @param {object} serverEvents An unprocessed search result from the server. * - * @ return {object} A response object that combines the events from the + * @return {object} A response object that combines the events from the * different event sources. * */ @@ -230,6 +323,9 @@ function combineEvents(previousSearchResult, localEvents = undefined, serverEven response.highlights = previousSearchResult.highlights; if (localEvents && serverEvents) { + // This is a first search call, combine the events from the server and + // the local index. Note where our oldest event came from, we shall + // fetch the next batch of events from the other source. if (compareOldestEvents(localEvents, serverEvents) < 0) { oldestEventFrom = "local"; } @@ -237,18 +333,28 @@ function combineEvents(previousSearchResult, localEvents = undefined, serverEven combineEventSources(previousSearchResult, response, localEvents.results, serverEvents.results); response.highlights = localEvents.highlights.concat(serverEvents.highlights); } else if (localEvents) { + // This is a pagination call fetching more events from the local index, + // meaning that our oldest event was on the server. + // Change the source of the oldest event if our local event is older + // than the cached one. if (compareOldestEvents(localEvents, cachedEvents) < 0) { oldestEventFrom = "local"; } combineEventSources(previousSearchResult, response, localEvents.results, cachedEvents); } else if (serverEvents) { + // This is a pagination call fetching more events from the server, + // meaning that our oldest event was in the local index. + // Change the source of the oldest event if our server event is older + // than the cached one. if (compareOldestEvents(serverEvents, cachedEvents) < 0) { oldestEventFrom = "server"; } combineEventSources(previousSearchResult, response, serverEvents.results, cachedEvents); } else { + // This is a pagination call where we exhausted both of our event + // sources, let's push the remaining cached events. response.results = cachedEvents; - delete previousSearchResult.cachedEvents; + previousSearchResult.cachedEvents = []; } previousSearchResult.oldestEventFrom = oldestEventFrom; @@ -258,8 +364,18 @@ function combineEvents(previousSearchResult, localEvents = undefined, serverEven /** * Combine the local and server search responses + * + * @param {object} previousSearchResult A search result from a previous search + * call. + * @param {object} localEvents An unprocessed search result from the event + * index. + * @param {object} serverEvents An unprocessed search result from the server. + * + * @return {object} A response object that combines the events from the + * different event sources. */ function combineResponses(previousSearchResult, localEvents = undefined, serverEvents = undefined) { + // Combine our events first. const response = combineEvents(previousSearchResult, localEvents, serverEvents); // Our first search will contain counts from both sources, subsequent @@ -293,7 +409,7 @@ function combineResponses(previousSearchResult, localEvents = undefined, serverE // pagination request. // // Provide a fake next batch token for that case. - if (!response.next_batch && previousSearchResult.cachedEvents && previousSearchResult.cachedEvents.length > 0) { + if (!response.next_batch && previousSearchResult.cachedEvents.length > 0) { response.next_batch = "cached"; } @@ -310,13 +426,15 @@ async function combinedPagination(searchResult) { let localResult; let serverSideResult; - if ((searchArgs.next_batch && oldestEventFrom === "server") || - (!searchResult.serverSideNextBatch && searchArgs.next_batch)) { + // Fetch events from the local index if we have a token for itand if it's + // the local indexes turn or the server has exhausted its results. + if (searchArgs.next_batch && (!searchResult.serverSideNextBatch || oldestEventFrom === "server")) { localResult = await eventIndex.search(searchArgs); } - if ((searchResult.serverSideNextBatch && oldestEventFrom === "local") || - (!searchArgs.next_batch && searchResult.serverSideNextBatch)) { + // Fetch events from the server if we have a token for it and if it's the + // local indexes turn or the local index has exhausted its results. + if (searchResult.serverSideNextBatch && (oldestEventFrom === "local" || !searchArgs.next_batch)) { const body = {body: searchResult._query, next_batch: searchResult.serverSideNextBatch}; serverSideResult = await client.search(body); } @@ -327,6 +445,7 @@ async function combinedPagination(searchResult) { serverEvents = serverSideResult.search_categories.room_events; } + // Combine our events. const combinedResult = combineResponses(searchResult, localResult, serverEvents); const response = { @@ -335,6 +454,7 @@ async function combinedPagination(searchResult) { }, }; + // Let the client process the combined result. const result = client._processRoomEventsSearch(searchResult, response); searchResult.pendingRequest = null; From 632f54d126737a5cfe3fdd15055cc86e671d0710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Jun 2020 10:09:16 +0200 Subject: [PATCH 13/19] Searching: Split out the search methods. This splits out the search methods out into ones that process the search result and ones that do not instead of toggling the return format using a boolean. --- src/Searching.js | 80 +++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index d2fd67fa27..be5e61aca7 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -19,7 +19,7 @@ import {MatrixClientPeg} from "./MatrixClientPeg"; const SEARCH_LIMIT = 10; -async function serverSideSearch(term, roomId = undefined, processResult = true) { +async function serverSideSearch(term, roomId = undefined) { const client = MatrixClientPeg.get(); const filter = { @@ -45,16 +45,6 @@ async function serverSideSearch(term, roomId = undefined, processResult = true) const response = await client.search({body: body}); - if (processResult) { - const searchResult = { - _query: body, - results: [], - highlights: [], - }; - - return client._processRoomEventsSearch(searchResult, response); - } - const result = { response: response, query: body, @@ -63,6 +53,19 @@ async function serverSideSearch(term, roomId = undefined, processResult = true) return result; } +async function serverSideSearchProcess(term, roomId = undefined) { + const client = MatrixClientPeg.get(); + const result = await serverSideSearch(term, roomId); + + const searchResult = { + _query: result.query, + results: [], + highlights: [], + }; + + return client._processRoomEventsSearch(searchResult, result.response); +} + function compareEvents(a, b) { const aEvent = a.result; const bEvent = b.result; @@ -78,8 +81,8 @@ async function combinedSearch(searchTerm) { // Create two promises, one for the local search, one for the // server-side search. - const serverSidePromise = serverSideSearch(searchTerm, undefined, false); - const localPromise = localSearch(searchTerm, undefined, false); + const serverSidePromise = serverSideSearch(searchTerm, undefined); + const localPromise = localSearch(searchTerm, undefined); // Wait for both promises to resolve. await Promise.all([serverSidePromise, localPromise]); @@ -121,6 +124,8 @@ async function combinedSearch(searchTerm) { } async function localSearch(searchTerm, roomId = undefined, processResult = true) { + const eventIndex = EventIndexPeg.get(); + const searchArgs = { search_term: searchTerm, before_limit: 1, @@ -134,30 +139,8 @@ async function localSearch(searchTerm, roomId = undefined, processResult = true) searchArgs.room_id = roomId; } - const emptyResult = { - seshatQuery: searchArgs, - results: [], - highlights: [], - }; - - if (searchTerm === "") return emptyResult; - - const eventIndex = EventIndexPeg.get(); - const localResult = await eventIndex.search(searchArgs); - if (processResult) { - emptyResult.seshatQuery.next_batch = localResult.next_batch; - - const response = { - search_categories: { - room_events: localResult, - }, - }; - - return MatrixClientPeg.get()._processRoomEventsSearch(emptyResult, response); - } - searchArgs.next_batch = localResult.next_batch; const result = { @@ -168,6 +151,27 @@ async function localSearch(searchTerm, roomId = undefined, processResult = true) return result; } +async function localSearchProcess(searchTerm, roomId = undefined) { + const emptyResult = { + results: [], + highlights: [], + }; + + if (searchTerm === "") return emptyResult; + + const result = await localSearch(searchTerm, roomId); + + emptyResult.seshatQuery = result.query; + + const response = { + search_categories: { + room_events: result.response, + }, + }; + + return MatrixClientPeg.get()._processRoomEventsSearch(emptyResult, response); +} + async function localPagination(searchResult) { const eventIndex = EventIndexPeg.get(); @@ -469,11 +473,11 @@ function eventIndexSearch(term, roomId = undefined) { if (MatrixClientPeg.get().isRoomEncrypted(roomId)) { // The search is for a single encrypted room, use our local // search method. - searchPromise = localSearch(term, roomId); + searchPromise = localSearchProcess(term, roomId); } else { // The search is for a single non-encrypted room, use the // server-side search. - searchPromise = serverSideSearch(term, roomId); + searchPromise = serverSideSearchProcess(term, roomId); } } else { // Search across all rooms, combine a server side search and a @@ -523,6 +527,6 @@ export function searchPagination(searchResult) { export default function eventSearch(term, roomId = undefined) { const eventIndex = EventIndexPeg.get(); - if (eventIndex === null) return serverSideSearch(term, roomId); + if (eventIndex === null) return serverSideSearchProcess(term, roomId); else return eventIndexSearch(term, roomId); } From daef8f329bfd705ded000a5e3ca365d307d561ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Jun 2020 10:34:11 +0200 Subject: [PATCH 14/19] Searching: Explain what the integer lists in the combineEvents docs mean. --- src/Searching.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Searching.js b/src/Searching.js index be5e61aca7..e580102232 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -234,6 +234,11 @@ function combineEventSources(previousSearchResult, response, a, b) { * need keep on looking for the oldest event. We are implementing a variation of * a sliding window. * + * The event sources are here represented as two sorted lists where the lowest + * number represents the newest event. The two sorted lists need to be merged + * so they can be shown as one search result. We first fetch SEARCH_LIMIT events + * from both sources. + * * If we set SEARCH_LIMIT to 3: * * Server events [01, 02, 04, 06, 07, 08, 11, 13] From 4c361bfeb71387664fb46382a1eb59041928fc28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Jun 2020 10:34:47 +0200 Subject: [PATCH 15/19] Searching: Explain why _query and the server next batch stored as it is. --- src/Searching.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Searching.js b/src/Searching.js index e580102232..f5e373da61 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -57,6 +57,9 @@ async function serverSideSearchProcess(term, roomId = undefined) { const client = MatrixClientPeg.get(); const result = await serverSideSearch(term, roomId); + // The js-sdk method backPaginateRoomEventsSearch() uses _query internally + // so we're reusing the concept here since we wan't to delegate the + // pagination back to backPaginateRoomEventsSearch() in some cases. const searchResult = { _query: result.query, results: [], @@ -98,6 +101,16 @@ async function combinedSearch(searchTerm) { const localResponse = localResult.response; // Store our queries for later on so we can support pagination. + // + // We're reusing _query here again to not introduce separate code paths and + // concepts for our different pagination methods. We're storing the + // server-side next batch separately since the query is the json body of + // the request and next_batch needs to be a query parameter. + // + // We can't put it in the final result that _processRoomEventsSearch() + // returns since that one can be either a server-side one, a local one or a + // fake one to fetch the remaining cached events. See the docs for + // combineEvents() for an explanation why we need to cache events. const emptyResult = { seshatQuery: localQuery, _query: serverQuery, From 67cad2807b26bfca4fd33bd88b35304399c49c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Jun 2020 10:44:38 +0200 Subject: [PATCH 16/19] Searching: Better wording for what's going on in combineEvents(). --- src/Searching.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index f5e373da61..0cbedf2305 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -247,10 +247,10 @@ function combineEventSources(previousSearchResult, response, a, b) { * need keep on looking for the oldest event. We are implementing a variation of * a sliding window. * - * The event sources are here represented as two sorted lists where the lowest - * number represents the newest event. The two sorted lists need to be merged - * so they can be shown as one search result. We first fetch SEARCH_LIMIT events - * from both sources. + * The event sources are here represented as two sorted lists where the smallest + * number represents the newest event. The two lists need to be merged in a way + * that preserves the sorted property so they can be shown as one search result. + * We first fetch SEARCH_LIMIT events from both sources. * * If we set SEARCH_LIMIT to 3: * From 398655f1a9bdc180d9b22931b8e32d7512964af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Jun 2020 11:45:32 +0200 Subject: [PATCH 17/19] Searching: Remove some redundant undefined function arguments. --- src/Searching.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 0cbedf2305..5fe89f8199 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -84,8 +84,8 @@ async function combinedSearch(searchTerm) { // Create two promises, one for the local search, one for the // server-side search. - const serverSidePromise = serverSideSearch(searchTerm, undefined); - const localPromise = localSearch(searchTerm, undefined); + const serverSidePromise = serverSideSearch(searchTerm); + const localPromise = localSearch(searchTerm); // Wait for both promises to resolve. await Promise.all([serverSidePromise, localPromise]); From 0e1e949c49573baa1aaf4a8b464cdd62edba16e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Jun 2020 14:25:30 +0200 Subject: [PATCH 18/19] Searching: Refactor out the e2ee status restoring logic. --- src/Searching.js | 52 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 6e6279555c..6c31a5ca1a 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -182,7 +182,11 @@ async function localSearchProcess(searchTerm, roomId = undefined) { }, }; - return MatrixClientPeg.get()._processRoomEventsSearch(emptyResult, response); + const processedResult = MatrixClientPeg.get()._processRoomEventsSearch(emptyResult, response); + // Restore our encryption info so we can properly re-verify the events. + restoreEncryptionInfo(processedResult.results); + + return processedResult; } async function localPagination(searchResult) { @@ -438,6 +442,31 @@ function combineResponses(previousSearchResult, localEvents = undefined, serverE return response; } +function restoreEncryptionInfo(searchResultSlice) { + for (let i = 0; i < searchResultSlice.length; i++) { + const timeline = searchResultSlice[i].context.getTimeline(); + + for (let j = 0; j < timeline.length; j++) { + const ev = timeline[j]; + + if (ev.event.curve25519Key) { + ev.makeEncrypted( + "m.room.encrypted", + { algorithm: ev.event.algorithm }, + ev.event.curve25519Key, + ev.event.ed25519Key, + ); + ev._forwardingCurve25519KeyChain = ev.event.forwardingCurve25519KeyChain; + + delete ev.event.curve25519Key; + delete ev.event.ed25519Key; + delete ev.event.algorithm; + delete ev.event.forwardingCurve25519KeyChain; + } + } + } +} + async function combinedPagination(searchResult) { const eventIndex = EventIndexPeg.get(); const client = MatrixClientPeg.get(); @@ -482,27 +511,6 @@ async function combinedPagination(searchResult) { searchResult.pendingRequest = null; // Restore our encryption info so we can properly re-verify the events. - for (let i = 0; i < result.results.length; i++) { - const timeline = result.results[i].context.getTimeline(); - - for (let j = 0; j < timeline.length; j++) { - const ev = timeline[j]; - if (ev.event.curve25519Key) { - ev.makeEncrypted( - "m.room.encrypted", - { algorithm: ev.event.algorithm }, - ev.event.curve25519Key, - ev.event.ed25519Key, - ); - ev._forwardingCurve25519KeyChain = ev.event.forwardingCurve25519KeyChain; - - delete ev.event.curve25519Key; - delete ev.event.ed25519Key; - delete ev.event.algorithm; - delete ev.event.forwardingCurve25519KeyChain; - } - } - } return result; } From a0934329fec605d67da635b2f2797d6b074d4829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Jun 2020 15:41:55 +0200 Subject: [PATCH 19/19] Searching: Restore the encryption state for the paginated results. --- src/Searching.js | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 6c31a5ca1a..b1507e6a49 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -133,6 +133,9 @@ async function combinedSearch(searchTerm) { const result = client._processRoomEventsSearch(emptyResult, response); + // Restore our encryption info so we can properly re-verify the events. + restoreEncryptionInfo(result.results); + return result; } @@ -197,6 +200,10 @@ async function localPagination(searchResult) { const localResult = await eventIndex.search(searchArgs); searchResult.seshatQuery.next_batch = localResult.next_batch; + // We only need to restore the encryption state for the new results, so + // remember how many of them we got. + const newResultCount = localResult.results.length; + const response = { search_categories: { room_events: localResult, @@ -204,6 +211,11 @@ async function localPagination(searchResult) { }; const result = MatrixClientPeg.get()._processRoomEventsSearch(searchResult, response); + + // Restore our encryption info so we can properly re-verify the events. + const newSlice = result.results.slice(Math.max(result.results.length - newResultCount, 0)); + restoreEncryptionInfo(newSlice); + searchResult.pendingRequest = null; return result; @@ -505,12 +517,17 @@ async function combinedPagination(searchResult) { }, }; + const oldResultCount = searchResult.results.length; + // Let the client process the combined result. const result = client._processRoomEventsSearch(searchResult, response); - searchResult.pendingRequest = null; - // Restore our encryption info so we can properly re-verify the events. + const newResultCount = result.results.length - oldResultCount; + const newSlice = result.results.slice(Math.max(result.results.length - newResultCount, 0)); + restoreEncryptionInfo(newSlice); + + searchResult.pendingRequest = null; return result; }