From 7c4ce957c76a187a5ef8fd79646ce77457137138 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 3 Sep 2014 17:37:08 +0100 Subject: [PATCH 1/9] Unindent list in specification to remove blockquote --- docs/specification.rst | 66 ++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/docs/specification.rst b/docs/specification.rst index e1c83bed78..c5b6fe29c8 100644 --- a/docs/specification.rst +++ b/docs/specification.rst @@ -30,48 +30,52 @@ ecosystem to communicate with one another. The principles that Matrix attempts to follow are: - - Pragmatic Web-friendly APIs (i.e. JSON over REST) - - Keep It Simple & Stupid +- Pragmatic Web-friendly APIs (i.e. JSON over REST) +- Keep It Simple & Stupid - + provide a simple architecture with minimal third-party dependencies. + + provide a simple architecture with minimal third-party dependencies. - - Fully open: +- Fully open: - + Fully open federation - anyone should be able to participate in the global Matrix network - + Fully open standard - publicly documented standard with no IP or patent licensing encumbrances - + Fully open source reference implementation - liberally-licensed example implementations with no - IP or patent licensing encumbrances + + Fully open federation - anyone should be able to participate in the global + Matrix network + + Fully open standard - publicly documented standard with no IP or patent + licensing encumbrances + + Fully open source reference implementation - liberally-licensed example + implementations with no IP or patent licensing encumbrances - - Empowering the end-user +- Empowering the end-user - + The user should be able to choose the server and clients they use - + The user should be control how private their communication is - + The user should know precisely where their data is stored + + The user should be able to choose the server and clients they use + + The user should be control how private their communication is + + The user should know precisely where their data is stored - - Fully decentralised - no single points of control over conversations or the network as a whole - - Learning from history to avoid repeating it +- Fully decentralised - no single points of control over conversations or the + network as a whole +- Learning from history to avoid repeating it - + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP whilst trying to avoid their failings + + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP + whilst trying to avoid their failings The functionality that Matrix provides includes: - - Creation and management of fully distributed chat rooms with no - single points of control or failure - - Eventually-consistent cryptographically secure synchronisation of room - state across a global open network of federated servers and services - - Sending and receiving extensible messages in a room with (optional) - end-to-end encryption - - Extensible user management (inviting, joining, leaving, kicking, banning) - mediated by a power-level based user privilege system. - - Extensible room state management (room naming, aliasing, topics, bans) - - Extensible user profile management (avatars, displaynames, etc) - - Managing user accounts (registration, login, logout) - - Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, - Facebook accounts to authenticate, identify and discover users on Matrix. - - Trusted federation of Identity servers for: +- Creation and management of fully distributed chat rooms with no + single points of control or failure +- Eventually-consistent cryptographically secure synchronisation of room + state across a global open network of federated servers and services +- Sending and receiving extensible messages in a room with (optional) + end-to-end encryption +- Extensible user management (inviting, joining, leaving, kicking, banning) + mediated by a power-level based user privilege system. +- Extensible room state management (room naming, aliasing, topics, bans) +- Extensible user profile management (avatars, displaynames, etc) +- Managing user accounts (registration, login, logout) +- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, + Facebook accounts to authenticate, identify and discover users on Matrix. +- Trusted federation of Identity servers for: - + Publishing user public keys for PKI - + Mapping of 3PIDs to Matrix IDs + + Publishing user public keys for PKI + + Mapping of 3PIDs to Matrix IDs The end goal of Matrix is to be a ubiquitous messaging layer for synchronising arbitrary data between sets of people, devices and services - be that for instant From 1b491e50c9ea7bd389d558c89b30540b386dcea5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Sep 2014 17:46:37 +0100 Subject: [PATCH 2/9] Implement a kick api --- synapse/rest/room.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 37b6b8cbc6..308b447090 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -388,7 +388,7 @@ class RoomMembershipRestServlet(RestServlet): def register(self, http_server): # /rooms/$roomid/[invite|join|leave] PATTERN = ("/rooms/(?P[^/]*)/" + - "(?Pjoin|invite|leave|ban)") + "(?Pjoin|invite|leave|ban|kick)") register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks @@ -399,11 +399,14 @@ class RoomMembershipRestServlet(RestServlet): # target user is you unless it is an invite state_key = user.to_string() - if membership_action in ["invite", "ban"]: + if membership_action in ["invite", "ban", "kick"]: if "user_id" not in content: raise SynapseError(400, "Missing user_id key.") state_key = content["user_id"] + if membership_action == "kick": + membership_action = "leave" + event = self.event_factory.create_event( etype=RoomMemberEvent.TYPE, content={"membership": unicode(membership_action)}, From 79bf9d25db8758a0368c0de568918cc72074965c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 3 Sep 2014 17:48:01 +0100 Subject: [PATCH 3/9] Added more terms. --- docs/specification.rst | 72 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/docs/specification.rst b/docs/specification.rst index c5b6fe29c8..d331f36b4b 100644 --- a/docs/specification.rst +++ b/docs/specification.rst @@ -1809,12 +1809,76 @@ Glossary .. NOTE:: This section is a work in progress. -.. TODO - - domain specific words/acronyms with definitions +Backfilling: + The process of synchronising historic state from one home server to another, + to backfill the event storage so that scrollback can be presented to the + client(s). Not to be confused with pagination. + +Context: + A single human-level entity of interest (currently, a chat room) + +EDU (Ephemeral Data Unit): + A message that relates directly to a given pair of home servers that are + exchanging it. EDUs are short-lived messages that related only to one single + pair of servers; they are not persisted for a long time and are not forwarded + on to other servers. Because of this, they have no internal ID nor previous + EDUs reference chain. + +Event: + A record of activity that records a single thing that happened on to a context + (currently, a chat room). These are the "chat messages" that Synapse makes + available. + +PDU (Persistent Data Unit): + A message that relates to a single context, irrespective of the server that + is communicating it. PDUs either encode a single Event, or a single State + change. A PDU is referred to by its PDU ID; the pair of its origin server + and local reference from that server. + +PDU ID: + The pair of PDU Origin and PDU Reference, that together globally uniquely + refers to a specific PDU. + +PDU Origin: + The name of the origin server that generated a given PDU. This may not be the + server from which it has been received, due to the way they are copied around + from server to server. The origin always records the original server that + created it. + +PDU Reference: + A local ID used to refer to a specific PDU from a given origin server. These + references are opaque at the protocol level, but may optionally have some + structured meaning within a given origin server or implementation. + +Presence: + The concept of whether a user is currently online, how available they declare + they are, and so on. See also: doc/model/presence + +Profile: + A set of metadata about a user, such as a display name, provided for the + benefit of other users. See also: doc/model/profiles + +Room ID: + An opaque string (of as-yet undecided format) that identifies a particular + room and used in PDUs referring to it. + +Room Alias: + A human-readable string of the form #name:some.domain that users can use as a + pointer to identify a room; a Directory Server will map this to its Room ID + +State: + A set of metadata maintained about a Context, which is replicated among the + servers in addition to the history of Events. User ID: - An opaque ID which identifies an end-user, which consists of some opaque - localpart combined with the domain name of their home server. + A string of the form @localpart:domain.name that identifies a user for + wire-protocol purposes. The localpart is meaningless outside of a particular + home server. This takes a human-readable form that end-users can use directly + if they so wish, avoiding the 3PIDs. + +Transaction: + A message which relates to the communication between a given pair of servers. + A transaction contains possibly-empty lists of PDUs and EDUs. .. Links through the external API docs are below From 1a32b1f002bcc436070bacd931e6abbd40a793d1 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Wed, 3 Sep 2014 18:55:27 +0200 Subject: [PATCH 4/9] Normalise users power levels so that the user with the higher power level will have a bar covering 100% of the width of his avatar --- webclient/room/room-controller.js | 23 +++++++++++++++++++++++ webclient/room/room.html | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 0ffbe8761f..52c57856ee 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -253,6 +253,29 @@ angular.module('RoomController', ['ngSanitize', 'mFileInput']) var member = $scope.members[user_id]; if (member) { member.powerLevel = matrixService.getUserPowerLevel($scope.room_id, user_id); + + normaliseMembersPowerLevels(); + } + } + + // Normalise users power levels so that the user with the higher power level + // will have a bar covering 100% of the width of his avatar + var normaliseMembersPowerLevels = function() { + // Find the max power level + var maxPowerLevel = 0; + for (var i in $scope.members) { + var member = $scope.members[i]; + if (member.powerLevel) { + maxPowerLevel = Math.max(maxPowerLevel, member.powerLevel); + } + } + + // Normalized them on a 0..100% scale to be use in css width + if (maxPowerLevel) { + for (var i in $scope.members) { + var member = $scope.members[i]; + member.powerLevelNorm = (member.powerLevel * 100) / maxPowerLevel; + } } } diff --git a/webclient/room/room.html b/webclient/room/room.html index 05e87387a5..e29f511ecf 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -24,7 +24,7 @@ title="{{ member.id }}" width="80" height="80"/> -
+
{{ member.displayname || member.id.substr(0, member.id.indexOf(':')) }}
{{ member.displayname ? "" : member.id.substr(member.id.indexOf(':')) }}
From f68dbbd3dad4fd589197d3a3f55136eef2b43acd Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 3 Sep 2014 17:59:54 +0100 Subject: [PATCH 5/9] More explanation of federation keys. --- docs/specification.rst | 132 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 8 deletions(-) diff --git a/docs/specification.rst b/docs/specification.rst index d331f36b4b..44eddcaf87 100644 --- a/docs/specification.rst +++ b/docs/specification.rst @@ -1510,6 +1510,31 @@ Each transaction has: - An origin and destination server name. - A list of "previous IDs". - A list of PDUs and EDUs - the actual message payload that the Transaction carries. + +``origin`` + Type: + String + Description: + DNS name of homeserver making this transaction. + +``ts`` + Type: + Integer + Description: + Timestamp in milliseconds on originating homeserver when this transaction + started. + +``previous_ids`` + Type: + List of strings + Description: + List of transactions that were sent immediately prior to this transaction. + +``pdus`` + Type: + List of Objects. + Description: + List of updates contained in this transaction. :: @@ -1551,8 +1576,98 @@ All PDUs have: - A list of other PDU IDs that have been seen recently on that context (regardless of which origin sent them) -[[TODO(paul): Update this structure so that 'pdu_id' is a two-element -[origin,ref] pair like the prev_pdus are]] +``context`` + Type: + String + Description: + Event context identifier + +``origin`` + Type: + String + Description: + DNS name of homeserver that created this PDU. + +``pdu_id`` + Type: + String + Description: + Unique identifier for PDU within the context for the originating homeserver + +``ts`` + Type: + Integer + Description: + Timestamp in milliseconds on originating homeserver when this PDU was created. + +``pdu_type`` + Type: + String + Description: + PDU event type. + +``prev_pdus`` + Type: + List of pairs of strings + Description: + The originating homeserver and PDU ids of the most recent PDUs the + homeserver was aware of for this context when it made this PDU. + +``depth`` + Type: + Integer + Description: + The maximum depth of the previous PDUs plus one. + + +.. TODO paul + [[TODO(paul): Update this structure so that 'pdu_id' is a two-element + [origin,ref] pair like the prev_pdus are]] + + +For state updates: + +``is_state`` + Type: + Boolean + Description: + True if this PDU is updating state. + +``state_key`` + Type: + String + Description: + Optional key identifying the updated state within the context. + +``power_level`` + Type: + Integer + Description: + The asserted power level of the user performing the update. + +``min_update`` + Type: + Integer + Description: + The required power level needed to replace this update. + +``prev_state_id`` + Type: + String + Description: + PDU event type. + +``prev_state_origin`` + Type: + String + Description: + The PDU id of the update this replaces. + +``user`` + Type: + String + Description: + The user updating the state. :: @@ -1593,12 +1708,13 @@ keys exist to support this: "prev_state_id":TODO "prev_state_origin":TODO} -[[TODO(paul): At this point we should probably have a long description of how -State management works, with descriptions of clobbering rules, power levels, etc -etc... But some of that detail is rather up-in-the-air, on the whiteboard, and -so on. This part needs refining. And writing in its own document as the details -relate to the server/system as a whole, not specifically to server-server -federation.]] +.. TODO paul + [[TODO(paul): At this point we should probably have a long description of how + State management works, with descriptions of clobbering rules, power levels, etc + etc... But some of that detail is rather up-in-the-air, on the whiteboard, and + so on. This part needs refining. And writing in its own document as the details + relate to the server/system as a whole, not specifically to server-server + federation.]] EDUs, by comparison to PDUs, do not have an ID, a context, or a list of "previous" IDs. The only mandatory fields for these are the type, origin and From 5391ccdfe67fa21493f5402830e7e494a9bc4787 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 3 Sep 2014 18:02:33 +0100 Subject: [PATCH 6/9] Marked docs/client-server/specification.rst as old. --- .../{specification.rst => OLD_specification.rst} | 14 ++++++++++++++ 1 file changed, 14 insertions(+) rename docs/client-server/{specification.rst => OLD_specification.rst} (99%) diff --git a/docs/client-server/specification.rst b/docs/client-server/OLD_specification.rst similarity index 99% rename from docs/client-server/specification.rst rename to docs/client-server/OLD_specification.rst index 2f6645ceb9..47fba5eeac 100644 --- a/docs/client-server/specification.rst +++ b/docs/client-server/OLD_specification.rst @@ -2,6 +2,20 @@ Matrix Client-Server API ======================== + +.. WARNING:: + This specification is old. Please see /docs/specification.rst instead. + + + + + + + + + + + The following specification outlines how a client can send and receive data from a home server. From 0d75b9fa96250cb6ee0759f7d23806dde0c50302 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Sep 2014 18:02:51 +0100 Subject: [PATCH 7/9] Fix a few cases where we used user_id instead of state_key --- webclient/app-filter.js | 10 +++++----- webclient/components/matrix/event-handler-service.js | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/webclient/app-filter.js b/webclient/app-filter.js index 31d4ac4421..27f435674f 100644 --- a/webclient/app-filter.js +++ b/webclient/app-filter.js @@ -107,18 +107,18 @@ angular.module('matrixWebClient') if (2 === Object.keys(room.members).length) { for (var i in room.members) { var member = room.members[i]; - if (member.user_id !== matrixService.config().user_id) { + if (member.state_key !== matrixService.config().user_id) { - if (member.user_id in $rootScope.presence) { + if (member.state_key in $rootScope.presence) { // If the user is available in presence, use the displayname there // as it is the most uptodate - roomName = $rootScope.presence[member.user_id].content.displayname; + roomName = $rootScope.presence[member.state_key].content.displayname; } else if (member.content.displayname) { roomName = member.content.displayname; } else { - roomName = member.user_id; + roomName = member.state_key; } } } @@ -145,7 +145,7 @@ angular.module('matrixWebClient') roomName = $rootScope.presence[userID].content.displayname; } else { - roomName = member.user_id; + roomName = userID; } } } diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index a2fbad796d..ee478d2eb0 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -97,7 +97,7 @@ angular.module('eventHandlerService', []) } } - $rootScope.events.rooms[event.room_id].members[event.user_id] = event; + $rootScope.events.rooms[event.room_id].members[event.state_key] = event; $rootScope.$broadcast(MEMBER_EVENT, event, isLiveEvent); }; From f196d77f6602fc61fa6df3790d217aa2c3b9c5f3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 3 Sep 2014 18:07:36 +0100 Subject: [PATCH 8/9] Added federation protocol urls section from other docs. --- docs/specification.rst | 73 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/docs/specification.rst b/docs/specification.rst index 44eddcaf87..239e51b4f3 100644 --- a/docs/specification.rst +++ b/docs/specification.rst @@ -1726,6 +1726,79 @@ destination home server names, and the actual nested content. "origin":"blue", "destination":"orange", "content":...} + + +Protocol URLs +============= +.. WARNING:: + This section may be misleading or inaccurate. + +All these URLs are namespaced within a prefix of:: + + /_matrix/federation/v1/... + +For active pushing of messages representing live activity "as it happens":: + + PUT .../send/:transaction_id/ + Body: JSON encoding of a single Transaction + Response: TODO + +The transaction_id path argument will override any ID given in the JSON body. +The destination name will be set to that of the receiving server itself. Each +embedded PDU in the transaction body will be processed. + + +To fetch a particular PDU:: + + GET .../pdu/:origin/:pdu_id/ + Response: JSON encoding of a single Transaction containing one PDU + +Retrieves a given PDU from the server. The response will contain a single new +Transaction, inside which will be the requested PDU. + + +To fetch all the state of a given context:: + + GET .../state/:context/ + Response: JSON encoding of a single Transaction containing multiple PDUs + +Retrieves a snapshot of the entire current state of the given context. The +response will contain a single Transaction, inside which will be a list of +PDUs that encode the state. + +To backfill events on a given context:: + + GET .../backfill/:context/ + Query args: v, limit + Response: JSON encoding of a single Transaction containing multiple PDUs + +Retrieves a sliding-window history of previous PDUs that occurred on the +given context. Starting from the PDU ID(s) given in the "v" argument, the +PDUs that preceeded it are retrieved, up to a total number given by the +"limit" argument. These are then returned in a new Transaction containing all +off the PDUs. + + +To stream events all the events:: + + GET .../pull/ + Query args: origin, v + Response: JSON encoding of a single Transaction consisting of multiple PDUs + +Retrieves all of the transactions later than any version given by the "v" +arguments. + + +To make a query:: + + GET .../query/:query_type + Query args: as specified by the individual query types + Response: JSON encoding of a response object + +Performs a single query request on the receiving home server. The Query Type +part of the path specifies the kind of query being made, and its query +arguments have a meaning specific to that kind of query. The response is a +JSON-encoded object whose meaning also depends on the kind of query. Backfilling ----------- From d703e712f7b6f6741010360617d88dfb067b349d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 3 Sep 2014 18:18:35 +0100 Subject: [PATCH 9/9] Add support to _simple_insert() to do INSERT OR REPLACE --- synapse/storage/_base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index aaa09f47d0..bae50e7d1f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -79,19 +79,21 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values): + def _simple_insert(self, table, values, or_replace=False): """Executes an INSERT query on the named table. Args: table : string giving the table name values : dict of new column names and values for them + or_replace : bool; if True performs an INSERT OR REPLACE """ return self._db_pool.runInteraction( - self._simple_insert_txn, table, values, + self._simple_insert_txn, table, values, or_replace=or_replace ) - def _simple_insert_txn(self, txn, table, values): - sql = "INSERT INTO %s (%s) VALUES(%s)" % ( + def _simple_insert_txn(self, txn, table, values, or_replace=False): + sql = "%s INTO %s (%s) VALUES(%s)" % ( + ("INSERT OR REPLACE" if or_replace else "INSERT"), table, ", ".join(k for k in values), ", ".join("?" for k in values)