# Copyright 2015, 2016 OpenMarket Ltd # Copyright 2022 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import urllib.parse from typing import ( TYPE_CHECKING, Any, Awaitable, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, TypeVar, ) from prometheus_client import Counter from typing_extensions import Concatenate, ParamSpec, TypeGuard from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind from synapse.api.errors import CodeMessageException, HttpResponseException from synapse.appservice import ( ApplicationService, TransactionOneTimeKeysCount, TransactionUnusedFallbackKeys, ) from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig, serialize_event from synapse.http.client import SimpleHttpClient, is_unknown_endpoint from synapse.types import DeviceListUpdates, JsonDict, ThirdPartyInstanceID from synapse.util.caches.response_cache import ResponseCache if TYPE_CHECKING: from synapse.server import HomeServer logger = logging.getLogger(__name__) sent_transactions_counter = Counter( "synapse_appservice_api_sent_transactions", "Number of /transactions/ requests sent", ["service"], ) failed_transactions_counter = Counter( "synapse_appservice_api_failed_transactions", "Number of /transactions/ requests that failed to send", ["service"], ) sent_events_counter = Counter( "synapse_appservice_api_sent_events", "Number of events sent to the AS", ["service"] ) sent_ephemeral_counter = Counter( "synapse_appservice_api_sent_ephemeral", "Number of ephemeral events sent to the AS", ["service"], ) sent_todevice_counter = Counter( "synapse_appservice_api_sent_todevice", "Number of todevice messages sent to the AS", ["service"], ) HOUR_IN_MS = 60 * 60 * 1000 APP_SERVICE_PREFIX = "/_matrix/app/v1" APP_SERVICE_UNSTABLE_PREFIX = "/_matrix/app/unstable" P = ParamSpec("P") R = TypeVar("R") def _is_valid_3pe_metadata(info: JsonDict) -> bool: if "instances" not in info: return False if not isinstance(info["instances"], list): return False return True def _is_valid_3pe_result(r: object, field: str) -> TypeGuard[JsonDict]: if not isinstance(r, dict): return False for k in (field, "protocol"): if k not in r: return False if not isinstance(r[k], str): return False if "fields" not in r: return False fields = r["fields"] if not isinstance(fields, dict): return False return True class ApplicationServiceApi(SimpleHttpClient): """This class manages HS -> AS communications, including querying and pushing. """ def __init__(self, hs: "HomeServer"): super().__init__(hs) self.clock = hs.get_clock() self.protocol_meta_cache: ResponseCache[Tuple[str, str]] = ResponseCache( hs.get_clock(), "as_protocol_meta", timeout_ms=HOUR_IN_MS ) async def _send_with_fallbacks( self, service: "ApplicationService", prefixes: List[str], path: str, func: Callable[Concatenate[str, P], Awaitable[R]], *args: P.args, **kwargs: P.kwargs, ) -> R: """ Attempt to call an application service with multiple paths, falling back until one succeeds. Args: service: The appliacation service, this provides the base URL. prefixes: A last of paths to try in order for the requests. path: A suffix to append to each prefix. func: The function to call, the first argument will be the full endpoint to fetch. Other arguments are provided by args/kwargs. Returns: The return value of func. """ for i, prefix in enumerate(prefixes, start=1): uri = f"{service.url}{prefix}{path}" try: return await func(uri, *args, **kwargs) except HttpResponseException as e: # If an error is received that is due to an unrecognised path, # fallback to next path (if one exists). Otherwise, consider it # a legitimate error and raise. if i < len(prefixes) and is_unknown_endpoint(e): continue raise except Exception: # Unexpected exceptions get sent to the caller. raise # The function should always exit via the return or raise above this. raise RuntimeError("Unexpected fallback behaviour. This should never be seen.") async def query_user(self, service: "ApplicationService", user_id: str) -> bool: if service.url is None: return False # This is required by the configuration. assert service.hs_token is not None try: response = await self._send_with_fallbacks( service, [APP_SERVICE_PREFIX, ""], f"/users/{urllib.parse.quote(user_id)}", self.get_json, {"access_token": service.hs_token}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) if response is not None: # just an empty json object return True except CodeMessageException as e: if e.code == 404: return False logger.warning("query_user to %s received %s", service.url, e.code) except Exception as ex: logger.warning("query_user to %s threw exception %s", service.url, ex) return False async def query_alias(self, service: "ApplicationService", alias: str) -> bool: if service.url is None: return False # This is required by the configuration. assert service.hs_token is not None try: response = await self._send_with_fallbacks( service, [APP_SERVICE_PREFIX, ""], f"/rooms/{urllib.parse.quote(alias)}", self.get_json, {"access_token": service.hs_token}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) if response is not None: # just an empty json object return True except CodeMessageException as e: logger.warning("query_alias to %s received %s", service.url, e.code) if e.code == 404: return False except Exception as ex: logger.warning("query_alias to %s threw exception %s", service.url, ex) return False async def query_3pe( self, service: "ApplicationService", kind: str, protocol: str, fields: Dict[bytes, List[bytes]], ) -> List[JsonDict]: if kind == ThirdPartyEntityKind.USER: required_field = "userid" elif kind == ThirdPartyEntityKind.LOCATION: required_field = "alias" else: raise ValueError("Unrecognised 'kind' argument %r to query_3pe()", kind) if service.url is None: return [] # This is required by the configuration. assert service.hs_token is not None try: args: Mapping[Any, Any] = { **fields, b"access_token": service.hs_token, } response = await self._send_with_fallbacks( service, [APP_SERVICE_PREFIX, APP_SERVICE_UNSTABLE_PREFIX], f"/thirdparty/{kind}/{urllib.parse.quote(protocol)}", self.get_json, args=args, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) if not isinstance(response, list): logger.warning( "query_3pe to %s returned an invalid response %r", service.url, response, ) return [] ret = [] for r in response: if _is_valid_3pe_result(r, field=required_field): ret.append(r) else: logger.warning( "query_3pe to %s returned an invalid result %r", service.url, r ) return ret except Exception as ex: logger.warning("query_3pe to %s threw exception %s", service.url, ex) return [] async def get_3pe_protocol( self, service: "ApplicationService", protocol: str ) -> Optional[JsonDict]: if service.url is None: return {} async def _get() -> Optional[JsonDict]: # This is required by the configuration. assert service.hs_token is not None try: info = await self._send_with_fallbacks( service, [APP_SERVICE_PREFIX, APP_SERVICE_UNSTABLE_PREFIX], f"/thirdparty/protocol/{urllib.parse.quote(protocol)}", self.get_json, {"access_token": service.hs_token}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) if not _is_valid_3pe_metadata(info): logger.warning( "query_3pe_protocol to %s did not return a valid result", service.url, ) return None for instance in info.get("instances", []): network_id = instance.get("network_id", None) if network_id is not None: instance["instance_id"] = ThirdPartyInstanceID( service.id, network_id ).to_string() return info except Exception as ex: logger.warning( "query_3pe_protocol to %s threw exception %s", service.url, ex ) return None key = (service.id, protocol) return await self.protocol_meta_cache.wrap(key, _get) async def ping(self, service: "ApplicationService", txn_id: Optional[str]) -> None: # The caller should check that url is set assert service.url is not None, "ping called without URL being set" # This is required by the configuration. assert service.hs_token is not None await self.post_json_get_json( uri=f"{service.url}{APP_SERVICE_UNSTABLE_PREFIX}/fi.mau.msc2659/ping", post_json={"transaction_id": txn_id}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) async def push_bulk( self, service: "ApplicationService", events: Sequence[EventBase], ephemeral: List[JsonDict], to_device_messages: List[JsonDict], one_time_keys_count: TransactionOneTimeKeysCount, unused_fallback_keys: TransactionUnusedFallbackKeys, device_list_summary: DeviceListUpdates, txn_id: Optional[int] = None, ) -> bool: """ Push data to an application service. Args: service: The application service to send to. events: The persistent events to send. ephemeral: The ephemeral events to send. to_device_messages: The to-device messages to send. txn_id: An unique ID to assign to this transaction. Application services should deduplicate transactions received with identitical IDs. Returns: True if the task succeeded, False if it failed. """ if service.url is None: return True # This is required by the configuration. assert service.hs_token is not None serialized_events = self._serialize(service, events) if txn_id is None: logger.warning( "push_bulk: Missing txn ID sending events to %s", service.url ) txn_id = 0 # Never send ephemeral events to appservices that do not support it body: JsonDict = {"events": serialized_events} if service.supports_ephemeral: body.update( { # TODO: Update to stable prefixes once MSC2409 completes FCP merge. "de.sorunome.msc2409.ephemeral": ephemeral, "de.sorunome.msc2409.to_device": to_device_messages, } ) # TODO: Update to stable prefixes once MSC3202 completes FCP merge if service.msc3202_transaction_extensions: if one_time_keys_count: body[ "org.matrix.msc3202.device_one_time_key_counts" ] = one_time_keys_count body[ "org.matrix.msc3202.device_one_time_keys_count" ] = one_time_keys_count if unused_fallback_keys: body[ "org.matrix.msc3202.device_unused_fallback_key_types" ] = unused_fallback_keys if device_list_summary: body["org.matrix.msc3202.device_lists"] = { "changed": list(device_list_summary.changed), "left": list(device_list_summary.left), } try: await self._send_with_fallbacks( service, [APP_SERVICE_PREFIX, ""], f"/transactions/{urllib.parse.quote(str(txn_id))}", self.put_json, json_body=body, args={"access_token": service.hs_token}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) if logger.isEnabledFor(logging.DEBUG): logger.debug( "push_bulk to %s succeeded! events=%s", service.url, [event.get("event_id") for event in events], ) sent_transactions_counter.labels(service.id).inc() sent_events_counter.labels(service.id).inc(len(serialized_events)) sent_ephemeral_counter.labels(service.id).inc(len(ephemeral)) sent_todevice_counter.labels(service.id).inc(len(to_device_messages)) return True except CodeMessageException as e: logger.warning( "push_bulk to %s received code=%s msg=%s", service.url, e.code, e.msg, exc_info=logger.isEnabledFor(logging.DEBUG), ) except Exception as ex: logger.warning( "push_bulk to %s threw exception(%s) %s args=%s", service.url, type(ex).__name__, ex, ex.args, exc_info=logger.isEnabledFor(logging.DEBUG), ) failed_transactions_counter.labels(service.id).inc() return False async def claim_client_keys( self, service: "ApplicationService", query: List[Tuple[str, str, str, int]] ) -> Tuple[ Dict[str, Dict[str, Dict[str, JsonDict]]], List[Tuple[str, str, str, int]] ]: """Claim one time keys from an application service. Note that any error (including a timeout) is treated as the application service having no information. Args: service: The application service to query. query: An iterable of tuples of (user ID, device ID, algorithm). Returns: A tuple of: A map of user ID -> a map device ID -> a map of key ID -> JSON dict. A copy of the input which has not been fulfilled because the appservice doesn't support this endpoint or has not returned data for that tuple. """ if service.url is None: return {}, query # This is required by the configuration. assert service.hs_token is not None # Create the expected payload shape. body: Dict[str, Dict[str, List[str]]] = {} for user_id, device, algorithm, count in query: body.setdefault(user_id, {}).setdefault(device, []).extend( [algorithm] * count ) uri = f"{service.url}/_matrix/app/unstable/org.matrix.msc3983/keys/claim" try: response = await self.post_json_get_json( uri, body, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) except HttpResponseException as e: # The appservice doesn't support this endpoint. if is_unknown_endpoint(e): return {}, query logger.warning("claim_keys to %s received %s", uri, e.code) return {}, query except Exception as ex: logger.warning("claim_keys to %s threw exception %s", uri, ex) return {}, query # Check if the appservice fulfilled all of the queried user/device/algorithms # or if some are still missing. # # TODO This places a lot of faith in the response shape being correct. missing = [] for user_id, device, algorithm, count in query: # Count the number of keys in the response for this algorithm by # checking which key IDs start with the algorithm. This uses that # True == 1 in Python to generate a count. response_count = sum( key_id.startswith(f"{algorithm}:") for key_id in response.get(user_id, {}).get(device, {}) ) count -= response_count # If the appservice responds with fewer keys than requested, then # consider the request unfulfilled. if count > 0: missing.append((user_id, device, algorithm, count)) return response, missing async def query_keys( self, service: "ApplicationService", query: Dict[str, List[str]] ) -> Dict[str, Dict[str, Dict[str, JsonDict]]]: """Query the application service for keys. Note that any error (including a timeout) is treated as the application service having no information. Args: service: The application service to query. query: An iterable of tuples of (user ID, device ID, algorithm). Returns: A map of device_keys/master_keys/self_signing_keys/user_signing_keys: device_keys is a map of user ID -> a map device ID -> device info. """ if service.url is None: return {} # This is required by the configuration. assert service.hs_token is not None uri = f"{service.url}/_matrix/app/unstable/org.matrix.msc3984/keys/query" try: response = await self.post_json_get_json( uri, query, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) except HttpResponseException as e: # The appservice doesn't support this endpoint. if is_unknown_endpoint(e): return {} logger.warning("query_keys to %s received %s", uri, e.code) return {} except Exception as ex: logger.warning("query_keys to %s threw exception %s", uri, ex) return {} return response def _serialize( self, service: "ApplicationService", events: Iterable[EventBase] ) -> List[JsonDict]: time_now = self.clock.time_msec() return [ serialize_event( e, time_now, config=SerializeEventConfig( as_client_event=True, # If this is an invite or a knock membership event, and we're interested # in this user, then include any stripped state alongside the event. include_stripped_room_state=( e.type == EventTypes.Member and ( e.membership == Membership.INVITE or e.membership == Membership.KNOCK ) and service.is_interested_in_user(e.state_key) ), ), ) for e in events ]