Attempt to be more performant on PyPy (#3462)

pull/3467/head
Amber Brown 2018-06-28 14:49:57 +01:00 committed by GitHub
parent 72d2143ea8
commit 6350bf925e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 79 additions and 62 deletions

1
changelog.d/3462.feature Normal file
View File

@ -0,0 +1 @@
Synapse now uses the best performing JSON encoder/decoder according to your runtime (simplejson on CPython, stdlib json on PyPy).

View File

@ -17,7 +17,8 @@
import logging import logging
import simplejson as json from canonicaljson import json
from six import iteritems from six import iteritems
from six.moves import http_client from six.moves import http_client

View File

@ -17,7 +17,8 @@ from synapse.storage.presence import UserPresenceState
from synapse.types import UserID, RoomID from synapse.types import UserID, RoomID
from twisted.internet import defer from twisted.internet import defer
import simplejson as json from canonicaljson import json
import jsonschema import jsonschema
from jsonschema import FormatChecker from jsonschema import FormatChecker

View File

@ -18,7 +18,7 @@ from twisted.web.http import HTTPClient
from twisted.internet.protocol import Factory from twisted.internet.protocol import Factory
from twisted.internet import defer, reactor from twisted.internet import defer, reactor
from synapse.http.endpoint import matrix_federation_endpoint from synapse.http.endpoint import matrix_federation_endpoint
import simplejson as json from canonicaljson import json
import logging import logging

View File

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
import logging import logging
import simplejson as json from canonicaljson import json
from twisted.internet import defer from twisted.internet import defer
from synapse.api.errors import AuthError, FederationError, SynapseError, NotFoundError from synapse.api.errors import AuthError, FederationError, SynapseError, NotFoundError

View File

@ -16,6 +16,8 @@
from twisted.internet import defer, threads from twisted.internet import defer, threads
from canonicaljson import json
from ._base import BaseHandler from ._base import BaseHandler
from synapse.api.constants import LoginType from synapse.api.constants import LoginType
from synapse.api.errors import ( from synapse.api.errors import (
@ -32,7 +34,6 @@ from twisted.web.client import PartialDownloadError
import logging import logging
import bcrypt import bcrypt
import pymacaroons import pymacaroons
import simplejson
import attr import attr
import synapse.util.stringutils as stringutils import synapse.util.stringutils as stringutils
@ -403,7 +404,7 @@ class AuthHandler(BaseHandler):
except PartialDownloadError as pde: except PartialDownloadError as pde:
# Twisted is silly # Twisted is silly
data = pde.response data = pde.response
resp_body = simplejson.loads(data) resp_body = json.loads(data)
if 'success' in resp_body: if 'success' in resp_body:
# Note that we do NOT check the hostname here: we explicitly # Note that we do NOT check the hostname here: we explicitly

View File

@ -14,10 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import simplejson as json
import logging import logging
from canonicaljson import encode_canonical_json from canonicaljson import encode_canonical_json, json
from twisted.internet import defer from twisted.internet import defer
from six import iteritems from six import iteritems
@ -357,7 +356,7 @@ def _exception_to_failure(e):
# include ConnectionRefused and other errors # include ConnectionRefused and other errors
# #
# Note that some Exceptions (notably twisted's ResponseFailed etc) don't # Note that some Exceptions (notably twisted's ResponseFailed etc) don't
# give a string for e.message, which simplejson then fails to serialize. # give a string for e.message, which json then fails to serialize.
return { return {
"status": 503, "message": str(e.message), "status": 503, "message": str(e.message),
} }

View File

@ -19,7 +19,7 @@
import logging import logging
import simplejson as json from canonicaljson import json
from twisted.internet import defer from twisted.internet import defer

View File

@ -14,10 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import simplejson
import sys import sys
from canonicaljson import encode_canonical_json from canonicaljson import encode_canonical_json, json
import six import six
from six import string_types, itervalues, iteritems from six import string_types, itervalues, iteritems
from twisted.internet import defer from twisted.internet import defer
@ -797,7 +796,7 @@ class EventCreationHandler(object):
# Ensure that we can round trip before trying to persist in db # Ensure that we can round trip before trying to persist in db
try: try:
dump = frozendict_json_encoder.encode(event.content) dump = frozendict_json_encoder.encode(event.content)
simplejson.loads(dump) json.loads(dump)
except Exception: except Exception:
logger.exception("Failed to encode content: %r", event.content) logger.exception("Failed to encode content: %r", event.content)
raise raise

View File

@ -42,7 +42,7 @@ from twisted.web._newclient import ResponseDone
from six import StringIO from six import StringIO
from prometheus_client import Counter from prometheus_client import Counter
import simplejson as json from canonicaljson import json
import logging import logging
import urllib import urllib

View File

@ -27,7 +27,7 @@ from synapse.util import logcontext
from synapse.util.logcontext import make_deferred_yieldable from synapse.util.logcontext import make_deferred_yieldable
import synapse.util.retryutils import synapse.util.retryutils
from canonicaljson import encode_canonical_json from canonicaljson import encode_canonical_json, json
from synapse.api.errors import ( from synapse.api.errors import (
SynapseError, Codes, HttpResponseException, FederationDeniedError, SynapseError, Codes, HttpResponseException, FederationDeniedError,
@ -36,7 +36,6 @@ from synapse.api.errors import (
from signedjson.sign import sign_json from signedjson.sign import sign_json
import cgi import cgi
import simplejson as json
import logging import logging
import random import random
import sys import sys

View File

@ -29,7 +29,7 @@ import synapse.metrics
import synapse.events import synapse.events
from canonicaljson import ( from canonicaljson import (
encode_canonical_json, encode_pretty_printed_json encode_canonical_json, encode_pretty_printed_json, json
) )
from twisted.internet import defer from twisted.internet import defer
@ -41,7 +41,6 @@ from twisted.web.util import redirectTo
import collections import collections
import logging import logging
import urllib import urllib
import simplejson
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -410,7 +409,7 @@ def respond_with_json(request, code, json_object, send_cors=False,
if canonical_json or synapse.events.USE_FROZEN_DICTS: if canonical_json or synapse.events.USE_FROZEN_DICTS:
json_bytes = encode_canonical_json(json_object) json_bytes = encode_canonical_json(json_object)
else: else:
json_bytes = simplejson.dumps(json_object) json_bytes = json.dumps(json_object)
return respond_with_json_bytes( return respond_with_json_bytes(
request, code, json_bytes, request, code, json_bytes,

View File

@ -18,7 +18,9 @@
from synapse.api.errors import SynapseError, Codes from synapse.api.errors import SynapseError, Codes
import logging import logging
import simplejson
from canonicaljson import json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -171,7 +173,7 @@ def parse_json_value_from_request(request, allow_empty_body=False):
return None return None
try: try:
content = simplejson.loads(content_bytes) content = json.loads(content_bytes)
except Exception as e: except Exception as e:
logger.warn("Unable to parse JSON: %s", e) logger.warn("Unable to parse JSON: %s", e)
raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON)

View File

@ -147,7 +147,8 @@ class GCCounts(object):
yield cm yield cm
REGISTRY.register(GCCounts()) if not running_on_pypy:
REGISTRY.register(GCCounts())
# #
# Twisted reactor metrics # Twisted reactor metrics

View File

@ -19,13 +19,17 @@ allowed to be sent by which side.
""" """
import logging import logging
import simplejson import platform
if platform.python_implementation() == "PyPy":
import json
_json_encoder = json.JSONEncoder()
else:
import simplejson as json
_json_encoder = json.JSONEncoder(namedtuple_as_object=False)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_json_encoder = simplejson.JSONEncoder(namedtuple_as_object=False)
class Command(object): class Command(object):
"""The base command class. """The base command class.
@ -102,7 +106,7 @@ class RdataCommand(Command):
return cls( return cls(
stream_name, stream_name,
None if token == "batch" else int(token), None if token == "batch" else int(token),
simplejson.loads(row_json) json.loads(row_json)
) )
def to_line(self): def to_line(self):
@ -300,7 +304,7 @@ class InvalidateCacheCommand(Command):
def from_line(cls, line): def from_line(cls, line):
cache_func, keys_json = line.split(" ", 1) cache_func, keys_json = line.split(" ", 1)
return cls(cache_func, simplejson.loads(keys_json)) return cls(cache_func, json.loads(keys_json))
def to_line(self): def to_line(self):
return " ".join(( return " ".join((
@ -329,7 +333,7 @@ class UserIpCommand(Command):
def from_line(cls, line): def from_line(cls, line):
user_id, jsn = line.split(" ", 1) user_id, jsn = line.split(" ", 1)
access_token, ip, user_agent, device_id, last_seen = simplejson.loads(jsn) access_token, ip, user_agent, device_id, last_seen = json.loads(jsn)
return cls( return cls(
user_id, access_token, ip, user_agent, device_id, last_seen user_id, access_token, ip, user_agent, device_id, last_seen

View File

@ -23,7 +23,8 @@ from synapse.util.msisdn import phone_number_to_msisdn
from .base import ClientV1RestServlet, client_path_patterns from .base import ClientV1RestServlet, client_path_patterns
import simplejson as json from canonicaljson import json
import urllib import urllib
from six.moves.urllib import parse as urlparse from six.moves.urllib import parse as urlparse

View File

@ -31,7 +31,7 @@ from synapse.http.servlet import (
from six.moves.urllib import parse as urlparse from six.moves.urllib import parse as urlparse
import logging import logging
import simplejson as json from canonicaljson import json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -33,7 +33,7 @@ from ._base import set_timeline_upper_limit
import itertools import itertools
import logging import logging
import simplejson as json from canonicaljson import json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -22,8 +22,9 @@ from synapse.api.errors import (
from twisted.protocols.basic import FileSender from twisted.protocols.basic import FileSender
from twisted.web import server, resource from twisted.web import server, resource
from canonicaljson import json
import base64 import base64
import simplejson as json
import logging import logging
import os import os
import re import re

View File

@ -23,7 +23,8 @@ import re
import shutil import shutil
import sys import sys
import traceback import traceback
import simplejson as json
from canonicaljson import json
from six.moves import urllib_parse as urlparse from six.moves import urllib_parse as urlparse
from six import string_types from six import string_types

View File

@ -22,8 +22,9 @@ from synapse.storage.util.id_generators import StreamIdGenerator
from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.caches.stream_change_cache import StreamChangeCache
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
from canonicaljson import json
import abc import abc
import simplejson as json
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -15,8 +15,8 @@
# limitations under the License. # limitations under the License.
import logging import logging
import re import re
import simplejson as json
from twisted.internet import defer from twisted.internet import defer
from canonicaljson import json
from synapse.appservice import AppServiceTransaction from synapse.appservice import AppServiceTransaction
from synapse.config.appservice import load_appservices from synapse.config.appservice import load_appservices

View File

@ -18,7 +18,8 @@ from . import engines
from twisted.internet import defer from twisted.internet import defer
import simplejson as json from canonicaljson import json
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -14,7 +14,8 @@
# limitations under the License. # limitations under the License.
import logging import logging
import simplejson
from canonicaljson import json
from twisted.internet import defer from twisted.internet import defer
@ -85,7 +86,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
) )
rows = [] rows = []
for destination, edu in remote_messages_by_destination.items(): for destination, edu in remote_messages_by_destination.items():
edu_json = simplejson.dumps(edu) edu_json = json.dumps(edu)
rows.append((destination, stream_id, now_ms, edu_json)) rows.append((destination, stream_id, now_ms, edu_json))
txn.executemany(sql, rows) txn.executemany(sql, rows)
@ -177,7 +178,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
" WHERE user_id = ?" " WHERE user_id = ?"
) )
txn.execute(sql, (user_id,)) txn.execute(sql, (user_id,))
message_json = simplejson.dumps(messages_by_device["*"]) message_json = json.dumps(messages_by_device["*"])
for row in txn: for row in txn:
# Add the message for all devices for this user on this # Add the message for all devices for this user on this
# server. # server.
@ -199,7 +200,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
# Only insert into the local inbox if the device exists on # Only insert into the local inbox if the device exists on
# this server # this server
device = row[0] device = row[0]
message_json = simplejson.dumps(messages_by_device[device]) message_json = json.dumps(messages_by_device[device])
messages_json_for_user[device] = message_json messages_json_for_user[device] = message_json
if messages_json_for_user: if messages_json_for_user:
@ -253,7 +254,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
messages = [] messages = []
for row in txn: for row in txn:
stream_pos = row[0] stream_pos = row[0]
messages.append(simplejson.loads(row[1])) messages.append(json.loads(row[1]))
if len(messages) < limit: if len(messages) < limit:
stream_pos = current_stream_id stream_pos = current_stream_id
return (messages, stream_pos) return (messages, stream_pos)
@ -389,7 +390,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
messages = [] messages = []
for row in txn: for row in txn:
stream_pos = row[0] stream_pos = row[0]
messages.append(simplejson.loads(row[1])) messages.append(json.loads(row[1]))
if len(messages) < limit: if len(messages) < limit:
stream_pos = current_stream_id stream_pos = current_stream_id
return (messages, stream_pos) return (messages, stream_pos)

View File

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import simplejson as json
from twisted.internet import defer from twisted.internet import defer
@ -21,6 +20,8 @@ from synapse.api.errors import StoreError
from ._base import SQLBaseStore, Cache from ._base import SQLBaseStore, Cache
from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks
from canonicaljson import json
from six import itervalues, iteritems from six import itervalues, iteritems
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -16,8 +16,7 @@ from twisted.internet import defer
from synapse.util.caches.descriptors import cached from synapse.util.caches.descriptors import cached
from canonicaljson import encode_canonical_json from canonicaljson import encode_canonical_json, json
import simplejson as json
from ._base import SQLBaseStore from ._base import SQLBaseStore

View File

@ -19,7 +19,8 @@ from twisted.internet import defer
from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.util.caches.descriptors import cachedInlineCallbacks
import logging import logging
import simplejson as json
from canonicaljson import json
from six import iteritems from six import iteritems

View File

@ -19,7 +19,8 @@ from functools import wraps
import itertools import itertools
import logging import logging
import simplejson as json from canonicaljson import json
from twisted.internet import defer from twisted.internet import defer
from synapse.storage.events_worker import EventsWorkerStore from synapse.storage.events_worker import EventsWorkerStore

View File

@ -29,7 +29,8 @@ from synapse.api.errors import SynapseError
from collections import namedtuple from collections import namedtuple
import logging import logging
import simplejson as json
from canonicaljson import json
# these are only included to make the type annotations work # these are only included to make the type annotations work
from synapse.events import EventBase # noqa: F401 from synapse.events import EventBase # noqa: F401

View File

@ -19,8 +19,7 @@ from ._base import SQLBaseStore
from synapse.api.errors import SynapseError, Codes from synapse.api.errors import SynapseError, Codes
from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.util.caches.descriptors import cachedInlineCallbacks
from canonicaljson import encode_canonical_json from canonicaljson import encode_canonical_json, json
import simplejson as json
class FilteringStore(SQLBaseStore): class FilteringStore(SQLBaseStore):

View File

@ -20,7 +20,7 @@ from synapse.api.errors import SynapseError
from ._base import SQLBaseStore from ._base import SQLBaseStore
import simplejson as json from canonicaljson import json
# The category ID for the "default" category. We don't store as null in the # The category ID for the "default" category. We don't store as null in the

View File

@ -25,9 +25,10 @@ from synapse.push.baserules import list_with_base_rules
from synapse.api.constants import EventTypes from synapse.api.constants import EventTypes
from twisted.internet import defer from twisted.internet import defer
from canonicaljson import json
import abc import abc
import logging import logging
import simplejson as json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -17,12 +17,11 @@
from ._base import SQLBaseStore from ._base import SQLBaseStore
from twisted.internet import defer from twisted.internet import defer
from canonicaljson import encode_canonical_json from canonicaljson import encode_canonical_json, json
from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
import logging import logging
import simplejson as json
import types import types
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -21,9 +21,10 @@ from synapse.util.caches.stream_change_cache import StreamChangeCache
from twisted.internet import defer from twisted.internet import defer
from canonicaljson import json
import abc import abc
import logging import logging
import simplejson as json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -20,9 +20,10 @@ from synapse.storage._base import SQLBaseStore
from synapse.storage.search import SearchStore from synapse.storage.search import SearchStore
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
from canonicaljson import json
import collections import collections
import logging import logging
import simplejson as json
import re import re
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -28,7 +28,7 @@ from synapse.api.constants import Membership, EventTypes
from synapse.types import get_domain_from_id from synapse.types import get_domain_from_id
import logging import logging
import simplejson as json from canonicaljson import json
from six import itervalues, iteritems from six import itervalues, iteritems

View File

@ -16,7 +16,7 @@
from collections import namedtuple from collections import namedtuple
import logging import logging
import re import re
import simplejson as json from canonicaljson import json
from six import string_types from six import string_types

View File

@ -19,7 +19,8 @@ from synapse.storage.account_data import AccountDataWorkerStore
from synapse.util.caches.descriptors import cached from synapse.util.caches.descriptors import cached
from twisted.internet import defer from twisted.internet import defer
import simplejson as json from canonicaljson import json
import logging import logging
from six.moves import range from six.moves import range

View File

@ -19,12 +19,11 @@ from synapse.util.caches.descriptors import cached
from twisted.internet import defer from twisted.internet import defer
import six import six
from canonicaljson import encode_canonical_json from canonicaljson import encode_canonical_json, json
from collections import namedtuple from collections import namedtuple
import logging import logging
import simplejson as json
# py2 sqlite has buffer hardcoded as only binary type, so we must use it, # py2 sqlite has buffer hardcoded as only binary type, so we must use it,
# despite being deprecated and removed in favor of memoryview # despite being deprecated and removed in favor of memoryview

View File

@ -14,7 +14,7 @@
# limitations under the License. # limitations under the License.
from frozendict import frozendict from frozendict import frozendict
import simplejson as json from canonicaljson import json
from six import string_types from six import string_types