",),
40 | ]
41 | )
42 | def test_invalid_email_returns_400(self, address: str) -> None:
43 | self.sydent.run()
44 |
45 | with patch("sydent.http.servlets.store_invite_servlet.authV2") as authV2:
46 | authV2.return_value = Account(self.sender, 0, None)
47 |
48 | request, channel = make_request(
49 | self.sydent.reactor,
50 | self.sydent.clientApiHttpServer.factory,
51 | "POST",
52 | "/_matrix/identity/v2/store-invite",
53 | content={
54 | "address": address,
55 | "medium": "email",
56 | "room_id": "!myroom:test",
57 | "sender": self.sender,
58 | },
59 | )
60 |
61 | self.assertEqual(channel.code, 400)
62 | self.assertEqual(channel.json_body["errcode"], "M_INVALID_EMAIL")
63 |
--------------------------------------------------------------------------------
/sydent/http/servlets/bulklookupservlet.py:
--------------------------------------------------------------------------------
1 | # Copyright 2017 OpenMarket Ltd
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import logging
16 | from typing import TYPE_CHECKING
17 |
18 | from twisted.web.server import Request
19 |
20 | from sydent.db.threepid_associations import GlobalAssociationStore
21 | from sydent.http.servlets import (
22 | MatrixRestError,
23 | SydentResource,
24 | get_args,
25 | jsonwrap,
26 | send_cors,
27 | )
28 | from sydent.types import JsonDict
29 |
30 | if TYPE_CHECKING:
31 | from sydent.sydent import Sydent
32 |
33 | logger = logging.getLogger(__name__)
34 |
35 |
36 | class BulkLookupServlet(SydentResource):
37 | isLeaf = True
38 |
39 | def __init__(self, syd: "Sydent") -> None:
40 | super().__init__()
41 | self.sydent = syd
42 |
43 | @jsonwrap
44 | def render_POST(self, request: Request) -> JsonDict:
45 | """
46 | Bulk-lookup for threepids.
47 | Params: 'threepids': list of threepids, each of which is a list of medium, address
48 | Returns: Object with key 'threepids', which is a list of results where each result
49 | is a 3 item list of medium, address, mxid
50 | Note that results are not streamed to the client.
51 | Threepids for which no mapping is found are omitted.
52 | """
53 | send_cors(request)
54 |
55 | args = get_args(request, ("threepids",))
56 |
57 | threepids = args["threepids"]
58 | if not isinstance(threepids, list):
59 | raise MatrixRestError(400, "M_INVALID_PARAM", "threepids must be a list")
60 |
61 | logger.info("Bulk lookup of %d threepids", len(threepids))
62 |
63 | globalAssocStore = GlobalAssociationStore(self.sydent)
64 | results = globalAssocStore.getMxids(threepids)
65 |
66 | return {"threepids": results}
67 |
68 | def render_OPTIONS(self, request: Request) -> bytes:
69 | send_cors(request)
70 | return b""
71 |
--------------------------------------------------------------------------------
/sydent/http/servlets/pubkeyservlets.py:
--------------------------------------------------------------------------------
1 | # Copyright 2014 OpenMarket Ltd
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | from typing import TYPE_CHECKING
16 |
17 | from twisted.web.server import Request
18 | from unpaddedbase64 import encode_base64
19 |
20 | from sydent.db.invite_tokens import JoinTokenStore
21 | from sydent.http.servlets import SydentResource, get_args, jsonwrap
22 | from sydent.types import JsonDict
23 |
24 | if TYPE_CHECKING:
25 | from sydent.sydent import Sydent
26 |
27 |
28 | class Ed25519Servlet(SydentResource):
29 | isLeaf = True
30 |
31 | def __init__(self, syd: "Sydent") -> None:
32 | super().__init__()
33 | self.sydent = syd
34 |
35 | @jsonwrap
36 | def render_GET(self, request: Request) -> JsonDict:
37 | pubKey = self.sydent.keyring.ed25519.verify_key
38 | pubKeyBase64 = encode_base64(pubKey.encode())
39 |
40 | return {"public_key": pubKeyBase64}
41 |
42 |
43 | class PubkeyIsValidServlet(SydentResource):
44 | isLeaf = True
45 |
46 | def __init__(self, syd: "Sydent") -> None:
47 | super().__init__()
48 | self.sydent = syd
49 |
50 | @jsonwrap
51 | def render_GET(self, request: Request) -> JsonDict:
52 | args = get_args(request, ("public_key",))
53 |
54 | pubKey = self.sydent.keyring.ed25519.verify_key
55 | pubKeyBase64 = encode_base64(pubKey.encode())
56 |
57 | return {"valid": args["public_key"] == pubKeyBase64}
58 |
59 |
60 | class EphemeralPubkeyIsValidServlet(SydentResource):
61 | isLeaf = True
62 |
63 | def __init__(self, syd: "Sydent") -> None:
64 | super().__init__()
65 | self.joinTokenStore = JoinTokenStore(syd)
66 |
67 | @jsonwrap
68 | def render_GET(self, request: Request) -> JsonDict:
69 | args = get_args(request, ("public_key",))
70 | publicKey = args["public_key"]
71 |
72 | return {
73 | "valid": self.joinTokenStore.validateEphemeralPublicKey(publicKey),
74 | }
75 |
--------------------------------------------------------------------------------
/stubs/twisted/web/http.pyi:
--------------------------------------------------------------------------------
1 | import typing
2 | from typing import AnyStr, Dict, List, Optional
3 |
4 | from twisted.internet import protocol
5 | from twisted.internet.defer import Deferred
6 | from twisted.internet.interfaces import IAddress, ITCPTransport
7 | from twisted.logger import Logger
8 | from twisted.web.http_headers import Headers
9 | from twisted.web.iweb import IRequest
10 | from zope.interface import implementer
11 |
12 | class HTTPFactory(protocol.ServerFactory): ...
13 | class HTTPChannel: ...
14 |
15 | # Type ignore: I don't want to respecify the methods on the interface that we
16 | # don't use.
17 | @implementer(IRequest) # type: ignore[misc]
18 | class Request:
19 | # Instance attributes mentioned in the docstring
20 | method: bytes
21 | uri: bytes
22 | path: bytes
23 | args: Dict[bytes, List[bytes]]
24 | content: typing.BinaryIO
25 | cookies: List[bytes]
26 | requestHeaders: Headers
27 | responseHeaders: Headers
28 | notifications: List[Deferred[None]]
29 | _disconnected: bool
30 | _log: Logger
31 |
32 | # Other instance attributes set in __init__
33 | channel: HTTPChannel
34 | client: IAddress
35 | # This was hard to derive.
36 | # - `transport` is `self.channel.transport`
37 | # - `self.channel` is set in the constructor, and looks like it's always
38 | # an `HTTPChannel`.
39 | # - `HTTPChannel` is a `LineReceiver` is a `Protocol` is a `BaseProtocol`.
40 | # - `BaseProtocol` sets `self.transport` to initially `None`.
41 | #
42 | # Note that `transport` is set to an ITransport in makeConnection,
43 | # so is almost certainly not None by the time it reaches our code.
44 | #
45 | # I've narrowed this to ITCPTransport because
46 | # - we use `self.transport.abortConnection`, which belongs to that interface
47 | # - twisted does too! in its implementation of HTTPChannel.forceAbortClient
48 | transport: Optional[ITCPTransport]
49 | def __init__(self, channel: HTTPChannel): ...
50 | def getHeader(self, key: AnyStr) -> Optional[AnyStr]: ...
51 | def handleContentChunk(self, data: bytes) -> None: ...
52 | def setResponseCode(self, code: int, message: Optional[bytes] = ...) -> None: ...
53 | def setHeader(self, k: AnyStr, v: AnyStr) -> None: ...
54 | def write(self, data: bytes) -> None: ...
55 | def finish(self) -> None: ...
56 | def getClientAddress(self) -> IAddress: ...
57 |
58 | class PotentialDataLoss(Exception): ...
59 |
60 | CACHED: object
61 |
62 | def stringToDatetime(dateString: bytes) -> int: ...
63 |
--------------------------------------------------------------------------------
/sydent/validators/common.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from typing import TYPE_CHECKING, Dict
3 |
4 | from sydent.db.valsession import ThreePidValSessionStore
5 | from sydent.util import time_msec
6 | from sydent.validators import (
7 | THREEPID_SESSION_VALIDATION_TIMEOUT_MS,
8 | IncorrectClientSecretException,
9 | IncorrectSessionTokenException,
10 | InvalidSessionIdException,
11 | SessionExpiredException,
12 | )
13 |
14 | if TYPE_CHECKING:
15 | from sydent.sydent import Sydent
16 |
17 | logger = logging.getLogger(__name__)
18 |
19 |
20 | def validateSessionWithToken(
21 | sydent: "Sydent", sid: int, clientSecret: str, token: str
22 | ) -> Dict[str, bool]:
23 | """
24 | Attempt to validate a session, identified by the sid, using
25 | the token from out-of-band. The client secret is given to
26 | prevent attempts to guess the token for a sid.
27 |
28 | :param sid: The ID of the session to validate.
29 | :param clientSecret: The client secret to validate.
30 | :param token: The token to validate.
31 |
32 | :return: A dict with a "success" key which is True if the session
33 | was successfully validated, False otherwise.
34 |
35 | :raise IncorrectClientSecretException: The provided client_secret is incorrect.
36 | :raise SessionExpiredException: The session has expired.
37 | :raise InvalidSessionIdException: The session ID couldn't be matched with an
38 | existing session.
39 | :raise IncorrectSessionTokenException: The provided token is incorrect
40 | """
41 | valSessionStore = ThreePidValSessionStore(sydent)
42 | result = valSessionStore.getTokenSessionById(sid)
43 | if not result:
44 | logger.info("Session ID %s not found", sid)
45 | raise InvalidSessionIdException()
46 |
47 | session, token_info = result
48 |
49 | if not clientSecret == session.client_secret:
50 | logger.info("Incorrect client secret", sid)
51 | raise IncorrectClientSecretException()
52 |
53 | if session.mtime + THREEPID_SESSION_VALIDATION_TIMEOUT_MS < time_msec():
54 | logger.info("Session expired")
55 | raise SessionExpiredException()
56 |
57 | # TODO once we can validate the token oob
58 | # if tokenObj.validated and clientSecret == tokenObj.clientSecret:
59 | # return True
60 |
61 | if token_info.token == token:
62 | logger.info("Setting session %s as validated", session.id)
63 | valSessionStore.setValidated(session.id, True)
64 |
65 | return {"success": True}
66 | else:
67 | logger.info("Incorrect token submitted")
68 | raise IncorrectSessionTokenException()
69 |
--------------------------------------------------------------------------------
/tests/test_auth.py:
--------------------------------------------------------------------------------
1 | # Copyright 2020 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | from twisted.trial import unittest
16 |
17 | from sydent.http.auth import tokenFromRequest
18 | from tests.utils import make_request, make_sydent
19 |
20 |
21 | class AuthTestCase(unittest.TestCase):
22 | """Tests Sydent's auth code"""
23 |
24 | def setUp(self):
25 | # Create a new sydent
26 | self.sydent = make_sydent()
27 | self.test_token = "testingtoken"
28 |
29 | # Inject a fake OpenID token into the database
30 | cur = self.sydent.db.cursor()
31 | cur.execute(
32 | "INSERT INTO accounts (user_id, created_ts, consent_version)"
33 | "VALUES (?, ?, ?)",
34 | ("@bob:localhost", 101010101, "asd"),
35 | )
36 | cur.execute(
37 | "INSERT INTO tokens (user_id, token)" "VALUES (?, ?)",
38 | ("@bob:localhost", self.test_token),
39 | )
40 |
41 | self.sydent.db.commit()
42 |
43 | def test_can_read_token_from_headers(self):
44 | """Tests that Sydent correctly extracts an auth token from request headers"""
45 | self.sydent.run()
46 |
47 | request, _ = make_request(
48 | self.sydent.reactor,
49 | self.sydent.clientApiHttpServer.factory,
50 | "GET",
51 | "/_matrix/identity/v2/hash_details",
52 | )
53 | request.requestHeaders.addRawHeader(
54 | b"Authorization", b"Bearer " + self.test_token.encode("ascii")
55 | )
56 |
57 | token = tokenFromRequest(request)
58 |
59 | self.assertEqual(token, self.test_token)
60 |
61 | def test_can_read_token_from_query_parameters(self):
62 | """Tests that Sydent correctly extracts an auth token from query parameters"""
63 | self.sydent.run()
64 |
65 | request, _ = make_request(
66 | self.sydent.reactor,
67 | self.sydent.clientApiHttpServer.factory,
68 | "GET",
69 | "/_matrix/identity/v2/hash_details?access_token=" + self.test_token,
70 | )
71 |
72 | token = tokenFromRequest(request)
73 |
74 | self.assertEqual(token, self.test_token)
75 |
--------------------------------------------------------------------------------
/tests/test_threepidunbind.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | from http import HTTPStatus
15 | from unittest.mock import patch
16 |
17 | import twisted.internet.error
18 | import twisted.web.client
19 | from parameterized import parameterized
20 | from twisted.trial import unittest
21 |
22 | from tests.utils import make_request, make_sydent
23 |
24 |
25 | class ThreepidUnbindTestCase(unittest.TestCase):
26 | """Tests Sydent's threepidunbind servlet"""
27 |
28 | def setUp(self) -> None:
29 | # Create a new sydent
30 | self.sydent = make_sydent()
31 |
32 | # Duplicated from TestRegisterServelet. Is there a way for us to keep
33 | # ourselves DRY?
34 | @parameterized.expand(
35 | [
36 | (twisted.internet.error.DNSLookupError(),),
37 | (twisted.internet.error.TimeoutError(),),
38 | (twisted.internet.error.ConnectionRefusedError(),),
39 | # Naughty: strictly we're supposed to initialise a ResponseNeverReceived
40 | # with a list of 1 or more failures.
41 | (twisted.web.client.ResponseNeverReceived([]),),
42 | ]
43 | )
44 | def test_connection_failure(self, exc: Exception) -> None:
45 | """Check we respond sensibly if we can't contact the homeserver."""
46 | self.sydent.run()
47 | with patch.object(
48 | self.sydent.sig_verifier, "authenticate_request", side_effect=exc
49 | ):
50 | request, channel = make_request(
51 | self.sydent.reactor,
52 | self.sydent.clientApiHttpServer.factory,
53 | "POST",
54 | "/_matrix/identity/v2/3pid/unbind",
55 | content={
56 | "mxid": "@alice:wonderland",
57 | "threepid": {
58 | "address": "alice.cooper@wonderland.biz",
59 | "medium": "email",
60 | },
61 | },
62 | )
63 | self.assertEqual(channel.code, HTTPStatus.INTERNAL_SERVER_ERROR)
64 | self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN")
65 | self.assertIn("contact", channel.json_body["error"])
66 |
--------------------------------------------------------------------------------
/sydent/config/http.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | from configparser import ConfigParser
16 | from typing import Optional
17 |
18 | from sydent.config._base import BaseConfig
19 |
20 |
21 | class HTTPConfig(BaseConfig):
22 | def parse_config(self, cfg: "ConfigParser") -> bool:
23 | """
24 | Parse the http section of the config
25 |
26 | :param cfg: the configuration to be parsed
27 | """
28 | # This option is deprecated
29 | self.verify_response_template = cfg.get(
30 | "http", "verify_response_template", fallback=None
31 | )
32 |
33 | self.client_bind_address = cfg.get("http", "clientapi.http.bind_address")
34 | self.client_port = cfg.getint("http", "clientapi.http.port")
35 |
36 | # internal port is allowed to be set to an empty string in the config
37 | internal_api_port = cfg.get("http", "internalapi.http.port")
38 | self.internal_bind_address = cfg.get(
39 | "http", "internalapi.http.bind_address", fallback="::1"
40 | )
41 | self.internal_port: Optional[int] = None
42 | if internal_api_port != "":
43 | self.internal_port = int(internal_api_port)
44 |
45 | self.cert_file = cfg.get("http", "replication.https.certfile")
46 | self.ca_cert_file = cfg.get("http", "replication.https.cacert")
47 |
48 | self.replication_bind_address = cfg.get(
49 | "http", "replication.https.bind_address"
50 | )
51 | self.replication_port = cfg.getint("http", "replication.https.port")
52 |
53 | self.obey_x_forwarded_for = cfg.getboolean("http", "obey_x_forwarded_for")
54 |
55 | self.verify_federation_certs = cfg.getboolean("http", "federation.verifycerts")
56 |
57 | self.server_http_url_base = cfg.get("http", "client_http_base")
58 |
59 | self.base_replication_urls = {}
60 |
61 | for section in cfg.sections():
62 | if section.startswith("peer."):
63 | # peer name is all the characters after 'peer.'
64 | peer = section[5:]
65 | if cfg.has_option(section, "base_replication_url"):
66 | base_url = cfg.get(section, "base_replication_url")
67 | self.base_replication_urls[peer] = base_url
68 |
69 | return False
70 |
--------------------------------------------------------------------------------
/sydent/http/servlets/blindlysignstuffservlet.py:
--------------------------------------------------------------------------------
1 | # Copyright 2016 OpenMarket Ltd
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import logging
16 | from typing import TYPE_CHECKING
17 |
18 | import signedjson.key
19 | import signedjson.sign
20 | from twisted.web.server import Request
21 |
22 | from sydent.db.invite_tokens import JoinTokenStore
23 | from sydent.http.auth import authV2
24 | from sydent.http.servlets import (
25 | MatrixRestError,
26 | SydentResource,
27 | get_args,
28 | jsonwrap,
29 | send_cors,
30 | )
31 | from sydent.types import JsonDict
32 |
33 | if TYPE_CHECKING:
34 | from sydent.sydent import Sydent
35 |
36 | logger = logging.getLogger(__name__)
37 |
38 |
39 | class BlindlySignStuffServlet(SydentResource):
40 | isLeaf = True
41 |
42 | def __init__(self, syd: "Sydent", require_auth: bool = False) -> None:
43 | super().__init__()
44 | self.sydent = syd
45 | self.server_name = syd.config.general.server_name
46 | self.tokenStore = JoinTokenStore(syd)
47 | self.require_auth = require_auth
48 |
49 | @jsonwrap
50 | def render_POST(self, request: Request) -> JsonDict:
51 | send_cors(request)
52 |
53 | if self.require_auth:
54 | authV2(self.sydent, request)
55 |
56 | args = get_args(request, ("private_key", "token", "mxid"))
57 |
58 | private_key_base64 = args["private_key"]
59 | token = args["token"]
60 | mxid = args["mxid"]
61 |
62 | sender = self.tokenStore.getSenderForToken(token)
63 | if sender is None:
64 | raise MatrixRestError(404, "M_UNRECOGNIZED", "Didn't recognize token")
65 |
66 | to_sign = {
67 | "mxid": mxid,
68 | "sender": sender,
69 | "token": token,
70 | }
71 | try:
72 | private_key = signedjson.key.decode_signing_key_base64(
73 | "ed25519", "0", private_key_base64
74 | )
75 | signed: JsonDict = signedjson.sign.sign_json(
76 | to_sign, self.server_name, private_key
77 | )
78 | except Exception:
79 | logger.exception("signing failed")
80 | raise MatrixRestError(500, "M_UNKNOWN", "Internal Server Error")
81 |
82 | return signed
83 |
84 | def render_OPTIONS(self, request: Request) -> bytes:
85 | send_cors(request)
86 | return b""
87 |
--------------------------------------------------------------------------------
/sydent/config/crypto.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | from configparser import ConfigParser
16 |
17 | import nacl.encoding
18 | import nacl.signing
19 | import signedjson.key
20 | import signedjson.types
21 |
22 | from sydent.config._base import BaseConfig
23 |
24 |
25 | class CryptoConfig(BaseConfig):
26 | def parse_config(self, cfg: "ConfigParser") -> bool:
27 | """
28 | Parse the crypto section of the config
29 | :param cfg: the configuration to be parsed
30 | """
31 |
32 | signing_key_str = cfg.get("crypto", "ed25519.signingkey")
33 | signing_key_parts = signing_key_str.split(" ")
34 |
35 | save_key = False
36 |
37 | # N.B. `signedjson` expects `nacl.signing.SigningKey` instances which
38 | # have been monkeypatched to include new `alg` and `version` attributes.
39 | # This is captured by the `signedjson.types.SigningKey` protocol.
40 | self.signing_key: signedjson.types.SigningKey
41 |
42 | if signing_key_str == "":
43 | print(
44 | "INFO: This server does not yet have an ed25519 signing key. "
45 | "Creating one and saving it in the config file."
46 | )
47 |
48 | self.signing_key = signedjson.key.generate_signing_key("0")
49 |
50 | save_key = True
51 | elif len(signing_key_parts) == 1:
52 | # old format key
53 | print("INFO: Updating signing key format: brace yourselves")
54 |
55 | self.signing_key = nacl.signing.SigningKey(
56 | signing_key_str.encode("ascii"), encoder=nacl.encoding.HexEncoder
57 | )
58 | self.signing_key.version = "0"
59 | self.signing_key.alg = signedjson.key.NACL_ED25519
60 |
61 | save_key = True
62 | else:
63 | self.signing_key = signedjson.key.decode_signing_key_base64(
64 | signing_key_parts[0], signing_key_parts[1], signing_key_parts[2]
65 | )
66 |
67 | if save_key:
68 | signing_key_str = "%s %s %s" % (
69 | self.signing_key.alg,
70 | self.signing_key.version,
71 | signedjson.key.encode_signing_key_base64(self.signing_key),
72 | )
73 | cfg.set("crypto", "ed25519.signingkey", signing_key_str)
74 | return True
75 | else:
76 | return False
77 |
--------------------------------------------------------------------------------
/sydent/http/servlets/termsservlet.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import logging
16 | from typing import TYPE_CHECKING
17 |
18 | from twisted.web.server import Request
19 |
20 | from sydent.db.accounts import AccountStore
21 | from sydent.db.terms import TermsStore
22 | from sydent.http.auth import authV2
23 | from sydent.http.servlets import (
24 | MatrixRestError,
25 | SydentResource,
26 | get_args,
27 | jsonwrap,
28 | send_cors,
29 | )
30 | from sydent.terms.terms import get_terms
31 | from sydent.types import JsonDict
32 |
33 | if TYPE_CHECKING:
34 | from sydent.sydent import Sydent
35 |
36 | logger = logging.getLogger(__name__)
37 |
38 |
39 | class TermsServlet(SydentResource):
40 | isLeaf = True
41 |
42 | def __init__(self, syd: "Sydent") -> None:
43 | super().__init__()
44 | self.sydent = syd
45 |
46 | @jsonwrap
47 | def render_GET(self, request: Request) -> JsonDict:
48 | """
49 | Get the terms that must be agreed to in order to use this service
50 | Returns: Object describing the terms that require agreement
51 | """
52 | send_cors(request)
53 |
54 | terms = get_terms(self.sydent)
55 |
56 | return terms.getForClient()
57 |
58 | @jsonwrap
59 | def render_POST(self, request: Request) -> JsonDict:
60 | """
61 | Mark a set of terms and conditions as having been agreed to
62 | """
63 | send_cors(request)
64 |
65 | account = authV2(self.sydent, request, False)
66 |
67 | args = get_args(request, ("user_accepts",))
68 |
69 | user_accepts = args["user_accepts"]
70 |
71 | terms = get_terms(self.sydent)
72 | unknown_urls = list(set(user_accepts) - terms.getUrlSet())
73 | if len(unknown_urls) > 0:
74 | raise MatrixRestError(
75 | 400, "M_UNKNOWN", "Unrecognised URLs: %s" % (", ".join(unknown_urls),)
76 | )
77 |
78 | termsStore = TermsStore(self.sydent)
79 | termsStore.addAgreedUrls(account.userId, user_accepts)
80 |
81 | all_accepted_urls = termsStore.getAgreedUrls(account.userId)
82 |
83 | if terms.urlListIsSufficient(all_accepted_urls):
84 | accountStore = AccountStore(self.sydent)
85 | accountStore.setConsentVersion(account.userId, terms.getMasterVersion())
86 |
87 | return {}
88 |
89 | def render_OPTIONS(self, request: Request) -> bytes:
90 | send_cors(request)
91 | return b""
92 |
--------------------------------------------------------------------------------
/res/matrix-org/verification_template.eml:
--------------------------------------------------------------------------------
1 | Date: %(date)s
2 | From: %(from)s
3 | To: %(to)s
4 | Message-ID: %(messageid)s
5 | Subject: Confirm your email address for Matrix
6 | MIME-Version: 1.0
7 | Content-Type: multipart/alternative;
8 | boundary="%(multipart_boundary)s"
9 |
10 | --%(multipart_boundary)s
11 | Content-Type: text/plain; charset=UTF-8
12 | Content-Disposition: inline
13 |
14 | Hello,
15 |
16 | We have received a request to use this email address with a matrix.org identity
17 | server. If this was you who made this request, you may use the following link
18 | to complete the verification of your email address:
19 |
20 | %(link)s
21 |
22 | If your client requires a code, the code is %(token)s
23 |
24 | If you aren't aware of making such a request, please disregard this email.
25 |
26 |
27 | About Matrix:
28 |
29 | Matrix is an open standard for interoperable, decentralised, real-time communication
30 | over IP. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet
31 | of Things communication - or anywhere you need a standard HTTP API for publishing and
32 | subscribing to data whilst tracking the conversation history.
33 |
34 | Matrix defines the standard, and provides open source reference implementations of
35 | Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you
36 | create new communication solutions or extend the capabilities and reach of existing ones.
37 |
38 | --%(multipart_boundary)s
39 | Content-Type: text/html; charset=UTF-8
40 | Content-Disposition: inline
41 |
42 |
43 |
44 |
45 |
46 |
47 |
54 |
55 |
56 | Hello,
57 |
58 | We have received a request to use this email address with a matrix.org
59 | identity server. If this was you who made this request, you may use the
60 | following link to complete the verification of your email address:
61 |
62 | Complete email verification
63 |
64 | ...or copy this link into your web browser:
65 |
66 | %(link)s
67 |
68 | If your client requires a code, the code is %(token)s
69 |
70 | If you aren't aware of making such a request, please disregard this
71 | email.
72 |
73 |
74 | About Matrix:
75 |
76 | Matrix is an open standard for interoperable, decentralised, real-time communication
77 | over IP. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet
78 | of Things communication - or anywhere you need a standard HTTP API for publishing and
79 | subscribing to data whilst tracking the conversation history.
80 |
81 | Matrix defines the standard, and provides open source reference implementations of
82 | Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you
83 | create new communication solutions or extend the capabilities and reach of existing ones.
84 |
85 |
86 |
87 |
88 | --%(multipart_boundary)s--
89 |
--------------------------------------------------------------------------------
/res/matrix-org/verification_template.eml.j2:
--------------------------------------------------------------------------------
1 | Date: {{ date|safe }}
2 | From: {{ from|safe }}
3 | To: {{ to|safe }}
4 | Message-ID: {{ messageid|safe }}
5 | Subject: Confirm your email address for Matrix
6 | MIME-Version: 1.0
7 | Content-Type: multipart/alternative;
8 | boundary="{{ multipart_boundary|safe }}"
9 |
10 | --{{ multipart_boundary|safe }}
11 | Content-Type: text/plain; charset=UTF-8
12 | Content-Disposition: inline
13 |
14 | Hello,
15 |
16 | We have received a request to use this email address with a matrix.org identity
17 | server. If this was you who made this request, you may use the following link
18 | to complete the verification of your email address:
19 |
20 | {{ link|safe }}
21 |
22 | If your client requires a code, the code is {{ token|safe }}
23 |
24 | If you aren't aware of making such a request, please disregard this email.
25 |
26 |
27 | About Matrix:
28 |
29 | Matrix is an open standard for interoperable, decentralised, real-time communication
30 | over IP. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet
31 | of Things communication - or anywhere you need a standard HTTP API for publishing and
32 | subscribing to data whilst tracking the conversation history.
33 |
34 | Matrix defines the standard, and provides open source reference implementations of
35 | Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you
36 | create new communication solutions or extend the capabilities and reach of existing ones.
37 |
38 | --{{ multipart_boundary|safe }}
39 | Content-Type: text/html; charset=UTF-8
40 | Content-Disposition: inline
41 |
42 |
43 |
44 |
45 |
46 |
47 |
54 |
55 |
56 | Hello,
57 |
58 | We have received a request to use this email address with a matrix.org
59 | identity server. If this was you who made this request, you may use the
60 | following link to complete the verification of your email address:
61 |
62 | Complete email verification
63 |
64 | ...or copy this link into your web browser:
65 |
66 | {{ link }}
67 |
68 | If your client requires a code, the code is {{ token }}
69 |
70 | If you aren't aware of making such a request, please disregard this
71 | email.
72 |
73 |
74 | About Matrix:
75 |
76 | Matrix is an open standard for interoperable, decentralised, real-time communication
77 | over IP. It can be used to power Instant Messaging, VoIP/WebRTC signalling, Internet
78 | of Things communication - or anywhere you need a standard HTTP API for publishing and
79 | subscribing to data whilst tracking the conversation history.
80 |
81 | Matrix defines the standard, and provides open source reference implementations of
82 | Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you
83 | create new communication solutions or extend the capabilities and reach of existing ones.
84 |
85 |
86 |
87 |
88 | --{{ multipart_boundary|safe }}--
89 |
--------------------------------------------------------------------------------
/sydent/http/auth.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import logging
16 | from typing import TYPE_CHECKING, Optional
17 |
18 | from twisted.web.server import Request
19 |
20 | from sydent.db.accounts import AccountStore
21 | from sydent.http.servlets import MatrixRestError, get_args
22 | from sydent.terms.terms import get_terms
23 |
24 | if TYPE_CHECKING:
25 | from sydent.sydent import Sydent
26 | from sydent.users.accounts import Account
27 |
28 | logger = logging.getLogger(__name__)
29 |
30 |
31 | def tokenFromRequest(request: Request) -> Optional[str]:
32 | """Extract token from header of query parameter.
33 |
34 | :param request: The request to look for an access token in.
35 |
36 | :return: The token or None if not found
37 | """
38 | token = None
39 | # check for Authorization header first
40 | authHeader = request.getHeader("Authorization")
41 | if authHeader is not None and authHeader.startswith("Bearer "):
42 | token = authHeader[len("Bearer ") :]
43 |
44 | # no? try access_token query param
45 | if token is None:
46 | args = get_args(request, ("access_token",), required=False)
47 | token = args.get("access_token")
48 |
49 | return token
50 |
51 |
52 | def authV2(
53 | sydent: "Sydent",
54 | request: Request,
55 | requireTermsAgreed: bool = True,
56 | ) -> "Account":
57 | """For v2 APIs check that the request has a valid access token associated with it
58 |
59 | :param sydent: The Sydent instance to use.
60 | :param request: The request to look for an access token in.
61 | :param requireTermsAgreed: Whether to deny authentication if the user hasn't accepted
62 | the terms of service.
63 |
64 | :returns Account: The account object if there is correct auth
65 | :raises MatrixRestError: If the request is v2 but could not be authed or the user has
66 | not accepted terms.
67 | """
68 | token = tokenFromRequest(request)
69 |
70 | if token is None:
71 | raise MatrixRestError(401, "M_UNAUTHORIZED", "Unauthorized")
72 |
73 | accountStore = AccountStore(sydent)
74 |
75 | account = accountStore.getAccountByToken(token)
76 | if account is None:
77 | raise MatrixRestError(401, "M_UNAUTHORIZED", "Unauthorized")
78 |
79 | if requireTermsAgreed:
80 | terms = get_terms(sydent)
81 | if (
82 | terms.getMasterVersion() is not None
83 | and account.consentVersion != terms.getMasterVersion()
84 | ):
85 | raise MatrixRestError(403, "M_TERMS_NOT_SIGNED", "Terms not signed")
86 |
87 | return account
88 |
--------------------------------------------------------------------------------
/tests/test_ratelimiter.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 |
16 | from twisted.test.proto_helpers import MemoryReactorClock
17 | from twisted.trial import unittest
18 |
19 | from sydent.util.ratelimiter import LimitExceededException, Ratelimiter
20 |
21 |
22 | class RatelimiterTest(unittest.TestCase):
23 | def setUp(self) -> None:
24 | self.clock = MemoryReactorClock()
25 | self.ratelimiter = Ratelimiter(self.clock, burst=5, rate_hz=0.5)
26 |
27 | def test_simple(self) -> None:
28 | """Test that a request doesn't get ratelimited to start off with"""
29 | key = "key"
30 |
31 | # This should not raise as we're below the ratelimit
32 | self.ratelimiter.ratelimit(key)
33 |
34 | def test_burst(self) -> None:
35 | """Test that we can send `burst` number of messages before getting
36 | ratelimited
37 | """
38 | key = "key"
39 |
40 | # This should not raise as we're below the ratelimit
41 | for _ in range(5):
42 | self.ratelimiter.ratelimit(key)
43 |
44 | with self.assertRaises(LimitExceededException):
45 | self.ratelimiter.ratelimit(key)
46 |
47 | def test_burst_reset(self) -> None:
48 | """Test that once we hit the ratelimit we can wait a while and we'll be
49 | able to send requests again
50 | """
51 | key = "key"
52 |
53 | # This should not raise as we're below the ratelimit
54 | for _ in range(5):
55 | self.ratelimiter.ratelimit(key)
56 |
57 | with self.assertRaises(LimitExceededException):
58 | self.ratelimiter.ratelimit(key)
59 |
60 | self.clock.pump([2.0] * 5)
61 |
62 | for _ in range(5):
63 | self.ratelimiter.ratelimit(key)
64 |
65 | with self.assertRaises(LimitExceededException):
66 | self.ratelimiter.ratelimit(key)
67 |
68 | def test_average_rate(self):
69 | """Test that sending requests at a rate higher than the maximum rate
70 | gets ratelimited.
71 | """
72 | key = "key"
73 |
74 | with self.assertRaises(LimitExceededException):
75 | for _ in range(100):
76 | self.clock.advance(1)
77 | self.ratelimiter.ratelimit(key)
78 |
79 | def test_average_rate_burst(self):
80 | """Test that if we go above the maximum rate we'll get ratelimited"""
81 | key = "key"
82 |
83 | for _ in range(5):
84 | self.ratelimiter.ratelimit(key)
85 |
86 | for _ in range(100):
87 | self.clock.advance(2)
88 | self.ratelimiter.ratelimit(key)
89 |
90 | with self.assertRaises(LimitExceededException):
91 | self.ratelimiter.ratelimit(key)
92 |
--------------------------------------------------------------------------------
/sydent/util/ratelimiter.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 The Matrix.org Foundation C.I.C.
2 | # Licensed under the Apache License, Version 2.0 (the "License");
3 | # you may not use this file except in compliance with the License.
4 | # You may obtain a copy of the License at
5 | #
6 | # http://www.apache.org/licenses/LICENSE-2.0
7 | #
8 | # Unless required by applicable law or agreed to in writing, software
9 | # distributed under the License is distributed on an "AS IS" BASIS,
10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | # See the License for the specific language governing permissions and
12 | # limitations under the License.
13 |
14 | import logging
15 | from http import HTTPStatus
16 | from typing import Dict, Generic, Optional, TypeVar
17 |
18 | from twisted.internet import task
19 | from twisted.internet.interfaces import IReactorTime
20 |
21 | from sydent.http.servlets import MatrixRestError
22 |
23 | logger = logging.getLogger(__name__)
24 |
25 | K = TypeVar("K")
26 |
27 |
28 | class LimitExceededException(MatrixRestError):
29 | def __init__(self, error: Optional[str] = None) -> None:
30 | if error is None:
31 | error = "Too many requests"
32 |
33 | super().__init__(HTTPStatus.TOO_MANY_REQUESTS, "M_UNKNOWN", error)
34 |
35 |
36 | class Ratelimiter(Generic[K]):
37 | """A ratelimiter based on leaky token bucket algorithm.
38 |
39 | Args:
40 | reactor
41 | burst: the number of requests that can happen at once before we start
42 | ratelimiting
43 | rate_hz: The maximum average sustained rate in hertz of requests we'll
44 | accept.
45 | """
46 |
47 | def __init__(self, reactor: IReactorTime, burst: int, rate_hz: float) -> None:
48 | # The "burst" count (or the capacity of each bucket in leaky bucket
49 | # algorithm).
50 | self._burst = burst
51 |
52 | # A map from key to number of tokens in its bucket. We ratelimit when
53 | # the number of tokens is greater than `burst`.
54 | #
55 | # Entries are removed when token count hits zero.
56 | self._buckets: Dict[K, int] = {}
57 |
58 | # We remove tokens from all buckets at `rate_hz` hertz.
59 | call = task.LoopingCall(self._periodic_call)
60 | call.clock = reactor
61 | call.start(1 / rate_hz)
62 |
63 | def _periodic_call(self) -> None:
64 | # Take one away from all active buckets. If a bucket reaches zero then
65 | # remove it from the dict.
66 | self._buckets = {
67 | key: tokens - 1 for key, tokens in self._buckets.items() if tokens > 1
68 | }
69 |
70 | def ratelimit(self, key: K, error: Optional[str] = None) -> None:
71 | """Check if we should ratelimit the request with the given key.
72 |
73 | Raises:
74 | LimitExceededException: if the request should be denied.
75 | """
76 | if error is None:
77 | error = "Too many requests"
78 |
79 | # We get the current token count and compare it with the `burst`.
80 | current_tokens = self._buckets.get(key, 0)
81 | if current_tokens >= self._burst:
82 | logger.warning("Ratelimit hit: %s: %s", error, key)
83 | raise LimitExceededException(error)
84 |
85 | self._buckets[key] = current_tokens + 1
86 |
--------------------------------------------------------------------------------
/sydent/http/servlets/getvalidated3pidservlet.py:
--------------------------------------------------------------------------------
1 | # Copyright 2014 OpenMarket Ltd
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | from typing import TYPE_CHECKING
16 |
17 | from twisted.web.server import Request
18 |
19 | from sydent.db.valsession import ThreePidValSessionStore
20 | from sydent.http.auth import authV2
21 | from sydent.http.servlets import SydentResource, get_args, jsonwrap, send_cors
22 | from sydent.types import JsonDict
23 | from sydent.util.stringutils import is_valid_client_secret
24 | from sydent.validators import (
25 | IncorrectClientSecretException,
26 | InvalidSessionIdException,
27 | SessionExpiredException,
28 | SessionNotValidatedException,
29 | )
30 |
31 | if TYPE_CHECKING:
32 | from sydent.sydent import Sydent
33 |
34 |
35 | class GetValidated3pidServlet(SydentResource):
36 | isLeaf = True
37 |
38 | def __init__(self, syd: "Sydent", require_auth: bool = False) -> None:
39 | super().__init__()
40 | self.sydent = syd
41 | self.require_auth = require_auth
42 |
43 | @jsonwrap
44 | def render_GET(self, request: Request) -> JsonDict:
45 | send_cors(request)
46 | if self.require_auth:
47 | authV2(self.sydent, request)
48 |
49 | args = get_args(request, ("sid", "client_secret"))
50 |
51 | sid = args["sid"]
52 | clientSecret = args["client_secret"]
53 |
54 | if not is_valid_client_secret(clientSecret):
55 | request.setResponseCode(400)
56 | return {
57 | "errcode": "M_INVALID_PARAM",
58 | "error": "Invalid client_secret provided",
59 | }
60 |
61 | valSessionStore = ThreePidValSessionStore(self.sydent)
62 |
63 | noMatchError = {
64 | "errcode": "M_NO_VALID_SESSION",
65 | "error": "No valid session was found matching that sid and client secret",
66 | }
67 |
68 | try:
69 | s = valSessionStore.getValidatedSession(sid, clientSecret)
70 | except (IncorrectClientSecretException, InvalidSessionIdException):
71 | request.setResponseCode(404)
72 | return noMatchError
73 | except SessionExpiredException:
74 | request.setResponseCode(400)
75 | return {
76 | "errcode": "M_SESSION_EXPIRED",
77 | "error": "This validation session has expired: call requestToken again",
78 | }
79 | except SessionNotValidatedException:
80 | request.setResponseCode(400)
81 | return {
82 | "errcode": "M_SESSION_NOT_VALIDATED",
83 | "error": "This validation session has not yet been completed",
84 | }
85 |
86 | return {"medium": s.medium, "address": s.address, "validated_at": s.mtime}
87 |
88 | def render_OPTIONS(self, request: Request) -> bytes:
89 | send_cors(request)
90 | return b""
91 |
--------------------------------------------------------------------------------
/tests/test_email.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The Matrix.org Foundation C.I.C.
2 | # Licensed under the Apache License, Version 2.0 (the "License");
3 | # you may not use this file except in compliance with the License.
4 | # You may obtain a copy of the License at
5 | #
6 | # http://www.apache.org/licenses/LICENSE-2.0
7 | #
8 | # Unless required by applicable law or agreed to in writing, software
9 | # distributed under the License is distributed on an "AS IS" BASIS,
10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | # See the License for the specific language governing permissions and
12 | # limitations under the License.
13 |
14 | from typing import Optional
15 | from unittest.mock import Mock, patch
16 |
17 | from twisted.trial import unittest
18 |
19 | from sydent.types import JsonDict
20 | from tests.utils import make_request, make_sydent
21 |
22 |
23 | class TestRequestCode(unittest.TestCase):
24 | def setUp(self) -> None:
25 | # Create a new sydent
26 | self.sydent = make_sydent()
27 |
28 | def _make_request(self, url: str, body: Optional[JsonDict] = None) -> Mock:
29 | # Patch out the email sending so we can investigate the resulting email.
30 | with patch("sydent.util.emailutils.smtplib") as smtplib:
31 | request, channel = make_request(
32 | self.sydent.reactor,
33 | self.sydent.clientApiHttpServer.factory,
34 | "POST",
35 | url,
36 | body,
37 | )
38 |
39 | self.assertEqual(channel.code, 200)
40 |
41 | # Fish out the SMTP object and return it.
42 | smtp = smtplib.SMTP.return_value
43 | smtp.sendmail.assert_called_once()
44 |
45 | return smtp
46 |
47 | def test_request_code(self) -> None:
48 | self.sydent.run()
49 |
50 | smtp = self._make_request(
51 | "/_matrix/identity/api/v1/validate/email/requestToken",
52 | {
53 | "email": "test@test",
54 | "client_secret": "oursecret",
55 | "send_attempt": 0,
56 | },
57 | )
58 |
59 | # Ensure the email is as expected.
60 | email_contents = smtp.sendmail.call_args[0][2].decode("utf-8")
61 | self.assertIn("Confirm your email address for Matrix", email_contents)
62 |
63 | def test_request_code_via_url_query_params(self) -> None:
64 | self.sydent.run()
65 | url = (
66 | "/_matrix/identity/api/v1/validate/email/requestToken?"
67 | "email=test@test"
68 | "&client_secret=oursecret"
69 | "&send_attempt=0"
70 | )
71 | smtp = self._make_request(url)
72 |
73 | # Ensure the email is as expected.
74 | email_contents = smtp.sendmail.call_args[0][2].decode("utf-8")
75 | self.assertIn("Confirm your email address for Matrix", email_contents)
76 |
77 | def test_branded_request_code(self) -> None:
78 | self.sydent.run()
79 |
80 | smtp = self._make_request(
81 | "/_matrix/identity/api/v1/validate/email/requestToken?brand=vector-im",
82 | {
83 | "email": "test@test",
84 | "client_secret": "oursecret",
85 | "send_attempt": 0,
86 | },
87 | )
88 |
89 | # Ensure the email is as expected.
90 | email_contents = smtp.sendmail.call_args[0][2].decode("utf-8")
91 | self.assertIn("Confirm your email address for Element", email_contents)
92 |
--------------------------------------------------------------------------------
/docs/casefold_migration.md:
--------------------------------------------------------------------------------
1 | # This documentation is out of date!
2 |
3 | This documentation page is for the versions of Sydent maintained by the _Matrix.org Foundation_ ([github.com/matrix-org/sydent](https://github.com/matrix-org/sydent)), available under the Apache 2.0 licence.
4 |
5 | If you are interested in the documentation for a later version of Sydent, please refer to [github.com/element-hq/sydent](https://github.com/element-hq/sydent/tree/main/docs/casefold_migration.md).
6 |
7 | # Migrating to case-insensitive email addresses
8 |
9 | **Note: the operation described in this documentation is only needed if your server was
10 | running a version of Sydent earlier than 2.4.0 at some point, and only needs to be run
11 | once. If the first version of Sydent you have set up is 2.4.0 or later, or if you have
12 | already run this operation, you don't need to do it again.**
13 |
14 | In the past, the Matrix specification would consider email addresses as case-sensitive. This means
15 | `alice@example.com` and `Alice@example.com` would be seen as two different email addresses
16 | which could each be associated with a different Matrix user ID.
17 |
18 | With [MSC2265](https://github.com/matrix-org/matrix-doc/pull/2265), the Matrix
19 | specification was updated so that email addresses are considered without any case sensitivity (so the two
20 | addresses mentioned in the previous paragraph would be considered as being one and the
21 | same).
22 |
23 | As of version 2.4.0, Sydent supports this change by processing each new association
24 | without case sensitivity. However, some data might remain in the database from earlier
25 | versions when Sydent would support multiple associations for a given email address (by
26 | using variations of the same address with a different case). This means some addresses in
27 | your identity server's database might not have been stored in a format that allows for
28 | case-insensitive processing, or might have duplicate associations.
29 |
30 | To correct this, Sydent 2.4.0 introduces a [script](https://github.com/matrix-org/sydent/blob/main/scripts/casefold_db.py)
31 | that inspects an identity server's database and fixes it to be compatible with this change:
32 |
33 | ```
34 | Usage: /path/to/sydent/scripts/casefold_db.py [--no-email] [--dry-run] /path/to/sydent.conf
35 |
36 | Arguments:
37 | * --no-email: don't send out emails when deleting associations due to duplicates
38 | * --dry-run: don't update database rows and don't send out emails
39 | ```
40 |
41 | If the script finds a duplicate (i.e. an email address with multiple associations), it
42 | keeps the most recent association and deletes the others. If one or more of the Matrix
43 | user IDs that are being dissociated don't match the one being kept, the script also sends an
44 | email to the address to inform the user of the dissocation.
45 |
46 | The default template for this email can be found [here](https://github.com/matrix-org/sydent/blob/main/res/matrix-org/migration_template.eml.j2)
47 | and can be overriden by configuring a custom template directory (by changing the
48 | `templates.path` configuration setting). The custom template must be named `migration_template.eml.j2`
49 | (or `migration_template.eml` if not using Jinja 2 syntax), and will be given the Matrix
50 | user ID being dissociated at render through the variable `mxid`.
51 |
52 | This script is safe to run whilst Sydent is running.
53 |
54 | If the script is not run, there may be associations in your database that can no
55 | longer be looked up and duplicate associations may be registered.
56 |
--------------------------------------------------------------------------------
/.github/workflows/pipeline.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 | on:
3 | pull_request:
4 | push:
5 | branches: ["main"]
6 | workflow_dispatch:
7 |
8 | concurrency:
9 | group: ${{ github.workflow }}-${{ github.ref }}
10 | cancel-in-progress: true
11 |
12 | jobs:
13 | check-newsfile:
14 | name: Check PR has a changelog
15 | if: ${{ (github.base_ref == 'main' || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }}
16 | runs-on: ubuntu-latest
17 | steps:
18 | - uses: actions/checkout@v2
19 | with:
20 | fetch-depth: 0
21 | ref: ${{github.event.pull_request.head.sha}}
22 | - uses: actions/setup-python@v2
23 | with:
24 | python-version: "3.11"
25 | - run: python -m pip install towncrier
26 | - run: "scripts-dev/check_newsfragment.sh ${{ github.event.number }}"
27 |
28 | checks:
29 | runs-on: ubuntu-latest
30 | steps:
31 | - uses: actions/checkout@v3
32 |
33 | - uses: matrix-org/setup-python-poetry@v1
34 | with:
35 | python-version: 3.11
36 | install-project: false
37 |
38 | - name: Import order (isort)
39 | run: poetry run isort --check --diff .
40 |
41 | - name: Code style (black)
42 | run: poetry run black --check --diff .
43 |
44 | - name: Semantic checks (ruff)
45 | # --quiet suppresses the update check.
46 | run: poetry run ruff --quiet .
47 |
48 | - name: Restore/persist mypy's cache
49 | uses: actions/cache@v3
50 | with:
51 | path: |
52 | .mypy_cache
53 | key: mypy-cache-${{ github.context.sha }}
54 | restore-keys: mypy-cache-
55 |
56 | - name: Typechecking (mypy)
57 | run: poetry run mypy
58 |
59 | packaging:
60 | uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1"
61 |
62 | docker:
63 | # Sanity check that we can build the x64 image
64 | runs-on: ubuntu-latest
65 | steps:
66 | - uses: actions/checkout@v2
67 | - name: Set up Docker Buildx
68 | uses: docker/setup-buildx-action@v2
69 |
70 | - name: Build image
71 | uses: docker/build-push-action@v4
72 | with:
73 | cache-from: type=gha
74 | cache-to: type=gha,mode=max
75 | context: .
76 | push: false
77 |
78 | run-tests:
79 | name: Tests
80 | if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
81 | needs: [check-newsfile, checks, packaging]
82 | runs-on: ubuntu-latest
83 | strategy:
84 | matrix:
85 | python-version: ['3.7', '3.11']
86 | test-dir: ['tests', 'matrix_is_tester']
87 |
88 | steps:
89 | - uses: actions/checkout@v2
90 | - uses: matrix-org/setup-python-poetry@v1
91 | with:
92 | python-version: ${{ matrix.python-version }}
93 | - run: poetry run trial ${{ matrix.test-dir }}
94 | env:
95 | RUN_DESPITE_UNSUPPORTED: "Y"
96 |
97 | # a job which runs once all the other jobs are complete, thus allowing PRs to
98 | # be merged.
99 | tests-done:
100 | if: ${{ always() }}
101 | needs:
102 | - check-newsfile
103 | - checks
104 | - packaging
105 | - run-tests
106 | runs-on: ubuntu-latest
107 | steps:
108 | - uses: matrix-org/done-action@v2
109 | with:
110 | needs: ${{ toJSON(needs) }}
111 | # The newsfile lint may be skipped on non PR builds or on dependabot builds
112 | skippable:
113 | check-newsfile
114 |
115 |
--------------------------------------------------------------------------------
/matrix_is_test/launcher.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import os
16 | import shutil
17 | import tempfile
18 | import time
19 | from subprocess import Popen
20 |
21 | CFG_TEMPLATE = """
22 | [http]
23 | clientapi.http.bind_address = localhost
24 | clientapi.http.port = {port}
25 | client_http_base = http://localhost:{port}
26 | federation.verifycerts = False
27 |
28 | [db]
29 | db.file = :memory:
30 |
31 | [general]
32 | server.name = test.local
33 | terms.path = {terms_path}
34 | templates.path = {testsubject_path}/res
35 | brand.default = is-test
36 |
37 |
38 | ip.whitelist = 127.0.0.1
39 |
40 | [email]
41 | email.tlsmode = 0
42 | email.invite.subject = %(sender_display_name)s has invited you to chat
43 | email.invite.subject_space = %(sender_display_name)s has invited you to a space
44 | email.smtphost = localhost
45 | email.from = Sydent Validation
46 | email.smtpport = 9925
47 | email.subject = Your Validation Token
48 | email.ratelimit_sender.burst = 100000
49 | email.ratelimit_sender.rate_hz = 100000
50 | """
51 |
52 |
53 | class MatrixIsTestLauncher:
54 | def __init__(self, with_terms):
55 | self.with_terms = with_terms
56 |
57 | def launch(self):
58 | sydent_path = os.path.abspath(
59 | os.path.join(
60 | os.path.dirname(__file__),
61 | "..",
62 | )
63 | )
64 | testsubject_path = os.path.join(
65 | sydent_path,
66 | "matrix_is_test",
67 | )
68 | terms_path = (
69 | os.path.join(testsubject_path, "terms.yaml") if self.with_terms else ""
70 | )
71 | port = 8099 if self.with_terms else 8098
72 |
73 | self.tmpdir = tempfile.mkdtemp(prefix="sydenttest")
74 |
75 | with open(os.path.join(self.tmpdir, "sydent.conf"), "w") as cfgfp:
76 | cfgfp.write(
77 | CFG_TEMPLATE.format(
78 | testsubject_path=testsubject_path,
79 | terms_path=terms_path,
80 | port=port,
81 | )
82 | )
83 |
84 | newEnv = os.environ.copy()
85 | newEnv.update(
86 | {
87 | "PYTHONPATH": sydent_path,
88 | }
89 | )
90 |
91 | stderr_fp = open(os.path.join(testsubject_path, "sydent.stderr"), "w")
92 |
93 | pybin = os.getenv("SYDENT_PYTHON", "python")
94 |
95 | self.process = Popen(
96 | args=[pybin, "-m", "sydent.sydent"],
97 | cwd=self.tmpdir,
98 | env=newEnv,
99 | stderr=stderr_fp,
100 | )
101 | # XXX: wait for startup in a sensible way
102 | time.sleep(2)
103 |
104 | self._baseUrl = "http://localhost:%d" % (port,)
105 |
106 | def tearDown(self):
107 | print("Stopping sydent...")
108 | self.process.terminate()
109 | shutil.rmtree(self.tmpdir)
110 |
111 | def get_base_url(self):
112 | return self._baseUrl
113 |
--------------------------------------------------------------------------------
/sydent/config/sms.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | from configparser import ConfigParser
16 | from typing import Dict, List
17 |
18 | from sydent.config._base import BaseConfig
19 | from sydent.config.exceptions import ConfigError
20 |
21 |
22 | class SMSConfig(BaseConfig):
23 | def parse_config(self, cfg: "ConfigParser") -> bool:
24 | """
25 | Parse the sms section of the config
26 |
27 | :param cfg: the configuration to be parsed
28 | """
29 | self.body_template = cfg.get("sms", "bodyTemplate")
30 |
31 | # Make sure username and password are bytes otherwise we can't use them with
32 | # b64encode.
33 | self.api_username = cfg.get("sms", "username").encode("UTF-8")
34 | self.api_password = cfg.get("sms", "password").encode("UTF-8")
35 |
36 | self.originators: Dict[str, List[Dict[str, str]]] = {}
37 | self.smsRules = {}
38 |
39 | for opt in cfg.options("sms"):
40 | if opt.startswith("originators."):
41 | country = opt.split(".")[1]
42 | rawVal = cfg.get("sms", opt)
43 | rawList = [i.strip() for i in rawVal.split(",")]
44 |
45 | self.originators[country] = []
46 | for origString in rawList:
47 | parts = origString.split(":")
48 | if len(parts) != 2:
49 | raise ConfigError(
50 | "Originators must be in form: long:, short: or alpha:, separated by commas"
51 | )
52 | if parts[0] not in ["long", "short", "alpha"]:
53 | raise ConfigError(
54 | "Invalid originator type: valid types are long, short and alpha"
55 | )
56 | self.originators[country].append(
57 | {
58 | "type": parts[0],
59 | "text": parts[1],
60 | }
61 | )
62 | elif opt.startswith("smsrule."):
63 | country = opt.split(".")[1]
64 | action = cfg.get("sms", opt)
65 |
66 | if action not in ["allow", "reject"]:
67 | raise ConfigError(
68 | "Invalid SMS rule action: %s, expecting 'allow' or 'reject'"
69 | % action
70 | )
71 |
72 | self.smsRules[country] = action
73 |
74 | self.msisdn_ratelimit_burst = cfg.getint(
75 | "sms", "msisdn.ratelimit.burst", fallback=5
76 | )
77 | self.msisdn_ratelimit_rate_hz = cfg.getfloat(
78 | "sms", "msisdn.ratelimit.rate_hz", fallback=1.0 / (60.0 * 60.0)
79 | )
80 |
81 | self.country_ratelimit_burst = cfg.getint(
82 | "sms", "country.ratelimit.burst", fallback=50
83 | )
84 | self.country_ratelimit_rate_hz = cfg.getfloat(
85 | "sms", "country.ratelimit.rate_hz", fallback=1.0 / 60.0
86 | )
87 |
88 | return False
89 |
--------------------------------------------------------------------------------
/sydent/http/httpsclient.py:
--------------------------------------------------------------------------------
1 | # Copyright 2014 OpenMarket Ltd
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import json
16 | import logging
17 | from io import BytesIO
18 | from typing import TYPE_CHECKING, Optional
19 |
20 | from twisted.internet.defer import Deferred
21 | from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
22 | from twisted.internet.ssl import optionsForClientTLS
23 | from twisted.web.client import Agent, FileBodyProducer, Response
24 | from twisted.web.http_headers import Headers
25 | from twisted.web.iweb import IPolicyForHTTPS
26 | from zope.interface import implementer
27 |
28 | from sydent.types import JsonDict
29 |
30 | if TYPE_CHECKING:
31 | from sydent.sydent import Sydent
32 |
33 | logger = logging.getLogger(__name__)
34 |
35 |
36 | class ReplicationHttpsClient:
37 | """
38 | An HTTPS client specifically for talking replication to other Matrix Identity Servers
39 | (ie. presents our replication SSL certificate and validates peer SSL certificates as we would in the
40 | replication HTTPS server)
41 | """
42 |
43 | def __init__(self, sydent: "Sydent") -> None:
44 | self.sydent = sydent
45 | self.agent: Optional[Agent] = None
46 |
47 | if self.sydent.sslComponents.myPrivateCertificate:
48 | # We will already have logged a warn if this is absent, so don't do it again
49 | # cert = self.sydent.sslComponents.myPrivateCertificate
50 | # self.certOptions = twisted.internet.ssl.CertificateOptions(privateKey=cert.privateKey.original,
51 | # certificate=cert.original,
52 | # trustRoot=self.sydent.sslComponents.trustRoot)
53 | self.agent = Agent(self.sydent.reactor, SydentPolicyForHTTPS(self.sydent))
54 |
55 | def postJson(
56 | self, uri: str, jsonObject: JsonDict
57 | ) -> Optional["Deferred[Response]"]:
58 | """
59 | Sends an POST request over HTTPS.
60 |
61 | :param uri: The URI to send the request to.
62 | :param jsonObject: The request's body.
63 |
64 | :return: The request's response.
65 | """
66 | logger.debug("POSTing request to %s", uri)
67 | if not self.agent:
68 | logger.error("HTTPS post attempted but HTTPS is not configured")
69 | return None
70 |
71 | headers = Headers(
72 | {"Content-Type": ["application/json"], "User-Agent": ["Sydent"]}
73 | )
74 |
75 | json_bytes = json.dumps(jsonObject).encode("utf8")
76 | reqDeferred = self.agent.request(
77 | b"POST", uri.encode("utf8"), headers, FileBodyProducer(BytesIO(json_bytes))
78 | )
79 |
80 | return reqDeferred
81 |
82 |
83 | @implementer(IPolicyForHTTPS)
84 | class SydentPolicyForHTTPS:
85 | def __init__(self, sydent: "Sydent") -> None:
86 | self.sydent = sydent
87 |
88 | def creatorForNetloc(
89 | self, hostname: bytes, port: int
90 | ) -> IOpenSSLClientConnectionCreator:
91 | return optionsForClientTLS(
92 | hostname.decode("ascii"),
93 | trustRoot=self.sydent.sslComponents.trustRoot,
94 | clientCertificate=self.sydent.sslComponents.myPrivateCertificate,
95 | )
96 |
--------------------------------------------------------------------------------
/sydent/http/servlets/lookupservlet.py:
--------------------------------------------------------------------------------
1 | # Copyright 2014,2017 OpenMarket Ltd
2 | # Copyright 2019 The Matrix.org Foundation C.I.C.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | import logging
17 | from typing import TYPE_CHECKING
18 |
19 | import signedjson.sign
20 | from twisted.web.server import Request
21 |
22 | from sydent.db.threepid_associations import GlobalAssociationStore
23 | from sydent.http.servlets import SydentResource, get_args, jsonwrap, send_cors
24 | from sydent.types import JsonDict
25 | from sydent.util import json_decoder
26 |
27 | if TYPE_CHECKING:
28 | from sydent.sydent import Sydent
29 |
30 | logger = logging.getLogger(__name__)
31 |
32 |
33 | class LookupServlet(SydentResource):
34 | isLeaf = True
35 |
36 | def __init__(self, syd: "Sydent") -> None:
37 | super().__init__()
38 | self.sydent = syd
39 |
40 | @jsonwrap
41 | def render_GET(self, request: Request) -> JsonDict:
42 | """
43 | Look up an individual threepid.
44 |
45 | ** DEPRECATED **
46 |
47 | Params: 'medium': the medium of the threepid
48 | 'address': the address of the threepid
49 | Returns: A signed association if the threepid has a corresponding mxid, otherwise the empty object.
50 | """
51 | send_cors(request)
52 |
53 | args = get_args(request, ("medium", "address"))
54 |
55 | medium = args["medium"]
56 | address = args["address"]
57 |
58 | globalAssocStore = GlobalAssociationStore(self.sydent)
59 |
60 | sgassoc_raw = globalAssocStore.signedAssociationStringForThreepid(
61 | medium, address
62 | )
63 |
64 | if not sgassoc_raw:
65 | return {}
66 |
67 | # TODO validate this really is a dict
68 | sgassoc: JsonDict = json_decoder.decode(sgassoc_raw)
69 | if self.sydent.config.general.server_name not in sgassoc["signatures"]:
70 | # We have not yet worked out what the proper trust model should be.
71 | #
72 | # Maybe clients implicitly trust a server they talk to (and so we
73 | # should sign every assoc we return as ourselves, so they can
74 | # verify this).
75 | #
76 | # Maybe clients really want to know what server did the original
77 | # verification, and want to only know exactly who signed the assoc.
78 | #
79 | # Until we work out what we should do, sign all assocs we return as
80 | # ourself. This is vaguely ok because there actually is only one
81 | # identity server, but it happens to have two names (matrix.org and
82 | # vector.im), and so we're not really lying too much.
83 | #
84 | # We do this when we return assocs, not when we receive them over
85 | # replication, so that we can undo this decision in the future if
86 | # we wish, without having destroyed the raw underlying data.
87 | sgassoc = signedjson.sign.sign_json(
88 | sgassoc,
89 | self.sydent.config.general.server_name,
90 | self.sydent.keyring.ed25519,
91 | )
92 | return sgassoc
93 |
94 | def render_OPTIONS(self, request: Request) -> bytes:
95 | send_cors(request)
96 | return b""
97 |
--------------------------------------------------------------------------------
/stubs/twisted/web/iweb.pyi:
--------------------------------------------------------------------------------
1 | from typing import Any, AnyStr, BinaryIO, Dict, List, Mapping, Optional, Tuple
2 |
3 | from twisted.internet.defer import Deferred
4 | from twisted.internet.interfaces import (
5 | IAddress,
6 | IConsumer,
7 | IOpenSSLClientConnectionCreator,
8 | IProtocol,
9 | IPushProducer,
10 | IStreamClientEndpoint,
11 | )
12 | from twisted.python import urlpath
13 | from twisted.web.client import URI
14 | from twisted.web.http_headers import Headers
15 | from typing_extensions import Literal
16 | from zope.interface import Interface
17 |
18 | class IClientRequest(Interface):
19 | method: bytes
20 | absoluteURI: Optional[bytes]
21 | headers: Headers
22 |
23 | class IRequest(Interface):
24 | method: bytes
25 | uri: bytes
26 | path: bytes
27 | args: Mapping[bytes, List[bytes]]
28 | prepath: List[bytes]
29 | postpath: List[bytes]
30 | requestHeaders: Headers
31 | content: BinaryIO
32 | responseHeaders: Headers
33 | def getHeader(key: AnyStr) -> Optional[AnyStr]: ...
34 | def getCookie(key: bytes) -> Optional[bytes]: ...
35 | def getAllHeaders() -> Dict[bytes, bytes]: ...
36 | def getRequestHostname() -> bytes: ...
37 | def getHost() -> IAddress: ...
38 | def getClientAddress() -> IAddress: ...
39 | def getClientIP() -> Optional[str]: ...
40 | def getUser() -> str: ...
41 | def getPassword() -> str: ...
42 | def isSecure() -> bool: ...
43 | def getSession(sessionInterface: Any | None = ...) -> Any: ...
44 | def URLPath() -> urlpath.URLPath: ...
45 | def prePathURL() -> bytes: ...
46 | def rememberRootURL() -> None: ...
47 | def getRootURL() -> bytes: ...
48 | def finish() -> None: ...
49 | def write(data: bytes) -> None: ...
50 | def addCookie(
51 | k: AnyStr,
52 | v: AnyStr,
53 | expires: Optional[AnyStr] = ...,
54 | domain: Optional[AnyStr] = ...,
55 | path: Optional[AnyStr] = ...,
56 | max_age: Optional[AnyStr] = ...,
57 | comment: Optional[AnyStr] = ...,
58 | secure: Optional[bool] = ...,
59 | ) -> None: ...
60 | def setResponseCode(code: int, message: Optional[bytes] = ...) -> None: ...
61 | def setHeader(k: AnyStr, v: AnyStr) -> None: ...
62 | def redirect(url: AnyStr) -> None: ...
63 | # returns http.CACHED or False. http.CACHED is a string constant, but we
64 | # treat it as an opaque object, similar to UNKNOWN_LENGTH.
65 | def setLastModified(when: float) -> object | Literal[False]: ...
66 | def setETag(etag: str) -> object | Literal[False]: ...
67 | def setHost(host: bytes, port: int, ssl: bool = ...) -> None: ...
68 |
69 | class IBodyProducer(IPushProducer):
70 | # Length is either `int` or the opaque object UNKNOWN_LENGTH.
71 | length: int | object
72 | def startProducing(consumer: IConsumer) -> Deferred[None]: ...
73 | def stopProducing() -> None: ...
74 |
75 | class IResponse(Interface):
76 | version: Tuple[str, int, int]
77 | code: int
78 | phrase: str
79 | headers: Headers
80 | length: int | object
81 | request: IClientRequest
82 | previousResponse: Optional[IResponse]
83 | def deliverBody(protocol: IProtocol) -> None: ...
84 | def setPreviousResponse(response: IResponse) -> None: ...
85 |
86 | class IAgent(Interface):
87 | def request(
88 | method: bytes,
89 | uri: bytes,
90 | headers: Optional[Headers] = ...,
91 | bodyProducer: Optional[IBodyProducer] = ...,
92 | ) -> Deferred[IResponse]: ...
93 |
94 | class IPolicyForHTTPS(Interface):
95 | def creatorForNetloc(
96 | hostname: bytes, port: int
97 | ) -> IOpenSSLClientConnectionCreator: ...
98 |
99 | class IAgentEndpointFactory(Interface):
100 | def endpointForURI(uri: URI) -> IStreamClientEndpoint: ...
101 |
102 | UNKNOWN_LENGTH: object
103 |
--------------------------------------------------------------------------------
/sydent/http/servlets/threepidbindservlet.py:
--------------------------------------------------------------------------------
1 | # Copyright 2014 OpenMarket Ltd
2 | # Copyright 2019 The Matrix.org Foundation C.I.C.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | from typing import TYPE_CHECKING
17 |
18 | from twisted.web.server import Request
19 |
20 | from sydent.db.valsession import ThreePidValSessionStore
21 | from sydent.http.auth import authV2
22 | from sydent.http.servlets import (
23 | MatrixRestError,
24 | SydentResource,
25 | get_args,
26 | jsonwrap,
27 | send_cors,
28 | )
29 | from sydent.types import JsonDict
30 | from sydent.util.stringutils import is_valid_client_secret
31 | from sydent.validators import (
32 | IncorrectClientSecretException,
33 | InvalidSessionIdException,
34 | SessionExpiredException,
35 | SessionNotValidatedException,
36 | )
37 |
38 | if TYPE_CHECKING:
39 | from sydent.sydent import Sydent
40 |
41 |
42 | class ThreePidBindServlet(SydentResource):
43 | def __init__(self, sydent: "Sydent", require_auth: bool = False) -> None:
44 | super().__init__()
45 | self.sydent = sydent
46 | self.require_auth = require_auth
47 |
48 | @jsonwrap
49 | def render_POST(self, request: Request) -> JsonDict:
50 | send_cors(request)
51 |
52 | account = None
53 | if self.require_auth:
54 | account = authV2(self.sydent, request)
55 |
56 | args = get_args(request, ("sid", "client_secret", "mxid"))
57 |
58 | sid = args["sid"]
59 | mxid = args["mxid"]
60 | clientSecret = args["client_secret"]
61 |
62 | if not is_valid_client_secret(clientSecret):
63 | raise MatrixRestError(
64 | 400, "M_INVALID_PARAM", "Invalid client_secret provided"
65 | )
66 |
67 | if account:
68 | # This is a v2 API so only allow binding to the logged in user id
69 | if account.userId != mxid:
70 | raise MatrixRestError(
71 | 403,
72 | "M_UNAUTHORIZED",
73 | "This user is prohibited from binding to the mxid",
74 | )
75 |
76 | try:
77 | valSessionStore = ThreePidValSessionStore(self.sydent)
78 | s = valSessionStore.getValidatedSession(sid, clientSecret)
79 | except (IncorrectClientSecretException, InvalidSessionIdException):
80 | # Return the same error for not found / bad client secret otherwise
81 | # people can get information about sessions without knowing the
82 | # secret.
83 | raise MatrixRestError(
84 | 404,
85 | "M_NO_VALID_SESSION",
86 | "No valid session was found matching that sid and client secret",
87 | )
88 | except SessionExpiredException:
89 | raise MatrixRestError(
90 | 400,
91 | "M_SESSION_EXPIRED",
92 | "This validation session has expired: call requestToken again",
93 | )
94 | except SessionNotValidatedException:
95 | raise MatrixRestError(
96 | 400,
97 | "M_SESSION_NOT_VALIDATED",
98 | "This validation session has not yet been completed",
99 | )
100 |
101 | res = self.sydent.threepidBinder.addBinding(s.medium, s.address, mxid)
102 | return res
103 |
104 | def render_OPTIONS(self, request: Request) -> bytes:
105 | send_cors(request)
106 | return b""
107 |
--------------------------------------------------------------------------------
/sydent/util/ip_range.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The Matrix.org Foundation C.I.C.
2 | # Licensed under the Apache License, Version 2.0 (the "License");
3 | # you may not use this file except in compliance with the License.
4 | # You may obtain a copy of the License at
5 | #
6 | # http://www.apache.org/licenses/LICENSE-2.0
7 | #
8 | # Unless required by applicable law or agreed to in writing, software
9 | # distributed under the License is distributed on an "AS IS" BASIS,
10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | # See the License for the specific language governing permissions and
12 | # limitations under the License.
13 |
14 | import itertools
15 | from typing import Iterable, Optional
16 |
17 | from netaddr import AddrFormatError, IPNetwork, IPSet
18 |
19 | # IP ranges that are considered private / unroutable / don't make sense.
20 | DEFAULT_IP_RANGE_BLACKLIST = [
21 | # Localhost
22 | "127.0.0.0/8",
23 | # Private networks.
24 | "10.0.0.0/8",
25 | "172.16.0.0/12",
26 | "192.168.0.0/16",
27 | # Carrier grade NAT.
28 | "100.64.0.0/10",
29 | # Address registry.
30 | "192.0.0.0/24",
31 | # Link-local networks.
32 | "169.254.0.0/16",
33 | # Formerly used for 6to4 relay.
34 | "192.88.99.0/24",
35 | # Testing networks.
36 | "198.18.0.0/15",
37 | "192.0.2.0/24",
38 | "198.51.100.0/24",
39 | "203.0.113.0/24",
40 | # Multicast.
41 | "224.0.0.0/4",
42 | # Localhost
43 | "::1/128",
44 | # Link-local addresses.
45 | "fe80::/10",
46 | # Unique local addresses.
47 | "fc00::/7",
48 | # Testing networks.
49 | "2001:db8::/32",
50 | # Multicast.
51 | "ff00::/8",
52 | # Site-local addresses
53 | "fec0::/10",
54 | ]
55 |
56 |
57 | def generate_ip_set(
58 | ip_addresses: Optional[Iterable[str]],
59 | extra_addresses: Optional[Iterable[str]] = None,
60 | config_path: Optional[Iterable[str]] = None,
61 | ) -> IPSet:
62 | """
63 | Generate an IPSet from a list of IP addresses or CIDRs.
64 |
65 | Additionally, for each IPv4 network in the list of IP addresses, also
66 | includes the corresponding IPv6 networks.
67 |
68 | This includes:
69 |
70 | * IPv4-Compatible IPv6 Address (see RFC 4291, section 2.5.5.1)
71 | * IPv4-Mapped IPv6 Address (see RFC 4291, section 2.5.5.2)
72 | * 6to4 Address (see RFC 3056, section 2)
73 |
74 | Args:
75 | ip_addresses: An iterable of IP addresses or CIDRs.
76 | extra_addresses: An iterable of IP addresses or CIDRs.
77 | config_path: The path in the configuration for error messages.
78 |
79 | Returns:
80 | A new IP set.
81 | """
82 | result = IPSet()
83 | for ip in itertools.chain(ip_addresses or (), extra_addresses or ()):
84 | try:
85 | network = IPNetwork(ip)
86 | except AddrFormatError as e:
87 | raise Exception(
88 | "Invalid IP range provided: %s." % (ip,), config_path
89 | ) from e
90 | result.add(network)
91 |
92 | # It is possible that these already exist in the set, but that's OK.
93 | if ":" not in str(network):
94 | result.add(IPNetwork(network).ipv6(ipv4_compatible=True))
95 | result.add(IPNetwork(network).ipv6(ipv4_compatible=False))
96 | result.add(_6to4(network))
97 |
98 | return result
99 |
100 |
101 | def _6to4(network: IPNetwork) -> IPNetwork:
102 | """Convert an IPv4 network into a 6to4 IPv6 network per RFC 3056."""
103 |
104 | # 6to4 networks consist of:
105 | # * 2002 as the first 16 bits
106 | # * The first IPv4 address in the network hex-encoded as the next 32 bits
107 | # * The new prefix length needs to include the bits from the 2002 prefix.
108 | hex_network = hex(network.first)[2:]
109 | hex_network = ("0" * (8 - len(hex_network))) + hex_network
110 | return IPNetwork(
111 | "2002:%s:%s::/%d"
112 | % (
113 | hex_network[:4],
114 | hex_network[4:],
115 | 16 + network.prefixlen,
116 | )
117 | )
118 |
--------------------------------------------------------------------------------
/stubs/twisted/web/client.pyi:
--------------------------------------------------------------------------------
1 | from typing import BinaryIO, Optional, Sequence, Type, TypeVar
2 |
3 | from twisted.internet.defer import Deferred
4 | from twisted.internet.interfaces import IConsumer, IProtocol
5 | from twisted.internet.task import Cooperator
6 | from twisted.python.failure import Failure
7 | from twisted.web.http_headers import Headers
8 | from twisted.web.iweb import (
9 | IAgent,
10 | IAgentEndpointFactory,
11 | IBodyProducer,
12 | IPolicyForHTTPS,
13 | IResponse,
14 | )
15 | from zope.interface import implementer
16 |
17 | _C = TypeVar("_C")
18 |
19 | class ResponseFailed(Exception):
20 | def __init__(
21 | self, reasons: Sequence[Failure], response: Optional[Response] = ...
22 | ): ...
23 |
24 | class HTTPConnectionPool:
25 | persistent: bool
26 | maxPersistentPerHost: int
27 | cachedConnectionTimeout: float
28 | retryAutomatically: bool
29 | def __init__(self, reactor: object, persistent: bool = ...): ...
30 |
31 | @implementer(IAgent)
32 | class Agent:
33 | # Here and in `usingEndpointFactory`, reactor should be a "provider of
34 | # L{IReactorTCP}, L{IReactorTime} and either
35 | # L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}."
36 | # I don't know how to encode that in the type system; see also
37 | # https://github.com/Shoobx/mypy-zope/issues/58
38 | def __init__(
39 | self,
40 | reactor: object,
41 | contextFactory: IPolicyForHTTPS = ...,
42 | connectTimeout: Optional[float] = ...,
43 | bindAddress: Optional[bytes] = ...,
44 | pool: Optional[HTTPConnectionPool] = ...,
45 | ): ...
46 | def request(
47 | self,
48 | method: bytes,
49 | uri: bytes,
50 | headers: Optional[Headers] = ...,
51 | bodyProducer: Optional[IBodyProducer] = ...,
52 | ) -> Deferred[IResponse]: ...
53 | @classmethod
54 | def usingEndpointFactory(
55 | cls: Type[_C],
56 | reactor: object,
57 | endpointFactory: IAgentEndpointFactory,
58 | pool: Optional[HTTPConnectionPool] = ...,
59 | ) -> _C: ...
60 |
61 | @implementer(IBodyProducer)
62 | class FileBodyProducer:
63 | def __init__(
64 | self,
65 | inputFile: BinaryIO,
66 | cooperator: Cooperator = ...,
67 | readSize: int = ...,
68 | ): ...
69 | # Length is either `int` or the opaque object UNKNOWN_LENGTH.
70 | length: int | object
71 | def startProducing(self, consumer: IConsumer) -> Deferred[None]: ...
72 | def stopProducing(self) -> None: ...
73 | def pauseProducing(self) -> None: ...
74 | def resumeProducing(self) -> None: ...
75 |
76 | def readBody(response: IResponse) -> Deferred[bytes]: ...
77 |
78 | # Type ignore: I don't want to respecify the methods on the interface that we
79 | # don't use.
80 | @implementer(IResponse) # type: ignore[misc]
81 | class Response:
82 | code: int
83 | headers: Headers
84 | # Length is either `int` or the opaque object UNKNOWN_LENGTH.
85 | length: int | object
86 | def deliverBody(self, protocol: IProtocol) -> None: ...
87 |
88 | class ResponseDone: ...
89 |
90 | class URI:
91 | scheme: bytes
92 | netloc: bytes
93 | host: bytes
94 | port: int
95 | path: bytes
96 | params: bytes
97 | query: bytes
98 | fragment: bytes
99 | def __init__(
100 | self,
101 | scheme: bytes,
102 | netloc: bytes,
103 | host: bytes,
104 | port: int,
105 | path: bytes,
106 | params: bytes,
107 | query: bytes,
108 | fragment: bytes,
109 | ): ...
110 | @classmethod
111 | def fromBytes(
112 | cls: Type[_C], uri: bytes, defaultPort: Optional[int] = ...
113 | ) -> _C: ...
114 |
115 | @implementer(IAgent)
116 | class RedirectAgent:
117 | def __init__(self, agent: Agent, redirectLimit: int = ...): ...
118 | def request(
119 | self,
120 | method: bytes,
121 | uri: bytes,
122 | headers: Optional[Headers] = ...,
123 | bodyProducer: Optional[IBodyProducer] = ...,
124 | ) -> Deferred[IResponse]: ...
125 |
--------------------------------------------------------------------------------
/sydent/db/accounts.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | from typing import TYPE_CHECKING, Optional, Tuple
16 |
17 | from sydent.users.accounts import Account
18 |
19 | if TYPE_CHECKING:
20 | from sydent.sydent import Sydent
21 |
22 |
23 | class AccountStore:
24 | def __init__(self, sydent: "Sydent") -> None:
25 | self.sydent = sydent
26 |
27 | def getAccountByToken(self, token: str) -> Optional[Account]:
28 | """
29 | Select the account matching the given token, if any.
30 |
31 | :param token: The token to identify the account, if any.
32 |
33 | :return: The account matching the token, or None if no account matched.
34 | """
35 | cur = self.sydent.db.cursor()
36 | res = cur.execute(
37 | "select a.user_id, a.created_ts, a.consent_version from accounts a, tokens t "
38 | "where t.user_id = a.user_id and t.token = ?",
39 | (token,),
40 | )
41 |
42 | row: Optional[Tuple[str, int, Optional[str]]] = res.fetchone()
43 | if row is None:
44 | return None
45 |
46 | return Account(*row)
47 |
48 | def storeAccount(
49 | self, user_id: str, creation_ts: int, consent_version: Optional[str]
50 | ) -> None:
51 | """
52 | Stores an account for the given user ID.
53 |
54 | :param user_id: The Matrix user ID to create an account for.
55 | :param creation_ts: The timestamp in milliseconds.
56 | :param consent_version: The version of the terms of services that the user last
57 | accepted.
58 | """
59 | cur = self.sydent.db.cursor()
60 | cur.execute(
61 | "insert or ignore into accounts (user_id, created_ts, consent_version) "
62 | "values (?, ?, ?)",
63 | (user_id, creation_ts, consent_version),
64 | )
65 | self.sydent.db.commit()
66 |
67 | def setConsentVersion(self, user_id: str, consent_version: Optional[str]) -> None:
68 | """
69 | Saves that the given user has agreed to all of the terms in the document of the
70 | given version.
71 |
72 | :param user_id: The Matrix ID of the user that has agreed to the terms.
73 | :param consent_version: The version of the document the user has agreed to.
74 | """
75 | cur = self.sydent.db.cursor()
76 | cur.execute(
77 | "update accounts set consent_version = ? where user_id = ?",
78 | (consent_version, user_id),
79 | )
80 | self.sydent.db.commit()
81 |
82 | def addToken(self, user_id: str, token: str) -> None:
83 | """
84 | Stores the authentication token for a given user.
85 |
86 | :param user_id: The Matrix user ID to save the given token for.
87 | :param token: The token to store for that user ID.
88 | """
89 | cur = self.sydent.db.cursor()
90 | cur.execute(
91 | "insert into tokens (user_id, token) values (?, ?)",
92 | (user_id, token),
93 | )
94 | self.sydent.db.commit()
95 |
96 | def delToken(self, token: str) -> int:
97 | """
98 | Deletes an authentication token from the database.
99 |
100 | :param token: The token to delete from the database.
101 | """
102 | cur = self.sydent.db.cursor()
103 | cur.execute(
104 | "delete from tokens where token = ?",
105 | (token,),
106 | )
107 | deleted = cur.rowcount
108 | self.sydent.db.commit()
109 | return deleted
110 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [tool.towncrier]
2 | package = "sydent"
3 | filename = "CHANGELOG.md"
4 | directory = "changelog.d"
5 | issue_format = "[\\#{issue}](https://github.com/matrix-org/sydent/issues/{issue})"
6 |
7 | [[tool.towncrier.type]]
8 | directory = "feature"
9 | name = "Features"
10 | showcontent = true
11 |
12 | [[tool.towncrier.type]]
13 | directory = "bugfix"
14 | name = "Bugfixes"
15 | showcontent = true
16 |
17 | [[tool.towncrier.type]]
18 | directory = "docker"
19 | name = "Updates to the Docker image"
20 | showcontent = true
21 |
22 | [[tool.towncrier.type]]
23 | directory = "doc"
24 | name = "Improved Documentation"
25 | showcontent = true
26 |
27 | [[tool.towncrier.type]]
28 | directory = "removal"
29 | name = "Deprecations and Removals"
30 | showcontent = true
31 |
32 | [[tool.towncrier.type]]
33 | directory = "misc"
34 | name = "Internal Changes"
35 | showcontent = true
36 |
37 | [tool.isort]
38 | profile = "black"
39 |
40 | [tool.black]
41 | target-version = ['py36']
42 |
43 | [tool.mypy]
44 | plugins = "mypy_zope:plugin"
45 | show_error_codes = true
46 | namespace_packages = true
47 | strict = true
48 |
49 | files = [
50 | # Find files that pass with
51 | # find sydent tests -type d -not -name __pycache__ -exec bash -c "mypy --strict '{}' > /dev/null" \; -print
52 | "sydent"
53 | # TODO the rest of CI checks these---mypy ought to too.
54 | # "tests",
55 | # "matrix_is_test",
56 | # "scripts",
57 | # "setup.py",
58 | ]
59 | mypy_path = "stubs"
60 |
61 | [[tool.mypy.overrides]]
62 | module = [
63 | "idna",
64 | "netaddr",
65 | "signedjson.*",
66 | "sortedcontainers",
67 | ]
68 | ignore_missing_imports = true
69 |
70 | [tool.poetry]
71 | name = "matrix-sydent"
72 | version = "2.6.1"
73 | description = "Reference Matrix Identity Verification and Lookup Server"
74 | authors = ["Matrix.org Team and Contributors "]
75 | license = "Apache-2.0"
76 | readme = "README.rst"
77 | repository = "https://github.com/matrix-org/sydent"
78 | packages = [
79 | { include = "sydent" },
80 | ]
81 |
82 | include = [
83 | { path = "matrix-sydent.service" },
84 | { path = "res" },
85 | { path = "scripts" },
86 | { path = "matrix_is_test", format = "sdist" },
87 | { path = "scripts-dev", format = "sdist" },
88 | { path = "setup.cfg", format = "sdist" },
89 | { path = "tests", format = "sdist" },
90 | ]
91 | classifiers = [
92 | "Development Status :: 5 - Production/Stable",
93 | ]
94 |
95 | [tool.poetry.dependencies]
96 | python = "^3.7"
97 | attrs = ">=19.1.0"
98 | jinja2 = ">=3.0.0"
99 | netaddr = ">=0.7.0"
100 | matrix-common = "^1.1.0"
101 | phonenumbers = ">=8.12.32"
102 | # prometheus-client's lower bound is copied from Synapse.
103 | prometheus-client = ">=0.4.0"
104 | pynacl = ">=1.2.1"
105 | pyOpenSSL = ">=16.0.0"
106 | pyyaml = ">=3.11"
107 | # sentry-sdk's lower bound is copied from Synapse.
108 | sentry-sdk = { version = ">=0.7.2", optional = true }
109 | # twisted warns about about the absence of service-identity
110 | service-identity = ">=1.0.0"
111 | signedjson = "==1.1.1"
112 | sortedcontainers = ">=2.1.0"
113 | twisted = ">=18.4.0"
114 | typing-extensions = ">=3.7.4"
115 | unpaddedbase64 = ">=1.1.0"
116 | "zope.interface" = ">=4.6.0"
117 |
118 | [tool.poetry.dev-dependencies]
119 | black = "==21.6b0"
120 | ruff = "0.0.189"
121 | isort = "==5.8.0"
122 | matrix-is-tester = {git = "https://github.com/matrix-org/matrix-is-tester", rev = "main"}
123 | mypy = ">=0.902"
124 | mypy-zope = ">=0.3.1"
125 | parameterized = "==0.8.1"
126 | # sentry-sdk is required for typechecking.
127 | sentry-sdk = "*"
128 | types-Jinja2 = "2.11.9"
129 | types-mock = "4.0.8"
130 | types-PyOpenSSL = "21.0.3"
131 | types-PyYAML = "6.0.3"
132 | towncrier = "^21.9.0"
133 |
134 | [tool.poetry.extras]
135 | sentry = ["sentry-sdk"]
136 | prometheus = ["prometheus-client"]
137 |
138 | [tool.poetry.scripts]
139 | sydent = "sydent.sydent:main"
140 |
141 | [tool.ruff]
142 | line-length = 88
143 |
144 | ignore = [
145 | "E501",
146 | "F401",
147 | "F821",
148 | ]
149 | select = [
150 | # pycodestyle checks.
151 | "E",
152 | "W",
153 | # pyflakes checks.
154 | "F",
155 | ]
156 |
157 | [build-system]
158 | requires = ["poetry-core>=1.0.0"]
159 | build-backend = "poetry.core.masonry.api"
160 |
--------------------------------------------------------------------------------
/sydent/http/federation_tls_options.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 New Vector Ltd
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | import logging
15 | from typing import Callable
16 |
17 | from OpenSSL import SSL
18 | from twisted.internet import ssl
19 | from twisted.internet.abstract import isIPAddress, isIPv6Address
20 | from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
21 | from twisted.protocols.tls import TLSMemoryBIOProtocol
22 | from twisted.python.failure import Failure
23 | from zope.interface import implementer
24 |
25 | logger = logging.getLogger(__name__)
26 |
27 | F = Callable[[SSL.Connection, int, int], None]
28 |
29 |
30 | def _tolerateErrors(wrapped: F) -> F:
31 | """
32 | Wrap up an info_callback for pyOpenSSL so that if something goes wrong
33 | the error is immediately logged and the connection is dropped if possible.
34 | This is a copy of twisted.internet._sslverify._tolerateErrors. For
35 | documentation, see the twisted documentation.
36 | """
37 |
38 | def infoCallback(connection: SSL.Connection, where: int, ret: int) -> None:
39 | try:
40 | return wrapped(connection, where, ret)
41 | except BaseException:
42 | f = Failure()
43 | logger.exception("Error during info_callback")
44 | connection.get_app_data().failVerification(f)
45 |
46 | return infoCallback
47 |
48 |
49 | def _idnaBytes(text: str) -> bytes:
50 | """
51 | Convert some text typed by a human into some ASCII bytes. This is a
52 | copy of twisted.internet._idna._idnaBytes. For documentation, see the
53 | twisted documentation.
54 | """
55 | try:
56 | import idna
57 | except ImportError:
58 | return text.encode("idna")
59 | else:
60 | return idna.encode(text)
61 |
62 |
63 | @implementer(IOpenSSLClientConnectionCreator)
64 | class ClientTLSOptions:
65 | """
66 | Client creator for TLS without certificate identity verification. This is a
67 | copy of twisted.internet._sslverify.ClientTLSOptions with the identity
68 | verification left out. For documentation, see the twisted documentation.
69 | """
70 |
71 | def __init__(self, hostname: str, ctx: SSL.Context):
72 | self._ctx = ctx
73 |
74 | if isIPAddress(hostname) or isIPv6Address(hostname):
75 | self._hostnameBytes = hostname.encode("ascii")
76 | self._sendSNI = False
77 | else:
78 | self._hostnameBytes = _idnaBytes(hostname)
79 | self._sendSNI = True
80 |
81 | ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
82 |
83 | def clientConnectionForTLS(
84 | self, tlsProtocol: TLSMemoryBIOProtocol
85 | ) -> SSL.Connection:
86 | context = self._ctx
87 | connection = SSL.Connection(context, None)
88 | connection.set_app_data(tlsProtocol)
89 | return connection
90 |
91 | def _identityVerifyingInfoCallback(
92 | self, connection: SSL.Connection, where: int, ret: int
93 | ) -> None:
94 | # Literal IPv4 and IPv6 addresses are not permitted
95 | # as host names according to the RFCs
96 | if where & SSL.SSL_CB_HANDSHAKE_START and self._sendSNI:
97 | connection.set_tlsext_host_name(self._hostnameBytes)
98 |
99 |
100 | class ClientTLSOptionsFactory:
101 | """Factory for Twisted ClientTLSOptions that are used to make connections
102 | to remote servers for federation."""
103 |
104 | def __init__(self, verify_requests: bool):
105 | if verify_requests:
106 | self._options = ssl.CertificateOptions(trustRoot=ssl.platformTrust())
107 | else:
108 | self._options = ssl.CertificateOptions()
109 |
110 | def get_options(self, host: str) -> ClientTLSOptions:
111 | # Use _makeContext so that we get a fresh OpenSSL CTX each time.
112 | return ClientTLSOptions(host, self._options._makeContext())
113 |
--------------------------------------------------------------------------------
/sydent/config/email.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | import socket
15 | from configparser import ConfigParser
16 | from typing import Optional
17 |
18 | from sydent.config._base import BaseConfig
19 | from sydent.config.exceptions import ConfigError
20 | from sydent.util.emailutils import EmailAddressException, check_valid_email_address
21 |
22 |
23 | class EmailConfig(BaseConfig):
24 | def parse_config(self, cfg: ConfigParser) -> bool:
25 | """
26 | Parse the email section of the config
27 |
28 | :param cfg: the configuration to be parsed
29 | """
30 |
31 | # These two options are deprecated
32 | self.template: Optional[str] = cfg.get("email", "email.template", fallback=None)
33 |
34 | self.invite_template = cfg.get("email", "email.invite_template", fallback=None)
35 |
36 | # This isn't used anywhere...
37 | self.validation_subject = cfg.get("email", "email.subject")
38 |
39 | self.invite_subject = cfg.get("email", "email.invite.subject", raw=True)
40 | self.invite_subject_space = cfg.get(
41 | "email", "email.invite.subject_space", raw=True
42 | )
43 |
44 | self.smtp_server = cfg.get("email", "email.smtphost")
45 | self.smtp_port = cfg.get("email", "email.smtpport")
46 | self.smtp_username = cfg.get("email", "email.smtpusername")
47 | self.smtp_password = cfg.get("email", "email.smtppassword")
48 | self.tls_mode = cfg.get("email", "email.tlsmode")
49 |
50 | # This is the fully qualified domain name for SMTP HELO/EHLO
51 | self.host_name = cfg.get("email", "email.hostname")
52 | if self.host_name == "":
53 | self.host_name = socket.getfqdn()
54 |
55 | self.sender = cfg.get("email", "email.from")
56 | try:
57 | check_valid_email_address(self.sender, allow_description=True)
58 | except EmailAddressException as e:
59 | raise ConfigError(f"Invalid email address '{self.sender}'") from e
60 |
61 | self.default_web_client_location = cfg.get(
62 | "email", "email.default_web_client_location"
63 | )
64 |
65 | self.username_obfuscate_characters = cfg.getint(
66 | "email", "email.third_party_invite_username_obfuscate_characters"
67 | )
68 |
69 | self.domain_obfuscate_characters = cfg.getint(
70 | "email", "email.third_party_invite_domain_obfuscate_characters"
71 | )
72 |
73 | third_party_invite_homeserver_blocklist = cfg.get(
74 | "email", "email.third_party_invite_homeserver_blocklist", fallback=""
75 | )
76 | third_party_invite_room_blocklist = cfg.get(
77 | "email", "email.third_party_invite_room_blocklist", fallback=""
78 | )
79 | third_party_invite_keyword_blocklist = cfg.get(
80 | "email", "email.third_party_invite_keyword_blocklist", fallback=""
81 | )
82 | self.third_party_invite_homeserver_blocklist = {
83 | server
84 | for server in third_party_invite_homeserver_blocklist.split("\n")
85 | if server # filter out empty lines
86 | }
87 | self.third_party_invite_room_blocklist = {
88 | room_id
89 | for room_id in third_party_invite_room_blocklist.split("\n")
90 | if room_id # filter out empty lines
91 | }
92 | self.third_party_invite_keyword_blocklist = {
93 | keyword.casefold()
94 | for keyword in third_party_invite_keyword_blocklist.split("\n")
95 | if keyword
96 | }
97 |
98 | self.email_sender_ratelimit_burst = cfg.getint(
99 | "email", "email.ratelimit_sender.burst", fallback=5
100 | )
101 | self.email_sender_ratelimit_rate_hz = cfg.getfloat(
102 | "email", "email.ratelimit_sender.rate_hz", fallback=1.0 / (5 * 60.0)
103 | )
104 |
105 | return False
106 |
--------------------------------------------------------------------------------
/tests/test_msisdn.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 The Matrix.org Foundation C.I.C.
2 | # Licensed under the Apache License, Version 2.0 (the "License");
3 | # you may not use this file except in compliance with the License.
4 | # You may obtain a copy of the License at
5 | #
6 | # http://www.apache.org/licenses/LICENSE-2.0
7 | #
8 | # Unless required by applicable law or agreed to in writing, software
9 | # distributed under the License is distributed on an "AS IS" BASIS,
10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | # See the License for the specific language governing permissions and
12 | # limitations under the License.
13 | import asyncio
14 | import os.path
15 | from typing import Optional
16 | from unittest.mock import Mock, patch
17 |
18 | import attr
19 | from twisted.trial import unittest
20 |
21 | from sydent.types import JsonDict
22 | from tests.utils import make_request, make_sydent
23 |
24 |
25 | @attr.s(auto_attribs=True)
26 | class FakeHeader:
27 | """
28 | A fake header object
29 | """
30 |
31 | headers: dict
32 |
33 | def getAllRawHeaders(self):
34 | return self.headers
35 |
36 |
37 | @attr.s(auto_attribs=True)
38 | class FakeResponse:
39 | """A fake twisted.web.IResponse object"""
40 |
41 | # HTTP response code
42 | code: int
43 |
44 | # Fake Header
45 | headers: FakeHeader
46 |
47 |
48 | class TestRequestCode(unittest.TestCase):
49 | def setUp(self) -> None:
50 | # Create a new sydent
51 | config = {
52 | "general": {
53 | "templates.path": os.path.join(
54 | os.path.dirname(os.path.dirname(__file__)), "res"
55 | ),
56 | },
57 | }
58 | self.sydent = make_sydent(test_config=config)
59 |
60 | def _make_request(self, url: str, body: Optional[JsonDict] = None) -> Mock:
61 | # Patch out the email sending so we can investigate the resulting email.
62 | with patch("sydent.sms.openmarket.OpenMarketSMS.sendTextSMS") as sendTextSMS:
63 | # We can't use AsyncMock until Python 3.8. Instead, mock the
64 | # function as returning a future.
65 | f = asyncio.Future()
66 | f.set_result(Mock())
67 | sendTextSMS.return_value = f
68 |
69 | request, channel = make_request(
70 | self.sydent.reactor,
71 | self.sydent.clientApiHttpServer.factory,
72 | "POST",
73 | url,
74 | body,
75 | )
76 | self.assertEqual(channel.code, 200)
77 |
78 | return sendTextSMS
79 |
80 | def test_request_code(self) -> None:
81 | self.sydent.run()
82 |
83 | sendSMS_mock = self._make_request(
84 | "/_matrix/identity/api/v1/validate/msisdn/requestToken",
85 | {
86 | "phone_number": "447700900750",
87 | "country": "GB",
88 | "client_secret": "oursecret",
89 | "send_attempt": 0,
90 | },
91 | )
92 | sendSMS_mock.assert_called_once()
93 |
94 | def test_request_code_via_url_query_params(self) -> None:
95 | self.sydent.run()
96 | url = (
97 | "/_matrix/identity/api/v1/validate/msisdn/requestToken?"
98 | "phone_number=447700900750"
99 | "&country=GB"
100 | "&client_secret=oursecret"
101 | "&send_attempt=0"
102 | )
103 | sendSMS_mock = self._make_request(url)
104 | sendSMS_mock.assert_called_once()
105 |
106 | @patch("sydent.http.httpclient.HTTPClient.post_json_maybe_get_json")
107 | def test_bad_api_response_raises_exception(self, post_json: Mock) -> None:
108 | """Test that an error response from OpenMarket raises an exception
109 | and that the requester receives an error code."""
110 |
111 | header = FakeHeader({})
112 | resp = FakeResponse(code=400, headers=header), {}
113 | post_json.return_value = resp
114 | self.sydent.run()
115 | request, channel = make_request(
116 | self.sydent.reactor,
117 | self.sydent.clientApiHttpServer.factory,
118 | "POST",
119 | "/_matrix/identity/api/v1/validate/msisdn/requestToken",
120 | {
121 | "phone_number": "447700900750",
122 | "country": "GB",
123 | "client_secret": "oursecret",
124 | "send_attempt": 0,
125 | },
126 | )
127 | self.assertEqual(channel.code, 500)
128 |
--------------------------------------------------------------------------------
/sydent/util/stringutils.py:
--------------------------------------------------------------------------------
1 | # Copyright 2020 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import re
16 | from typing import Optional, Tuple
17 |
18 | from twisted.internet.abstract import isIPAddress, isIPv6Address
19 |
20 | # https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-register-email-requesttoken
21 | CLIENT_SECRET_REGEX = re.compile(r"^[0-9a-zA-Z\.=_\-]+$")
22 |
23 | # hostname/domain name
24 | # https://regex101.com/r/OyN1lg/2
25 | hostname_regex = re.compile(
26 | r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$",
27 | flags=re.IGNORECASE,
28 | )
29 |
30 | # it's unclear what the maximum length of an email address is. RFC3696 (as corrected
31 | # by errata) says:
32 | # the upper limit on address lengths should normally be considered to be 254.
33 | #
34 | # In practice, mail servers appear to be more tolerant and allow 400 characters
35 | # or so. Let's allow 500, which should be plenty for everyone.
36 | #
37 | MAX_EMAIL_ADDRESS_LENGTH = 500
38 |
39 |
40 | def is_valid_client_secret(client_secret: str) -> bool:
41 | """Validate that a given string matches the client_secret regex defined by the spec
42 |
43 | :param client_secret: The client_secret to validate
44 |
45 | :return: Whether the client_secret is valid
46 | """
47 | return (
48 | 0 < len(client_secret) <= 255
49 | and CLIENT_SECRET_REGEX.match(client_secret) is not None
50 | )
51 |
52 |
53 | def is_valid_hostname(string: str) -> bool:
54 | """Validate that a given string is a valid hostname or domain name.
55 |
56 | For domain names, this only validates that the form is right (for
57 | instance, it doesn't check that the TLD is valid).
58 |
59 | :param string: The string to validate
60 |
61 | :return: Whether the input is a valid hostname
62 | """
63 |
64 | return hostname_regex.match(string) is not None
65 |
66 |
67 | def parse_server_name(server_name: str) -> Tuple[str, Optional[str]]:
68 | """Split a server name into host/port parts.
69 |
70 | No validation is done on the host part. The port part is validated to be
71 | a valid port number.
72 |
73 | Args:
74 | server_name: server name to parse
75 |
76 | Returns:
77 | host/port parts.
78 |
79 | Raises:
80 | ValueError if the server name could not be parsed.
81 | """
82 | try:
83 | if server_name[-1] == "]":
84 | # ipv6 literal, hopefully
85 | return server_name, None
86 |
87 | host_port = server_name.rsplit(":", 1)
88 | host = host_port[0]
89 | port = host_port[1] if host_port[1:] else None
90 |
91 | if port:
92 | port_num = int(port)
93 |
94 | # exclude things like '08090' or ' 8090'
95 | if port != str(port_num) or not (1 <= port_num < 65536):
96 | raise ValueError("Invalid port")
97 |
98 | return host, port
99 | except Exception:
100 | raise ValueError("Invalid server name '%s'" % server_name)
101 |
102 |
103 | def is_valid_matrix_server_name(string: str) -> bool:
104 | """Validate that the given string is a valid Matrix server name.
105 |
106 | A string is a valid Matrix server name if it is one of the following, plus
107 | an optional port:
108 |
109 | a. IPv4 address
110 | b. IPv6 literal (`[IPV6_ADDRESS]`)
111 | c. A valid hostname
112 |
113 | :param string: The string to validate
114 |
115 | :return: Whether the input is a valid Matrix server name
116 | """
117 |
118 | try:
119 | host, port = parse_server_name(string)
120 | except ValueError:
121 | return False
122 |
123 | valid_ipv4_addr = isIPAddress(host)
124 | valid_ipv6_literal = (
125 | host[0] == "[" and host[-1] == "]" and isIPv6Address(host[1:-1])
126 | )
127 |
128 | return valid_ipv4_addr or valid_ipv6_literal or is_valid_hostname(host)
129 |
130 |
131 | def normalise_address(address: str, medium: str) -> str:
132 | if medium == "email":
133 | return address.casefold()
134 | else:
135 | return address
136 |
--------------------------------------------------------------------------------
/sydent/replication/pusher.py:
--------------------------------------------------------------------------------
1 | # Copyright 2014 OpenMarket Ltd
2 | # Copyright 2019 The Matrix.org Foundation C.I.C.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | import logging
17 | from typing import TYPE_CHECKING, List, Tuple
18 |
19 | import twisted.internet.reactor
20 | import twisted.internet.task
21 | from twisted.internet import defer
22 |
23 | from sydent.db.peers import PeerStore
24 | from sydent.db.threepid_associations import LocalAssociationStore
25 | from sydent.replication.peer import LocalPeer, RemotePeer
26 | from sydent.util import time_msec
27 |
28 | if TYPE_CHECKING:
29 | from sydent.sydent import Sydent
30 |
31 | logger = logging.getLogger(__name__)
32 |
33 | # Maximum amount of signed associations to replicate to a peer at a time
34 | ASSOCIATIONS_PUSH_LIMIT = 100
35 |
36 |
37 | class Pusher:
38 | def __init__(self, sydent: "Sydent") -> None:
39 | self.sydent = sydent
40 | self.pushing = False
41 | self.peerStore = PeerStore(self.sydent)
42 | self.local_assoc_store = LocalAssociationStore(self.sydent)
43 |
44 | def setup(self) -> None:
45 | cb = twisted.internet.task.LoopingCall(Pusher.scheduledPush, self)
46 | cb.clock = self.sydent.reactor
47 | cb.start(10.0)
48 |
49 | def doLocalPush(self) -> None:
50 | """
51 | Synchronously push local associations to this server (ie. copy them to globals table)
52 | The local server is essentially treated the same as any other peer except we don't do
53 | the network round-trip and this function can be used so the association goes into the
54 | global table before the http call returns (so clients know it will be available on at
55 | least the same ID server they used)
56 | """
57 | localPeer = LocalPeer(self.sydent)
58 |
59 | signedAssocs, _ = self.local_assoc_store.getSignedAssociationsAfterId(
60 | localPeer.lastId, None
61 | )
62 |
63 | localPeer.pushUpdates(signedAssocs)
64 |
65 | def scheduledPush(self) -> "defer.Deferred[List[Tuple[bool, None]]]":
66 | """Push pending updates to all known remote peers. To be called regularly.
67 |
68 | :returns a deferred.DeferredList of defers, one per peer we're pushing to that will
69 | resolve when pushing to that peer has completed, successfully or otherwise
70 | """
71 | peers = self.peerStore.getAllPeers()
72 |
73 | # Push to all peers in parallel
74 | dl = []
75 | for p in peers:
76 | dl.append(defer.ensureDeferred(self._push_to_peer(p)))
77 | return defer.DeferredList(dl)
78 |
79 | async def _push_to_peer(self, p: "RemotePeer") -> None:
80 | """
81 | For a given peer, retrieves the list of associations that were created since
82 | the last successful push to this peer (limited to ASSOCIATIONS_PUSH_LIMIT) and
83 | sends them.
84 |
85 | :param p: The peer to send associations to.
86 | """
87 | logger.debug("Looking for updates to push to %s", p.servername)
88 |
89 | # Check if a push operation is already active. If so, don't start another
90 | if p.is_being_pushed_to:
91 | logger.debug(
92 | "Waiting for %s to finish pushing...", p.replication_url_origin
93 | )
94 | return
95 |
96 | p.is_being_pushed_to = True
97 |
98 | try:
99 | # Push associations
100 | (
101 | assocs,
102 | latest_assoc_id,
103 | ) = self.local_assoc_store.getSignedAssociationsAfterId(
104 | p.lastSentVersion, ASSOCIATIONS_PUSH_LIMIT
105 | )
106 |
107 | # If there are no updates left to send, break the loop
108 | if not assocs:
109 | return
110 |
111 | logger.info(
112 | "Pushing %d updates to %s", len(assocs), p.replication_url_origin
113 | )
114 | result = await p.pushUpdates(assocs)
115 |
116 | self.peerStore.setLastSentVersionAndPokeSucceeded(
117 | p.servername, latest_assoc_id, time_msec()
118 | )
119 |
120 | logger.info(
121 | "Pushed updates to %s with result %d %s",
122 | p.replication_url_origin,
123 | result.code,
124 | result.phrase,
125 | )
126 | except Exception:
127 | logger.exception("Error pushing updates to %s", p.replication_url_origin)
128 | finally:
129 | # Whether pushing completed or an error occurred, signal that pushing has finished
130 | p.is_being_pushed_to = False
131 |
--------------------------------------------------------------------------------
/res/matrix-org/migration_template.eml.j2:
--------------------------------------------------------------------------------
1 | Date: {{ date|safe }}
2 | From: {{ from|safe }}
3 | To: {{ to|safe }}
4 | Message-ID: {{ messageid|safe }}
5 | Subject: We have changed the way your Matrix account and email address are associated
6 | MIME-Version: 1.0
7 | Content-Type: multipart/alternative;
8 | boundary="{{ multipart_boundary|safe }}"
9 |
10 | --{{ multipart_boundary|safe }}
11 | Content-Type: text/plain; charset=UTF-8
12 | Content-Disposition: inline
13 |
14 | Hello,
15 |
16 | We’ve recently improved how people discover your Matrix account.
17 | In the past, identity services took capitalisation into account when storing email
18 | addresses. This means Alice@example.com and alice@example.com would be considered to be
19 | two different addresses, and could be associated with different Matrix accounts. We’ve
20 | now updated this behaviour so anyone can find you, no matter how your email is capitalised.
21 | As part of this recent update, we've dissociated the Matrix account {{ mxid|safe }} from
22 | this e-mail address.
23 | No action is needed on your part. This doesn’t affect any passwords or password reset
24 | options on your account.
25 |
26 |
27 | About Matrix:
28 |
29 | Matrix.org is an open standard for interoperable, decentralised, real-time communication
30 | over IP, supporting group chat, file transfer, voice and video calling, integrations to
31 | other apps, bridges to other communication systems and much more. It can be used to power
32 | Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere
33 | you need a standard HTTP API for publishing and subscribing to data whilst tracking the
34 | conversation history.
35 |
36 | Matrix defines the standard, and provides open source reference implementations of
37 | Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you
38 | create new communication solutions or extend the capabilities and reach of existing ones.
39 |
40 | Thanks,
41 |
42 | Matrix
43 |
44 | --{{ multipart_boundary|safe }}
45 | Content-Type: text/html; charset=UTF-8
46 | Content-Disposition: inline
47 |
48 |
49 |
50 |
51 |
89 |
90 |
91 |
92 |
93 | | |
94 |
95 |
104 |
105 | Hello,
106 |
107 |
108 | We’ve recently improved how people discover your Matrix account.
109 | In the past, identity services took capitalisation into account when storing email
110 | addresses. This means Alice@example.com and alice@example.com would be considered to
111 | be two different addresses, and could be associated with different Matrix accounts.
112 | We’ve now updated this behaviour so anyone can find you, no matter how your email is
113 | capitalised. As part of this recent update, we've dissociated the Matrix account
114 | {{ mxid|safe }} from this e-mail address.
115 | No action is needed on your part. This doesn’t affect any passwords or password reset
116 | options on your account.
117 |
118 |
119 |
120 | About Matrix:
121 |
122 | Matrix.org is an open standard for interoperable, decentralised, real-time communication
123 | over IP, supporting group chat, file transfer, voice and video calling, integrations to
124 | other apps, bridges to other communication systems and much more. It can be used to power
125 | Instant Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere
126 | you need a standard HTTP API for publishing and subscribing to data whilst tracking the
127 | conversation history.
128 |
129 | Matrix defines the standard, and provides open source reference implementations of
130 | Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you
131 | create new communication solutions or extend the capabilities and reach of existing ones.
132 |
133 | Thanks,
134 |
135 | Matrix
136 | |
137 | |
138 |
139 |
140 |
141 |
142 |
143 | --{{ multipart_boundary|safe }}--
144 |
--------------------------------------------------------------------------------
/sydent/util/ttlcache.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 New Vector Ltd
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import enum
16 | import logging
17 | import time
18 | from typing import Callable, Dict, Generic, Tuple, TypeVar, Union
19 |
20 | import attr
21 | from sortedcontainers import SortedList
22 | from typing_extensions import Literal
23 |
24 | logger = logging.getLogger(__name__)
25 |
26 |
27 | class Sentinel(enum.Enum):
28 | token = enum.auto()
29 |
30 |
31 | K = TypeVar("K")
32 | V = TypeVar("V")
33 |
34 |
35 | class TTLCache(Generic[K, V]):
36 | """A key/value cache implementation where each entry has its own TTL"""
37 |
38 | def __init__(self, cache_name: str, timer: Callable[[], float] = time.time):
39 | self._data: Dict[K, _CacheEntry[K, V]] = {}
40 |
41 | # the _CacheEntries, sorted by expiry time
42 | self._expiry_list: SortedList[_CacheEntry] = SortedList()
43 |
44 | self._timer = timer
45 |
46 | def set(self, key: K, value: V, ttl: float) -> None:
47 | """Add/update an entry in the cache
48 |
49 | :param key: Key for this entry.
50 |
51 | :param value: Value for this entry.
52 |
53 | :param ttl: TTL for this entry, in seconds.
54 | """
55 | expiry = self._timer() + ttl
56 |
57 | self.expire()
58 | e = self._data.pop(key, Sentinel.token)
59 | if e != Sentinel.token:
60 | self._expiry_list.remove(e)
61 |
62 | entry = _CacheEntry(expiry_time=expiry, key=key, value=value)
63 | self._data[key] = entry
64 | self._expiry_list.add(entry)
65 |
66 | def get(
67 | self, key: K, default: Union[V, Literal[Sentinel.token]] = Sentinel.token
68 | ) -> V:
69 | """Get a value from the cache
70 |
71 | :param key: The key to look up.
72 | :param default: default value to return, if key is not found. If not
73 | set, and the key is not found, a KeyError will be raised.
74 |
75 | :returns a value from the cache, or the default.
76 | """
77 | self.expire()
78 | e = self._data.get(key, Sentinel.token)
79 | if e is Sentinel.token:
80 | if default is Sentinel.token:
81 | raise KeyError(key)
82 | return default
83 | return e.value
84 |
85 | def get_with_expiry(self, key: K) -> Tuple[V, float]:
86 | """Get a value, and its expiry time, from the cache
87 |
88 | :param key: key to look up
89 |
90 | :returns The value from the cache, and the expiry time.
91 | :rtype: Tuple[Any, float]
92 |
93 | Raises:
94 | KeyError if the entry is not found
95 | """
96 | self.expire()
97 | try:
98 | e = self._data[key]
99 | except KeyError:
100 | raise
101 | return e.value, e.expiry_time
102 |
103 | def pop(
104 | self, key: K, default: Union[V, Literal[Sentinel.token]] = Sentinel.token
105 | ) -> V:
106 | """Remove a value from the cache
107 |
108 | If key is in the cache, remove it and return its value, else return default.
109 | If default is not given and key is not in the cache, a KeyError is raised.
110 |
111 | :param key: key to look up
112 | :param default: default value to return, if key is not found. If not
113 | set, and the key is not found, a KeyError will be raised
114 |
115 | :returns a value from the cache, or the default
116 | """
117 | self.expire()
118 | e = self._data.pop(key, Sentinel.token)
119 | if e is Sentinel.token:
120 | if default == Sentinel.token:
121 | raise KeyError(key)
122 | return default
123 | self._expiry_list.remove(e)
124 | return e.value
125 |
126 | def __getitem__(self, key: K) -> V:
127 | return self.get(key)
128 |
129 | def __delitem__(self, key: K) -> None:
130 | self.pop(key)
131 |
132 | def __contains__(self, key: K) -> bool:
133 | return key in self._data
134 |
135 | def __len__(self) -> int:
136 | self.expire()
137 | return len(self._data)
138 |
139 | def expire(self) -> None:
140 | """Run the expiry on the cache. Any entries whose expiry times are due will
141 | be removed
142 | """
143 | now = self._timer()
144 | while self._expiry_list:
145 | first_entry = self._expiry_list[0]
146 | if first_entry.expiry_time - now > 0.0:
147 | break
148 | del self._data[first_entry.key]
149 | del self._expiry_list[0]
150 |
151 |
152 | @attr.s(frozen=True)
153 | class _CacheEntry(Generic[K, V]):
154 | """TTLCache entry"""
155 |
156 | # expiry_time is the first attribute, so that entries are sorted by expiry.
157 | expiry_time: float = attr.ib()
158 | key: K = attr.ib()
159 | value: V = attr.ib()
160 |
--------------------------------------------------------------------------------
/sydent/db/peers.py:
--------------------------------------------------------------------------------
1 | # Copyright 2014 OpenMarket Ltd
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
16 |
17 | from sydent.replication.peer import RemotePeer
18 |
19 | if TYPE_CHECKING:
20 | from sydent.sydent import Sydent
21 |
22 |
23 | class PeerStore:
24 | def __init__(self, sydent: "Sydent") -> None:
25 | self.sydent = sydent
26 |
27 | def getPeerByName(self, name: str) -> Optional[RemotePeer]:
28 | """
29 | Retrieves a remote peer using it's server name.
30 |
31 | :param name: The server name of the peer.
32 |
33 | :return: The retrieved peer.
34 | """
35 | cur = self.sydent.db.cursor()
36 | res = cur.execute(
37 | "select p.name, p.port, p.lastSentVersion, pk.alg, pk.key from peers p, peer_pubkeys pk "
38 | "where p.name = ? and pk.peername = p.name and p.active = 1",
39 | (name,),
40 | )
41 |
42 | # Type safety: if the query returns no rows, pubkeys will be empty
43 | # and we'll return None before using serverName. Otherwise, we'll read
44 | # at least one row and assign serverName a string value, because the
45 | # `name` column is declared `not null` in the DB.
46 | serverName: str = None # type: ignore[assignment]
47 | port: Optional[int] = None
48 | lastSentVer: Optional[int] = None
49 | pubkeys: Dict[str, str] = {}
50 |
51 | row: Tuple[str, Optional[int], Optional[int], str, str]
52 | for row in res.fetchall():
53 | serverName = row[0]
54 | port = row[1]
55 | lastSentVer = row[2]
56 | pubkeys[row[3]] = row[4]
57 |
58 | if len(pubkeys) == 0:
59 | return None
60 |
61 | p = RemotePeer(self.sydent, serverName, port, pubkeys, lastSentVer)
62 |
63 | return p
64 |
65 | def getAllPeers(self) -> List[RemotePeer]:
66 | """
67 | Retrieve all of the remote peers from the database.
68 |
69 | :return: A list of the remote peers this server knows about.
70 | """
71 | cur = self.sydent.db.cursor()
72 | res = cur.execute(
73 | "select p.name, p.port, p.lastSentVersion, pk.alg, pk.key from peers p, peer_pubkeys pk "
74 | "where pk.peername = p.name and p.active = 1"
75 | )
76 |
77 | peers = []
78 |
79 | # Safety: we need to convince ourselves that `peername` will be not None
80 | # when passed to `RemotePeer`.
81 | #
82 | # If `res` is empty, then `pubkeys` will start empty and never be written to.
83 | # So we will never create a `RemotePeer`. That's fine.
84 | #
85 | # Otherwise we process at least one row. The first row we process will
86 | # satisfy `row[0] is not None` because `name` is nonnull in the schema.
87 | # `pubkeys` will be empty, so we skip the innermost `if` and assign peername
88 | # to be a string. There are no further assignments of `None` to `peername`;
89 | # it will be a string whenever we use it.
90 | peername: str = None # type: ignore[assignment]
91 | port = None
92 | lastSentVer = None
93 | pubkeys: Dict[str, str] = {}
94 |
95 | row: Tuple[str, Optional[int], Optional[int], str, str]
96 | for row in res.fetchall():
97 | if row[0] != peername:
98 | if len(pubkeys) > 0:
99 | p = RemotePeer(self.sydent, peername, port, pubkeys, lastSentVer)
100 | peers.append(p)
101 | pubkeys = {}
102 | peername = row[0]
103 | port = row[1]
104 | lastSentVer = row[2]
105 | pubkeys[row[3]] = row[4]
106 |
107 | if len(pubkeys) > 0:
108 | p = RemotePeer(self.sydent, peername, port, pubkeys, lastSentVer)
109 | peers.append(p)
110 |
111 | return peers
112 |
113 | def setLastSentVersionAndPokeSucceeded(
114 | self,
115 | peerName: str,
116 | lastSentVersion: Optional[int],
117 | lastPokeSucceeded: Optional[int],
118 | ) -> None:
119 | """
120 | Sets the ID of the last association sent to a given peer and the time of the
121 | last successful request sent to that peer.
122 |
123 | :param peerName: The server name of the peer.
124 | :param lastSentVersion: The ID of the last association sent to that peer.
125 | :param lastPokeSucceeded: The timestamp in milliseconds of the last successful
126 | request sent to that peer.
127 | """
128 | cur = self.sydent.db.cursor()
129 | cur.execute(
130 | "update peers set lastSentVersion = ?, lastPokeSucceededAt = ? "
131 | "where name = ?",
132 | (lastSentVersion, lastPokeSucceeded, peerName),
133 | )
134 | self.sydent.db.commit()
135 |
--------------------------------------------------------------------------------
/sydent/http/servlets/registerservlet.py:
--------------------------------------------------------------------------------
1 | # Copyright 2019 The Matrix.org Foundation C.I.C.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import logging
16 | import urllib
17 | from http import HTTPStatus
18 | from json import JSONDecodeError
19 | from typing import TYPE_CHECKING, Dict
20 |
21 | from twisted.internet.error import ConnectError, DNSLookupError
22 | from twisted.web.client import ResponseFailed
23 | from twisted.web.server import Request
24 |
25 | from sydent.http.httpclient import FederationHttpClient
26 | from sydent.http.servlets import SydentResource, asyncjsonwrap, get_args, send_cors
27 | from sydent.types import JsonDict
28 | from sydent.users.tokens import issueToken
29 | from sydent.util.stringutils import is_valid_matrix_server_name
30 |
31 | if TYPE_CHECKING:
32 | from sydent.sydent import Sydent
33 |
34 | logger = logging.getLogger(__name__)
35 |
36 |
37 | class RegisterServlet(SydentResource):
38 | isLeaf = True
39 |
40 | def __init__(self, syd: "Sydent") -> None:
41 | super().__init__()
42 | self.sydent = syd
43 | self.client = FederationHttpClient(self.sydent)
44 |
45 | @asyncjsonwrap
46 | async def render_POST(self, request: Request) -> JsonDict:
47 | """
48 | Register with the Identity Server
49 | """
50 | send_cors(request)
51 |
52 | args = get_args(request, ("matrix_server_name", "access_token"))
53 |
54 | matrix_server = args["matrix_server_name"].lower()
55 |
56 | if self.sydent.config.general.homeserver_allow_list:
57 | if matrix_server not in self.sydent.config.general.homeserver_allow_list:
58 | request.setResponseCode(403)
59 | return {
60 | "errcode": "M_UNAUTHORIZED",
61 | "error": "This homeserver is not authorized to access this server.",
62 | }
63 |
64 | if not is_valid_matrix_server_name(matrix_server):
65 | request.setResponseCode(400)
66 | return {
67 | "errcode": "M_INVALID_PARAM",
68 | "error": "matrix_server_name must be a valid Matrix server name (IP address or hostname)",
69 | }
70 |
71 | def federation_request_problem(error: str) -> Dict[str, str]:
72 | logger.warning(error)
73 | request.setResponseCode(HTTPStatus.INTERNAL_SERVER_ERROR)
74 | return {
75 | "errcode": "M_UNKNOWN",
76 | "error": error,
77 | }
78 |
79 | try:
80 | result = await self.client.get_json(
81 | "matrix://%s/_matrix/federation/v1/openid/userinfo?access_token=%s"
82 | % (
83 | matrix_server,
84 | urllib.parse.quote(args["access_token"]),
85 | ),
86 | 1024 * 5,
87 | )
88 | except (DNSLookupError, ConnectError, ResponseFailed) as e:
89 | return federation_request_problem(
90 | f"Unable to contact the Matrix homeserver ({type(e).__name__})"
91 | )
92 | except JSONDecodeError:
93 | return federation_request_problem(
94 | "The Matrix homeserver returned invalid JSON"
95 | )
96 |
97 | if "sub" not in result:
98 | return federation_request_problem(
99 | "The Matrix homeserver did not include 'sub' in its response",
100 | )
101 |
102 | user_id = result["sub"]
103 |
104 | if not isinstance(user_id, str):
105 | return federation_request_problem(
106 | "The Matrix homeserver returned a malformed reply"
107 | )
108 |
109 | user_id_components = user_id.split(":", 1)
110 |
111 | # Ensure there's a localpart and domain in the returned user ID.
112 | if len(user_id_components) != 2:
113 | return federation_request_problem(
114 | "The Matrix homeserver returned an invalid MXID"
115 | )
116 |
117 | user_id_server = user_id_components[1]
118 |
119 | if not is_valid_matrix_server_name(user_id_server):
120 | return federation_request_problem(
121 | "The Matrix homeserver returned an invalid MXID"
122 | )
123 |
124 | if user_id_server != matrix_server:
125 | return federation_request_problem(
126 | "The Matrix homeserver returned a MXID belonging to another homeserver"
127 | )
128 |
129 | tok = issueToken(self.sydent, user_id)
130 |
131 | # XXX: `token` is correct for the spec, but we released with `access_token`
132 | # for a substantial amount of time. Serve both to make spec-compliant clients
133 | # happy.
134 | return {
135 | "access_token": tok,
136 | "token": tok,
137 | }
138 |
139 | def render_OPTIONS(self, request: Request) -> bytes:
140 | send_cors(request)
141 | return b""
142 |
--------------------------------------------------------------------------------
44 | ![[matrix]](http://matrix.org/img/matrix-120x51.png)
101 |