Merge branch 'braid/integration-dind' into 'main'

feat: migrate e2ee test to DinD

See merge request famedly/company/frontend/famedlysdk!1179

.gitlab-ci.yml

@@ -61,14 +61,39 @@ coverage_without_olm:
 
 e2ee_test:
   stage: test
-  image: registry.gitlab.com/famedly/company/frontend/flutter-dockerimages/flutter-linux/stable:${FLUTTER_IMAGE_TAG}
+  image:
+    name: registry.gitlab.com/famedly/company/frontend/flutter-dockerimages/flutter/stable:${FLUTTER_IMAGE_TAG}
+    pull_policy: if-not-present
+  services:
+    - name: docker:dind
+      alias: docker
+  variables:
+    # activate container-to-container networking
+    FF_NETWORK_PER_BUILD: "true"
+    # Tell docker CLI how to talk to Docker daemon.
+    DOCKER_HOST: tcp://docker:2375/
+    # Use the overlayfs driver for improved performance.
+    DOCKER_DRIVER: overlay2
+    # Disable TLS since we're running inside local network.
+    DOCKER_TLS_CERTDIR: ""
   dependencies: []
+  parallel:
+    matrix:
+      - HOMESERVER:
+          - synapse
+          - dendrite
+          - conduit
+  before_script:
+    - scripts/integration-prepare-alpine.sh
+    # create Synapse instance
+    - scripts/integration-server-$HOMESERVER.sh
+    # properly set the homeserver IP and create test users
+    - scripts/integration-prepare-homeserver.sh
+    # setup OLM
+    - scripts/prepare.sh
   script:
-    - printf "abstract class TestUser {\n  static const String homeserver = '$TEST_HOMESERVER';\n  static const String username = '$TEST_USER1';\n  static const String username2 = '$TEST_USER2';\n  static const String displayname = '$TEST_USER1_NAME';\n  static const String displayname2 = '$TEST_USER2_NAME';\n  static const String password = '$TEST_USER_PASSWORD';\n}" > ./test_driver/test_config.dart
-    - ./scripts/prepare.sh
     - ./scripts/test_driver.sh
-  timeout: 16m
-  resource_group: e2ee_test
+  timeout: 10m
 
 dry-run:
   stage: deploy

lib/src/utils/pushrule_evaluator.dart

@@ -51,6 +51,7 @@ class EvaluatedPushRuleAction {
 
 class _PatternCondition {
   RegExp pattern = RegExp('');
+
   // what field to match on, i.e. content.body
   String field = '';
 
@@ -140,6 +141,7 @@ class _MemberCountCondition {
       count = int.parse(is_);
     }
   }
+
   bool match(int memberCount) {
     switch (op) {
       case _CountComparisonOp.ge:
@@ -234,11 +236,19 @@ class PushruleEvaluator {
   PushruleEvaluator.fromRuleset(PushRuleSet ruleset) {
     for (final o in ruleset.override ?? []) {
       if (!o.enabled) continue;
-      _override.add(_OptimizedRules.fromRule(o));
+      try {
+        _override.add(_OptimizedRules.fromRule(o));
+      } catch (e) {
+        Logs().d('Error parsing push rule $o', e);
+      }
     }
     for (final u in ruleset.underride ?? []) {
       if (!u.enabled) continue;
-      _underride.add(_OptimizedRules.fromRule(u));
+      try {
+        _underride.add(_OptimizedRules.fromRule(u));
+      } catch (e) {
+        Logs().d('Error parsing push rule $u', e);
+      }
     }
     for (final c in ruleset.content ?? []) {
       if (!c.enabled) continue;
@@ -252,7 +262,11 @@ class PushruleEvaluator {
         default$: c.default$,
         enabled: c.enabled,
       );
-      _content_rules.add(_OptimizedRules.fromRule(rule));
+      try {
+        _content_rules.add(_OptimizedRules.fromRule(rule));
+      } catch (e) {
+        Logs().d('Error parsing push rule $rule', e);
+      }
     }
     for (final r in ruleset.room ?? []) {
       if (r.enabled) {

scripts/integration-prepare-alpine.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+apk update && apk add docker drill grep olm cmake

scripts/integration-prepare-homeserver.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+IP_ADDRESS="$(drill docker | grep -m 1 -P "\d+\.\d+\.\d+.\d+" | awk -F ' ' '{print $NF}')"
+
+echo "Homeserver IP found at $IP_ADDRESS."
+
+echo "Waiting for homeserver to be available..."
+
+while ! curl -XGET "http://$IP_ADDRESS/_matrix/client/v3/login" >/dev/null 2>/dev/null; do
+  sleep 2
+done
+echo "Homeserver is up."
+
+sed -i "s/localhost/$IP_ADDRESS/g" test_driver/test_config.dart
+
+curl -XPOST -d '{"username":"alice", "password":"AliceInWonderland", "inhibit_login":true, "auth": {"type":"m.login.dummy"}}' "http://$IP_ADDRESS/_matrix/client/r0/register"
+curl -XPOST -d '{"username":"bob", "password":"JoWirSchaffenDas", "inhibit_login":true, "auth": {"type":"m.login.dummy"}}' "http://$IP_ADDRESS/_matrix/client/r0/register"
+curl -XPOST -d '{"username":"trudy", "password":"HaveIBeenPwned", "inhibit_login":true, "auth": {"type":"m.login.dummy"}}' "http://$IP_ADDRESS/_matrix/client/r0/register"

scripts/integration-server-conduit.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+ docker run -d \
+  -e CONDUIT_SERVER_NAME="famedlysdk.test" \
+  -e CONDUIT_PORT="80" \
+  -e CONDUIT_DATABASE_BACKEND="rocksdb" \
+  -e CONDUIT_ALLOW_REGISTRATION=true \
+  -e CONDUIT_ALLOW_FEDERATION=true \
+  -e CONDUIT_MAX_REQUEST_SIZE="20000000" \
+  -e CONDUIT_TRUSTED_SERVERS="[\"conduit.rs\"]" \
+  -e CONDUIT_MAX_CONCURRENT_REQUESTS="100" \
+  -e CONDUIT_LOG="info,rocket=off,_=off,sled=off" \
+  --name conduit -p 80:80 matrixconduit/matrix-conduit:latest

scripts/integration-server-dendrite.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+chown -R 991:991 test_driver/dendrite
+
+# creating integration test SSL certificates
+docker run --rm --entrypoint="" \
+  --volume="$(pwd)/test_driver/dendrite/data":/mnt:rw \
+  matrixdotorg/dendrite-monolith:latest \
+  /usr/bin/generate-keys \
+  -private-key /mnt/matrix_key.pem \
+  -tls-cert /mnt/server.crt \
+  -tls-key /mnt/server.key
+
+ docker run -d --volume="$(pwd)/test_driver/dendrite/data":/etc/dendrite:rw \
+  --name dendrite -p 80:8008 matrixdotorg/dendrite-monolith:latest -really-enable-open-registration

scripts/integration-server-synapse.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+chown -R 991:991 test_driver/synapse
+docker run -d --name synapse --user 991:991 --volume="$(pwd)/test_driver/synapse/data":/data:rw -p 80:80 matrixdotorg/synapse:latest

scripts/prepare.sh

@@ -1,27 +1,37 @@
-#!/bin/sh
+#!/usr/bin/env bash
+
+ENTRYPOINT="$(pwd)"
 
 mkdir js
 cd js
+
 curl -O 'https://packages.matrix.org/npm/olm/olm-3.1.4.tgz'
 tar xaf olm-3.1.4.tgz
+
 cd ..
 
-if [ -f /usr/lib/x86_64-linux-gnu/libolm.so.3 ]
-then
-mkdir -p ffi/olm/
-ln -sf /usr/lib/x86_64-linux-gnu/libolm.so.3 ffi/olm/libolm.so
+if [ -f /usr/lib/x86_64-linux-gnu/libolm.so.3 ]; then
+  mkdir -p ffi/olm/
+  ln -sf /usr/lib/x86_64-linux-gnu/libolm.so.3 ffi/olm/libolm.so
+# alpine specific location
+elif [ -f /usr/lib/libolm.so.3 ]; then
+  mkdir -p ffi/olm
+  ln -sf /usr/lib/libolm.so.3 ffi/olm/libolm.so
 else
-cd ffi
-pushd ffi
-git clone --depth 1 https://gitlab.matrix.org/matrix-org/olm.git
-cd olm
-cmake -DCMAKE_BUILD_TYPE=Release .
-cmake --build .
-cd ..
+  mkdir ffi
+  cd ffi
+  cd ..
+  git clone --depth 1 https://gitlab.matrix.org/matrix-org/olm.git
+  cd olm
+  cmake -DCMAKE_BUILD_TYPE=Release .
+  cmake --build .
+  cd ..
 fi
 
+cd "$ENTRYPOINT"
+
 if which flutter >/dev/null; then
-    flutter pub get
+  flutter pub get
 else
-    dart pub get
+  dart pub get
 fi

test_driver/dendrite/data/dendrite.yaml

@@ -0,0 +1,327 @@
+# This is the Dendrite configuration file.
+#
+# The configuration is split up into sections - each Dendrite component has a
+# configuration section, in addition to the "global" section which applies to
+# all components.
+
+# The version of the configuration file.
+version: 2
+
+# Global Matrix configuration. This configuration applies to all components.
+global:
+  # The domain name of this homeserver.
+  server_name: famedlysdk.test
+
+  # The path to the signing private key file, used to sign requests and events.
+  # Note that this is NOT the same private key as used for TLS! To generate a
+  # signing key, use "./bin/generate-keys --private-key matrix_key.pem".
+  private_key: matrix_key.pem
+
+  # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
+  # to old signing private keys that were formerly in use on this domain. These
+  # keys will not be used for federation request or event signing, but will be
+  # provided to any other homeserver that asks when trying to verify old events.
+  old_private_keys:
+  #  - private_key: old_matrix_key.pem
+  #    expired_at: 1601024554498
+
+  # How long a remote server can cache our server signing key before requesting it
+  # again. Increasing this number will reduce the number of requests made by other
+  # servers for our key but increases the period that a compromised key will be
+  # considered valid by other homeservers.
+  key_validity_period: 168h0m0s
+
+  # Global database connection pool, for PostgreSQL monolith deployments only. If
+  # this section is populated then you can omit the "database" blocks in all other
+  # sections. For polylith deployments, or monolith deployments using SQLite databases,
+  # you must configure the "database" block for each component instead.
+  database:
+    connection_string:
+    max_open_conns:
+    max_idle_conns:
+    conn_max_lifetime:
+
+  # Configuration for in-memory caches. Caches can often improve performance by
+  # keeping frequently accessed items (like events, identifiers etc.) in memory
+  # rather than having to read them from the database.
+  cache:
+    # The estimated maximum size for the global cache in bytes, or in terabytes,
+    # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or
+    # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a
+    # memory limit for the entire process. A cache that is too small may ultimately
+    # provide little or no benefit.
+    max_size_estimated: 1gb
+
+    # The maximum amount of time that a cache entry can live for in memory before
+    # it will be evicted and/or refreshed from the database. Lower values result in
+    # easier admission of new cache entries but may also increase database load in
+    # comparison to higher values, so adjust conservatively. Higher values may make
+    # it harder for new items to make it into the cache, e.g. if new rooms suddenly
+    # become popular.
+    max_age: 1h
+
+  # The server name to delegate server-server communications to, with optional port
+  # e.g. localhost:443
+  well_known_server_name: ""
+
+  # Lists of domains that the server will trust as identity servers to verify third
+  # party identifiers such as phone numbers and email addresses.
+  trusted_third_party_id_servers:
+    - matrix.org
+    - vector.im
+
+  # Disables federation. Dendrite will not be able to communicate with other servers
+  # in the Matrix federation and the federation API will not be exposed.
+  disable_federation: false
+
+  # Configures the handling of presence events. Inbound controls whether we receive
+  # presence events from other servers, outbound controls whether we send presence
+  # events for our local users to other servers.
+  presence:
+    enable_inbound: false
+    enable_outbound: false
+
+  # Configures phone-home statistics reporting. These statistics contain the server
+  # name, number of active users and some information on your deployment config.
+  # We use this information to understand how Dendrite is being used in the wild.
+  report_stats:
+    enabled: false
+    endpoint: https://matrix.org/report-usage-stats/push
+
+  # Server notices allows server admins to send messages to all users on the server.
+  server_notices:
+    enabled: false
+    # The local part, display name and avatar URL (as a mxc:// URL) for the user that
+    # will send the server notices. These are visible to all users on the deployment.
+    local_part: "_server"
+    display_name: "Server Alerts"
+    avatar_url: ""
+    # The room name to be used when sending server notices. This room name will
+    # appear in user clients.
+    room_name: "Server Alerts"
+
+  # Configuration for NATS JetStream
+  jetstream:
+    # A list of NATS Server addresses to connect to. If none are specified, an
+    # internal NATS server will be started automatically when running Dendrite in
+    # monolith mode. For polylith deployments, it is required to specify the address
+    # of at least one NATS Server node.
+    addresses:
+    # - localhost:4222
+
+    # Persistent directory to store JetStream streams in. This directory should be
+    # preserved across Dendrite restarts.
+    storage_path: ./
+
+    # The prefix to use for stream names for this homeserver - really only useful
+    # if you are running more than one Dendrite server on the same NATS deployment.
+    topic_prefix: Dendrite
+
+  # Configuration for Prometheus metric collection.
+  metrics:
+    enabled: false
+    basic_auth:
+      username: metrics
+      password: metrics
+
+  # Optional DNS cache. The DNS cache may reduce the load on DNS servers if there
+  # is no local caching resolver available for use.
+  dns_cache:
+    enabled: false
+    cache_size: 256
+    cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration
+
+# Configuration for the Appservice API.
+app_service_api:
+  database:
+    connection_string: file:app_service_api.db
+
+  # Disable the validation of TLS certificates of appservices. This is
+  # not recommended in production since it may allow appservice traffic
+  # to be sent to an insecure endpoint.
+  disable_tls_validation: true
+
+  # Appservice configuration files to load into this homeserver.
+  config_files:
+  #  - /path/to/appservice_registration.yaml
+
+# Configuration for the Client API.
+client_api:
+  # Prevents new users from being able to register on this homeserver, except when
+  # using the registration shared secret below.
+  registration_disabled: false
+
+  # Prevents new guest accounts from being created. Guest registration is also
+  # disabled implicitly by setting 'registration_disabled' above.
+  guests_disabled: true
+
+  # If set, allows registration by anyone who knows the shared secret, regardless
+  # of whether registration is otherwise disabled.
+  registration_shared_secret: ""
+
+  # Whether to require reCAPTCHA for registration. If you have enabled registration
+  # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used
+  # for coordinated spam attacks.
+  enable_registration_captcha: false
+
+  # Settings for ReCAPTCHA.
+  recaptcha_public_key: ""
+  recaptcha_private_key: ""
+  recaptcha_bypass_secret: ""
+  recaptcha_siteverify_api: ""
+
+  # TURN server information that this homeserver should send to clients.
+  turn:
+    turn_user_lifetime: ""
+    turn_uris:
+    #  - turn:turn.server.org?transport=udp
+    #  - turn:turn.server.org?transport=tcp
+    turn_shared_secret: ""
+    turn_username: ""
+    turn_password: ""
+
+  # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
+  # number of "slots" have been taken by requests from a specific host. Each "slot"
+  # will be released after the cooloff time in milliseconds. Server administrators
+  # and appservice users are exempt from rate limiting by default.
+  rate_limiting:
+    enabled: true
+    threshold: 5
+    cooloff_ms: 500
+    exempt_user_ids:
+    #  - "@user:domain.com"
+
+# Configuration for the Federation API.
+federation_api:
+  database:
+    connection_string: file:federation_api.db
+
+  # How many times we will try to resend a failed transaction to a specific server. The
+  # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once
+  # the max retries are exceeded, Dendrite will no longer try to send transactions to
+  # that server until it comes back to life and connects to us again.
+  send_max_retries: 16
+
+  # Disable the validation of TLS certificates of remote federated homeservers. Do not
+  # enable this option in production as it presents a security risk!
+  disable_tls_validation: false
+
+  # Perspective keyservers to use as a backup when direct key fetches fail. This may
+  # be required to satisfy key requests for servers that are no longer online when
+  # joining some rooms.
+  key_perspectives:
+    - server_name: matrix.org
+      keys:
+        - key_id: ed25519:auto
+          public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
+        - key_id: ed25519:a_RXGa
+          public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
+
+  # This option will control whether Dendrite will prefer to look up keys directly
+  # or whether it should try perspective servers first, using direct fetches as a
+  # last resort.
+  prefer_direct_fetch: false
+
+# Configuration for the Media API.
+media_api:
+  database:
+    connection_string: file:media_api.db
+  
+  # Storage path for uploaded media. May be relative or absolute.
+  base_path: ./media_store
+
+  # The maximum allowed file size (in bytes) for media uploads to this homeserver
+  # (0 = unlimited). If using a reverse proxy, ensure it allows requests at least
+  #this large (e.g. the client_max_body_size setting in nginx).
+  max_file_size_bytes: 10485760
+
+  # Whether to dynamically generate thumbnails if needed.
+  dynamic_thumbnails: false
+
+  # The maximum number of simultaneous thumbnail generators to run.
+  max_thumbnail_generators: 10
+
+  # A list of thumbnail sizes to be generated for media content.
+  thumbnail_sizes:
+    - width: 32
+      height: 32
+      method: crop
+    - width: 96
+      height: 96
+      method: crop
+    - width: 640
+      height: 480
+      method: scale
+
+# Configuration for enabling experimental MSCs on this homeserver.
+mscs:
+  database:
+    connection_string: file:mscs.db
+  mscs:
+  #  - msc2836  # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
+  #  - msc2946  # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
+
+# Configuration for the Sync API.
+sync_api:
+  # This option controls which HTTP header to inspect to find the real remote IP
+  # address of the client. This is likely required if Dendrite is running behind
+  # a reverse proxy server.
+  # real_ip_header: X-Real-IP
+  database:
+    connection_string: file:sync_api.db
+
+key_server:
+  database:
+   connection_string: file:key_server.db
+
+room_server:
+  database:
+    connection_string: file:room_server.db
+  
+
+# Configuration for the User API.
+user_api:
+  account_database:
+    connection_string: file:user_api.db
+
+  # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31
+  # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information.
+  # Setting this lower makes registration/login consume less CPU resources at the cost
+  # of security should the database be compromised. Setting this higher makes registration/login
+  # consume more CPU resources but makes it harder to brute force password hashes. This value
+  # can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds).
+  bcrypt_cost: 10
+
+  # The length of time that a token issued for a relying party from
+  # /_matrix/client/r0/user/{userId}/openid/request_token endpoint
+  # is considered to be valid in milliseconds.
+  # The default lifetime is 3600000ms (60 minutes).
+  # openid_token_lifetime_ms: 3600000
+
+# Configuration for Opentracing.
+# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
+# how this works and how to set it up.
+tracing:
+  enabled: false
+  jaeger:
+    serviceName: ""
+    disabled: false
+    rpc_metrics: false
+    tags: []
+    sampler: null
+    reporter: null
+    headers: null
+    baggage_restrictions: null
+    throttler: null
+
+# Logging configuration. The "std" logging type controls the logs being sent to
+# stdout. The "file" logging type controls logs being written to a log folder on
+# the disk. Supported log levels are "debug", "info", "warn", "error".
+logging:
+  - type: std
+    level: info
+  - type: file
+    level: info
+    params:
+      path: ./logs
+

test_driver/matrixsdk_test.dart

@@ -44,21 +44,19 @@ void test() async {
     Logs().i('++++ Login Alice at ++++');
     testClientA = Client('TestClientA', databaseBuilder: getDatabase);
     await testClientA.checkHomeserver(Uri.parse(TestUser.homeserver));
-    // Workaround: Logging in with displayname instead of mxid because of a bug
-    // in the UIA proxy: https://gitlab.com/famedly/company/backend/services/uia-proxy/-/issues/27
-    // When this gets fixed, we no longer need the displayname login at all.
     await testClientA.login(LoginType.mLoginPassword,
-        identifier: AuthenticationUserIdentifier(user: TestUser.displayname),
+        identifier:
+            AuthenticationUserIdentifier(user: TestUser.username.localpart!),
         password: TestUser.password);
     assert(testClientA.encryptionEnabled);
 
     Logs().i('++++ Login Bob ++++');
     testClientB = Client('TestClientB', databaseBuilder: getDatabase);
     await testClientB.checkHomeserver(Uri.parse(TestUser.homeserver));
-    // Workaround: Logging in with displayname instead of mxid
     await testClientB.login(LoginType.mLoginPassword,
-        identifier: AuthenticationUserIdentifier(user: TestUser.displayname2),
-        password: TestUser.password);
+        identifier:
+            AuthenticationUserIdentifier(user: TestUser.username2.localpart!),
+        password: TestUser.password2);
     assert(testClientB.encryptionEnabled);
 
     Logs().i('++++ (Alice) Leave all rooms ++++');
@@ -101,7 +99,10 @@ void test() async {
         .deviceKeys[testClientB.deviceID!]!.blocked);
 
     Logs().i('++++ (Alice) Create room and invite Bob ++++');
-    await testClientA.createRoom(invite: [TestUser.username2]);
+    await testClientA.startDirectChat(
+      TestUser.username2,
+      enableEncryption: false,
+    );
     await Future.delayed(Duration(seconds: 1));
     final room = testClientA.rooms.first;
     final roomId = room.id;
@@ -254,7 +255,7 @@ void test() async {
     // Workaround: Logging in with displayname instead of mxid
     await testClientC.login(LoginType.mLoginPassword,
         identifier: AuthenticationUserIdentifier(user: TestUser.displayname2),
-        password: TestUser.password);
+        password: TestUser.password2);
     await Future.delayed(Duration(seconds: 3));
 
     Logs().i("++++ (Alice) Send again encrypted message: '$testMessage4' ++++");

test_driver/synapse/data/.gitignore

@@ -0,0 +1,2 @@
+localhost.signing.key
+homeserver.db*

test_driver/synapse/data/localhost.log.config

@@ -0,0 +1,27 @@
+version: 1
+
+formatters:
+  precise:
+    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+handlers:
+
+
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+
+loggers:
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: INFO
+
+root:
+    level: INFO
+
+
+    handlers: [console]
+
+
+disable_existing_loggers: false

test_driver/test_config.dart

@@ -1,8 +1,9 @@
 class TestUser {
-  static const String homeserver = 'https://enter-your-server.here';
-  static const String username = 'alice';
-  static const String username2 = 'bob';
+  static const String homeserver = 'http://localhost';
+  static const String username = '@alice:famedlysdk.test';
+  static const String username2 = '@bob:famedlysdk.test';
   static const String displayname = 'Alice';
   static const String displayname2 = 'Bob';
-  static const String password = '1234';
+  static const String password = 'AliceInWonderland';
+  static const String password2 = 'JoWirSchaffenDas';
 }