Merge pull request #1519 from famedly/nico/github-actions

feat: Use github actions

.github/workflows/app.yml

@@ -0,0 +1,98 @@
+name: "All the sdk specific jobs"
+
+on:
+  workflow_call:
+    inputs:
+      flutter_version:
+        description: "The flutter version used for tests and builds"
+        type: string
+        required: true
+      dart_version:
+        description: "The dart version used for tests and builds"
+        type: string
+        required: true
+
+jobs:
+  e2ee_test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        homeserver: [synapse, dendrite, conduit]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run tests
+        run: |
+          export NETWORK='--network mynet'
+          docker network create mynet
+          # deploy homeserver instance
+          scripts/integration-server-${{matrix.homeserver}}.sh
+          docker run $NETWORK --env GITHUB_ACTIONS="${GITHUB_ACTIONS}" --env HOMESERVER_IMPLEMENTATION="${{matrix.homeserver}}" --env HOMESERVER="${{startsWith('dendrite', matrix.homeserver) && format('{0}:8008', matrix.homeserver) || matrix.homeserver }}" --volume="$(pwd):/workdir" --workdir /workdir ghcr.io/famedly/container-image-flutter/flutter:${{inputs.flutter_version}} /bin/bash -c "set -e
+          scripts/integration-prepare-alpine.sh
+          # create test user environment variables
+          source scripts/integration-create-environment-variables.sh
+          # properly set the homeserver IP and create test users
+          scripts/integration-prepare-homeserver.sh 
+          # setup OLM
+          scripts/prepare.sh
+          scripts/test_driver.sh"
+
+# coverage is done on the flutter image because we setup olm there
+# coverage_without_olm is done on dart images because why not :D
+  coverage:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/famedly/container-image-flutter/flutter:${{inputs.flutter_version}}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run tests
+        run: |
+          sed -i 's/#flutter_test/flutter_test/g' pubspec.yaml
+          rm -r example
+          ./scripts/prepare.sh
+          ./scripts/test.sh
+
+  coverage_without_olm:
+    runs-on: ubuntu-latest
+    container:
+      image: dart:${{inputs.dart_version}}
+    env:
+      NO_OLM: 1
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run tests
+        run: |
+          apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl lcov python3 python3-distutils
+          curl -o /bin/lcov_cobertura.py https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py && sed 's/env python/env python3/' -i /bin/lcov_cobertura.py && chmod +x /bin/lcov_cobertura.py
+          dart pub get
+          ./scripts/test.sh
+
+  pub-dev-dry-run:
+    runs-on: ubuntu-latest
+    container:
+      image: dart:${{inputs.dart_version}}
+    steps:
+      - uses: actions/checkout@v3
+      - name: pub.dev publish dry run
+        run: |
+          dart pub get
+          dart pub publish --dry-run
+
+  pub-dev:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+    container:
+      image: dart:${{inputs.dart_version}}
+    steps:
+      - uses: actions/checkout@v3
+      - name: pub.dev publish
+        run: |
+          if [ -z ${{ secrets.PUB_DEV_CREDENTIALS }}" ]; then
+            echo "Missing PUB_DEV_CREDENTIALS environment variable"
+            exit 1
+          fi
+          
+          mkdir -p ~/.config/dart/
+          cp "${{ secrets.PUB_DEV_CREDENTIALS }}" ~/.config/dart/pub-credentials.json
+
+          dart pub get
+          dart pub publish --force

.github/workflows/issue-triage.yml

@@ -0,0 +1,18 @@
+name: Add issues to Product Management Project.
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v0.5.0
+        with:
+          # You can target a repository in a different organization
+          # to the issue
+          project-url: https://github.com/orgs/famedly/projects/4
+          github-token: ${{ secrets.ADD_ISSUE_TO_PROJECT_PAT }}

.github/workflows/main.yml

@@ -1,18 +1,55 @@
-name: Add issues to Product Management Project.
+name: CI
 
 on:
-  issues:
+  push:
-    types:
+    branches:
-      - opened
+      - main
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+  pull_request:
+  merge_group:
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  FLUTTER_VERSION:  3.10.6
+  DART_VERSION: 3.0.6
 
 jobs:
-  add-to-project:
+  # because there is no easy way to pass env variables to jobs
-    name: Add issue to project
+  versions:
     runs-on: ubuntu-latest
+    outputs:
+      flutter_version: ${{ steps.flutterver.outputs.FLUTTER_VERSION }}
+      dart_version: ${{ steps.dartver.outputs.DART_VERSION }}
     steps:
-      - uses: actions/add-to-project@v0.5.0
+      - id: flutterver
+        run: echo "FLUTTER_VERSION=${{ env.FLUTTER_VERSION }}" >> "$GITHUB_OUTPUT"
+      - id: dartver
+        run: echo "DART_VERSION=${{ env.DART_VERSION }}" >> "$GITHUB_OUTPUT"
+
+  dart:
+    permissions:
+      contents: read
+    uses: famedly/frontend-ci-templates/.github/workflows/dart.yml@main
+    needs: [versions]
     with:
-          # You can target a repository in a different organization
+      flutter_version: ${{ needs.versions.outputs.flutter_version }}
-          # to the issue
+      dart_version: ${{ needs.versions.outputs.dart_version }}
-          project-url: https://github.com/orgs/famedly/projects/4
+    secrets:
-          github-token: ${{ secrets.ADD_ISSUE_TO_PROJECT_PAT }}
+      ssh_key: "${{ secrets.CI_SSH_PRIVATE_KEY }}"
+
+  general:
+    permissions:
+      contents: read
+    uses: famedly/frontend-ci-templates/.github/workflows/general.yml@main
+
+  app_jobs:
+    secrets: inherit
+    uses: ./.github/workflows/app.yml
+    needs: [versions]
+    with:
+      flutter_version: ${{ needs.versions.outputs.flutter_version }}
+      dart_version: ${{ needs.versions.outputs.dart_version }}

scripts/integration-server-conduit.sh

@@ -9,4 +9,5 @@
   -e CONDUIT_TRUSTED_SERVERS="[\"conduit.rs\"]" \
   -e CONDUIT_MAX_CONCURRENT_REQUESTS="100" \
   -e CONDUIT_LOG="info,rocket=off,_=off,sled=off" \
+  $NETWORK \
   --name conduit -p 80:80 matrixconduit/matrix-conduit:latest

scripts/integration-server-dendrite.sh

@@ -11,4 +11,4 @@ docker run --rm --entrypoint="" \
   -tls-key /mnt/server.key
 
 docker run -d --volume="$(pwd)/test_driver/dendrite/data":/etc/dendrite:rw \
-  --name dendrite -p 80:8008 matrixdotorg/dendrite-monolith:latest -really-enable-open-registration
+  --name dendrite $NETWORK -p 80:8008 matrixdotorg/dendrite-monolith:latest -really-enable-open-registration

scripts/integration-server-synapse.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-docker run -d --name synapse --tmpfs /data \
+
+docker run -d --name synapse --tmpfs /data $NETWORK \
     --volume="$(pwd)/test_driver/synapse/data/homeserver.yaml":/data/homeserver.yaml:rw \
     --volume="$(pwd)/test_driver/synapse/data/localhost.log.config":/data/localhost.log.config:rw \
     -p 80:80 matrixdotorg/synapse:latest

scripts/test.sh

@@ -1,38 +1,26 @@
 #!/bin/bash
 if which flutter >/dev/null; then
     flutter pub global activate junitreport
-    flutter test --coverage --machine | tee TEST-report.json
+    flutter test --coverage
-    TEST_CODE=${PIPESTATUS[0]}
+    TEST_CODE=$?
-    
-    # junit report
-    flutter pub global run junitreport:tojunit --input TEST-report.json --output TEST-report.xml
-    # remove shell escapes since those are invalid xml
-    sed 's///g' -i TEST-report.xml
     
     # coverage
     flutter pub global activate remove_from_coverage
     flutter pub global run remove_from_coverage:remove_from_coverage -f coverage/lcov.info -r '\.g\.dart$'
 else
-    dart pub global activate junitreport
+    dart test --coverage=coverage
-    # Test coverage using dart only is broken: https://github.com/dart-lang/test/issues/1698
-    #dart test --coverage=coverage --file-reporter='json:TEST-report.json'
     dart test --file-reporter='json:TEST-report.json'
     TEST_CODE=$?
     
-    # junit report
-    dart pub global run junitreport:tojunit --input TEST-report.json --output TEST-report.xml
-    # remove shell escapes since those are invalid xml
-    sed 's///g' -i TEST-report.xml
-    
     # coverage -> broken see https://github.com/dart-lang/test/issues/1698
-    #dart pub global activate coverage
+    dart pub global activate coverage
 
     #reporton="--report-on=lib/"
-    #if [ -n "$NO_OLM" ]; then reporton="--report-on=lib/src --report-on=lib/msc_extensions"; fi
+    if [ -n "$NO_OLM" ]; then reporton="--report-on=lib/src --report-on=lib/msc_extensions"; fi
 
-    #dart pub global run coverage:format_coverage -i coverage/  --lcov -o coverage/lcov.info $reporton
+    dart pub global run coverage:format_coverage -i coverage/  --lcov -o coverage/lcov.info $reporton
-    #dart pub global activate remove_from_coverage
+    dart pub global activate remove_from_coverage
-    #dart pub global run remove_from_coverage:remove_from_coverage -f coverage/lcov.info -r '\.g\.dart$'
+    dart pub global run remove_from_coverage:remove_from_coverage -f coverage/lcov.info -r '\.g\.dart$'
 fi
 
 # coverage html report

scripts/test_driver.sh

@@ -2,5 +2,5 @@
 if which flutter >/dev/null; then
     flutter --no-version-check test test_driver/matrixsdk_test.dart --dart-define=HOMESERVER=$HOMESERVER --dart-define=USER1_NAME=$USER1_NAME --dart-define=USER2_NAME=$USER2_NAME --dart-define=USER3_NAME=$USER3_NAME --dart-define=USER1_PW=$USER1_PW --dart-define=USER2_PW=$USER2_PW --dart-define=USER3_PW=$USER3_PW
 else
-    dart run -D HOMESERVER=$HOMESERVER -D USER1_NAME=$USER1_NAME -D USER2_NAME=$USER2_NAME -D USER3_NAME=$USER3_NAME -D USER1_PW=$USER1_PW -D USER2_PW=$USER2_PW -D USER3_PW=$USER3_PW test_driver/matrixsdk_test.dart -p vm
+    dart -D HOMESERVER=$HOMESERVER -D USER1_NAME=$USER1_NAME -D USER2_NAME=$USER2_NAME -D USER3_NAME=$USER3_NAME -D USER1_PW=$USER1_PW -D USER2_PW=$USER2_PW -D USER3_PW=$USER3_PW test test_driver/matrixsdk_test.dart -p vm
 fi

test_driver/synapse/data/homeserver.yaml

@@ -6,7 +6,6 @@
 #
 # [1] https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html
 
-
 ## Modules ##
 
 # Server admins can expand Synapse's functionality with external modules.
@@ -21,7 +20,6 @@ modules:
   #- module: my_other_super_module.SomeClass
   #  config: {}
 
-
 ## Server ##
 
 # The public-facing domain of the server
@@ -350,7 +348,6 @@ manhole_settings:
 #
 #dummy_events_threshold: 5
 
-
 ## Homeserver blocking ##
 
 # How to reach the server admin, used in ResourceLimitError
@@ -528,7 +525,6 @@ templates:
 #exclude_rooms_from_sync:
 #    - !foo:example.com
 
-
 # Message retention policy at the server level.
 #
 # Room admins and mods can define a retention period for their rooms using the
@@ -598,7 +594,6 @@ retention:
   #  - shortest_max_lifetime: 3d
   #    interval: 1d
 
-
 ## TLS ##
 
 # PEM-encoded X509 certificate for TLS.
@@ -658,7 +653,6 @@ retention:
 #  - myCA2.pem
 #  - myCA3.pem
 
-
 ## Federation ##
 
 # Restrict federation to the following whitelist of domains.
@@ -695,7 +689,6 @@ retention:
 #
 #allow_device_name_lookup_over_federation: true
 
-
 ## Caching ##
 
 # Caching can be configured through the following options.
@@ -760,7 +753,6 @@ caches:
   #
   #sync_response_cache_duration: 2m
 
-
 ## Database ##
 
 # The 'database' setting defines the database that synapse uses to store all of
@@ -816,7 +808,6 @@ database:
   args:
     database: /data/homeserver.db
 
-
 ## Logging ##
 
 # A yaml python logging config file as described by
@@ -824,7 +815,6 @@ database:
 #
 log_config: "/data/localhost.log.config"
 
-
 ## Ratelimiting ##
 
 # Ratelimiting settings for client actions (registration, login, messaging).
@@ -944,8 +934,6 @@ log_config: "/data/localhost.log.config"
 #
 #federation_rr_transactions_per_room_per_second: 50
 
-
-
 ## Media Store ##
 
 # Enable the media store service in the Synapse master. Uncomment the
@@ -1126,7 +1114,6 @@ media_store_path: "/data/media_store"
 url_preview_accept_language:
 #   - en
 
-
 # oEmbed allows for easier embedding content from a website. It can be
 # used for generating URLs previews of services which support it.
 #
@@ -1147,7 +1134,6 @@ oembed:
   #additional_providers:
   #  - oembed/my_providers.json
 
-
 ## Captcha ##
 # See docs/CAPTCHA_SETUP.md for full details of configuring this.
 
@@ -1172,7 +1158,6 @@ oembed:
 #
 #recaptcha_siteverify_api: "https://my.recaptcha.site"
 
-
 ## TURN ##
 
 # The public URIs of the TURN server to give to clients
@@ -1201,7 +1186,6 @@ oembed:
 #
 #turn_allow_guests: true
 
-
 ## Registration ##
 #
 # Registration can be rate-limited using the parameters in the "Ratelimiting"
@@ -1473,7 +1457,6 @@ account_threepid_delegates:
 #
 #inhibit_user_in_use_error: true
 
-
 ## Metrics ###
 
 # Enable collection and rendering of performance metrics
@@ -1509,7 +1492,6 @@ report_stats: false
 #
 #report_stats_endpoint: https://example.com/report-usage-stats/push
 
-
 ## API Configuration ##
 
 # Controls for the state that is shared with users who receive an invite
@@ -1555,7 +1537,6 @@ room_prejoin_state:
 #
 #track_puppeted_user_ips: true
 
-
 # A list of application service config files to use
 #
 #app_service_config_files:
@@ -1567,7 +1548,6 @@ room_prejoin_state:
 #
 #track_appservice_user_ips: true
 
-
 # a secret which is used to sign access tokens. If none is specified,
 # the registration_shared_secret is used, if one is given; otherwise,
 # a secret key is derived from the signing key.
@@ -1662,7 +1642,6 @@ trusted_key_servers:
 #
 #key_server_signing_keys_path: "key_server_signing_keys.key"
 
-
 ## Single sign-on integration ##
 
 # The following settings can be used to make Synapse use a single sign-on
@@ -1841,7 +1820,6 @@ saml2_config:
   #
   #idp_entityid: 'https://our_idp/entityid'
 
-
 # List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration
 # and login.
 #
@@ -2034,7 +2012,6 @@ oidc_providers:
   #    - attribute: userGroup
   #      value: "synapseUsers"
 
-
 # Enable Central Authentication Service (CAS) for registration and login.
 #
 cas_config:
@@ -2063,7 +2040,6 @@ cas_config:
   #  userGroup: "staff"
   #  department: None
 
-
 # Additional settings to use with single-sign on systems such as OpenID Connect,
 # SAML2 and CAS.
 #
@@ -2101,7 +2077,6 @@ sso:
   #
   #update_profile_information: true
 
-
 # JSON web token integration. The following settings can be used to make
 # Synapse JSON web tokens for authentication, instead of its internal
 # password database.
@@ -2118,51 +2093,50 @@ sso:
 # See https://matrix-org.github.io/synapse/latest/jwt.html.
 #
 #jwt_config:
-    # Uncomment the following to enable authorization using JSON web
+# Uncomment the following to enable authorization using JSON web
-    # tokens. Defaults to false.
+# tokens. Defaults to false.
-    #
+#
-    #enabled: true
+#enabled: true
 
-    # This is either the private shared secret or the public key used to
+# This is either the private shared secret or the public key used to
-    # decode the contents of the JSON web token.
+# decode the contents of the JSON web token.
-    #
+#
-    # Required if 'enabled' is true.
+# Required if 'enabled' is true.
-    #
+#
-    #secret: "provided-by-your-issuer"
+#secret: "provided-by-your-issuer"
 
-    # The algorithm used to sign the JSON web token.
+# The algorithm used to sign the JSON web token.
-    #
+#
-    # Supported algorithms are listed at
+# Supported algorithms are listed at
-    # https://pyjwt.readthedocs.io/en/latest/algorithms.html
+# https://pyjwt.readthedocs.io/en/latest/algorithms.html
-    #
+#
-    # Required if 'enabled' is true.
+# Required if 'enabled' is true.
-    #
+#
-    #algorithm: "provided-by-your-issuer"
+#algorithm: "provided-by-your-issuer"
 
-    # Name of the claim containing a unique identifier for the user.
+# Name of the claim containing a unique identifier for the user.
-    #
+#
-    # Optional, defaults to `sub`.
+# Optional, defaults to `sub`.
-    #
+#
-    #subject_claim: "sub"
+#subject_claim: "sub"
 
-    # The issuer to validate the "iss" claim against.
+# The issuer to validate the "iss" claim against.
-    #
+#
-    # Optional, if provided the "iss" claim will be required and
+# Optional, if provided the "iss" claim will be required and
-    # validated for all JSON web tokens.
+# validated for all JSON web tokens.
-    #
+#
-    #issuer: "provided-by-your-issuer"
+#issuer: "provided-by-your-issuer"
-
-    # A list of audiences to validate the "aud" claim against.
-    #
-    # Optional, if provided the "aud" claim will be required and
-    # validated for all JSON web tokens.
-    #
-    # Note that if the "aud" claim is included in a JSON web token then
-    # validation will fail without configuring audiences.
-    #
-    #audiences:
-    #    - "provided-by-your-issuer"
 
+# A list of audiences to validate the "aud" claim against.
+#
+# Optional, if provided the "aud" claim will be required and
+# validated for all JSON web tokens.
+#
+# Note that if the "aud" claim is included in a JSON web token then
+# validation will fail without configuring audiences.
+#
+#audiences:
+#    - "provided-by-your-issuer"
 
 password_config:
   # Uncomment to disable password login
@@ -2234,7 +2208,6 @@ ui_auth:
   #
   #session_timeout: "15s"
 
-
 # Configuration for sending emails from Synapse.
 #
 # Server admins can configure custom templates for email content. See
@@ -2383,8 +2356,6 @@ email:
   # ownership.
   #email_validation: "[%(server_name)s] Validate your email"
 
-
-
 ## Push ##
 
 push:
@@ -2415,7 +2386,6 @@ push:
   #
   #group_unread_count_by_room: false
 
-
 ## Rooms ##
 
 # Controls whether locally-created rooms should be end-to-end encrypted by
@@ -2435,7 +2405,6 @@ push:
 #
 #encryption_enabled_by_default_for_room_type: invite
 
-
 # Uncomment to allow non-server-admin users to create groups on this server
 #
 #enable_group_creation: true
@@ -2445,8 +2414,6 @@ push:
 #
 #group_creation_prefix: "unofficial_"
 
-
-
 # User Directory configuration
 #
 user_directory:
@@ -2483,7 +2450,6 @@ user_directory:
   #
   #prefer_local_users: true
 
-
 # User Consent configuration
 #
 # for detailed instructions, see
@@ -2534,8 +2500,6 @@ user_directory:
 #  policy_name: Privacy Policy
 #
 
-
-
 # Settings for local room and user statistics collection. See
 # https://matrix-org.github.io/synapse/latest/room_and_user_statistics.html.
 #
@@ -2546,7 +2510,6 @@ stats:
   #
   #enabled: false
 
-
 # Server Notices room configuration
 #
 # Uncomment this section to enable a room which can be used to send notices
@@ -2566,8 +2529,6 @@ stats:
 #  system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
 #  room_name: "Server Notices"
 
-
-
 # Uncomment to disable searching the public room list. When disabled
 # blocks searching local and remote room lists for local and remote
 # users by always returning an empty list for all queries.
@@ -2631,7 +2592,6 @@ stats:
 #    room_id: "*"
 #    action: allow
 
-
 ## Opentracing ##
 
 # These settings enable opentracing, which implements distributed tracing.
@@ -2679,7 +2639,6 @@ opentracing:
   #  logging:
   #    false
 
-
 ## Workers ##
 
 # Disables sending of outbound federation transactions on the main process.
@@ -2726,7 +2685,6 @@ opentracing:
 #
 #worker_replication_secret: ""
 
-
 # Configuration for Redis when using workers. This *must* be enabled when
 # using workers (unless using old style direct TCP configuration).
 #
@@ -2745,7 +2703,6 @@ redis:
   #
   #password: <secret_password>
 
-
 ## Background Updates ##
 
 # Background updates are database updates that are run in the background in batches.
@@ -2776,6 +2733,4 @@ background_updates:
   # Uncomment and set a size to change the default.
   #
   #default_batch_size: 50
-
-
 # vim:ft=yaml