From 2004143f142d58dc83c52d7f1bdfbaafcedbb91a Mon Sep 17 00:00:00 2001 From: Johanna Dorothea Reichmann Date: Mon, 27 Jul 2020 15:53:33 +0200 Subject: [PATCH 01/65] Bump version of mautrix-telegram to 0.8.2 fixes matrix users unable to delete messages --- roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index 43210c07..d2ab9909 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -4,7 +4,7 @@ matrix_mautrix_telegram_enabled: true # See: https://mau.dev/tulir/mautrix-telegram/container_registry -matrix_mautrix_telegram_docker_image: "dock.mau.dev/tulir/mautrix-telegram:v0.8.1" +matrix_mautrix_telegram_docker_image: "dock.mau.dev/tulir/mautrix-telegram:v0.8.2" matrix_mautrix_telegram_docker_image_force_pull: "{{ matrix_mautrix_telegram_docker_image.endswith(':latest') }}" matrix_mautrix_telegram_base_path: "{{ matrix_base_data_path }}/mautrix-telegram" From be5ca5258b035c668a2d3adda198a41b6b75c9ca Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 28 Jul 2020 19:03:45 +0300 Subject: [PATCH 02/65] Upgrade Element (1.7.1 -> 1.7.2) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index c61283a4..09a1990e 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/riot-web:v1.7.1" +matrix_client_element_docker_image: "vectorim/riot-web:v1.7.2" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" From 9b61fef271f85e245d6bac0f6e20f57219267b35 Mon Sep 17 00:00:00 2001 From: vractal Date: Tue, 28 Jul 2020 15:26:16 -0400 Subject: [PATCH 03/65] Replace gpg dependency for gnupg for better debian compatibility --- roles/matrix-base/tasks/server_base/setup_debian.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/server_base/setup_debian.yml b/roles/matrix-base/tasks/server_base/setup_debian.yml index b71d31ec..6d8d18fa 100644 --- a/roles/matrix-base/tasks/server_base/setup_debian.yml +++ b/roles/matrix-base/tasks/server_base/setup_debian.yml @@ -5,7 +5,7 @@ name: - apt-transport-https - ca-certificates - - gpg + - gnupg state: present update_cache: yes From 6539f2a1561634feb431da172423c9cce434b366 Mon Sep 17 00:00:00 2001 From: Benjamin Fichtner Date: Wed, 29 Jul 2020 12:37:05 +0200 Subject: [PATCH 04/65] Make ansible check mode runs silent, for all tasks which can't be idempotent --- roles/matrix-common-after/tasks/start.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/roles/matrix-common-after/tasks/start.yml b/roles/matrix-common-after/tasks/start.yml index 069fb346..4bb59bfe 100644 --- a/roles/matrix-common-after/tasks/start.yml +++ b/roles/matrix-common-after/tasks/start.yml @@ -1,21 +1,23 @@ --- -- name: Ensure systemd reloaded +- name: Ensure systemd is reloaded service: daemon_reload: yes -- name: Ensure Matrix services stopped +- name: Ensure Matrix services are stopped service: name: "{{ item }}" state: stopped with_items: "{{ matrix_systemd_services_list }}" + when: not ansible_check_mode -- name: Ensure Matrix services started +- name: Ensure Matrix services are started service: name: "{{ item }}" enabled: yes state: started with_items: "{{ matrix_systemd_services_list }}" + when: not ansible_check_mode # If we check service state immediately, we may succeed, # because it takes some time for the service to attempt to start and actually fail. From 627c225101852ca8526f47c4d2dd44fdd4440b4b Mon Sep 17 00:00:00 2001 From: vractal Date: Wed, 29 Jul 2020 12:38:07 -0400 Subject: [PATCH 05/65] Enable ma1sd hashing by default --- roles/matrix-ma1sd/defaults/main.yml | 3 +++ roles/matrix-ma1sd/templates/ma1sd.yaml.j2 | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml index 713dd3ed..63bc5a59 100644 --- a/roles/matrix-ma1sd/defaults/main.yml +++ b/roles/matrix-ma1sd/defaults/main.yml @@ -83,6 +83,9 @@ matrix_ma1sd_verbose_logging: false matrix_ma1sd_v1_enabled: true matrix_ma1sd_v2_enabled: true +# Fix for missing 3PIDS bug +matrix_ma1sd_hashing_enabled: true + # Default ma1sd configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 index 0deef3d8..8f3569b1 100644 --- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 +++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 @@ -63,6 +63,19 @@ threepid: {% endif %} {% endif %} +{% if matrix_ma1sd_hashing_enabled %} +hashing: + enabled: true # enable or disable the hash lookup MSC2140 (default is false) + pepperLength: 20 # length of the pepper value (default is 20) + rotationPolicy: per_requests # or `per_seconds` how often the hashes will be updating + hashStorageType: sql # or `in_memory` where the hashes will be stored + algorithms: + - none # the same as v1 bulk lookup + - sha256 # hash the 3PID and pepper. + delay: 2m # how often hashes will be updated if rotation policy = per_seconds (default is 10s) + requests: 10 +{% endif %} + synapseSql: enabled: {{ matrix_ma1sd_synapsesql_enabled }} type: {{ matrix_ma1sd_synapsesql_type }} From fad1f72df8b56245b1de3d79bf566560889b6e93 Mon Sep 17 00:00:00 2001 From: vaivars Date: Wed, 29 Jul 2020 19:53:39 +0300 Subject: [PATCH 06/65] Add a secondary option on how to get access token Potentially the CURLing could be a better approach, since then you don't leave spare sessions hanging. --- docs/configuring-playbook-dimension.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/configuring-playbook-dimension.md b/docs/configuring-playbook-dimension.md index 9b9c09e7..10276e7c 100644 --- a/docs/configuring-playbook-dimension.md +++ b/docs/configuring-playbook-dimension.md @@ -37,7 +37,8 @@ We recommend that you create a dedicated Matrix user for Dimension (`dimension` Follow our [Registering users](registering-users.md) guide to learn how to register **a regular (non-admin) user**. You are required to specify an access token (belonging to this new user) for Dimension to work. -To get an access token for the Dimension user, follow these steps: +To get an access token for the Dimension user, you can follow one of two options: +*Through an interactive login*: 1. In a private browsing session (incognito window), open Element. 2. Log in with the `dimension` user and its password. @@ -46,6 +47,17 @@ To get an access token for the Dimension user, follow these steps: 3. Copy the highlighted text to your configuration. 4. Close the private browsing session. **Do not log out**. Logging out will invalidate the token, making it not work. +*With CURL* + +``` +curl -X POST --header 'Content-Type: application/json' -d '{ + "identifier": { "type": "m.id.user", "user": "YourDimensionUsername" }, + "password": "YourDimensionPassword", + "type": "m.login.password" +}' 'http://localhost:8008/_matrix/client/r0/login' +``` +*Change the "YourDimensionUser/Pass" accordingly* + **Access tokens are sensitive information. Do not include them in any bug reports, messages, or logs. Do not share the access token with anyone.** Add access token to your configuration file (`inventory/host_vars/matrix./vars.yml`): From e73b8633295655e06c59a19f635e2cc16d1d6b3c Mon Sep 17 00:00:00 2001 From: vaivars Date: Thu, 30 Jul 2020 10:52:49 +0300 Subject: [PATCH 07/65] Change the url to reflect the most common usecase --- docs/configuring-playbook-dimension.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-dimension.md b/docs/configuring-playbook-dimension.md index 10276e7c..2b6b30ed 100644 --- a/docs/configuring-playbook-dimension.md +++ b/docs/configuring-playbook-dimension.md @@ -54,9 +54,9 @@ curl -X POST --header 'Content-Type: application/json' -d '{ "identifier": { "type": "m.id.user", "user": "YourDimensionUsername" }, "password": "YourDimensionPassword", "type": "m.login.password" -}' 'http://localhost:8008/_matrix/client/r0/login' +}' 'https://matrix.YOURDOMAIN/_matrix/client/r0/login' ``` -*Change the "YourDimensionUser/Pass" accordingly* +*Change the "YourDimensionUser/Pass" URL accordigly* **Access tokens are sensitive information. Do not include them in any bug reports, messages, or logs. Do not share the access token with anyone.** From f78a5d4ee84e76f9bbe47c3cdc0d600abc54dc21 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 30 Jul 2020 14:20:57 +0300 Subject: [PATCH 08/65] Upgrade Synapse (v1.17.0 -> v1.18.0) --- roles/matrix-synapse/defaults/main.yml | 2 +- .../templates/synapse/homeserver.yaml.j2 | 205 ++++++++++++++---- 2 files changed, 167 insertions(+), 40 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index f20c8a74..b1b7ff4e 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.17.0" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.18.0" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 5ed2524b..fe155cc9 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -89,7 +89,9 @@ default_room_version: {{ matrix_synapse_default_room_version|to_json }} #gc_thresholds: [700, 10, 10] # Set the limit on the returned events in the timeline in the get -# and sync operations. The default value is -1, means no upper limit. +# and sync operations. The default value is 100. -1 means no upper limit. +# +# Uncomment the following to increase the limit to 5000. # #filter_timeline_limit: 5000 @@ -105,41 +107,6 @@ default_room_version: {{ matrix_synapse_default_room_version|to_json }} # #enable_search: false -# Restrict federation to the following whitelist of domains. -# N.B. we recommend also firewalling your federation listener to limit -# inbound federation traffic as early as possible, rather than relying -# purely on this application-layer restriction. If not specified, the -# default is to whitelist everything. -# -#federation_domain_whitelist: -# - lon.example.com -# - nyc.example.com -# - syd.example.com -{% if matrix_synapse_federation_domain_whitelist is not none %} -{# Cannot use `|to_nice_yaml` here, as an empty list does not get serialized properly by it. #} -federation_domain_whitelist: {{ matrix_synapse_federation_domain_whitelist|to_json }} -{% endif %} - -# Prevent federation requests from being sent to the following -# blacklist IP address CIDR ranges. If this option is not specified, or -# specified with an empty list, no ip range blacklist will be enforced. -# -# As of Synapse v1.4.0 this option also affects any outbound requests to identity -# servers provided by user input. -# -# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly -# listed here, since they correspond to unroutable addresses.) -# -federation_ip_range_blacklist: - - '127.0.0.0/8' - - '10.0.0.0/8' - - '172.16.0.0/12' - - '192.168.0.0/16' - - '100.64.0.0/10' - - '169.254.0.0/16' - - '::1/128' - - 'fe80::/64' - - 'fc00::/7' # List of ports that Synapse should listen on, their purpose and their # configuration. @@ -169,7 +136,7 @@ federation_ip_range_blacklist: # names: a list of names of HTTP resources. See below for a list of # valid resource names. # -# compress: set to true to enable HTTP comression for this resource. +# compress: set to true to enable HTTP compression for this resource. # # additional_resources: Only valid for an 'http' listener. A map of # additional endpoints which should be loaded via dynamic modules. @@ -610,6 +577,43 @@ acme: +# Restrict federation to the following whitelist of domains. +# N.B. we recommend also firewalling your federation listener to limit +# inbound federation traffic as early as possible, rather than relying +# purely on this application-layer restriction. If not specified, the +# default is to whitelist everything. +# +#federation_domain_whitelist: +# - lon.example.com +# - nyc.example.com +# - syd.example.com +{% if matrix_synapse_federation_domain_whitelist is not none %} +{# Cannot use `|to_nice_yaml` here, as an empty list does not get serialized properly by it. #} +federation_domain_whitelist: {{ matrix_synapse_federation_domain_whitelist|to_json }} +{% endif %} + +# Prevent federation requests from being sent to the following +# blacklist IP address CIDR ranges. If this option is not specified, or +# specified with an empty list, no ip range blacklist will be enforced. +# +# As of Synapse v1.4.0 this option also affects any outbound requests to identity +# servers provided by user input. +# +# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +# listed here, since they correspond to unroutable addresses.) +# +federation_ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '169.254.0.0/16' + - '::1/128' + - 'fe80::/64' + - 'fc00::/7' + + ## Caching ## # Caching can be configured through the following options. @@ -1788,6 +1792,9 @@ sso: # Each JSON Web Token needs to contain a "sub" (subject) claim, which is # used as the localpart of the mxid. # +# Additionally, the expiration time ("exp"), not before time ("nbf"), +# and issued at ("iat") claims are validated if present. +# # Note that this is a non-standard login type and client support is # expected to be non-existant. # @@ -1879,8 +1886,8 @@ email: #notif_from: "Your Friendly %(app)s homeserver " notif_from: {{ matrix_synapse_email_notif_from|string|to_json }} - # app_name defines the default value for '%(app)s' in notif_from. It - # defaults to 'Matrix'. + # app_name defines the default value for '%(app)s' in notif_from and email + # subjects. It defaults to 'Matrix'. # #app_name: my_branded_matrix_server app_name: Matrix @@ -1946,6 +1953,73 @@ email: # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates # #template_dir: "res/templates" + + # Subjects to use when sending emails from Synapse. + # + # The placeholder '%(app)s' will be replaced with the value of the 'app_name' + # setting above, or by a value dictated by the Matrix client application. + # + # If a subject isn't overridden in this configuration file, the value used as + # its example will be used. + # + #subjects: + + # Subjects for notification emails. + # + # On top of the '%(app)s' placeholder, these can use the following + # placeholders: + # + # * '%(person)s', which will be replaced by the display name of the user(s) + # that sent the message(s), e.g. "Alice and Bob". + # * '%(room)s', which will be replaced by the name of the room the + # message(s) have been sent to, e.g. "My super room". + # + # See the example provided for each setting to see which placeholder can be + # used and how to use them. + # + # Subject to use to notify about one message from one or more user(s) in a + # room which has a name. + #message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + # + # Subject to use to notify about one message from one or more user(s) in a + # room which doesn't have a name. + #message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..." + # + # Subject to use to notify about multiple messages from one or more users in + # a room which doesn't have a name. + #messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..." + # + # Subject to use to notify about multiple messages in a room which has a + # name. + #messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..." + # + # Subject to use to notify about multiple messages in multiple rooms. + #messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + # + # Subject to use to notify about multiple messages from multiple persons in + # multiple rooms. This is similar to the setting above except it's used when + # the room in which the notification was triggered has no name. + #messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..." + # + # Subject to use to notify about an invite to a room which has a name. + #invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + # + # Subject to use to notify about an invite to a room which doesn't have a + # name. + #invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..." + + # Subject for emails related to account administration. + # + # On top of the '%(app)s' placeholder, these one can use the + # '%(server_name)s' placeholder, which will be replaced by the value of the + # 'server_name' setting in your Synapse configuration. + # + # Subject to use when sending a password reset email. + #password_reset: "[%(server_name)s] Password reset" + # + # Subject to use when sending a verification email to assert an address's + # ownership. + #email_validation: "[%(server_name)s] Validate your email" {% endif %} # Password providers allow homeserver administrators to integrate @@ -2303,4 +2377,57 @@ opentracing: # false +## Workers ## + +# Disables sending of outbound federation transactions on the main process. +# Uncomment if using a federation sender worker. +# +#send_federation: false + +# It is possible to run multiple federation sender workers, in which case the +# work is balanced across them. +# +# This configuration must be shared between all federation sender workers, and if +# changed all federation sender workers must be stopped at the same time and then +# started, to ensure that all instances are running with the same config (otherwise +# events may be dropped). +# +#federation_sender_instances: +# - federation_sender1 + +# When using workers this should be a map from `worker_name` to the +# HTTP replication listener of the worker, if configured. +# +#instance_map: +# worker1: +# host: localhost +# port: 8034 + +# Experimental: When using workers you can define which workers should +# handle event persistence and typing notifications. Any worker +# specified here must also be in the `instance_map`. +# +#stream_writers: +# events: worker1 +# typing: worker1 + + +# Configuration for Redis when using workers. This *must* be enabled when +# using workers (unless using old style direct TCP configuration). +# +redis: + # Uncomment the below to enable Redis support. + # + #enabled: true + + # Optional host and port to use to connect to redis. Defaults to + # localhost and 6379 + # + #host: localhost + #port: 6379 + + # Optional password if configured on the Redis instance + # + #password: + # vim:ft=yaml From 7755e5efd48c2c47abb63d2348b780de615e6652 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Thu, 30 Jul 2020 15:25:07 +0200 Subject: [PATCH 09/65] Update sms-bridge (0.3.1 -> 0.3.2) --- roles/matrix-bridge-sms/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 0784841a..25d2208c 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "folivonet/matrix-sms-bridge:0.3.1.RELEASE" +matrix_sms_bridge_docker_image: "folivonet/matrix-sms-bridge:0.3.2.RELEASE" matrix_sms_bridge_database_docker_image: "neo4j:latest" matrix_sms_bridge_database_docker_image_force_pull: "{{ matrix_sms_bridge_docker_image.endswith(':latest') }}" @@ -149,4 +149,4 @@ matrix_sms_bridge_registration_yaml: | sender_localpart: smsbot rate_limited: false -matrix_sms_bridge_registration: "{{ matrix_sms_bridge_registration_yaml|from_yaml }}" \ No newline at end of file +matrix_sms_bridge_registration: "{{ matrix_sms_bridge_registration_yaml|from_yaml }}" From 54195b22c7e800027b0c8adb20c2a876e68e6a75 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 4 Aug 2020 16:06:59 +0300 Subject: [PATCH 10/65] Allow framing Jitsi Hopefully fixes a regression caused by b106a9592ec1183811. Related to #597 (Github Pull Request). --- .../templates/nginx/conf.d/matrix-jitsi.conf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 index a6516dab..4cacf1f1 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 @@ -5,7 +5,6 @@ gzip_types text/plain application/json application/javascript text/css image/x-icon font/ttf image/gif; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; add_header X-Content-Type-Options nosniff; - add_header X-Frame-Options SAMEORIGIN; {% for configuration_block in matrix_nginx_proxy_proxy_jitsi_additional_server_configuration_blocks %} {{- configuration_block }} {% endfor %} From b22b593d83fd5769b4f434fd7eb765ac5ac39e57 Mon Sep 17 00:00:00 2001 From: Dennis Ciba Date: Tue, 4 Aug 2020 14:27:39 +0200 Subject: [PATCH 11/65] Changed setup of synapse-admin to allow for self-build images --- roles/matrix-synapse-admin/defaults/main.yml | 4 ++++ roles/matrix-synapse-admin/tasks/setup.yml | 21 +++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/roles/matrix-synapse-admin/defaults/main.yml b/roles/matrix-synapse-admin/defaults/main.yml index 01308d4a..aec38703 100644 --- a/roles/matrix-synapse-admin/defaults/main.yml +++ b/roles/matrix-synapse-admin/defaults/main.yml @@ -3,6 +3,10 @@ matrix_synapse_admin_enabled: true +matrix_synapse_admin_container_self_build: false +matrix_synapse_admin_docker_repo: "https://github.com/Awesome-Technologies/synapse-admin.git" +matrix_synapse_admin_docker_src_files_path: "{{ matrix_base_data_path }}/synapse-admin/docker-src" + matrix_synapse_admin_docker_image: "awesometechnologies/synapse-admin:0.4.1" matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-synapse-admin/tasks/setup.yml b/roles/matrix-synapse-admin/tasks/setup.yml index c1f0d240..65e5c0b9 100644 --- a/roles/matrix-synapse-admin/tasks/setup.yml +++ b/roles/matrix-synapse-admin/tasks/setup.yml @@ -10,7 +10,26 @@ source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_synapse_admin_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_synapse_admin_docker_image_force_pull }}" - when: matrix_synapse_admin_enabled|bool + when: "matrix_synapse_admin_enabled|bool and not matrix_synapse_admin_container_self_build|bool" + +- name: Ensure matrix-synapse-admin repository is present when self-building + git: + repo: "{{ matrix_synapse_admin_docker_repo }}" + dest: "{{ matrix_synapse_admin_docker_src_files_path }}" + force: "yes" + register: matrix_synapse_admin_git_pull_results + when: "matrix_synapse_admin_enabled|bool and matrix_synapse_admin_container_self_build|bool" + +- name: Ensure matrix-synapse-admin Docker image is build + docker_image: + name: "{{ matrix_synapse_admin_docker_image }}" + source: build + force_source: yes + build: + dockerfile: Dockerfile + path: "{{ matrix_synapse_admin_docker_src_files_path }}" + pull: yes + when: "matrix_synapse_admin_enabled|bool and matrix_synapse_admin_container_self_build|bool and matrix_synapse_admin_git_pull_results.changed" - name: Ensure matrix-synapse-admin.service installed template: From fffe9da5a9a750eb9fffeca320f4225d835df722 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 4 Aug 2020 17:24:27 +0300 Subject: [PATCH 12/65] Mention self-building support for synapse-admin Related to #620 (Github Pull Request). --- docs/self-building.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/self-building.md b/docs/self-building.md index bc0d097c..84d61fe0 100644 --- a/docs/self-building.md +++ b/docs/self-building.md @@ -11,6 +11,7 @@ To make use of self-building, you don't need to do anything besides change your Note that **not all components support self-building yet**. List of roles where self-building the Docker image is currently possible: - `matrix-synapse` +- `matrix-synapse-admin` - `matrix-client-element` - `matrix-coturn` - `matrix-ma1sd` From f57b2f6e925d9797a03f2fc3dc72c9b50705a20d Mon Sep 17 00:00:00 2001 From: merklaw Date: Wed, 5 Aug 2020 12:45:44 +0200 Subject: [PATCH 13/65] Better wording --- docs/configuring-playbook-client-element.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-client-element.md b/docs/configuring-playbook-client-element.md index 26536364..c53bb444 100644 --- a/docs/configuring-playbook-client-element.md +++ b/docs/configuring-playbook-client-element.md @@ -6,7 +6,7 @@ If that's okay, you can skip this document. ## Disabling Element -If you'd like for the playbook to not install (or to uninstall the previously installed Element), you can disable it in your configuration file (`inventory/host_vars/matrix./vars.yml`): +If you'd like for the playbook to not install Element (or to uninstall the previously installed Element), you can disable it in your configuration file (`inventory/host_vars/matrix./vars.yml`): ```yaml matrix_client_element_enabled: false From e167b80f94a69284fd8e2ad55abf66b7fcfdb1ba Mon Sep 17 00:00:00 2001 From: merklaw Date: Wed, 5 Aug 2020 12:46:30 +0200 Subject: [PATCH 14/65] Added notes about disabling ma1sd --- docs/configuring-playbook-ma1sd.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/configuring-playbook-ma1sd.md b/docs/configuring-playbook-ma1sd.md index aca858f2..a25db89b 100644 --- a/docs/configuring-playbook-ma1sd.md +++ b/docs/configuring-playbook-ma1sd.md @@ -6,6 +6,17 @@ This server is private by default, potentially at the expense of user discoverab ma1sd is a fork of [mxisd](https://github.com/kamax-io/mxisd) which was pronounced end of life 2019-06-21. + +## Disabling ma1sd + +ma1sd, being an Identity Server, is not strictly needed. It is only used for 3PIDs (3rd party identifiers like E-mail and phone numbers) and some [enhanced features](https://github.com/ma1uta/ma1sd/#features). + +If you'd like for the playbook to not install ma1sd (or to uninstall the previously installed ma1sd), you can disable it in your configuration file (`inventory/host_vars/matrix./vars.yml`): + +```yaml +matrix_ma1sd_enabled: false +``` + ## Matrix.org lookup forwarding To ensure maximum discovery, you can make your identity server also forward lookups to the central matrix.org Identity server (at the cost of potentially leaking all your contacts information). From 205c15a80bbf1823ccca05a98b2c88f07718a58d Mon Sep 17 00:00:00 2001 From: merklaw Date: Wed, 5 Aug 2020 12:47:32 +0200 Subject: [PATCH 15/65] Add note about certificate exceptions when using self-signed --- docs/configuring-playbook-ssl-certificates.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-playbook-ssl-certificates.md b/docs/configuring-playbook-ssl-certificates.md index 0b593f20..916f6838 100644 --- a/docs/configuring-playbook-ssl-certificates.md +++ b/docs/configuring-playbook-ssl-certificates.md @@ -28,6 +28,8 @@ If self-signed certificates are alright with you, you can ask the playbook to ge matrix_ssl_retrieval_method: self-signed ``` +If you get a `Cannot reach homeserver` error in Element, you will have to visit `https://matrix.` in your browser and agree to the certificate exception before you can login. + ## Using your own SSL certificates From 0cd243095d3e4ea963a26ccf103a1d4a02f5dd6a Mon Sep 17 00:00:00 2001 From: merklaw Date: Wed, 5 Aug 2020 12:49:40 +0200 Subject: [PATCH 16/65] The docker package is a requirement and was not previously mentioned --- docs/prerequisites.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 5f551eb4..1ea9b639 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -4,7 +4,7 @@ - `root` access to your server (or a user capable of elevating to `root` via `sudo`). -- [Python](https://www.python.org/) being installed on the server. Most distributions install Python by default, but some don't (e.g. Ubuntu 18.04) and require manual installation (something like `apt-get install python`). +- [Python](https://www.python.org/) and the [`docker`](https://pypi.org/project/docker/) package (requires [`pip`](https://packaging.python.org/guides/installing-using-linux-tools/)) being installed on the server. Most distributions install Python by default, but some don't (e.g. Ubuntu 18.04) and require manual installation (something like `apt-get install python`). - A `cron`-like tool installed on the server such as `cron` or `anacron` to automatically schedule the Let's Encrypt SSL certificates's renewal. *This can be ignored if you use your own SSL certificates.* From a460420b34ecdbc2f47f0224066a9d663fbb5a61 Mon Sep 17 00:00:00 2001 From: merklaw Date: Wed, 5 Aug 2020 15:59:52 +0200 Subject: [PATCH 17/65] Better wording --- docs/configuring-playbook-client-element.md | 2 +- docs/configuring-playbook-ma1sd.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-client-element.md b/docs/configuring-playbook-client-element.md index c53bb444..8b0eb331 100644 --- a/docs/configuring-playbook-client-element.md +++ b/docs/configuring-playbook-client-element.md @@ -6,7 +6,7 @@ If that's okay, you can skip this document. ## Disabling Element -If you'd like for the playbook to not install Element (or to uninstall the previously installed Element), you can disable it in your configuration file (`inventory/host_vars/matrix./vars.yml`): +If you'd like for the playbook to not install Element (or to uninstall it if it was previously installed), you can disable it in your configuration file (`inventory/host_vars/matrix./vars.yml`): ```yaml matrix_client_element_enabled: false diff --git a/docs/configuring-playbook-ma1sd.md b/docs/configuring-playbook-ma1sd.md index a25db89b..03208337 100644 --- a/docs/configuring-playbook-ma1sd.md +++ b/docs/configuring-playbook-ma1sd.md @@ -11,7 +11,7 @@ ma1sd is a fork of [mxisd](https://github.com/kamax-io/mxisd) which was pronounc ma1sd, being an Identity Server, is not strictly needed. It is only used for 3PIDs (3rd party identifiers like E-mail and phone numbers) and some [enhanced features](https://github.com/ma1uta/ma1sd/#features). -If you'd like for the playbook to not install ma1sd (or to uninstall the previously installed ma1sd), you can disable it in your configuration file (`inventory/host_vars/matrix./vars.yml`): +If you'd like for the playbook to not install ma1sd (or to uninstall it if it was previously installed), you can disable it in your configuration file (`inventory/host_vars/matrix./vars.yml`): ```yaml matrix_ma1sd_enabled: false From 48b93091f9cde37721e53cc7beea93b5bb952d18 Mon Sep 17 00:00:00 2001 From: merklaw Date: Wed, 5 Aug 2020 16:38:39 +0200 Subject: [PATCH 18/65] Revert notes about the docker package being needed --- docs/prerequisites.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 1ea9b639..5f551eb4 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -4,7 +4,7 @@ - `root` access to your server (or a user capable of elevating to `root` via `sudo`). -- [Python](https://www.python.org/) and the [`docker`](https://pypi.org/project/docker/) package (requires [`pip`](https://packaging.python.org/guides/installing-using-linux-tools/)) being installed on the server. Most distributions install Python by default, but some don't (e.g. Ubuntu 18.04) and require manual installation (something like `apt-get install python`). +- [Python](https://www.python.org/) being installed on the server. Most distributions install Python by default, but some don't (e.g. Ubuntu 18.04) and require manual installation (something like `apt-get install python`). - A `cron`-like tool installed on the server such as `cron` or `anacron` to automatically schedule the Let's Encrypt SSL certificates's renewal. *This can be ignored if you use your own SSL certificates.* From 87df15441c92abfbaa8092391a89c74c9404db43 Mon Sep 17 00:00:00 2001 From: merklaw Date: Wed, 5 Aug 2020 17:31:16 +0200 Subject: [PATCH 19/65] Add note about installilng 'docker' Python package if Docker installation is disabled --- roles/matrix-base/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 520f36f1..8e4c8079 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -85,7 +85,7 @@ matrix_vars_yml_snapshotting_src: "{{ inventory_dir }}/host_vars/{{ inventory_ho matrix_well_known_matrix_server_enabled: true # Controls whether Docker is automatically installed. -# If you change this to false you must install and update Docker manually. +# If you change this to false you must install and update Docker manually. You also need to install the [`docker`](https://pypi.org/project/docker/) Python package. matrix_docker_installation_enabled: true # Controls the Docker package that is installed. From fa6d85636f77e15d531b0ff70354f30e1d2edc5e Mon Sep 17 00:00:00 2001 From: merklaw Date: Wed, 5 Aug 2020 17:35:25 +0200 Subject: [PATCH 20/65] Add note about installing 'docker' Python package if Docker installation is disabled --- roles/matrix-base/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 8e4c8079..9ab8bb3c 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -85,7 +85,7 @@ matrix_vars_yml_snapshotting_src: "{{ inventory_dir }}/host_vars/{{ inventory_ho matrix_well_known_matrix_server_enabled: true # Controls whether Docker is automatically installed. -# If you change this to false you must install and update Docker manually. You also need to install the [`docker`](https://pypi.org/project/docker/) Python package. +# If you change this to false you must install and update Docker manually. You also need to install the docker (https://pypi.org/project/docker/) Python package. matrix_docker_installation_enabled: true # Controls the Docker package that is installed. From e6dd0fbaee1a0f113d0f66e53d3b4d1dea8adc2c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 6 Aug 2020 19:25:42 +0300 Subject: [PATCH 21/65] Upgrade Element (1.7.2 -> 1.7.3) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 09a1990e..6585d018 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/riot-web:v1.7.2" +matrix_client_element_docker_image: "vectorim/riot-web:v1.7.3" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" From 19cc5fe573cc81ba65a9f396d46d888b8c1e04b9 Mon Sep 17 00:00:00 2001 From: MatthewCroughan Date: Sat, 8 Aug 2020 17:41:32 +0100 Subject: [PATCH 22/65] Update README.md Stating "many times" makes me think there is a finite amount of times I can run it before it *will* start causing trouble. But this is not true. So just state it can me ran more than once without causing trouble :) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f6daa129..d571223a 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ This is similar to the [EMnify/matrix-synapse-auto-deploy](https://github.com/EM - this one installs and hooks together **a lot more Matrix-related services** for you (see above) -- this one **can be re-ran many times** without causing trouble +- this one **can be executed more than once** without causing trouble - works on various distros: **CentOS** (7.0+), Debian-based distributions (**Debian** 9/Stretch+, **Ubuntu** 16.04+), **Archlinux** From c5d18733d26647e80e38871285dddba7459788f9 Mon Sep 17 00:00:00 2001 From: Justin Croonenberghs Date: Sat, 8 Aug 2020 23:19:07 -0500 Subject: [PATCH 23/65] Update CORS for ma1sd Even with the v2 updates listed in #503 and partially addressed in #614, this is still needed to enable identity services to function with Element Desktop/Web. Testing on multiple clients with a clean config has confirmed this, at least for my installation. --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 8fd87958..8729bdc6 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -64,6 +64,7 @@ proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; + add_header Access-Control-Allow-Origin *; } {% endif %} From 31e2a1f06bf5620e2986b875191b3acd4e8c4c36 Mon Sep 17 00:00:00 2001 From: Justin Croonenberghs Date: Sun, 9 Aug 2020 14:20:37 -0500 Subject: [PATCH 24/65] Undo ill-advised change In #628 I proposed a CORS change that turns out not to be the root of the issue. Caffeine-addled diagnosis leads to sloppy thinking, and this change should be reverted. In fact, if left it will cause problems for new installations. --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 8729bdc6..8fd87958 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -64,7 +64,6 @@ proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; - add_header Access-Control-Allow-Origin *; } {% endif %} From 20eea648a57c771e1f35ad779b69e2f1bf5b349c Mon Sep 17 00:00:00 2001 From: Dan Arnfield Date: Sun, 16 Aug 2020 14:41:40 -0500 Subject: [PATCH 25/65] Update postgres versions (12.3 -> 12.4, etc) --- roles/matrix-postgres/defaults/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index e26c43e7..b5c6f6a8 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -8,10 +8,10 @@ matrix_postgres_db_name: "" matrix_postgres_base_path: "{{ matrix_base_data_path }}/postgres" matrix_postgres_data_path: "{{ matrix_postgres_base_path }}/data" -matrix_postgres_docker_image_v9: "postgres:9.6.18-alpine" -matrix_postgres_docker_image_v10: "postgres:10.13-alpine" -matrix_postgres_docker_image_v11: "postgres:11.8-alpine" -matrix_postgres_docker_image_v12: "postgres:12.3-alpine" +matrix_postgres_docker_image_v9: "postgres:9.6.19-alpine" +matrix_postgres_docker_image_v10: "postgres:10.14-alpine" +matrix_postgres_docker_image_v11: "postgres:11.9-alpine" +matrix_postgres_docker_image_v12: "postgres:12.4-alpine" matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v12 }}" # This variable is assigned at runtime. Overriding its value has no effect. From 8d373409b8adb437941cf5cbfb8ad8ddc7f5e57d Mon Sep 17 00:00:00 2001 From: Dan Arnfield Date: Sun, 16 Aug 2020 14:59:48 -0500 Subject: [PATCH 26/65] Update nginx (1.19.1 -> 1.19.2) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index ea3dd61d..d9c0a012 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -3,7 +3,7 @@ matrix_nginx_proxy_enabled: true # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but # that is frequently out of date. -matrix_nginx_proxy_docker_image: "nginx:1.19.1-alpine" +matrix_nginx_proxy_docker_image: "nginx:1.19.2-alpine" matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}" matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy" From c8754f422a863b2a8f13cffecfce705d8382887f Mon Sep 17 00:00:00 2001 From: Dan Arnfield Date: Sun, 16 Aug 2020 15:01:13 -0500 Subject: [PATCH 27/65] Update certbot (1.6.0 -> 1.7.0) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index ea3dd61d..e01040c8 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -227,7 +227,7 @@ matrix_ssl_domains_to_obtain_certificates_for: [] # Controls whether to obtain production or staging certificates from Let's Encrypt. matrix_ssl_lets_encrypt_staging: false -matrix_ssl_lets_encrypt_certbot_docker_image: "certbot/certbot:{{ matrix_ssl_architecture }}-v1.6.0" +matrix_ssl_lets_encrypt_certbot_docker_image: "certbot/certbot:{{ matrix_ssl_architecture }}-v1.7.0" matrix_ssl_lets_encrypt_certbot_docker_image_force_pull: "{{ matrix_ssl_lets_encrypt_certbot_docker_image.endswith(':latest') }}" matrix_ssl_lets_encrypt_certbot_standalone_http_port: 2402 matrix_ssl_lets_encrypt_support_email: ~ From 9952ec6c160e487695cebc6731d77b519d224dc0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Aug 2020 17:02:40 +0300 Subject: [PATCH 28/65] Upgrade Synapse (v1.18.0 -> v1.19.0) --- group_vars/matrix_servers | 5 +- roles/matrix-synapse/defaults/main.yml | 9 +--- .../matrix-synapse/tasks/validate_config.yml | 1 + .../templates/synapse/homeserver.yaml.j2 | 46 +++++++++++-------- roles/matrix-synapse/vars/main.yml | 2 - 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index bd221425..4a989f03 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -16,7 +16,7 @@ # ###################################################################### -matrix_identity_server_url: "{{ 'https://' + matrix_synapse_trusted_third_party_id_servers[0] if matrix_synapse_trusted_third_party_id_servers|length > 0 else None }}" +matrix_identity_server_url: "{{ ('https://' + matrix_server_fqn_matrix) if matrix_ma1sd_enabled else None }}" matrix_riot_jitsi_preferredDomain: "{{ matrix_server_fqn_jitsi if matrix_jitsi_enabled else '' }}" @@ -921,9 +921,6 @@ matrix_client_element_jitsi_preferredDomain: "{{ matrix_server_fqn_jitsi if matr matrix_synapse_container_image_self_build: "{{ matrix_architecture != 'amd64'}}" -# When ma1sd is enabled, we can use it instead of the default public Identity servers. -matrix_synapse_trusted_third_party_id_servers: "{{ [matrix_server_fqn_matrix] if matrix_ma1sd_enabled else matrix_synapse_id_servers_public }}" - # When ma1sd is enabled, we can use it to validate email addresses and phone numbers. # Synapse can validate email addresses by itself as well, but it's probably not what we want by default when we have an identity server. matrix_synapse_account_threepid_delegates_email: "{{ 'http://matrix-ma1sd:8090' if matrix_ma1sd_enabled else '' }}" diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index b1b7ff4e..8b2186a5 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.18.0" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.19.0" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" @@ -73,13 +73,6 @@ matrix_synapse_registration_shared_secret: "{{ matrix_synapse_macaroon_secret_ke matrix_synapse_allow_guest_access: false matrix_synapse_form_secret: "{{ matrix_synapse_macaroon_secret_key }}" -matrix_synapse_id_servers_public: ['matrix.org', 'vector.im'] - -# The list of identity servers to use for Synapse. -# We assume this role runs standalone without a local Identity server, so we point Synapse to public ones. -# This most likely gets overwritten later, so that a local Identity server is used. -matrix_synapse_trusted_third_party_id_servers: "{{ matrix_synapse_id_servers_public }}" - matrix_synapse_max_upload_size_mb: 10 # The tmpfs at /tmp needs to be large enough to handle multiple concurrent file uploads. diff --git a/roles/matrix-synapse/tasks/validate_config.yml b/roles/matrix-synapse/tasks/validate_config.yml index 01368cc8..f39ef3fe 100644 --- a/roles/matrix-synapse/tasks/validate_config.yml +++ b/roles/matrix-synapse/tasks/validate_config.yml @@ -32,3 +32,4 @@ - {'old': 'matrix_synapse_container_expose_federation_api_port', 'new': ''} - {'old': 'matrix_synapse_container_expose_metrics_port', 'new': ''} - {'old': 'matrix_synapse_cache_factor', 'new': 'matrix_synapse_caches_global_factor'} + - {'old': 'matrix_synapse_trusted_third_party_id_servers', 'new': ''} diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index fe155cc9..d02d743e 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -309,6 +309,10 @@ limit_remote_rooms: # #complexity_error: "This room is too complex." + # allow server admins to join complex rooms. Default is false. + # + #admins_can_join: true + # Whether to require a user to be in the room to add an alias to it. # Defaults to 'true'. # @@ -703,6 +707,10 @@ log_config: "/data/{{ matrix_server_fqn_matrix }}.log.config" # - one for ratelimiting redactions by room admins. If this is not explicitly # set then it uses the same ratelimiting as per rc_message. This is useful # to allow room admins to deal with abuse quickly. +# - two for ratelimiting number of rooms a user can join, "local" for when +# users are joining rooms the server is already in (this is cheap) vs +# "remote" for when users are trying to join rooms not on the server (which +# can be more expensive) # # The defaults are as shown below. # @@ -731,6 +739,14 @@ rc_login: {{ matrix_synapse_rc_login|to_json }} #rc_admin_redaction: # per_second: 1 # burst_count: 50 +# +#rc_joins: +# local: +# per_second: 0.1 +# burst_count: 3 +# remote: +# per_second: 0.01 +# burst_count: 3 # Ratelimiting settings for incoming federation @@ -1117,25 +1133,6 @@ allow_guest_access: {{ matrix_synapse_allow_guest_access|to_json }} # #default_identity_server: https://matrix.org -# The list of identity servers trusted to verify third party -# identifiers by this server. -# -# Also defines the ID server which will be called when an account is -# deactivated (one will be picked arbitrarily). -# -# Note: This option is deprecated. Since v0.99.4, Synapse has tracked which identity -# server a 3PID has been bound to. For 3PIDs bound before then, Synapse runs a -# background migration script, informing itself that the identity server all of its -# 3PIDs have been bound to is likely one of the below. -# -# As of Synapse v1.4.0, all other functionality of this option has been deprecated, and -# it is now solely used for the purposes of the background migration script, and can be -# removed once it has run. -{% if matrix_synapse_trusted_third_party_id_servers|length > 0 %} -trusted_third_party_id_servers: -{{ matrix_synapse_trusted_third_party_id_servers|to_nice_yaml }} -{% endif %} - # Handle threepid (email/phone etc) registration and password resets through a set of # *trusted* identity servers. Note that this allows the configured identity server to # reset passwords for accounts! @@ -1546,6 +1543,17 @@ saml2_config: # #grandfathered_mxid_source_attribute: upn + # It is possible to configure Synapse to only allow logins if SAML attributes + # match particular values. The requirements can be listed under + # `attribute_requirements` as shown below. All of the listed attributes must + # match for the login to be permitted. + # + #attribute_requirements: + # - attribute: userGroup + # value: "staff" + # - attribute: department + # value: "sales" + # Directory in which Synapse will try to find the template files below. # If not set, default templates from within the Synapse package will be used. # diff --git a/roles/matrix-synapse/vars/main.yml b/roles/matrix-synapse/vars/main.yml index b53dceb8..7c07145b 100644 --- a/roles/matrix-synapse/vars/main.yml +++ b/roles/matrix-synapse/vars/main.yml @@ -1,7 +1,5 @@ --- -matrix_synapse_id_servers_public: ['vector.im', 'matrix.org'] - matrix_synapse_client_api_url_endpoint_public: "https://{{ matrix_server_fqn_matrix }}/_matrix/client/versions" matrix_synapse_federation_api_url_endpoint_public: "https://{{ matrix_server_fqn_matrix }}:{{ matrix_federation_public_port }}/_matrix/federation/v1/version" From b4a549b772e228cd1dee8d8c7bdf03dfce5c858d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Aug 2020 17:03:19 +0300 Subject: [PATCH 29/65] Upgrade Element (1.7.3 -> 1.7.4) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 6585d018..8b1bfb2e 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/riot-web:v1.7.3" +matrix_client_element_docker_image: "vectorim/riot-web:v1.7.4" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" From 264fb60e996ef8c2313962bd997492cd71ca0cca Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Aug 2020 18:34:06 +0300 Subject: [PATCH 30/65] Fix broken URLs - add some missing slashes --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d571223a..180be14a 100644 --- a/README.md +++ b/README.md @@ -162,15 +162,15 @@ This playbook sets up your server using the following Docker images: - [folivonet/matrix-sms-bridge](https://hub.docker.com/repository/docker/folivonet/matrix-sms-bridge) - the [matrix-sms-brdige](https://github.com/benkuly/matrix-sms-bridge) (optional) -- [sorunome/mx-puppet-skype](https://hub.docker.com/r/sorunome/mx-puppet-skype) - the [mx-puppet-skype](https://github.com/Sorunome/mx-puppet-skype) bridge to [Skype](https:/www.skype.com) (optional) +- [sorunome/mx-puppet-skype](https://hub.docker.com/r/sorunome/mx-puppet-skype) - the [mx-puppet-skype](https://github.com/Sorunome/mx-puppet-skype) bridge to [Skype](https://www.skype.com) (optional) -- [sorunome/mx-puppet-slack](https://hub.docker.com/r/sorunome/mx-puppet-slack) - the [mx-puppet-slack](https://github.com/Sorunome/mx-puppet-slack) bridge to [Slack](https:/slack.com) (optional) +- [sorunome/mx-puppet-slack](https://hub.docker.com/r/sorunome/mx-puppet-slack) - the [mx-puppet-slack](https://github.com/Sorunome/mx-puppet-slack) bridge to [Slack](https://slack.com) (optional) - [sorunome/mx-puppet-instagram](https://hub.docker.com/r/sorunome/mx-puppet-instagram) - the [mx-puppet-instagram](https://github.com/Sorunome/mx-puppet-instagram) bridge to [Instagram](https://www.instagram.com) (optional) - [sorunome/mx-puppet-twitter](https://hub.docker.com/r/sorunome/mx-puppet-twitter) - the [mx-puppet-twitter](https://github.com/Sorunome/mx-puppet-twitter) bridge to [Twitter](https://twitter.com) (optional) -- [sorunome/mx-puppet-discord](https://hub.docker.com/r/sorunome/mx-puppet-discord) - the [mx-puppet-discord](https://github.com/matrix-discord/mx-puppet-discord) bridge to [Discord](https:/discordapp.com) (optional) +- [sorunome/mx-puppet-discord](https://hub.docker.com/r/sorunome/mx-puppet-discord) - the [mx-puppet-discord](https://github.com/matrix-discord/mx-puppet-discord) bridge to [Discord](https://discordapp.com) (optional) - [icewind1991/mx-puppet-steam](https://hub.docker.com/r/icewind1991/mx-puppet-steam) - the [mx-puppet-steam](https://github.com/icewind1991/mx-puppet-steam) bridge to [Steam](https://steampowered.com) (optional) From 073c96a3fd7df54031c5b5cd317ac8b147244236 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 19 Aug 2020 09:29:39 +0300 Subject: [PATCH 31/65] Be more clear that Corporal requires Shared Secret Auth --- docs/configuring-playbook-matrix-corporal.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuring-playbook-matrix-corporal.md b/docs/configuring-playbook-matrix-corporal.md index e983f472..6d7faad0 100644 --- a/docs/configuring-playbook-matrix-corporal.md +++ b/docs/configuring-playbook-matrix-corporal.md @@ -15,9 +15,15 @@ If you decide that you'd like to let this playbook install it for you, you'd nee ## Playbook configuration + You would then need some configuration like this: ```yaml +# The Shared Secret Auth password provider module is required for Corporal to work. +# See configuring-playbook-shared-secret-auth.md +matrix_synapse_ext_password_provider_shared_secret_auth_enabled: true +matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret: YOUR_SHARED_SECRET_GOES_HERE + matrix_corporal_enabled: true matrix_corporal_policy_provider_config: | From daf13107a03e5047ff30bceb57edfa76fd449eda Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Aug 2020 13:52:16 +0300 Subject: [PATCH 32/65] Add support for rust-synapse-compress-state --- CHANGELOG.md | 9 ++ docs/maintenance-postgres.md | 4 +- docs/maintenance-synapse.md | 73 ++++++----- roles/matrix-base/defaults/main.yml | 1 + roles/matrix-synapse/defaults/main.yml | 7 ++ roles/matrix-synapse/tasks/main.yml | 5 + .../compress_room.yml | 48 +++++++ .../rust-synapse-compress-state/main.yml | 118 ++++++++++++++++++ 8 files changed, 227 insertions(+), 38 deletions(-) create mode 100644 roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml create mode 100644 roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 140c4f70..674721cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# 2020-08-21 + +## rust-synapse-compress-state support + +The playbook can now help you use [rust-synapse-compress-state](https://github.com/matrix-org/rust-synapse-compress-state) to compress the state groups in your Synapse database. + +See our [Compressing state with rust-synapse-compress-state](docs/maintenance-synapse.md#compressing-state-with-rust-synapse-compress-state) documentation page to get started. + + # 2020-07-22 ## Synapse Admin support diff --git a/docs/maintenance-postgres.md b/docs/maintenance-postgres.md index da62509b..6f599de1 100644 --- a/docs/maintenance-postgres.md +++ b/docs/maintenance-postgres.md @@ -22,6 +22,8 @@ If you are using an [external Postgres server](configuring-playbook-external-pos ## Vacuuming PostgreSQL +Deleting lots data from Postgres does not make it release disk space, until you perform a `VACUUM` operation. + To perform a `FULL` Postgres [VACUUM](https://www.postgresql.org/docs/current/sql-vacuum.html), run the playbook with `--tags=run-postgres-vacuum`. Example: @@ -42,7 +44,7 @@ docker run \ --rm \ --network=matrix \ --env-file=/matrix/postgres/env-postgres-psql \ -postgres:12.1-alpine \ +postgres:12.4-alpine \ pg_dumpall -h matrix-postgres \ | gzip -c \ > /postgres.sql.gz diff --git a/docs/maintenance-synapse.md b/docs/maintenance-synapse.md index afbf86ed..79998299 100644 --- a/docs/maintenance-synapse.md +++ b/docs/maintenance-synapse.md @@ -9,75 +9,74 @@ Table of contents: - [Purging old data with the Purge History API](#purging-old-data-with-the-purge-history-api), for when you wish to delete in-use (but old) data from the Synapse database - [Synapse maintenance](#synapse-maintenance) - - [Purging unused data with synapse-janitor](#purging-unused-data-with-synapse-janitor) - - [Vacuuming Postgres](#vacuuming-postgres) - [Purging old data with the Purge History API](#purging-old-data-with-the-purge-history-api) - [Compressing state with rust-synapse-compress-state](#compressing-state-with-rust-synapse-compress-state) + - [Purging unused data with synapse-janitor](#purging-unused-data-with-synapse-janitor) - [Browse and manipulate the database](#browse-and-manipulate-the-database) - [Browse and manipulate the database](#browse-and-manipulate-the-database), for when you really need to take matters into your own hands -## Purging unused data with synapse-janitor -**NOTE**: There are [reports](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/465) that **synapse-janitor is dangerous to use and causes database corruption**. You may wish to refrain from using it. +## Purging old data with the Purge History API -When you **leave** and **forget** a room, Synapse can clean up its data, but currently doesn't. -This **unused and unreachable data** remains in your database forever. +You can use the **Purge History API** to delete in-use (but old) data. -There are external tools (like [synapse-janitor](https://github.com/xwiki-labs/synapse_scripts)), which are meant to solve this problem. +**This is destructive** (especially for non-federated rooms), because it means **people will no longer have access to history past a certain point**. -To ask the playbook to run synapse-janitor, execute: +Synapse's [Purge History API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst) can be used to purge on a per-room basis. -```bash -ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor,start -``` +To make use of this API, **you'll need an admin access token** first. You can find your access token in the setting of some clients (like Element). +Alternatively, you can log in and obtain a new access token like this: -**Note**: this will automatically stop Synapse temporarily and restart it later. +``` +curl \ +--data '{"identifier": {"type": "m.id.user", "user": "YOUR_MATRIX_USERNAME" }, "password": "YOUR_MATRIX_PASSWORD", "type": "m.login.password", "device_id": "Synapse-Purge-History-API"}' \ +https://matrix.DOMAIN/_matrix/client/r0/login +``` +Follow the [Purge History API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst) documentation page for the actual purging instructions. -### Vacuuming Postgres +After deleting data, you may wish to run a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql). -Running synapse-janitor potentially deletes a lot of data from the Postgres database. -However, disk space only ever gets released after a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql). -It's easiest if you ask the playbook to run both synapse-janitor and a `VACUUM FULL` in one call: +## Compressing state with rust-synapse-compress-state -```bash -ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor,run-postgres-vacuum,start -``` +[rust-synapse-compress-state](https://github.com/matrix-org/rust-synapse-compress-state) can be used to optimize some `_state` tables used by Synapse. -**Note**: this will automatically stop Synapse temporarily and restart it later. You'll also need plenty of available disk space in your Postgres data directory (usually `/matrix/postgres/data`). +This tool should be safe to use (even when Synapse is running), but it's always a good idea to [make Postgres backups](./maintenance-postgres.md#backing-up-postgresql) first. +To ask the playbook to run rust-synapse-compress-state, execute: -## Purging old data with the Purge History API +``` +ansible-playbook -i inventory/hosts setup.yml --tags=rust-synapse-compress-state +``` -If [purging unused and unreachable data](#purging-unused-data-with-synapse-janitor) is not enough for you, you can start deleting in-use (but old) data. +By default, all rooms with more than `100000` state group rows will be compressed. +If you need to adjust this, pass: `--extra-vars='matrix_synapse_rust_synapse_compress_state_min_state_groups_required=SOME_NUMBER_HERE'` to the command above. -**This is destructive** (especially for non-federated rooms), because it means **people will no longer have access to history past a certain point**. +After state compression, you may wish to run a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql). -Synapse provides a [Purge History API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst) that you can use to purge on a per-room basis. -To make use of this API, **you'll need an admin access token** first. You can find your access token in the setting of some clients (like Element). -Alternatively, you can log in and obtain a new access token like this: +## Purging unused data with synapse-janitor -``` -curl \ ---data '{"identifier": {"type": "m.id.user", "user": "YOUR_MATRIX_USERNAME" }, "password": "YOUR_MATRIX_PASSWORD", "type": "m.login.password", "device_id": "Synapse-Purge-History-API"}' \ -https://matrix.DOMAIN/_matrix/client/r0/login -``` +**NOTE**: There are [reports](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/465) that **synapse-janitor is dangerous to use and causes database corruption**. You may wish to refrain from using it. -Follow the [Purge History API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst) documentation page for the actual purging instructions. +When you **leave** and **forget** a room, Synapse can clean up its data, but currently doesn't. +This **unused and unreachable data** remains in your database forever. -Don't forget that disk space only ever gets released after a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql) - something the playbook can help you with. +There are external tools (like [synapse-janitor](https://github.com/xwiki-labs/synapse_scripts)), which are meant to solve this problem. +To ask the playbook to run synapse-janitor, execute: -## Compressing state with rust-synapse-compress-state +```bash +ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor,start +``` -[rust-synapse-compress-state](https://github.com/matrix-org/rust-synapse-compress-state) can be used to optimize some `_state` tables used by Synapse. +**Note**: this will automatically stop Synapse temporarily and restart it later. -Unfortunately, at this time the playbook can't help you run this **experimental tool**. +Running synapse-janitor potentially deletes a lot of data from the Postgres database. +You may wish to run a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql) after that. -Since it's also experimental, you may wish to stay away from it, or at least [make Postgres backups](./maintenance-postgres.md#backing-up-postgresql) first. ## Browse and manipulate the database diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 9ab8bb3c..c70781b3 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -101,6 +101,7 @@ run_postgres_vacuum: true run_synapse_register_user: true run_synapse_update_user_password: true run_synapse_import_media_store: true +run_synapse_rust_synapse_compress_state: true run_setup: true run_self_check: true run_start: true diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 8b2186a5..24dee1e2 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -364,6 +364,13 @@ matrix_synapse_redaction_retention_period: 7d matrix_synapse_user_ips_max_age: 28d + +matrix_synapse_rust_synapse_compress_state_docker_image: "devture/rust-synapse-compress-state:v0.1.0" +matrix_synapse_rust_synapse_compress_state_docker_image_force_pull: "{{ matrix_synapse_rust_synapse_compress_state_docker_image.endswith(':latest') }}" + +matrix_synapse_rust_synapse_compress_state_base_path: "{{ matrix_base_data_path }}/rust-synapse-compress-state" + + # Default Synapse configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-synapse/tasks/main.yml b/roles/matrix-synapse/tasks/main.yml index 585a95bb..48e72ace 100644 --- a/roles/matrix-synapse/tasks/main.yml +++ b/roles/matrix-synapse/tasks/main.yml @@ -43,6 +43,11 @@ tags: - update-user-password +- import_tasks: "{{ role_path }}/tasks/rust-synapse-compress-state/main.yml" + when: run_synapse_rust_synapse_compress_state|bool + tags: + - rust-synapse-compress-state + - name: Mark matrix-synapse role as executed set_fact: matrix_synapse_role_executed: true diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml new file mode 100644 index 00000000..ad7b91b4 --- /dev/null +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml @@ -0,0 +1,48 @@ +- debug: + msg: "Compressing room `{{ room_details.room_id }}` having {{ room_details.count }} state group rows" + +- name: Generate rust-synapse-compress-state room compression command + set_fact: + matrix_synapse_rust_synapse_compress_state_compress_room_command: >- + {{ matrix_host_command_docker }} run --rm --name matrix-rust-synapse-compress-state-compress-room + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + -v {{ matrix_synapse_rust_synapse_compress_state_base_path }}:/work + {{ matrix_synapse_rust_synapse_compress_state_docker_image }} + /synapse-compress-state -t -o /work/state-compressor.sql + -p "host={{ matrix_synapse_database_host }} user={{ matrix_synapse_database_user }} password={{ matrix_synapse_database_password }} dbname={{ matrix_synapse_database_database }}" + -r '{{ room_details.room_id }}' + +- name: Run rust-synapse-compress-state room compression command (SQL generation) + command: "{{ matrix_synapse_rust_synapse_compress_state_compress_room_command }}" + async: "{{ matrix_synapse_rust_synapse_compress_state_compress_room_time }}" + poll: 10 + register: matrix_synapse_rust_synapse_compress_state_compress_room_command_result + +- debug: var="matrix_synapse_rust_synapse_compress_state_compress_room_command_result" + +- name: Generate Postgres compression SQL import command + set_fact: + matrix_synapse_rust_synapse_compress_state_psql_import_command: >- + {{ matrix_host_command_docker }} run --rm --name matrix-rust-synapse-compress-state-psql-import + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql + -v {{ matrix_synapse_rust_synapse_compress_state_base_path }}:/work:ro + --entrypoint=/bin/sh + {{ matrix_postgres_docker_image_latest }} + -c "cat /work/state-compressor.sql | + psql -v ON_ERROR_STOP=1 -h matrix-postgres" + +- name: Import compression SQL into Postgres + command: "{{ matrix_synapse_rust_synapse_compress_state_psql_import_command }}" + async: "{{ matrix_synapse_rust_synapse_compress_state_psql_import_time }}" + poll: 10 + register: matrix_synapse_rust_synapse_compress_state_psql_import_command_result + +- name: Clean up + file: + path: "{{ matrix_synapse_rust_synapse_compress_state_base_path }}/state-compressor.sql" + state: absent diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml new file mode 100644 index 00000000..4ce02bc4 --- /dev/null +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml @@ -0,0 +1,118 @@ +# Pre-checks + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot use rust-synapse-compress-state." + when: "not matrix_postgres_enabled|bool" + + +# Defaults + +- name: Set matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time, if not provided + set_fact: + matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time: 15 + when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time|default('') == ''" + +- name: Set matrix_synapse_rust_synapse_compress_state_compress_room_time, if not provided + set_fact: + matrix_synapse_rust_synapse_compress_state_compress_room_time: 1800 + when: "matrix_synapse_rust_synapse_compress_state_compress_room_time|default('') == ''" + +- name: Set matrix_synapse_rust_synapse_compress_state_psql_import_time, if not provided + set_fact: + matrix_synapse_rust_synapse_compress_state_psql_import_time: 1800 + when: "matrix_synapse_rust_synapse_compress_state_psql_import_time|default('') == ''" + +- name: Set matrix_synapse_rust_synapse_compress_state_min_state_groups_required, if not provided + set_fact: + # The minimum number of state groups we're looking for before we consider a room eligible for compression. + # Rooms with a smaller state groups count will not be compressed. + matrix_synapse_rust_synapse_compress_state_min_state_groups_required: 100000 + when: "matrix_synapse_rust_synapse_compress_state_min_state_groups_required|default('') == ''" + + +# Actual compression work + +- name: Ensure rust-synapse-compress-state paths exist + file: + path: "{{ matrix_synapse_rust_synapse_compress_state_base_path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure rust-synapse-compress-state image is pulled + docker_image: + name: "{{ matrix_synapse_rust_synapse_compress_state_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_synapse_rust_synapse_compress_state_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_synapse_rust_synapse_compress_state_docker_image_force_pull }}" + +- name: Generate rust-synapse-compress-state room find command + set_fact: + matrix_synapse_rust_synapse_compress_state_find_rooms_command: >- + {{ matrix_host_command_docker }} run --rm --name matrix-rust-synapse-compress-state-find-rooms + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql + {{ matrix_postgres_docker_image_latest }} + psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -c + 'SELECT array_to_json(array_agg(row_to_json (r))) FROM (SELECT room_id, count(*) AS count FROM state_groups_state GROUP BY room_id HAVING count(*) > {{ matrix_synapse_rust_synapse_compress_state_min_state_groups_required }} ORDER BY count DESC) r;' + +- name: Find rooms eligible for compression with rust-synapse-compress-state + command: "{{ matrix_synapse_rust_synapse_compress_state_find_rooms_command }}" + async: "{{ matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time }}" + poll: 10 + register: matrix_synapse_rust_synapse_compress_state_find_rooms_command_result + +# We expect the output to be like this: +# +# "stdout_lines": [ +# " array_to_json ", +# "----------------------------------------------------------------------------------------------------------------------------", +# " [{\"room_id\":\"!some-id\",\"count\":2461329},{\"room_id\":\"!another-id\",\"count\":512017}]", +# "(1 row)" +# ] +# +# Row 3 (out of 4) contains the actual result. +# +# Row 3 contains a space when there's no result. + +- block: + - debug: var="matrix_synapse_rust_synapse_compress_state_find_rooms_command_result" + + - name: Fail if room find result is not what we expect + fail: + msg: >- + Expecting 4 lines in the "find rooms" result. + when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.failed or matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.stdout_lines|length != 4" + +- block: + # matrix_synapse_rust_synapse_compress_state_eligible_rooms is a list + # of dictionaries like this: {'room_id': '!some-id', 'count': 2461329} + - set_fact: + matrix_synapse_rust_synapse_compress_state_eligible_rooms: "{{ matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.stdout_lines[2] | from_json }}" + + - name: Display rooms that will be compressed + debug: + msg: >- + The following rooms contain more than {{ matrix_synapse_rust_synapse_compress_state_min_state_groups_required }} state group rows + (configurable via `matrix_synapse_rust_synapse_compress_state_min_state_groups_required`) + and will be compressed: + {{ matrix_synapse_rust_synapse_compress_state_eligible_rooms }} + + - name: Compress room state + include_tasks: "{{ role_path }}/tasks/rust-synapse-compress-state/compress_room.yml" + with_items: "{{ matrix_synapse_rust_synapse_compress_state_eligible_rooms }}" + loop_control: + loop_var: room_details + when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.stdout_lines[2] != ' '" + +- name: Show notice about lack of rooms to compress + debug: + msg: >- + No rooms were found to contain more than {{ matrix_synapse_rust_synapse_compress_state_min_state_groups_required }} state group rows + (configurable via `matrix_synapse_rust_synapse_compress_state_min_state_groups_required`), + so there's nothing to compress. + when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.stdout_lines[2] == ' '" From 6e9600ffeca3e06197b1614590ac79f133a81156 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Aug 2020 12:59:11 +0300 Subject: [PATCH 33/65] Upgrade Synapse (v1.19.0 -> v1.19.1) --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 24dee1e2..c8bc2fd5 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.19.0" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.19.1" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" From 3c285bc6f51371c44b8048747fa6eea0c7759d07 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Aug 2020 13:58:35 +0300 Subject: [PATCH 34/65] Install lsb-release on Debian distros if unavailable Certain more-minimal Debian installations may not have lsb-release installed, which makes the playbook fail. We need lsb-release on Debian, so that ansible_lsb could tell us if this is Debian or Raspbian. --- roles/matrix-base/tasks/server_base/setup.yml | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/roles/matrix-base/tasks/server_base/setup.yml b/roles/matrix-base/tasks/server_base/setup.yml index 2425c82b..64f461ef 100644 --- a/roles/matrix-base/tasks/server_base/setup.yml +++ b/roles/matrix-base/tasks/server_base/setup.yml @@ -3,11 +3,26 @@ - include_tasks: "{{ role_path }}/tasks/server_base/setup_centos.yml" when: ansible_distribution == 'CentOS' -- include_tasks: "{{ role_path }}/tasks/server_base/setup_debian.yml" - when: (ansible_os_family == 'Debian') and (ansible_lsb.id != 'Raspbian') +- block: + # ansible_lsb is only available if lsb-release is installed. + - name: Ensure lsb-release installed + apt: + name: + - lsb-release + state: present + update_cache: yes + register: lsb_release_installation_result -- include_tasks: "{{ role_path }}/tasks/server_base/setup_raspbian.yml" - when: (ansible_os_family == 'Debian') and (ansible_lsb.id == 'Raspbian') + - name: Reread ansible_lsb facts if lsb-release got installed + setup: filter=ansible_lsb* + when: lsb_release_installation_result.changed + + - include_tasks: "{{ role_path }}/tasks/server_base/setup_debian.yml" + when: (ansible_os_family == 'Debian') and (ansible_lsb.id != 'Raspbian') + + - include_tasks: "{{ role_path }}/tasks/server_base/setup_raspbian.yml" + when: (ansible_os_family == 'Debian') and (ansible_lsb.id == 'Raspbian') + when: ansible_os_family == 'Debian' - include_tasks: "{{ role_path }}/tasks/server_base/setup_archlinux.yml" when: ansible_distribution == 'Archlinux' From e3dca2f66f10b384d36d91f1dc29c278ef14db8a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Sep 2020 09:03:06 +0300 Subject: [PATCH 35/65] Try to avoid Docker logs growing too much for one-off containers We recently had a report of the Postgres backup container's log file growing the size of /var/lib/docker until it ran out of disk space. Trying to prevent similar problems in the future. --- docs/maintenance-postgres.md | 1 + roles/matrix-postgres/tasks/import_postgres.yml | 1 + roles/matrix-postgres/tasks/import_sqlite_db.yml | 1 + roles/matrix-postgres/tasks/upgrade_postgres.yml | 2 ++ 4 files changed, 5 insertions(+) diff --git a/docs/maintenance-postgres.md b/docs/maintenance-postgres.md index 6f599de1..d385bc9c 100644 --- a/docs/maintenance-postgres.md +++ b/docs/maintenance-postgres.md @@ -42,6 +42,7 @@ To make a back up of the current PostgreSQL database, make sure it's running and ```bash docker run \ --rm \ +--log-driver=none \ --network=matrix \ --env-file=/matrix/postgres/env-postgres-psql \ postgres:12.4-alpine \ diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index e58711f1..0753c292 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -64,6 +64,7 @@ set_fact: matrix_postgres_import_command: >- {{ matrix_host_command_docker }} run --rm --name matrix-postgres-import + --log-driver=none --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL --network={{ matrix_docker_network }} diff --git a/roles/matrix-postgres/tasks/import_sqlite_db.yml b/roles/matrix-postgres/tasks/import_sqlite_db.yml index 850ee370..fe0deba8 100644 --- a/roles/matrix-postgres/tasks/import_sqlite_db.yml +++ b/roles/matrix-postgres/tasks/import_sqlite_db.yml @@ -74,6 +74,7 @@ docker run --rm --name=matrix-synapse-migrate + --log-driver=none --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL --network={{ matrix_docker_network }} diff --git a/roles/matrix-postgres/tasks/upgrade_postgres.yml b/roles/matrix-postgres/tasks/upgrade_postgres.yml index 73263695..c1a01d3f 100644 --- a/roles/matrix-postgres/tasks/upgrade_postgres.yml +++ b/roles/matrix-postgres/tasks/upgrade_postgres.yml @@ -80,6 +80,7 @@ - name: Perform Postgres database dump command: >- {{ matrix_host_command_docker }} run --rm --name matrix-postgres-dump + --log-driver=none --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --network={{ matrix_docker_network }} --env-file={{ matrix_postgres_base_path }}/env-postgres-psql @@ -124,6 +125,7 @@ set_fact: matrix_postgres_import_command: >- {{ matrix_host_command_docker }} run --rm --name matrix-postgres-import + --log-driver=none --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL --network={{ matrix_docker_network }} From b117dc0cb76e548bb1f1badadf6c485a448f22a0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Sep 2020 11:44:44 +0300 Subject: [PATCH 36/65] Fix self-check certificate validation when not self-signed --- group_vars/matrix_servers | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 4a989f03..4cf4e851 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -737,7 +737,7 @@ matrix_ma1sd_threepid_medium_email_connectors_smtp_host: "matrix-mailer" matrix_ma1sd_threepid_medium_email_connectors_smtp_port: 8025 matrix_ma1sd_threepid_medium_email_connectors_smtp_tls: 0 -matrix_ma1sd_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else false }}" +matrix_ma1sd_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" matrix_ma1sd_systemd_wanted_services_list: | {{ @@ -799,7 +799,7 @@ matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled: "{{ matrix_ma1sd_ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_with_container: "{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container }}" matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_sans_container: "{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container }}" -matrix_nginx_proxy_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else false }}" +matrix_nginx_proxy_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" matrix_nginx_proxy_systemd_wanted_services_list: | {{ @@ -890,7 +890,7 @@ matrix_client_element_integrations_rest_url: "{{ matrix_dimension_integrations_r matrix_client_element_integrations_widgets_urls: "{{ matrix_dimension_integrations_widgets_urls if matrix_dimension_enabled else ['https://scalar.vector.im/api'] }}" matrix_client_element_integrations_jitsi_widget_url: "{{ matrix_dimension_integrations_jitsi_widget_url if matrix_dimension_enabled else 'https://scalar.vector.im/api/widgets/jitsi.html' }}" -matrix_client_element_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else false }}" +matrix_client_element_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" matrix_client_element_registration_enabled: "{{ matrix_synapse_enable_registration }}" @@ -985,7 +985,7 @@ matrix_synapse_turn_uris: | matrix_synapse_turn_shared_secret: "{{ matrix_coturn_turn_static_auth_secret if matrix_coturn_enabled else '' }}" -matrix_synapse_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else false }}" +matrix_synapse_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" matrix_synapse_systemd_required_services_list: | {{ From a456e3a9e73481ff6560f3d4237ab2eaadf81820 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Sep 2020 13:12:11 +0300 Subject: [PATCH 37/65] Surface certain messages at the end of playbook execution Fixes #106 (Github Issue). --- .../tasks/dump_runtime_results.yml | 6 ++++++ roles/matrix-common-after/tasks/main.yml | 4 ++++ roles/matrix-postgres/tasks/setup_postgres.yml | 13 ++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 roles/matrix-common-after/tasks/dump_runtime_results.yml diff --git a/roles/matrix-common-after/tasks/dump_runtime_results.yml b/roles/matrix-common-after/tasks/dump_runtime_results.yml new file mode 100644 index 00000000..9788bf84 --- /dev/null +++ b/roles/matrix-common-after/tasks/dump_runtime_results.yml @@ -0,0 +1,6 @@ +# Ansible outputs the message in the `item=` field. +# It's unnecessary to output it again in the actual message, so we don't. +- debug: + msg: "" + with_items: "{{ matrix_playbook_runtime_results }}" + when: "matrix_playbook_runtime_results is defined and matrix_playbook_runtime_results|length > 0" diff --git a/roles/matrix-common-after/tasks/main.yml b/roles/matrix-common-after/tasks/main.yml index fb942553..197ab6a9 100644 --- a/roles/matrix-common-after/tasks/main.yml +++ b/roles/matrix-common-after/tasks/main.yml @@ -7,3 +7,7 @@ when: run_stop|bool tags: - stop + +- import_tasks: "{{ role_path }}/tasks/dump_runtime_results.yml" + tags: + - always diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index e4d44392..f186bdca 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -18,9 +18,16 @@ matrix_postgres_docker_image_to_use: "{{ matrix_postgres_docker_image_latest if matrix_postgres_detected_version_corresponding_docker_image == '' else matrix_postgres_detected_version_corresponding_docker_image }}" when: matrix_postgres_enabled|bool -- name: Warn if on an old version of Postgres - debug: - msg: "NOTE: Your setup is on an old Postgres version ({{ matrix_postgres_docker_image_to_use }}), while {{ matrix_postgres_docker_image_latest }} is supported. You can upgrade using --tags=upgrade-postgres" +- name: Inject warning if on an old version of Postgres + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your setup is on an old Postgres version ({{ matrix_postgres_docker_image_to_use }}), while {{ matrix_postgres_docker_image_latest }} is supported. You can upgrade using --tags=upgrade-postgres" + ] + }} when: "matrix_postgres_enabled|bool and matrix_postgres_docker_image_to_use != matrix_postgres_docker_image_latest" # Even if we don't run the internal server, we still need this for running the CLI From da38a7869f91f01f7e023510864718177fb51299 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Sep 2020 13:46:05 +0300 Subject: [PATCH 38/65] Add matrix-registration support --- CHANGELOG.md | 9 ++ README.md | 16 ++- ...onfiguring-playbook-matrix-registration.md | 53 +++++++++ docs/configuring-playbook.md | 2 + docs/registering-users.md | 42 +++++-- docs/self-building.md | 1 + group_vars/matrix_servers | 31 ++++++ roles/matrix-registration/defaults/main.yml | 83 ++++++++++++++ .../tasks/generate_token.yml | 50 +++++++++ roles/matrix-registration/tasks/init.yml | 64 +++++++++++ roles/matrix-registration/tasks/main.yml | 19 ++++ roles/matrix-registration/tasks/setup.yml | 103 ++++++++++++++++++ .../tasks/validate_config.yml | 11 ++ .../templates/config.yaml.j2 | 30 +++++ .../systemd/matrix-registration.service.j2 | 40 +++++++ roles/matrix-synapse-admin/tasks/setup.yml | 2 +- setup.yml | 1 + 17 files changed, 542 insertions(+), 15 deletions(-) create mode 100644 docs/configuring-playbook-matrix-registration.md create mode 100644 roles/matrix-registration/defaults/main.yml create mode 100644 roles/matrix-registration/tasks/generate_token.yml create mode 100644 roles/matrix-registration/tasks/init.yml create mode 100644 roles/matrix-registration/tasks/main.yml create mode 100644 roles/matrix-registration/tasks/setup.yml create mode 100644 roles/matrix-registration/tasks/validate_config.yml create mode 100644 roles/matrix-registration/templates/config.yaml.j2 create mode 100644 roles/matrix-registration/templates/systemd/matrix-registration.service.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 674721cb..2aabee8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# 2020-09-01 + +## matrix-registration support + +The playbook can now help you set up [matrix-registration](https://github.com/ZerataX/matrix-registration) - an application that lets you keep your Matrix server's registration private, but still allow certain users (those having a unique registration link) to register by themselves. + +See our [Setting up matrix-registration](docs/configuring-playbook-matrix-registration.md) documentation page to get started. + + # 2020-08-21 ## rust-synapse-compress-state support diff --git a/README.md b/README.md index 180be14a..a9734142 100644 --- a/README.md +++ b/README.md @@ -60,17 +60,19 @@ Using this playbook, you can get the following services configured on your serve - (optional) the [mx-puppet-steam](https://github.com/icewind1991/mx-puppet-steam) bridge for [Steam](https://steamapp.com/)) - see [docs/configuring-playbook-bridge-mx-puppet-steam.md](docs/configuring-playbook-bridge-mx-puppet-steam.md) for setup documentation -- (optional) the [matrix-sms-bridge](https://github.com/benkuly/matrix-sms-bridge) for bridging your Matrix server to SMS +- (optional) the [matrix-sms-bridge](https://github.com/benkuly/matrix-sms-bridge) for bridging your Matrix server to SMS - see [docs/configuring-playbook-matrix-bridge-sms.md](docs/configuring-playbook-matrix-bridge-sms.md) for setup documentation -- (optional) [Email2Matrix](https://github.com/devture/email2matrix) for relaying email messages to Matrix rooms +- (optional) [Email2Matrix](https://github.com/devture/email2matrix) for relaying email messages to Matrix rooms - see [docs/configuring-playbook-email2matrix.md](docs/configuring-playbook-email2matrix.md) for setup documentation -- (optional) [Dimension](https://github.com/turt2live/matrix-dimension), an open source integrations manager for matrix clients +- (optional) [Dimension](https://github.com/turt2live/matrix-dimension), an open source integrations manager for matrix clients - see [docs/configuring-playbook-dimension.md](docs/configuring-playbook-dimension.md) for setup documentation -- (optional) [Jitsi](https://jitsi.org/), an open source video-conferencing platform +- (optional) [Jitsi](https://jitsi.org/), an open source video-conferencing platform - see [docs/configuring-playbook-jitsi.md](docs/configuring-playbook-jitsi.md) for setup documentation -- (optional) [matrix-reminder-bot](https://github.com/anoadragon453/matrix-reminder-bot) for scheduling one-off & recurring reminders and alarms +- (optional) [matrix-reminder-bot](https://github.com/anoadragon453/matrix-reminder-bot) for scheduling one-off & recurring reminders and alarms - see [docs/configuring-playbook-bot-matrix-reminder-bot.md](docs/configuring-playbook-bot-matrix-reminder-bot.md) for setup documentation -- (optional) [synapse-admin](https://github.com/Awesome-Technologies/synapse-admin), a web UI tool for administrating users and rooms on your Matrix server +- (optional) [synapse-admin](https://github.com/Awesome-Technologies/synapse-admin), a web UI tool for administrating users and rooms on your Matrix server - see [docs/configuring-playbook-synapse-admin.md](docs/configuring-playbook-synapse-admin.md) for setup documentation + +- (optional) [matrix-registration](https://github.com/ZerataX/matrix-registration), a simple python application to have a token based matrix registration - see [docs/configuring-playbook-matrix-registration.md](docs/configuring-playbook-matrix-registration.md) for setup documentation Basically, this playbook aims to get you up-and-running with all the basic necessities around Matrix, without you having to do anything else. @@ -140,6 +142,8 @@ This playbook sets up your server using the following Docker images: - [devture/matrix-corporal](https://hub.docker.com/r/devture/matrix-corporal/) - [Matrix Corporal](https://github.com/devture/matrix-corporal): reconciliator and gateway for a managed Matrix server (optional) +- [devture/zeratax-matrix-registration](https://hub.docker.com/r/devture/zeratax-matrix-registration/) - [matrix-registration](https://github.com/ZerataX/matrix-registration): a simple python application to have a token based matrix registration (optional) + - [nginx](https://hub.docker.com/_/nginx/) - the [nginx](http://nginx.org/) web server (optional) - [certbot/certbot](https://hub.docker.com/r/certbot/certbot/) - the [certbot](https://certbot.eff.org/) tool for obtaining SSL certificates from [Let's Encrypt](https://letsencrypt.org/) (optional) diff --git a/docs/configuring-playbook-matrix-registration.md b/docs/configuring-playbook-matrix-registration.md new file mode 100644 index 00000000..28616945 --- /dev/null +++ b/docs/configuring-playbook-matrix-registration.md @@ -0,0 +1,53 @@ +# Setting up matrix-registration (optional) + +The playbook can install and configure [matrix-registration](https://github.com/ZerataX/matrix-registration) for you. + +> matrix-registration is a simple python application to have a token based matrix registration. + +Use matrix-registration to **create unique registration links**, which people can use to register on your Matrix server. It allows you to **keep your server's registration closed (private)**, but still allow certain people (these having a special link) to register a user account. + +**matrix-registration** provides 2 things: + +- **an API for creating registration tokens** (unique registration links). This API can be used via `curl` or via the playbook (see [Usage](#usage) below) + +- **a user registration page**, where people can use these registration tokens. By default, exposed at `https:///matrix.DOMAIN/matrix-registration` + + +## Installing + +Adjust your playbook configuration (your `inventory/host_vars/matrix.DOMAIN/vars.yml` file): + +```yaml +matrix_registration_enabled: true + +# Generate a strong secret using: `pwgen -s 64 1`. +matrix_registration_admin_secret: "ENTER_SOME_SECRET_HERE" +``` + +Then, run the [installation](installing.md) command again: + +``` +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start +``` + + +## Usage + +**matrix-registration** gets exposed at `https:///matrix.DOMAIN/matrix-registration` + +It provides various [APIs](https://github.com/ZerataX/matrix-registration/wiki/api) - for creating registration tokens, listing tokens, disabling tokens, etc. To make use of all of its capabilities, consider using `curl`. + +We make the most common API (the one for creating unique registration tokens) easy to use via the playbook. + +**To create a new user registration token (link)**, use this command: + +``` +ansible-playbook -i inventory/hosts setup.yml \ +--tags=generate-matrix-registration-token \ +--extra-vars="one_time=yes ex_date=2021-12-31" +``` + +The above command creates and returns a **one-time use** token, which **expires** on the 31st of December 2021. +Adjust the `one_time` and `ex_date` variables as you see fit. + +Share the unique registration link (generated by the command above) with users to let them register on your Matrix server. diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 3714247e..e4d5b756 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -70,6 +70,8 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up Synapse Admin](configuring-playbook-synapse-admin.md) (optional) +- [Setting up matrix-registration](configuring-playbook-matrix-registration.md) (optional) + - [Setting up the REST authentication password provider module](configuring-playbook-rest-auth.md) (optional, advanced) - [Setting up the Shared Secret Auth password provider module](configuring-playbook-shared-secret-auth.md) (optional, advanced) diff --git a/docs/registering-users.md b/docs/registering-users.md index 09394450..cd77b6b6 100644 --- a/docs/registering-users.md +++ b/docs/registering-users.md @@ -1,6 +1,18 @@ # Registering users -Run this to create a new user account on your Matrix server. +This documentation page tells you how to create user account on your Matrix server. + +Table of contents: + +- [Registering users](#registering-users) + - [Registering users manually](#registering-users-manually) + - [Managing users via a Web UI](#managing-users-via-a-web-ui) + - [Letting certain users register on your private server](#letting-certain-users-register-on-your-private-server) + - [Enabling public user registration](#enabling-public-user-registration) + - [Adding/Removing Administrator privileges to an existing user](#addingremoving-administrator-privileges-to-an-existing-user) + + +## Registering users manually You can do it via this Ansible playbook (make sure to edit the `` and `` part below): @@ -22,10 +34,29 @@ ansible-playbook -i inventory/hosts setup.yml --extra-vars='username= <0/1> ``` - - -## Managing users via a Web UI - -To manage users more easily (via a web user-interace), you can install [Synapse Admin](configuring-playbook-synapse-admin.md). diff --git a/docs/self-building.md b/docs/self-building.md index 84d61fe0..169e4aa5 100644 --- a/docs/self-building.md +++ b/docs/self-building.md @@ -13,6 +13,7 @@ List of roles where self-building the Docker image is currently possible: - `matrix-synapse` - `matrix-synapse-admin` - `matrix-client-element` +- `matrix-registration` - `matrix-coturn` - `matrix-ma1sd` - `matrix-mailer` diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 4cf4e851..b82dbf23 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1029,3 +1029,34 @@ matrix_synapse_admin_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy # /matrix-synapse-admin # ###################################################################### + + + +###################################################################### +# +# matrix-registration +# +###################################################################### + +matrix_registration_enabled: false + +# Normally, matrix-nginx-proxy is enabled and nginx can reach matrix-registration over the container network. +# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose +# matrix-registration's HTTP port to the local host. +matrix_registration_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:8767' }}" + +matrix_registration_riot_instance: "{{ ('https://' + matrix_server_fqn_element) if matrix_client_element_enabled else 'https://riot.im/app/' }}" + +matrix_registration_shared_secret: "{{ matrix_synapse_registration_shared_secret if matrix_synapse_enabled else '' }}" + +matrix_registration_server_location: "{{ 'http://matrix-synapse:8008' if matrix_synapse_enabled else '' }}" + +matrix_registration_api_validate_certs: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" + +matrix_registration_container_image_self_build: "{{ matrix_architecture != 'amd64' }}" + +###################################################################### +# +# /matrix-registration +# +###################################################################### diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml new file mode 100644 index 00000000..7eb9340e --- /dev/null +++ b/roles/matrix-registration/defaults/main.yml @@ -0,0 +1,83 @@ +# matrix-registration is a simple python application to have a token based matrix registration +# See: https://zeratax.github.io/matrix-registration/ + +matrix_registration_enabled: true + +matrix_registration_container_image_self_build: false + +matrix_registration_base_path: "{{ matrix_base_data_path }}/matrix-registration" +matrix_registration_config_path: "{{ matrix_registration_base_path }}/config" +matrix_registration_data_path: "{{ matrix_registration_base_path }}/data" +matrix_registration_docker_src_files_path: "{{ matrix_registration_base_path }}/docker-src" + +matrix_registration_version: "v0.7.0" + +matrix_registration_docker_image: "devture/zeratax-matrix-registration:{{ matrix_registration_version }}" +matrix_registration_docker_image_force_pull: "{{ matrix_registration_docker_image.endswith(':latest') }}" +matrix_registration_docker_repo: "https://github.com/ZerataX/matrix-registration" + +# A list of extra arguments to pass to the container +matrix_registration_container_extra_arguments: [] + +# List of systemd services that matrix-registration.service depends on +matrix_registration_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-registration.service wants +matrix_registration_systemd_wanted_services_list: [] + +# Controls whether the matrix-registration container exposes its HTTP port (tcp/5000 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:8767"), or empty string to not expose. +matrix_registration_container_http_host_bind_port: '' + +# The path at which Matrix Registration will be exposed on `matrix.DOMAIN` +# (only applies when matrix-nginx-proxy is used). +matrix_registration_public_endpoint: /matrix-registration + +matrix_registration_api_register_endpoint: "{{ matrix_homeserver_url }}{{ matrix_registration_public_endpoint }}/register" +matrix_registration_api_token_endpoint: "{{ matrix_homeserver_url }}{{ matrix_registration_public_endpoint }}/token" + +matrix_registration_api_validate_certs: true + +# The URL to your homeserver (e.g.: `https://matrix.DOMAIN`). +# A local (in-container address) is preferable. +matrix_registration_server_location: "" + +matrix_registration_server_name: "{{ matrix_domain }}" + +# matrix_registration_shared_secret needs to match the homeserver's registration secret. +# For Synapse, that's the `registration_shared_secret` setting. +matrix_registration_shared_secret: "" + +# matrix_registration_admin_secret is your own admin secret for using matrix-registration (creating new tokens, etc.) +matrix_registration_admin_secret: "" + +matrix_registration_riot_instance: "https://riot.im/app/" + + +# Default matrix-registration configuration template which covers the generic use case. +# You can customize it by controlling the various variables inside it. +# +# For a more advanced customization, you can extend the default (see `matrix_registration_configuration_extension_yaml`) +# or completely replace this variable with your own template. +matrix_registration_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}" + +matrix_registration_configuration_extension_yaml: | + # Your custom YAML configuration for registration goes here. + # This configuration extends the default starting configuration (`matrix_registration_configuration_yaml`). + # + # You can override individual variables from the default configuration, or introduce new ones. + # + # If you need something more special, you can take full control by + # completely redefining `matrix_registration_configuration_yaml`. + # + # Example configuration extension follows: + # + # password: + # min_length: 12 + +matrix_registration_configuration_extension: "{{ matrix_registration_configuration_extension_yaml|from_yaml if matrix_registration_configuration_extension_yaml|from_yaml is mapping else {} }}" + +# Holds the final matrix-registration configuration (a combination of the default and its extension). +# You most likely don't need to touch this variable. Instead, see `matrix_registration_configuration_yaml`. +matrix_registration_configuration: "{{ matrix_registration_configuration_yaml|from_yaml|combine(matrix_registration_configuration_extension, recursive=True) }}" diff --git a/roles/matrix-registration/tasks/generate_token.yml b/roles/matrix-registration/tasks/generate_token.yml new file mode 100644 index 00000000..ae5bdf4c --- /dev/null +++ b/roles/matrix-registration/tasks/generate_token.yml @@ -0,0 +1,50 @@ +- name: Fail if playbook called incorrectly + fail: + msg: "The `one_time` variable needs to be provided to this playbook, via --extra-vars" + when: "one_time is not defined or one_time not in ['yes', 'no']" + +- name: Fail if playbook called incorrectly + fail: + msg: "The `ex_date` variable (expiration date) needs to be provided to this playbook, via --extra-vars" + when: "ex_date is not defined or ex_date == ''" + +- name: Call matrix-registration token creation API + uri: + url: "{{ matrix_registration_api_token_endpoint }}" + follow_redirects: none + validate_certs: "{{ matrix_registration_api_validate_certs }}" + headers: + Content-Type: application/json + Authorization: "SharedSecret {{ matrix_registration_admin_secret }}" + method: POST + body_format: json + body: | + { + "one_time": {{ 'true' if one_time == 'yes' else 'false' }}, + "ex_date": {{ ex_date|to_json }} + } + check_mode: no + register: matrix_registration_api_result + +- set_fact: + matrix_registration_api_result_message: >- + matrix-registration result: + + Direct registration link (with the token prefilled): + + {{ matrix_registration_api_register_endpoint }}?token={{ matrix_registration_api_result.json.name }} + + Full token details are: + + {{ matrix_registration_api_result.json }} + check_mode: no + +- name: Inject result message into matrix_playbook_runtime_results + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [matrix_registration_api_result_message] + }} + check_mode: no diff --git a/roles/matrix-registration/tasks/init.yml b/roles/matrix-registration/tasks/init.yml new file mode 100644 index 00000000..60d4b326 --- /dev/null +++ b/roles/matrix-registration/tasks/init.yml @@ -0,0 +1,64 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-registration'] }}" + when: matrix_registration_enabled|bool + +- block: + - name: Fail if matrix-nginx-proxy role already executed + fail: + msg: >- + Trying to append matrix-registration's reverse-proxying configuration to matrix-nginx-proxy, + but it's pointless since the matrix-nginx-proxy role had already executed. + To fix this, please change the order of roles in your plabook, + so that the matrix-nginx-proxy role would run after the matrix-registration role. + when: matrix_nginx_proxy_role_executed|default(False)|bool + + - name: Generate matrix-registration proxying configuration for matrix-nginx-proxy + set_fact: + matrix_registration_matrix_nginx_proxy_configuration: | + rewrite ^{{ matrix_registration_public_endpoint }}$ $scheme://$server_name{{ matrix_registration_public_endpoint }}/ permanent; + rewrite ^{{ matrix_registration_public_endpoint }}/$ $scheme://$server_name{{ matrix_registration_public_endpoint }}/register redirect; + + location ~ ^{{ matrix_registration_public_endpoint }}/(.*) { + {% if matrix_nginx_proxy_enabled|default(False) %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-registration:5000"; + proxy_pass http://$backend/$1; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://127.0.0.1:8767/$1; + {% endif %} + + {# + Workaround matrix-registration serving static files at /static + (see https://github.com/ZerataX/matrix-registration/issues/29) + + Also fixing the form, which goes to /register. + #} + sub_filter_once off; + sub_filter_types text/html; + sub_filter "/static/" "{{ matrix_registration_public_endpoint }}/static/"; + sub_filter "/register" "{{ matrix_registration_public_endpoint }}/register"; + } + + - name: Register matrix-registration proxying configuration with matrix-nginx-proxy + set_fact: + matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks: | + {{ + matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks|default([]) + + + [matrix_registration_matrix_nginx_proxy_configuration] + }} + tags: + - always + when: matrix_registration_enabled|bool + +- name: Warn about reverse-proxying if matrix-nginx-proxy not used + debug: + msg: >- + NOTE: You've enabled the matrix-registration tool but are not using the matrix-nginx-proxy + reverse proxy. + Please make sure that you're proxying the `{{ matrix_registration_public_endpoint }}` + URL endpoint to the matrix-registration container. + You can expose the container's port using the `matrix_registration_container_http_host_bind_port` variable. + when: "matrix_registration_enabled|bool and matrix_nginx_proxy_enabled is not defined" diff --git a/roles/matrix-registration/tasks/main.yml b/roles/matrix-registration/tasks/main.yml new file mode 100644 index 00000000..4a884ccd --- /dev/null +++ b/roles/matrix-registration/tasks/main.yml @@ -0,0 +1,19 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/validate_config.yml" + when: "run_setup|bool and matrix_registration_enabled|bool" + tags: + - setup-all + - setup-matrix-registration + +- import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + - setup-all + - setup-matrix-registration + +- import_tasks: "{{ role_path }}/tasks/generate_token.yml" + when: "run_setup|bool and matrix_registration_enabled|bool" + tags: + - generate-matrix-registration-token diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup.yml new file mode 100644 index 00000000..29b2347b --- /dev/null +++ b/roles/matrix-registration/tasks/setup.yml @@ -0,0 +1,103 @@ +--- + +# +# Tasks related to setting up matrix-registration +# + +- name: Ensure matrix-registration paths exist + file: + path: "{{ item.path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - { path: "{{ matrix_registration_base_path }}", when: true } + - { path: "{{ matrix_registration_config_path }}", when: true } + - { path: "{{ matrix_registration_data_path }}", when: true } + - { path: "{{ matrix_registration_docker_src_files_path }}", when: "{{ matrix_registration_container_image_self_build }}"} + when: matrix_registration_enabled|bool and item.when + +- name: Ensure matrix-registration image is pulled + docker_image: + name: "{{ matrix_registration_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_registration_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_registration_docker_image_force_pull }}" + when: "matrix_registration_enabled|bool and not matrix_registration_container_image_self_build|bool" + +- name: Ensure matrix-registration repository is present when self-building + git: + repo: "{{ matrix_registration_docker_repo }}" + dest: "{{ matrix_registration_docker_src_files_path }}" + version: "{{ matrix_registration_version }}" + force: "yes" + register: matrix_registration_git_pull_results + when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool" + +- name: Ensure matrix-registration Docker image is built + docker_image: + name: "{{ matrix_registration_docker_image }}" + source: build + force_source: yes + build: + dockerfile: Dockerfile + path: "{{ matrix_registration_docker_src_files_path }}" + pull: yes + when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool and matrix_registration_git_pull_results.changed" + +- name: Ensure matrix-registration config installed + copy: + content: "{{ matrix_registration_configuration|to_nice_yaml }}" + dest: "{{ matrix_registration_config_path }}/config.yaml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + when: matrix_registration_enabled|bool + +- name: Ensure matrix-registration.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-registration.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-registration.service" + mode: 0644 + register: matrix_registration_systemd_service_result + when: matrix_registration_enabled|bool + +- name: Ensure systemd reloaded after matrix-registration.service installation + service: + daemon_reload: yes + when: "matrix_registration_enabled|bool and matrix_registration_systemd_service_result.changed" + +# +# Tasks related to getting rid of matrix-registration (if it was previously enabled) +# + +- name: Check existence of matrix-registration service + stat: + path: "{{ matrix_systemd_path }}/matrix-registration.service" + register: matrix_registration_service_stat + +- name: Ensure matrix-registration is stopped + service: + name: matrix-registration + state: stopped + daemon_reload: yes + register: stopping_result + when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" + +- name: Ensure matrix-registration.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-registration.service" + state: absent + when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-registration.service removal + service: + daemon_reload: yes + when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" + +- name: Ensure matrix-registration Docker image doesn't exist + docker_image: + name: "{{ matrix_registration_docker_image }}" + state: absent + when: "not matrix_registration_enabled|bool" diff --git a/roles/matrix-registration/tasks/validate_config.yml b/roles/matrix-registration/tasks/validate_config.yml new file mode 100644 index 00000000..6b2f0277 --- /dev/null +++ b/roles/matrix-registration/tasks/validate_config.yml @@ -0,0 +1,11 @@ +--- + +- name: Fail if required matrix-registration settings not defined + fail: + msg: > + You need to define a required configuration setting (`{{ item }}`) for using matrix-registration. + when: "vars[item] == ''" + with_items: + - "matrix_registration_shared_secret" + - "matrix_registration_admin_secret" + - "matrix_registration_server_location" diff --git a/roles/matrix-registration/templates/config.yaml.j2 b/roles/matrix-registration/templates/config.yaml.j2 new file mode 100644 index 00000000..f3b1c57b --- /dev/null +++ b/roles/matrix-registration/templates/config.yaml.j2 @@ -0,0 +1,30 @@ +server_location: {{ matrix_registration_server_location|to_json }} +server_name: {{ matrix_registration_server_name|to_json }} +shared_secret: {{ matrix_registration_shared_secret|to_json }} +admin_secret: {{ matrix_registration_admin_secret|to_json }} +riot_instance: {{ matrix_registration_riot_instance|to_json }} +db: 'sqlite:////data/db.sqlite3' +host: '0.0.0.0' +port: 5000 +rate_limit: ["100 per day", "10 per minute"] +allow_cors: false +logging: + disable_existing_loggers: False + version: 1 + root: + level: DEBUG + handlers: [console] + formatters: + brief: + format: '%(name)s - %(levelname)s - %(message)s' + precise: + format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + handlers: + console: + class: logging.StreamHandler + level: INFO + formatter: brief + stream: ext://sys.stdout +# password requirements +password: + min_length: 8 diff --git a/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 b/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 new file mode 100644 index 00000000..38860729 --- /dev/null +++ b/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 @@ -0,0 +1,40 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=matrix-registration +{% for service in matrix_registration_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_registration_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} + +[Service] +Type=simple +ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-registration +ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-registration + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-registration \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + {% if matrix_registration_container_http_host_bind_port %} + -p {{ matrix_registration_container_http_host_bind_port }}:5000 \ + {% endif %} + -v {{ matrix_registration_config_path }}:/config:ro \ + -v {{ matrix_registration_data_path }}:/data \ + {% for arg in matrix_registration_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_registration_docker_image }} \ + serve + +ExecStop=-{{ matrix_host_command_docker }} kill matrix-registration +ExecStop=-{{ matrix_host_command_docker }} rm matrix-registration +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-registration + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-synapse-admin/tasks/setup.yml b/roles/matrix-synapse-admin/tasks/setup.yml index 65e5c0b9..0ee5e8d2 100644 --- a/roles/matrix-synapse-admin/tasks/setup.yml +++ b/roles/matrix-synapse-admin/tasks/setup.yml @@ -20,7 +20,7 @@ register: matrix_synapse_admin_git_pull_results when: "matrix_synapse_admin_enabled|bool and matrix_synapse_admin_container_self_build|bool" -- name: Ensure matrix-synapse-admin Docker image is build +- name: Ensure matrix-synapse-admin Docker image is built docker_image: name: "{{ matrix_synapse_admin_docker_image }}" source: build diff --git a/setup.yml b/setup.yml index 1c19d442..68c4e4ee 100755 --- a/setup.yml +++ b/setup.yml @@ -26,6 +26,7 @@ - matrix-bot-matrix-reminder-bot - matrix-synapse - matrix-synapse-admin + - matrix-registration - matrix-client-element - matrix-jitsi - matrix-ma1sd From 79012934383eba82153a6f22e583eafa59b92b49 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Sep 2020 13:47:58 +0300 Subject: [PATCH 39/65] Relocate docs page for consistency --- CHANGELOG.md | 2 +- README.md | 2 +- ...-sms.md => configuring-playbook-bridge-matrix-bridge-sms.md} | 0 docs/configuring-playbook.md | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename docs/{configuring-playbook-matrix-bridge-sms.md => configuring-playbook-bridge-matrix-bridge-sms.md} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aabee8b..3e1002b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,7 +96,7 @@ To reuse your existing rooms, invite `@smsbot:yourServer` to the room or write a Thanks to [benkuly](https://github.com/benkuly)'s efforts, the playbook now supports bridging to SMS (with one telephone number only) via [matrix-sms-bridge](https://github.com/benkuly/matrix-sms-bridge). -See our [Setting up Matrix SMS bridging](docs/configuring-playbook-matrix-bridge-sms.md) documentation page for getting started. +See our [Setting up Matrix SMS bridging](docs/configuring-playbook-bridge-matrix-bridge-sms.md) documentation page for getting started. # 2020-05-19 diff --git a/README.md b/README.md index a9734142..a8044354 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Using this playbook, you can get the following services configured on your serve - (optional) the [mx-puppet-steam](https://github.com/icewind1991/mx-puppet-steam) bridge for [Steam](https://steamapp.com/)) - see [docs/configuring-playbook-bridge-mx-puppet-steam.md](docs/configuring-playbook-bridge-mx-puppet-steam.md) for setup documentation -- (optional) the [matrix-sms-bridge](https://github.com/benkuly/matrix-sms-bridge) for bridging your Matrix server to SMS - see [docs/configuring-playbook-matrix-bridge-sms.md](docs/configuring-playbook-matrix-bridge-sms.md) for setup documentation +- (optional) the [matrix-sms-bridge](https://github.com/benkuly/matrix-sms-bridge) for bridging your Matrix server to SMS - see [docs/configuring-playbook-bridge-matrix-bridge-sms.md](docs/configuring-playbook-bridge-matrix-bridge-sms.md) for setup documentation - (optional) [Email2Matrix](https://github.com/devture/email2matrix) for relaying email messages to Matrix rooms - see [docs/configuring-playbook-email2matrix.md](docs/configuring-playbook-email2matrix.md) for setup documentation diff --git a/docs/configuring-playbook-matrix-bridge-sms.md b/docs/configuring-playbook-bridge-matrix-bridge-sms.md similarity index 100% rename from docs/configuring-playbook-matrix-bridge-sms.md rename to docs/configuring-playbook-bridge-matrix-bridge-sms.md diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index e4d5b756..b174637e 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -115,7 +115,7 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up Email2Matrix](configuring-playbook-email2matrix.md) (optional) -- [Setting up Matrix SMS bridging](configuring-playbook-matrix-bridge-sms.md) (optional) +- [Setting up Matrix SMS bridging](configuring-playbook-bridge-matrix-bridge-sms.md) (optional) ### Bots From 1fd81835b62d2265883b727d30d69cc23d4aefef Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Sep 2020 14:00:17 +0300 Subject: [PATCH 40/65] Mention CAPTCHA for public-registration servers --- docs/registering-users.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/registering-users.md b/docs/registering-users.md index cd77b6b6..90c065bf 100644 --- a/docs/registering-users.md +++ b/docs/registering-users.md @@ -55,6 +55,8 @@ matrix_synapse_enable_registration: true and running the [installation](installing.md) procedure once again. +If you're opening up registrations publicly like this, you might also wish to [configure CAPTCHA protection](configuring-captcha.md). + ## Adding/Removing Administrator privileges to an existing user From 6def66940ffe64b98cefa9e30952122622ace056 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Sep 2020 18:17:04 +0300 Subject: [PATCH 41/65] Fix broken cover photo for matrix-registration --- roles/matrix-registration/tasks/init.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-registration/tasks/init.yml b/roles/matrix-registration/tasks/init.yml index 60d4b326..8a7cdc75 100644 --- a/roles/matrix-registration/tasks/init.yml +++ b/roles/matrix-registration/tasks/init.yml @@ -36,7 +36,7 @@ Also fixing the form, which goes to /register. #} sub_filter_once off; - sub_filter_types text/html; + sub_filter_types text/html text/css; sub_filter "/static/" "{{ matrix_registration_public_endpoint }}/static/"; sub_filter "/register" "{{ matrix_registration_public_endpoint }}/register"; } From bed16fd0655c05ed02f1564834c023e97d3c605b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Sep 2020 20:51:51 +0300 Subject: [PATCH 42/65] Upgrade Element (1.7.4 -> 1.7.5) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 8b1bfb2e..8592c270 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/riot-web:v1.7.4" +matrix_client_element_docker_image: "vectorim/riot-web:v1.7.5" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" From 4ef873ceb021a985ee9b67a3da28f4339f1c6457 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 3 Sep 2020 16:31:27 +0300 Subject: [PATCH 43/65] Clarify Discord bridging docs --- docs/configuring-playbook-bridge-appservice-discord.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-bridge-appservice-discord.md b/docs/configuring-playbook-bridge-appservice-discord.md index 3e98d69e..13a5174b 100644 --- a/docs/configuring-playbook-bridge-appservice-discord.md +++ b/docs/configuring-playbook-bridge-appservice-discord.md @@ -23,7 +23,7 @@ matrix_appservice_discord_bot_token: "YOUR DISCORD APP BOT TOKEN" ``` 4. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. -5. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`) +5. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S). 6. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended. 7. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client. From 2a1ec38e3a0d8bf44a1b516d68a74107b01df148 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 6 Sep 2020 10:49:19 +0300 Subject: [PATCH 44/65] Stop using Ansible's cron module This is mainly to address SSL renewal not working for us due to: - https://github.com/ansible/ansible/issues/71213 - https://github.com/ansible/ansible/pull/71207 Using the cron module was hacky anyway. We shouldn't need an extra level of buggy abstraction to manage a cronjob file. --- roles/matrix-coturn/tasks/setup_coturn.yml | 18 +++---- .../cron.d/matrix-coturn-ssl-reload.j2 | 1 + .../tasks/ssl/setup_ssl_lets_encrypt.yml | 52 +++---------------- .../cron.d/matrix-ssl-lets-encrypt.j2 | 5 ++ 4 files changed, 19 insertions(+), 57 deletions(-) create mode 100644 roles/matrix-coturn/templates/cron.d/matrix-coturn-ssl-reload.j2 create mode 100644 roles/matrix-nginx-proxy/templates/cron.d/matrix-ssl-lets-encrypt.j2 diff --git a/roles/matrix-coturn/tasks/setup_coturn.yml b/roles/matrix-coturn/tasks/setup_coturn.yml index a83908d3..17b6137d 100644 --- a/roles/matrix-coturn/tasks/setup_coturn.yml +++ b/roles/matrix-coturn/tasks/setup_coturn.yml @@ -91,15 +91,10 @@ # We optimize for the common use-case though (short-lived Let's Encrypt certificates). # Reloading doesn't hurt anyway, so there's no need to make this more flexible. - name: Ensure periodic reloading of matrix-coturn is configured for SSL renewal (matrix-coturn-reload) - cron: - user: root - cron_file: matrix-coturn-ssl-reload - name: matrix-coturn-ssl-reload - state: present - hour: "4" - minute: "20" - day: "*/5" - job: "{{ matrix_host_command_systemctl }} reload matrix-coturn.service" + template: + src: "{{ role_path }}/templates/cron.d/matrix-coturn-ssl-reload.j2" + dest: /etc/cron.d/matrix-coturn-ssl-reload + mode: 0644 when: "matrix_coturn_enabled|bool and matrix_coturn_tls_enabled|bool" @@ -108,9 +103,8 @@ # - name: Ensure matrix-coturn-ssl-reload cronjob removed - cron: - user: root - cron_file: matrix-coturn-ssl-reload + file: + path: /etc/cron.d/matrix-coturn-ssl-reload state: absent when: "not matrix_coturn_enabled|bool or not matrix_coturn_tls_enabled|bool" diff --git a/roles/matrix-coturn/templates/cron.d/matrix-coturn-ssl-reload.j2 b/roles/matrix-coturn/templates/cron.d/matrix-coturn-ssl-reload.j2 new file mode 100644 index 00000000..1076f539 --- /dev/null +++ b/roles/matrix-coturn/templates/cron.d/matrix-coturn-ssl-reload.j2 @@ -0,0 +1 @@ +20 4 */5 * * root {{ matrix_host_command_systemctl }} reload matrix-coturn.service diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml index 08dc6e58..2946c111 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml @@ -55,37 +55,11 @@ mode: 0750 when: "matrix_ssl_retrieval_method == 'lets-encrypt'" -- block: - - name: Ensure periodic SSL renewal cronjob configured (MAILTO) - cron: - user: root - cron_file: matrix-ssl-lets-encrypt - env: yes - name: MAILTO - value: "{{ matrix_ssl_lets_encrypt_support_email }}" - - - name: Ensure periodic SSL renewal cronjob configured (matrix-ssl-lets-encrypt-certificates-renew) - cron: - user: root - cron_file: matrix-ssl-lets-encrypt - name: matrix-ssl-lets-encrypt-certificates-renew - state: present - hour: "4" - minute: "15" - day: "*" - job: "{{ matrix_local_bin_path }}/matrix-ssl-lets-encrypt-certificates-renew" - - - name: Ensure periodic reloading of matrix-nginx-proxy is configured for SSL renewal (matrix-nginx-proxy-reload) - cron: - user: root - cron_file: matrix-ssl-lets-encrypt - name: matrix-nginx-proxy-reload - state: present - hour: "5" - minute: "20" - day: "*" - job: "{{ matrix_host_command_systemctl }} reload matrix-nginx-proxy.service" - when: matrix_nginx_proxy_enabled|bool +- name: Ensure periodic SSL renewal cronjob configured + template: + src: "{{ role_path }}/templates/cron.d/matrix-ssl-lets-encrypt.j2" + dest: /etc/cron.d/matrix-ssl-lets-encrypt + mode: 0644 when: "matrix_ssl_retrieval_method == 'lets-encrypt'" @@ -93,21 +67,9 @@ # Tasks related to getting rid of Let's Encrypt's management of certificates # -# When nginx-proxy is disabled, make sure its reloading cronjob is gone. -# Other cronjobs can potentially remain there (see below). -- name: Ensure matrix-nginx-proxy-reload cronjob removed - cron: - user: root - cron_file: matrix-ssl-lets-encrypt - name: matrix-nginx-proxy-reload - state: absent - when: "not matrix_nginx_proxy_enabled|bool" - - name: Ensure matrix-ssl-lets-encrypt-renew cronjob removed - cron: - user: root - cron_file: matrix-ssl-lets-encrypt - name: matrix-ssl-lets-encrypt-certificates-renew + file: + path: /etc/cron.d/matrix-ssl-lets-encrypt state: absent when: "matrix_ssl_retrieval_method != 'lets-encrypt'" diff --git a/roles/matrix-nginx-proxy/templates/cron.d/matrix-ssl-lets-encrypt.j2 b/roles/matrix-nginx-proxy/templates/cron.d/matrix-ssl-lets-encrypt.j2 new file mode 100644 index 00000000..f58e37d4 --- /dev/null +++ b/roles/matrix-nginx-proxy/templates/cron.d/matrix-ssl-lets-encrypt.j2 @@ -0,0 +1,5 @@ +MAILTO="{{ matrix_ssl_lets_encrypt_support_email }}" +15 4 * * * root {{ matrix_local_bin_path }}/matrix-ssl-lets-encrypt-certificates-renew +{% if matrix_nginx_proxy_enabled %} +20 5 * * * root {{ matrix_host_command_systemctl }} reload matrix-nginx-proxy.service +{% endif %} From 5733f4d1575b1a0d5eaa679e4e3222e8463c24c0 Mon Sep 17 00:00:00 2001 From: Aki Salminen Date: Mon, 7 Sep 2020 17:14:34 +0300 Subject: [PATCH 45/65] Update configuring-playbook-telemetry.md Change anonymized to more proper term as server vice this is not anonymized. Server name is the first parameter that is collected. And if server happens to be for individual use these statistics would be at personal level without any anonymising. --- docs/configuring-playbook-telemetry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-telemetry.md b/docs/configuring-playbook-telemetry.md index fd7f1218..da583838 100644 --- a/docs/configuring-playbook-telemetry.md +++ b/docs/configuring-playbook-telemetry.md @@ -10,7 +10,7 @@ growth of the Matrix community, and helps to make Matrix a success. ## Enabling Telemetry -If you'd like to **help by enabling submission of anonymized usage statistics** for your homeserver, add this to your configuration file (`inventory/host_vars/matrix./vars.yml`): +If you'd like to **help by enabling submission of general usage statistics** for your homeserver, add this to your configuration file (`inventory/host_vars/matrix./vars.yml`): ```yaml matrix_synapse_report_stats: true From 5bb2c43502d85b418020254f5719cf45dda549b8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 10 Sep 2020 09:04:51 +0300 Subject: [PATCH 46/65] Add support for enabling Jitsi lobby Related to #643 (Github Issue) --- roles/matrix-jitsi/defaults/main.yml | 2 ++ roles/matrix-jitsi/templates/prosody/env.j2 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/roles/matrix-jitsi/defaults/main.yml b/roles/matrix-jitsi/defaults/main.yml index 13c36515..7486821d 100644 --- a/roles/matrix-jitsi/defaults/main.yml +++ b/roles/matrix-jitsi/defaults/main.yml @@ -49,6 +49,8 @@ matrix_jitsi_jibri_xmpp_password: '' matrix_jitsi_jibri_recorder_user: recorder matrix_jitsi_jibri_recorder_password: '' +matrix_jitsi_enable_lobby: false + matrix_jitsi_container_image_tag: "stable-4857" matrix_jitsi_web_docker_image: "jitsi/web:{{ matrix_jitsi_container_image_tag }}" diff --git a/roles/matrix-jitsi/templates/prosody/env.j2 b/roles/matrix-jitsi/templates/prosody/env.j2 index e4479b73..a09fa0f8 100644 --- a/roles/matrix-jitsi/templates/prosody/env.j2 +++ b/roles/matrix-jitsi/templates/prosody/env.j2 @@ -42,4 +42,6 @@ JIBRI_XMPP_PASSWORD={{ matrix_jitsi_jibri_xmpp_password }} JIBRI_RECORDER_USER={{ matrix_jitsi_jibri_recorder_user }} JIBRI_RECORDER_PASSWORD={{ matrix_jitsi_jibri_recorder_password }} +ENABLE_LOBBY={{ 1 if matrix_jitsi_enable_lobby else 0 }} + TZ={{ matrix_jitsi_timezone }} From abf70f777258de8c75aa4a758f0074ed07f3043f Mon Sep 17 00:00:00 2001 From: Olaf Schoenwald Date: Sat, 12 Sep 2020 10:52:25 +0200 Subject: [PATCH 47/65] Adds example for Caddy2 Caddyfile --- .DS_Store | Bin 0 -> 6148 bytes examples/caddy2/Caddyfile | 126 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 .DS_Store create mode 100644 examples/caddy2/Caddyfile diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..7ca3f0dd3376c60b62a895fce41d817aca687d8c GIT binary patch literal 6148 zcmeHK%Wl&^6ur~B)JcoT0;yekgT$hWQbCCjLQ2y_5Q&#Cf(4+|uHDp*kRH)J>e2&x46M_Tyb|wAiD!^UF?NWw=*M-oVK0hC9gekvNKnepT%+byX?5n1Ao%5T|J7U%n2et z5#+%0;qvN55P9*W9gia~730{l!!#>qrC-~f&2Aqw_pF?Ia50Xn3HCa z(mjReA$8~>aK%y)OCZnK5CP>!%BtY!Ju6tgK0?TYU#BS+O;~&z#zUWHpnB0w1>fAd z=tAiEYX!6dOA7G)!9wELQn-*Pwhm Date: Sat, 12 Sep 2020 11:04:16 +0200 Subject: [PATCH 48/65] Comment in host-cars --- examples/host-vars.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/host-vars.yml b/examples/host-vars.yml index 64d12097..e4e08bda 100644 --- a/examples/host-vars.yml +++ b/examples/host-vars.yml @@ -4,6 +4,9 @@ # Note: this playbook does not touch the server referenced here. # Installation happens on another server ("matrix."). # +# Plesae remember, if you've deployed the wrong URL, you have to run the Uninstalling step, +# cause you can't change the Domain after deployment. +# # Example value: example.com matrix_domain: YOUR_BARE_DOMAIN_NAME_HERE From 77635c452937750aac3d9e5b7508f740d7e033ad Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Sat, 12 Sep 2020 14:11:16 -0500 Subject: [PATCH 49/65] Fix links to using your own webserver doc --- docs/configuring-playbook-ssl-certificates.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-ssl-certificates.md b/docs/configuring-playbook-ssl-certificates.md index 916f6838..5b5c7cdc 100644 --- a/docs/configuring-playbook-ssl-certificates.md +++ b/docs/configuring-playbook-ssl-certificates.md @@ -3,7 +3,7 @@ By default, this playbook retrieves and auto-renews free SSL certificates from [Let's Encrypt](https://letsencrypt.org/) for the domains it needs (`matrix.` and possibly `element.`) Those certificates are used when configuring the nginx reverse proxy installed by this playbook. -They can also be used for configuring [your own webserver](docs/configuring-playbook-own-webserver.md), in case you're not using the integrated nginx server provided by the playbook. +They can also be used for configuring [your own webserver](configuring-playbook-own-webserver.md), in case you're not using the integrated nginx server provided by the playbook. If you need to retrieve certificates for other domains (e.g. your base domain) or more control over certificate retrieval, read below. @@ -13,7 +13,7 @@ Things discussed in this document: - [Using your own SSL certificates](#using-your-own-ssl-certificates), if you don't want to or can't use Let's Encrypt certificates, but are still interested in using the integrated nginx reverse proxy server -- [Not bothering with SSL certificates](#not-bothering-with-ssl-certificates), if you're using [your own webserver](docs/configuring-playbook-own-webserver.md) and would rather this playbook leaves SSL certificate management to you +- [Not bothering with SSL certificates](#not-bothering-with-ssl-certificates), if you're using [your own webserver](configuring-playbook-own-webserver.md) and would rather this playbook leaves SSL certificate management to you - [Obtaining SSL certificates for additional domains](#obtaining-ssl-certificates-for-additional-domains), if you'd like to host additional domains on the Matrix server and would like the playbook to help you obtain and renew certificates for those domains automatically From b0511603fd9f4cf129f5501ce7af6320d6161364 Mon Sep 17 00:00:00 2001 From: Dominik Zajac Date: Sat, 12 Sep 2020 22:10:41 +0200 Subject: [PATCH 50/65] Update configuring-playbook-jitsi.md modified example for jitsi ldap authentication to show all available variables. --- docs/configuring-playbook-jitsi.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/configuring-playbook-jitsi.md b/docs/configuring-playbook-jitsi.md index efc5eaaf..72402360 100644 --- a/docs/configuring-playbook-jitsi.md +++ b/docs/configuring-playbook-jitsi.md @@ -56,11 +56,19 @@ The default authentication mode of Jitsi is `internal`, however LDAP is also sup ```yaml matrix_jitsi_enable_auth: true matrix_jitsi_auth_type: ldap -matrix_jitsi_ldap_url: ldap://ldap.DOMAIN # or ldaps:// if using tls -matrix_jitsi_ldap_base: "OU=People,DC=DOMAIN" -matrix_jitsi_ldap_filter: "(&(uid=%u)(employeeType=active))" -matrix_jitsi_ldap_use_tls: false -matrix_jitsi_ldap_start_tls: true +matrix_jitsi_ldap_url: "ldap://ldap.DOMAIN" +matrix_jitsi_ldap_base: "OU=People,DC=DOMAIN +#matrix_jitsi_ldap_binddn: "" +#matrix_jitsi_ldap_bindpw: "" +matrix_jitsi_ldap_filter: "uid=%u" +matrix_jitsi_ldap_auth_method: "bind" +matrix_jitsi_ldap_version: "3" +matrix_jitsi_ldap_use_tls: true +matrix_jitsi_ldap_tls_ciphers: "" +matrix_jitsi_ldap_tls_check_peer: true +matrix_jitsi_ldap_tls_cacert_file: "/etc/ssl/certs/ca-certificates.crt" +matrix_jitsi_ldap_tls_cacert_dir: "/etc/ssl/certs" +matrix_jitsi_ldap_start_tls: false ``` For more information refer to the [docker-jitsi-meet](https://github.com/jitsi/docker-jitsi-meet#authentication-using-ldap) and the [saslauthd `LDAP_SASLAUTHD`](https://github.com/winlibs/cyrus-sasl/blob/master/saslauthd/LDAP_SASLAUTHD) documentation. From a49718632a9cd56372bf3d5aad20f9805b409210 Mon Sep 17 00:00:00 2001 From: 0hlov3 <36544727+0hlov3@users.noreply.github.com> Date: Sat, 12 Sep 2020 23:26:11 +0200 Subject: [PATCH 51/65] Delete .DS_Store Removes DS-Store --- .DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 7ca3f0dd3376c60b62a895fce41d817aca687d8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%Wl&^6ur~B)JcoT0;yekgT$hWQbCCjLQ2y_5Q&#Cf(4+|uHDp*kRH)J>e2&x46M_Tyb|wAiD!^UF?NWw=*M-oVK0hC9gekvNKnepT%+byX?5n1Ao%5T|J7U%n2et z5#+%0;qvN55P9*W9gia~730{l!!#>qrC-~f&2Aqw_pF?Ia50Xn3HCa z(mjReA$8~>aK%y)OCZnK5CP>!%BtY!Ju6tgK0?TYU#BS+O;~&z#zUWHpnB0w1>fAd z=tAiEYX!6dOA7G)!9wELQn-*Pwhm Date: Sat, 12 Sep 2020 23:26:27 +0200 Subject: [PATCH 52/65] Removes DS_Store --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 32ab139f..d6068088 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ !/inventory/host_vars/.gitkeep !/inventory/scripts /roles/*/files/scratchpad +.DS_Store From c366e2636009a9668611b8d2a61e1ec1038ad2a5 Mon Sep 17 00:00:00 2001 From: 0hlov3 Date: Sun, 13 Sep 2020 03:11:37 +0200 Subject: [PATCH 53/65] Updates exempes/caddy2/Caddyfile to SSL Grade A+ --- examples/caddy2/Caddyfile | 68 ++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/examples/caddy2/Caddyfile b/examples/caddy2/Caddyfile index 09b3367e..0abb25af 100644 --- a/examples/caddy2/Caddyfile +++ b/examples/caddy2/Caddyfile @@ -27,13 +27,17 @@ matrix.DOMAIN.tld { } header { - Access-Control-Allow-Origin * - Strict-Transport-Security "mag=age=31536000;" - X-Frame-Options "DENY" + # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + # Enable cross-site filter (XSS) and tell browser to block detected attacks + X-XSS-Protection "1; mode=block" + # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type + X-Content-Type-Options "nosniff" + # Disallow the site to be rendered within a frame (clickjacking protection) + X-Frame-Options "DENY" + # X-Robots-Tag + X-Robots-Tag "noindex, noarchive, nofollow" 167,9 79% - Strict-Transport-Security "mag=age=31536000;" - X-Frame-Options "DENY" - X-XSS-Protection "1; mode=block" } # Cache @@ -78,23 +82,36 @@ matrix.DOMAIN.tld { } } -:8448 { - handle { - encode zstd gzip +matrix.DOMAIN.tld:8448 { + handle { + encode zstd gzip - reverse_proxy localhost:8448 { - header_up X-Forwarded-Port {http.request.port} - header_up X-Forwarded-Proto {http.request.scheme} - header_up X-Forwarded-TlsProto {tls_protocol} - header_up X-Forwarded-TlsCipher {tls_cipher} - header_up X-Forwarded-HttpsProto {proto} - } - } + reverse_proxy 127.0.0.1:8048 { + header_up X-Forwarded-Port {http.request.port} + header_up X-Forwarded-Proto {http.request.scheme} + header_up X-Forwarded-TlsProto {tls_protocol} + header_up X-Forwarded-TlsCipher {tls_cipher} + header_up X-Forwarded-HttpsProto {proto} + } + } } dimension.DOMAIN.tld { - tls {$CADDY_TLS} + tls {$CADDY_TLS} + + header { + # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + # Enable cross-site filter (XSS) and tell browser to block detected attacks + X-XSS-Protection "1; mode=block" + # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type + X-Content-Type-Options "nosniff" + # Disallow the site to be rendered within a frame (clickjacking protection) + X-Frame-Options "DENY" + # X-Robots-Tag + X-Robots-Tag "noindex, noarchive, nofollow" + } handle { encode zstd gzip @@ -111,7 +128,20 @@ dimension.DOMAIN.tld { element.DOMAIN.tld { - tls {$CADDY_TLS} + tls {$CADDY_TLS} + + header { + # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + # Enable cross-site filter (XSS) and tell browser to block detected attacks + X-XSS-Protection "1; mode=block" + # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type + X-Content-Type-Options "nosniff" + # Disallow the site to be rendered within a frame (clickjacking protection) + X-Frame-Options "DENY" + # X-Robots-Tag + X-Robots-Tag "noindex, noarchive, nofollow" + } handle { encode zstd gzip From c19abe4a76cdb2d9285f63d2cbf43005658fefe7 Mon Sep 17 00:00:00 2001 From: 0hlov3 Date: Sun, 13 Sep 2020 04:19:19 +0200 Subject: [PATCH 54/65] Changes matrix_dimension_integrations_ui_url from /riot to /element https://dimension.t2bot.io/ --- roles/matrix-dimension/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index 770f5bcb..33e7212c 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -27,7 +27,7 @@ matrix_dimension_container_http_host_bind_port: '' # A list of extra arguments to pass to the container matrix_dimension_container_extra_arguments: [] -matrix_dimension_integrations_ui_url: "https://{{ matrix_server_fqn_dimension }}/riot" +matrix_dimension_integrations_ui_url: "https://{{ matrix_server_fqn_dimension }}/element" matrix_dimension_integrations_rest_url: "https://{{ matrix_server_fqn_dimension }}/api/v1/scalar" matrix_dimension_integrations_widgets_urls: ["https://{{ matrix_server_fqn_dimension }}/widgets"] matrix_dimension_integrations_jitsi_widget_url: "https://{{ matrix_server_fqn_dimension }}/widgets/jitsi" From 6fefbc248abbc8e2d1666d4a26b8caa8e3229dcb Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Sep 2020 09:51:04 +0300 Subject: [PATCH 55/65] Fix typo and wording --- examples/host-vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/host-vars.yml b/examples/host-vars.yml index e4e08bda..409f344a 100644 --- a/examples/host-vars.yml +++ b/examples/host-vars.yml @@ -4,8 +4,8 @@ # Note: this playbook does not touch the server referenced here. # Installation happens on another server ("matrix."). # -# Plesae remember, if you've deployed the wrong URL, you have to run the Uninstalling step, -# cause you can't change the Domain after deployment. +# If you've deployed using the wrong domain, you'll have to run the Uninstalling step, +# because you can't change the Domain after deployment. # # Example value: example.com matrix_domain: YOUR_BARE_DOMAIN_NAME_HERE From 6e8a39119bdeee039ec63a6e7fbc14122f81a226 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Sep 2020 10:19:47 +0300 Subject: [PATCH 56/65] Update matrix-reminder-bot (0.1.0 -> 0.2.0) --- roles/matrix-bot-matrix-reminder-bot/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml index a502690e..8f4a2c2f 100644 --- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml +++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml @@ -3,7 +3,7 @@ matrix_bot_matrix_reminder_bot_enabled: true -matrix_bot_matrix_reminder_bot_docker_image: "anoa/matrix-reminder-bot:release-0.1.0" +matrix_bot_matrix_reminder_bot_docker_image: "anoa/matrix-reminder-bot:release-v0.2.0" matrix_bot_matrix_reminder_bot_docker_image_force_pull: "{{ matrix_bot_matrix_reminder_bot_docker_image.endswith(':latest') }}" matrix_bot_matrix_reminder_bot_base_path: "{{ matrix_base_data_path }}/matrix-reminder-bot" From b24333dd0f8b22bbfe4f386596dd10a1b7c1055b Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Mon, 14 Sep 2020 11:11:30 -0700 Subject: [PATCH 57/65] Use the same naming convention as the other mx-puppet suite. --- .../templates/systemd/matrix-mx-puppet-instagram.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 index 4c4b74a5..b94c399a 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 +++ b/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 @@ -1,6 +1,6 @@ #jinja2: lstrip_blocks: "True" [Unit] -Description=Matrix mx-puppet-instagram bridge +Description=Matrix Mx Puppet Instagram server {% for service in matrix_mx_puppet_instagram_systemd_required_services_list %} Requires={{ service }} After={{ service }} From 8f41041f6db84a2b8b10ddd9e035c42d47962118 Mon Sep 17 00:00:00 2001 From: Daniel Wiegreffe Date: Tue, 15 Sep 2020 09:11:56 +0200 Subject: [PATCH 58/65] replacement of the docker image for appservice-slack to the officially maintained image --- roles/matrix-bridge-appservice-slack/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index 58dde6a0..8974d2e0 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -3,7 +3,7 @@ matrix_appservice_slack_enabled: true -matrix_appservice_slack_docker_image: "cadair/matrix-appservice-slack:cadair" +matrix_appservice_slack_docker_image: "matrixdotorg/matrix-appservice-slack" matrix_appservice_slack_docker_image_force_pull: "{{ matrix_appservice_slack_docker_image.endswith(':latest') }}" matrix_appservice_slack_base_path: "{{ matrix_base_data_path }}/appservice-slack" From faa96ca0c3aa8fb824cc1ddebb5eed93f40be710 Mon Sep 17 00:00:00 2001 From: Dan Arnfield Date: Tue, 15 Sep 2020 06:15:30 -0500 Subject: [PATCH 59/65] Update element (1.7.5 -> 1.7.7) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 8592c270..0aaa8a52 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/riot-web:v1.7.5" +matrix_client_element_docker_image: "vectorim/riot-web:v1.7.7" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" From 59e1abb07203263917dd46c3a6bf6c04afab0b1a Mon Sep 17 00:00:00 2001 From: 0hlov3 Date: Tue, 15 Sep 2020 14:54:47 +0200 Subject: [PATCH 60/65] Corrects the Caddyfile --- examples/caddy2/Caddyfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/caddy2/Caddyfile b/examples/caddy2/Caddyfile index 0abb25af..55fca3fd 100644 --- a/examples/caddy2/Caddyfile +++ b/examples/caddy2/Caddyfile @@ -49,7 +49,7 @@ matrix.DOMAIN.tld { # identity handle @identity { - reverse_proxy localhost:8090/_matrix/identity { + reverse_proxy localhost:8090 { header_up X-Forwarded-Port {http.request.port} header_up X-Forwarded-Proto {http.request.scheme} header_up X-Forwarded-TlsProto {tls_protocol} @@ -60,7 +60,7 @@ matrix.DOMAIN.tld { # search handle @search { - reverse_proxy localhost:8090/_matrix/client/r0/user_directory/search { + reverse_proxy localhost:8090 { header_up X-Forwarded-Port {http.request.port} header_up X-Forwarded-Proto {http.request.scheme} header_up X-Forwarded-TlsProto {tls_protocol} From 8029ca59c9dc20b057f85e9f0aeb52585a64cde0 Mon Sep 17 00:00:00 2001 From: jens quade Date: Wed, 16 Sep 2020 13:14:29 +0200 Subject: [PATCH 61/65] Update configuring-playbook-dimension.md Improved formatting for a header that was attached to the previous paragraph (...by inserting a newline). --- docs/configuring-playbook-dimension.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuring-playbook-dimension.md b/docs/configuring-playbook-dimension.md index 2b6b30ed..604e6aa6 100644 --- a/docs/configuring-playbook-dimension.md +++ b/docs/configuring-playbook-dimension.md @@ -38,6 +38,7 @@ Follow our [Registering users](registering-users.md) guide to learn how to regis You are required to specify an access token (belonging to this new user) for Dimension to work. To get an access token for the Dimension user, you can follow one of two options: + *Through an interactive login*: 1. In a private browsing session (incognito window), open Element. From e10e3e354d0da178852f0a033acc51f704244837 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 16 Sep 2020 16:35:12 +0300 Subject: [PATCH 62/65] Upgrade Synapse (v1.19.1 -> v1.19.2) --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index c8bc2fd5..0962f0b0 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.19.1" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.19.2" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" From 152c1636039902b60fa5831d3b1e18c47a0d62d6 Mon Sep 17 00:00:00 2001 From: tctovsli Date: Thu, 17 Sep 2020 10:20:41 +0200 Subject: [PATCH 63/65] Fixed dead link to mx-puppet-discord --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8044354..72018115 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Using this playbook, you can get the following services configured on your serve - (optional) the [mx-puppet-twitter](https://github.com/Sorunome/mx-puppet-twitter) bridge for Twitter-DMs ([Twitter](https://twitter.com/) - see [docs/configuring-playbook-bridge-mx-puppet-twitter.md](docs/configuring-playbook-bridge-mx-puppet-twitter.md) for setup documentation -- (optional) the [mx-puppet-discord](https://github.com/Sorunome/mx-puppet-discord) bridge for [Discord](https://discordapp.com/)) - see [docs/configuring-playbook-bridge-mx-puppet-discord.md](docs/configuring-playbook-bridge-mx-puppet-discord.md) for setup documentation +- (optional) the [mx-puppet-discord](https://github.com/matrix-discord/mx-puppet-discord) bridge for [Discord](https://discordapp.com/)) - see [docs/configuring-playbook-bridge-mx-puppet-discord.md](docs/configuring-playbook-bridge-mx-puppet-discord.md) for setup documentation - (optional) the [mx-puppet-steam](https://github.com/icewind1991/mx-puppet-steam) bridge for [Steam](https://steamapp.com/)) - see [docs/configuring-playbook-bridge-mx-puppet-steam.md](docs/configuring-playbook-bridge-mx-puppet-steam.md) for setup documentation From b3926e7ccad4e5ee42b65705d274a1bba15b7e9a Mon Sep 17 00:00:00 2001 From: Daniel Wiegreffe Date: Fri, 18 Sep 2020 13:26:07 +0200 Subject: [PATCH 64/65] Update main.yml --- roles/matrix-bridge-appservice-slack/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index 8974d2e0..eef7d070 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -3,7 +3,7 @@ matrix_appservice_slack_enabled: true -matrix_appservice_slack_docker_image: "matrixdotorg/matrix-appservice-slack" +matrix_appservice_slack_docker_image: "matrixdotorg/matrix-appservice-slack:release-1.5.0" matrix_appservice_slack_docker_image_force_pull: "{{ matrix_appservice_slack_docker_image.endswith(':latest') }}" matrix_appservice_slack_base_path: "{{ matrix_base_data_path }}/appservice-slack" From 65e22a6888a5ca1de5e699399d6d0c6213e46043 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 18 Sep 2020 17:36:48 +0300 Subject: [PATCH 65/65] Upgrade Synapse (v1.19.2 -> v1.19.3) --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 0962f0b0..812d06e8 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.19.2" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.19.3" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse"