jowj
/
matrix-docker-ansible-deploy
forked from mirrors/matrix-docker-ansible-deploy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'synapse-workers' into feature/add-worker-support

Browse Source
master
Max Klenk 4 years ago
parent 3c285bc6f5 46984a4f99
commit 567d0318b0
No known key found for this signature in database
GPG Key ID: 26EFB2972F02F380
10 changed files with 362 additions and 0 deletions
Show Stats Download Patch File Download Diff File

134
roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
Unescape View File

@ -101,6 +101,114 @@
}
{% endif %}
{% if synchrotron_workers %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #}
location /_matrix/client/r0/sync {
proxy_pass http://synchrotron$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
location /_matrix/client/r0/events {
proxy_pass http://synchrotron$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
location /_matrix/client/r0/initialSync {
proxy_pass http://synchrotron$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ {
proxy_pass http://synchrotron$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %}
{% if client_reader_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #}
location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ client_reader_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %}
{% if media_repository_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #}
location /_matrix/media/ {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ media_repository_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #}
location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ media_repository_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %}
{% if event_creator_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ event_creator_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %}
{% if frontend_proxy_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% if not matrix_synapse_use_presence %}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% endif %}
{% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %}
{% if user_dir_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ user_dir_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %}
{{- configuration_block }}
{% endfor %}
@ -174,6 +282,19 @@
}
{% endmacro %}
{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %}
{% if synchrotron_workers %}
upstream synchrotron {
# ensures that requests from the same client will always be passed
# to the same server (except when this server is unavailable)
ip_hash;
{% for synchrotron_worker in synchrotron_workers %}
server "matrix-synapse:{{ synchrotron_worker.port }}";
{% endfor %}
}
{% endif %}
server {
listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }};
server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }};
@ -255,6 +376,19 @@ server {
ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
{% endif %}
{% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %}
{% if federation_reader_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #}
location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ federation_reader_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
location / {
{% if matrix_nginx_proxy_enabled %}
{# Use the embedded DNS resolver in Docker containers to discover the service #}

37
roles/matrix-synapse/defaults/main.yml
Unescape View File

@ -260,6 +260,43 @@ matrix_synapse_metrics_port: 9100
# See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md
matrix_synapse_manhole_enabled: false
# Enable support for Synapse workers
matrix_synapse_workers_enabled: false
# List of workers to spawn
matrix_synapse_workers_enabled_list: []
# Default list of workers to spawn
matrix_synapse_workers_enabled_list:
- { worker: synchrotron, port: 18082 }
- { worker: synchrotron, port: 18083 }
- { worker: synchrotron, port: 18084 }
- { worker: appservice, port: 18085 }
- { worker: client_reader, port: 18086 }
- { worker: event_creator, port: 18087 }
- { worker: federation_reader, port: 18088 }
- { worker: federation_sender, port: 18089 }
- { worker: frontend_proxy, port: 18090 }
- { worker: media_repository, port: 18091 }
- { worker: pusher, port: 18092 }
- { worker: user_dir, port: 18093 }
# The list of available workers (2020-04-14)
matrix_synapse_workers_avail_list:
- appservice
- client_reader
- event_creator
- federation_reader
- federation_sender
- frontend_proxy
- media_repository
- pusher
- synchrotron
- user_dir
# Ports used for communication between main synapse process and workers
matrix_synapse_replication_tcp_port: 9092
matrix_synapse_replication_http_port: 9093
# Send ERROR logs to sentry.io for easier tracking
# To set this up: go to sentry.io, create a python project, and set

2
roles/matrix-synapse/tasks/setup_synapse.yml
Unescape View File

@ -18,6 +18,8 @@
- import_tasks: "{{ role_path }}/tasks/ext/setup.yml"
- import_tasks: "{{ role_path }}/tasks/workers/setup.yml"
- import_tasks: "{{ role_path }}/tasks/synapse/setup.yml"
- import_tasks: "{{ role_path }}/tasks/goofys/setup.yml"

8
roles/matrix-synapse/tasks/workers/setup.yml
Unescape View File

@ -0,0 +1,8 @@
---
# a negative when condition will not actually prevent ansible from executing loops in imported tasks!
- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"
- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml"
when: "not matrix_synapse_workers_enabled|bool"

42
roles/matrix-synapse/tasks/workers/setup_install.yml
Unescape View File

@ -0,0 +1,42 @@
---
- name: Ensure synapse worker base service file installed
template:
src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
mode: 0644
register: matrix_synapse_worker_systemd_service_result
- name: Ensure previous worker service symlinks are cleaned (FIXME)
file:
path: "{{ item.root + '/' + item.path }}"
state: absent
when:
- matrix_synapse_workers_enabled|bool
- item.state == 'link'
- item.path is match('matrix-synapse-worker@.*\\.service')
with_filetree:
- "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
- name: Ensure systemd reloaded the worker service unit
service:
daemon_reload: yes
- name: Ensure individual worker service symlinks exist
service:
name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
enabled: true
with_items: "{{ matrix_synapse_workers_enabled_list }}"
- name: Ensure creation of specific worker configs
template:
src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml"
with_list: "{{ matrix_synapse_workers_enabled_list }}"
- name: Add workers to synapse.wants list
set_fact:
matrix_synapse_systemd_wanted_services_list: >
{{ matrix_synapse_systemd_wanted_services_list +
['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }}
with_items: "{{ matrix_synapse_workers_enabled_list }}"

38
roles/matrix-synapse/tasks/workers/setup_uninstall.yml
Unescape View File

@ -0,0 +1,38 @@
---
- name: Populate service facts
service_facts:
- name: Ensure any worker services are stopped
service:
name: "{{ item.key }}"
state: stopped
with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}"
# As we cannot know the ports of workers removed from the enabled_list..
# => .. just kill them all (FIXME?)
- name: Ensure previous worker service symlinks are cleaned
file:
path: "{{ item.root + '/' + item.path }}"
state: absent
when:
- not matrix_synapse_workers_enabled|bool
- item.state == 'link'
- item.path is match('matrix-synapse-worker@.*\\.service')
with_filetree:
- "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
- name: Ensure synapse worker base service file gets removed
file:
path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
state: absent
register: matrix_synapse_worker_systemd_service_result
- name: Remove workers from synapse.wants list
set_fact:
matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}"
with_items: "{{ matrix_synapse_workers_avail_list }}"
- name: Ensure systemd noticed removal of worker service units
service:
daemon_reload: yes

38
roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
Unescape View File

@ -223,6 +223,44 @@ listeners:
type: manhole
{% endif %}
{% if matrix_synapse_workers_enabled %}
# c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md
# TCP replication: streaming data from the master to the workers
- port: {{ matrix_synapse_replication_tcp_port }}
bind_addresses: ['0.0.0.0']
type: replication
# HTTP replication: for the workers to send data to the main synapse process
- port: {{ matrix_synapse_replication_http_port }}
bind_addresses: ['0.0.0.0']
type: http
resources:
- names: [replication]
# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md
worker_app: synapse.app.homeserver
# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/
# reduce the main worker's offerings to core homeserver business
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list %}
notify_appservices: false
{% endif %}
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list %}
send_federation: false
{% endif %}
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %}
enable_media_repo: false
{% endif %}
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list %}
start_pushers: false
{% endif %}
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %}
update_user_directory: false
{% endif %}
# rather let systemd handle the forking
daemonize: false
{% endif %}
# Forward extremities can build up in a room due to networking delays between
# homeservers. Once this happens in a large room, calculation of the state of

29
roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2
Unescape View File

@ -0,0 +1,29 @@
#jinja2: lstrip_blocks: "True"
# c.f. https://github.com/matrix-org/synapse/pull/4662
[Unit]
Description=Synapse Matrix Worker
After=matrix-synapse.service
BindsTo=matrix-synapse.service
[Service]
Type=simple
# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
ExecStartPre=/bin/sleep 5
# systemd ftw 🤦‍♂️
# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923
ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
exec /usr/bin/docker exec \
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
matrix-synapse \
python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i
KillMode=process
Restart=always
RestartSec=10
SyslogIdentifier=matrix-synapse-%i
[Install]
WantedBy=matrix-synapse.service

5
roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
Unescape View File

@ -43,6 +43,11 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
{% endif %}
{% if matrix_synapse_workers_enabled %}
{% for worker in matrix_synapse_workers_enabled_list %}
-p {{ worker.port }}:{{ worker.port }} \
{% endfor %}
{% endif %}
-v {{ matrix_synapse_config_dir_path }}:/data:ro \
-v {{ matrix_synapse_storage_path }}:/matrix-media-store-parent:slave \
{% for volume in matrix_synapse_container_additional_volumes %}

29
roles/matrix-synapse/templates/synapse/worker.yaml.j2
Unescape View File

@ -0,0 +1,29 @@
#jinja2: lstrip_blocks: "True"
worker_app: synapse.app.{{ item.worker }}
worker_replication_host: 127.0.0.1
worker_replication_port: {{ matrix_synapse_replication_tcp_port }}
worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %}
worker_listeners:
- type: http
port: {{ item.port }}
resources:
- names:
{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %}
- client
{% elif item.worker in [ 'federation_reader' ] %}
- federation
{% elif item.worker in [ 'media_repository' ] %}
- media
{% endif %}
{% endif %}
{% if item.worker == 'frontend_proxy' %}
worker_main_http_uri: http://127.0.0.1:8008
{% endif %}
worker_daemonize: false
worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid
worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config
Write Preview
Loading…
Cancel
Save