From ea91ef7fb257a0c45ffa378adf59f04a96ccdcc6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 7 Sep 2017 12:12:31 +0300 Subject: [PATCH] Move media_store & logs out of /data. Allow logging to be configured The goal is to allow these to be on separate partitions (including remote ones in the future). Because the `silviof/docker-matrix` image chowns everything to MATRIX_UID:MATRIX_GID on startup, we definitely don't want to include `media_store` in it. If it's on a remote FS, it would cause a slow startup. Also, adding some safety checks to the "import media store" task, after passing a wrong path to it on multiple occassions and wondering what's wrong. Also, making logging configurable. The default of keeping 10x100MB log files is likely excessive and people may want to change that. --- roles/matrix-server/defaults/main.yml | 9 +- .../tasks/import_media_store.yml | 18 +++- .../matrix-server/tasks/import_sqlite_db.yml | 4 +- roles/matrix-server/tasks/setup_synapse.yml | 82 ++++++++++--------- .../systemd/matrix-synapse.service.j2 | 4 +- 5 files changed, 72 insertions(+), 45 deletions(-) diff --git a/roles/matrix-server/defaults/main.yml b/roles/matrix-server/defaults/main.yml index 1b1c5876e..9f298373f 100644 --- a/roles/matrix-server/defaults/main.yml +++ b/roles/matrix-server/defaults/main.yml @@ -24,7 +24,10 @@ matrix_postgres_db_name: "homeserver" matrix_base_data_path: "/matrix" matrix_environment_variables_data_path: "{{ matrix_base_data_path }}/environment-variables" -matrix_synapse_data_path: "{{ matrix_base_data_path }}/synapse" +matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" +matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config" +matrix_synapse_run_path: "{{ matrix_synapse_base_path }}/run" +matrix_synapse_media_store_path: "{{ matrix_synapse_base_path }}/media-store" matrix_postgres_data_path: "{{ matrix_base_data_path }}/postgres" matrix_nginx_proxy_data_path: "{{ matrix_base_data_path }}/nginx-proxy" matrix_nginx_proxy_confd_path: "{{ matrix_nginx_proxy_data_path }}/conf.d" @@ -46,4 +49,6 @@ matrix_coturn_turn_udp_max_port: 49172 matrix_coturn_turn_external_ip_address: "{{ ansible_host }}" -matrix_max_upload_size_mb: 10 \ No newline at end of file +matrix_max_upload_size_mb: 10 +matrix_max_log_file_size_mb: 100 +matrix_max_log_files_count: 10 \ No newline at end of file diff --git a/roles/matrix-server/tasks/import_media_store.yml b/roles/matrix-server/tasks/import_media_store.yml index 955309c06..be7a878a2 100644 --- a/roles/matrix-server/tasks/import_media_store.yml +++ b/roles/matrix-server/tasks/import_media_store.yml @@ -14,6 +14,22 @@ fail: msg="File cannot be found on the local machine at {{ local_path_media_store }}" when: "not local_path_media_store_stat.stat.exists or not local_path_media_store_stat.stat.isdir" +- name: Check if media store contains local_content + stat: path="{{ local_path_media_store }}/local_content" + delegate_to: 127.0.0.1 + become: false + register: local_path_media_store_local_content_stat + +- name: Check if media store contains remote_content + stat: path="{{ local_path_media_store }}/remote_content" + delegate_to: 127.0.0.1 + become: false + register: local_path_media_store_remote_content_stat + +- name: Fail if media_store directory doesn't look okay (lacking remote and local content) + fail: msg="{{ local_path_media_store }} contains neither local_content nor remote_content. It's most likely a mistake and is not a media store directory." + when: "not local_path_media_store_local_content_stat.stat.exists and not local_path_media_store_remote_content_stat.stat.exists" + - name: Ensure matrix-synapse is stopped service: name=matrix-synapse state=stopped daemon_reload=yes register: stopping_result @@ -21,7 +37,7 @@ - name: Ensure provided media_store directory is copied to the server synchronize: src: "{{ local_path_media_store }}/" - dest: "{{ matrix_synapse_data_path }}/media_store" + dest: "{{ matrix_synapse_media_store_path }}" delete: yes - name: Ensure Matrix Synapse is started (if it previously was) diff --git a/roles/matrix-server/tasks/import_sqlite_db.yml b/roles/matrix-server/tasks/import_sqlite_db.yml index 5809acd8e..0390e195c 100644 --- a/roles/matrix-server/tasks/import_sqlite_db.yml +++ b/roles/matrix-server/tasks/import_sqlite_db.yml @@ -66,7 +66,9 @@ command: "/usr/local/bin/synapse_port_db_with_patch --sqlite-database /scratchpad/homeserver.db --postgres-config /data/homeserver.yaml" user: "{{ matrix_user_uid }}:{{ matrix_user_gid }}" volumes: - - "{{ matrix_synapse_data_path }}:/data" + - "{{ matrix_synapse_config_dir_path }}:/data" + - "{{ matrix_synapse_run_path }}:/matrix-run" + - "{{ matrix_synapse_media_store_path }}:/matrix-media-store" - "{{ matrix_scratchpad_dir }}:/scratchpad" - "{{ matrix_scratchpad_dir }}/synapse_port_db_with_patch:/usr/local/bin/synapse_port_db_with_patch" links: diff --git a/roles/matrix-server/tasks/setup_synapse.yml b/roles/matrix-server/tasks/setup_synapse.yml index 5057feb3a..91329ba58 100644 --- a/roles/matrix-server/tasks/setup_synapse.yml +++ b/roles/matrix-server/tasks/setup_synapse.yml @@ -1,12 +1,17 @@ --- -- name: Ensure Matrix Synapse data path exists +- name: Ensure Matrix Synapse paths exists file: - path: "{{ matrix_synapse_data_path }}" + path: "{{ item }}" state: directory mode: 0750 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" + with_items: + - "{{ matrix_synapse_base_path }}" + - "{{ matrix_synapse_config_dir_path }}" + - "{{ matrix_synapse_run_path }}" + - "{{ matrix_synapse_media_store_path }}" - name: Ensure Matrix Docker image is pulled docker_image: @@ -14,7 +19,7 @@ - name: Check if a Matrix Synapse configuration exists stat: - path: "{{ matrix_synapse_data_path }}/homeserver.yaml" + path: "{{ matrix_synapse_config_dir_path }}/homeserver.yaml" register: matrix_synapse_config_stat - name: Generate initial Matrix config @@ -29,41 +34,44 @@ REPORT_STATS: "no" user: "{{ matrix_user_uid }}:{{ matrix_user_gid }}" volumes: - - "{{ matrix_synapse_data_path }}:/data" + - "{{ matrix_synapse_config_dir_path }}:/data" when: "not matrix_synapse_config_stat.stat.exists" -- name: Augment Matrix config (configure SSL fullchain location) - lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml" - args: - regexp: "^tls_certificate_path:" - line: 'tls_certificate_path: "/acmetool-certs/live/{{ hostname_matrix }}/fullchain"' +- name: Ensure self-signed certificates are removed + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ matrix_synapse_config_dir_path }}/{{ hostname_matrix }}.tls.crt" + - "{{ matrix_synapse_config_dir_path }}/{{ hostname_matrix }}.tls.key" -- name: Augment Matrix config (configure SSL private key location) - lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml" +- name: Augment Matrix log config + lineinfile: "dest={{ matrix_synapse_config_dir_path }}/{{ hostname_matrix }}.log.config" args: - regexp: "^tls_private_key_path:" - line: 'tls_private_key_path: "/acmetool-certs/live/{{ hostname_matrix }}/privkey"' + regexp: "{{ item.regexp }}" + line: '{{ item.line }}' + with_items: + - {"regexp": "^ filename:", "line": ' filename: /matrix-run/homeserver.log'} + - {"regexp": "^ maxBytes:", "line": ' maxBytes: {{ matrix_max_log_file_size_mb * 1024 * 1024 }}'} + - {"regexp": "^ backupCount:", "line": ' backupCount: {{ matrix_max_log_files_count }}'} -- name: Augment Matrix config (configure server name) - lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml" +- name: Augment Matrix config + lineinfile: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml" args: - regexp: "^server_name:" - line: 'server_name: "{{ hostname_identity }}"' - -- name: Augment Matrix config (disable TURN for guests) - lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml" - args: - regexp: "^turn_allow_guests:" - line: 'turn_allow_guests: False' - -- name: Augment Matrix config (enable URL previews) - lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml" - args: - regexp: "^url_preview_enabled:" - line: 'url_preview_enabled: True' + regexp: "{{ item.regexp }}" + line: '{{ item.line }}' + with_items: + - {"regexp": "^log_file:", "line": 'log_file: "/matrix-run/homeserver.log"'} + - {"regexp": "^tls_certificate_path:", "line": 'tls_certificate_path: "/acmetool-certs/live/{{ hostname_matrix }}/fullchain"'} + - {"regexp": "^tls_private_key_path:", "line": 'tls_private_key_path: "/acmetool-certs/live/{{ hostname_matrix }}/privkey"'} + - {"regexp": "^server_name:", "line": 'server_name: "{{ hostname_identity }}"'} + - {"regexp": "^turn_allow_guests:", "line": 'turn_allow_guests: False'} + - {"regexp": "^url_preview_enabled:", "line": 'url_preview_enabled: True'} + - {"regexp": "^max_upload_size:", "line": 'max_upload_size: "{{ matrix_max_upload_size_mb }}M"'} + - {"regexp": "^media_store_path:", "line": 'media_store_path: "/matrix-media-store"'} - name: Augment Matrix config (specify URL previews blacklist) - lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml" + lineinfile: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml" args: regexp: "^url_preview_ip_range_blacklist:" line: 'url_preview_ip_range_blacklist: ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10", "169.254.0.0/16"]' @@ -72,27 +80,27 @@ # We only wish to do this for the 8008 port and not for the 8448 port # (2nd instance of `x_forwarded` found in the config) - name: Augment Matrix config (mark 8008 plain traffic as forwarded) - replace: "dest={{ matrix_synapse_data_path }}/homeserver.yaml" + replace: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml" args: regexp: "8008((?:.|\n)*)x_forwarded(.*)" replace: '8008\g<1>x_forwarded: true' - name: Augment Matrix config (change database from SQLite to Postgres) lineinfile: - dest: "{{ matrix_synapse_data_path }}/homeserver.yaml" + dest: "{{ matrix_synapse_config_dir_path }}/homeserver.yaml" regexp: '(.*)name: "sqlite3"' line: '\1name: "psycopg2"' backrefs: yes - name: Augment Matrix config (add the Postgres connection parameters) lineinfile: - dest: "{{ matrix_synapse_data_path }}/homeserver.yaml" + dest: "{{ matrix_synapse_config_dir_path }}/homeserver.yaml" regexp: '(.*)database: "(.*)homeserver.db"' line: '\1user: "{{ matrix_postgres_connection_username }}"\n\1password: "{{ matrix_postgres_connection_password }}"\n\1database: "homeserver"\n\1host: "postgres"\n\1cp_min: 5\n\1cp_max: 10' backrefs: yes - name: Augment Matrix config (configure Coturn) - lineinfile: "dest={{ matrix_synapse_data_path }}/turnserver.conf" + lineinfile: "dest={{ matrix_synapse_config_dir_path }}/turnserver.conf" args: regexp: "^{{ item.variable }}=" line: '{{ item.variable }}={{ item.value }}' @@ -101,12 +109,6 @@ - {'variable': 'max-port', 'value': "{{ matrix_coturn_turn_udp_max_port }}"} - {'variable': 'external-ip', 'value': "{{ matrix_coturn_turn_external_ip_address }}"} -- name: Augment Matrix config (set max upload size) - lineinfile: "dest={{ matrix_synapse_data_path }}/homeserver.yaml" - args: - regexp: "^max_upload_size:" - line: 'max_upload_size: "{{ matrix_max_upload_size_mb }}M"' - - name: Allow access to Matrix ports in firewalld firewalld: port: "{{ item }}" diff --git a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 index 8b60fd9ad..c1f7b1f6d 100644 --- a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 @@ -16,7 +16,9 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \ -p 3478:3478 \ -p 3478:3478/udp \ -p {{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}:{{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}/udp \ - -v {{ matrix_synapse_data_path }}:/data \ + -v {{ matrix_synapse_config_dir_path }}:/data \ + -v {{ matrix_synapse_run_path }}:/matrix-run \ + -v {{ matrix_synapse_media_store_path }}:/matrix-media-store \ -v {{ ssl_certs_path }}:/acmetool-certs \ {{ docker_matrix_image }} ExecStop=-/usr/bin/docker kill matrix-synapse