From af3ea67bbaa699ad6f03db6633f1dd1902ab9290 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 17:16:25 +0200 Subject: [PATCH] Add (SQLite + Postgres) support and automatic migration to matrix-ma1sd --- group_vars/matrix_servers | 10 ++++++ roles/matrix-ma1sd/defaults/main.yml | 22 ++++++++++++ roles/matrix-ma1sd/tasks/setup_install.yml | 36 +++++++++++++++++++ roles/matrix-ma1sd/templates/ma1sd.yaml.j2 | 16 +++++++-- .../tasks/util/migrate_db_to_postgres.yml | 2 +- 5 files changed, 82 insertions(+), 4 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d6d6cc4a..735d1324 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -835,6 +835,10 @@ matrix_ma1sd_systemd_wanted_services_list: | (['matrix-mailer.service'] if matrix_mailer_enabled else []) }} +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_ma1sd_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_ma1sd_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ma1sd.db') | to_uuid }}" + ###################################################################### # # /matrix-ma1sd @@ -953,6 +957,12 @@ matrix_postgres_db_name: "homeserver" matrix_postgres_additional_databases: | {{ + ([{ + 'name': matrix_ma1sd_database_db_name, + 'username': matrix_ma1sd_database_username, + 'password': matrix_ma1sd_database_password, + }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else []) + + ([{ 'name': matrix_bot_matrix_reminder_bot_database_db_name, 'username': matrix_bot_matrix_reminder_bot_database_username, diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml index f7a0782d..42e36e92 100644 --- a/roles/matrix-ma1sd/defaults/main.yml +++ b/roles/matrix-ma1sd/defaults/main.yml @@ -39,6 +39,28 @@ matrix_ma1sd_systemd_wanted_services_list: [] # Enabling this is discouraged. Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/features/identity.md#lookups matrix_ma1sd_matrixorg_forwarding_enabled: false + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_ma1sd_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_ma1sd_postgres_*` variables +matrix_ma1sd_database_engine: 'sqlite' + +matrix_ma1sd_sqlite_database_path_local: "{{ matrix_ma1sd_data_path }}/ma1sd.db" +matrix_ma1sd_sqlite_database_path_in_container: "/var/ma1sd/ma1sd.db" + +matrix_ma1sd_database_username: 'matrix_ma1sd' +matrix_ma1sd_database_password: 'some-password' +matrix_ma1sd_database_hostname: 'matrix-postgres' +matrix_ma1sd_database_port: 5432 +matrix_ma1sd_database_db_name: 'matrix_ma1sd' + +matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}' + + # ma1sd has serveral supported identity stores. # One of them is storing identities directly in Synapse's database. # Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/stores/synapse.md diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml index aa11ae7a..9ae5f077 100644 --- a/roles/matrix-ma1sd/tasks/setup_install.yml +++ b/roles/matrix-ma1sd/tasks/setup_install.yml @@ -15,6 +15,36 @@ - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml" + +# These (SQLite -> Postgres) migration tasks are usually at the top, +# but we'd like to run them after `migrate_mxisd.yml`, which requires the ma1sd paths to exist. +- set_fact: + matrix_ma1sd_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_ma1sd_sqlite_database_path_local }}" + register: matrix_ma1sd_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_ma1sd_sqlite_database_path_local }}" + dst: "{{ matrix_ma1sd_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_ma1sd_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-ma1sd.service'] + pgloader_options: ['--with "quote identifiers"'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_ma1sd_requires_restart: true + when: "matrix_ma1sd_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_ma1sd_database_engine == 'postgres'" + - name: Ensure ma1sd image is pulled docker_image: name: "{{ matrix_ma1sd_docker_image }}" @@ -102,3 +132,9 @@ service: daemon_reload: yes when: "matrix_ma1sd_systemd_service_result.changed|bool" + +- name: Ensure matrix-ma1sd.service restarted, if necessary + service: + name: "matrix-ma1sd.service" + state: restarted + when: "matrix_ma1sd_requires_restart|bool" diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 index 84585707..43b4022d 100644 --- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 +++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 @@ -11,9 +11,19 @@ key: path: /var/ma1sd/sign.key storage: - provider: - sqlite: - database: /var/ma1sd/ma1sd.db + {% if matrix_ma1sd_database_engine == 'sqlite' %} + backend: sqlite + provider: + sqlite: + database: {{ matrix_ma1sd_sqlite_database_path_in_container|to_json }} + {% elif matrix_ma1sd_database_engine == 'postgres' %} + backend: postgresql + provider: + postgresql: + database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }} + username: {{ matrix_ma1sd_database_username|to_json }} + password: {{ matrix_ma1sd_database_password|to_json }} + {% endif %} {% if matrix_ma1sd_dns_overwrite_enabled %} dns: diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index b77ce4b0..5d917538 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -93,7 +93,7 @@ --entrypoint=/bin/sh {{ matrix_postgres_pgloader_docker_image }} -c - 'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}' + 'pgloader {{ matrix_postgres_db_migration_request.pgloader_options|default([])|join(' ') }} /in.db {{ matrix_postgres_db_migration_request.dst }}' - name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup) command: