diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml index e0e932bcc..9b89a3400 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml @@ -49,7 +49,7 @@ To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command. Alternatively, to migrate your existing neDB database to Postgres: 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`) + 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`) 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) when: "matrix_appservice_irc_nedb_stat_result.stat.exists" when: "matrix_appservice_irc_database_engine == 'postgres'" diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml new file mode 100644 index 000000000..cc1f9d788 --- /dev/null +++ b/roles/matrix-postgres/tasks/import_nedb.yml @@ -0,0 +1,117 @@ +--- + +# Pre-checks + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import." + when: "not matrix_postgres_enabled|bool" + +- name: Fail if playbook called incorrectly + fail: + msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars" + when: "nedb_database_path is not defined or nedb_database_path.startswith('<')" + +- name: Check if the provided nedb database file exists + stat: + path: "{{ nedb_database_path }}" + register: nedb_database_path_stat_result + +- name: Fail if provided SQLite database file doesn't exist + fail: + msg: "File cannot be found on the server at {{ nedb_database_path }}" + when: "not nedb_database_path_stat_result.stat.exists" + +# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string, +# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string. + +- block: + - name: Fail if postgres_connection_string_variable_name points to an undefined variable + fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" + when: "postgres_connection_string_variable_name not in vars" + + - name: Get Postgres connection string from variable + set_fact: + postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}" + when: 'postgres_connection_string_variable_name is defined' + +- name: Fail if playbook called incorrectly + fail: + msg: >- + Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`. + Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"` + when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')" + + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL +- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres + when: database == 'appservice_irc' + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro + --entrypoint=/bin/sh + {{ matrix_appservice_irc_docker_image }} + -c + './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' + +# No migration.sh available, but found this: +# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts +# Usage should be similar to appservice_irc +- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres + when: database == 'appservice_slack' + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro + --entrypoint=/bin/sh + {{ matrix_appservice_slack_docker_image }} + -c + 'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' + +- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup) + command: + cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup" + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." + ] + }} diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index b9c2ae7c9..518013147 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -32,6 +32,22 @@ tags: - import-generic-sqlite-db +# Imports appservice-irc NeDB into postgres +- import_tasks: "{{ role_path }}/tasks/import_nedb.yml" + vars: + database: appservice_irc + when: run_postgres_import_nedb|bool + tags: + - import-irc-nedb + +# Imports slacks neDB to postgres. +- import_tasks: "{{ role_path }}/tasks/import_nedb.yml" + vars: + database: appservice_slack + when: run_postgres_import_nedb|bool + tags: + - import-slack-nedb + - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml" when: run_postgres_upgrade|bool tags: