From 990a6369e13f7f70e5bc544f55ea99727c554aaa Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 17 Feb 2023 16:23:59 +0200 Subject: [PATCH] Switch to using an external Redis role --- CHANGELOG.md | 8 +++ group_vars/matrix_servers | 30 +++++++++--- playbooks/matrix.yml | 2 +- requirements.yml | 3 ++ roles/custom/matrix-redis/defaults/main.yml | 25 ---------- roles/custom/matrix-redis/tasks/main.yml | 17 ------- .../matrix-redis/tasks/setup_install.yml | 49 ------------------- .../matrix-redis/tasks/setup_uninstall.yml | 31 ------------ .../matrix-redis/templates/redis.conf.j2 | 4 -- .../templates/systemd/matrix-redis.service.j2 | 37 -------------- .../defaults/main.yml | 3 ++ .../tasks/validate_config.yml | 15 ++++++ 12 files changed, 52 insertions(+), 172 deletions(-) delete mode 100644 roles/custom/matrix-redis/defaults/main.yml delete mode 100644 roles/custom/matrix-redis/tasks/main.yml delete mode 100644 roles/custom/matrix-redis/tasks/setup_install.yml delete mode 100644 roles/custom/matrix-redis/tasks/setup_uninstall.yml delete mode 100644 roles/custom/matrix-redis/templates/redis.conf.j2 delete mode 100644 roles/custom/matrix-redis/templates/systemd/matrix-redis.service.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 493315b72..5fea8b789 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # 2023-02-17 +## The matrix-redis role lives independently now + +**TLDR**: the `matrix-redis` role is now included from another repository. Some variables have been renamed. All functionality remains intact. + +The `matrix-redis` role (which configures [Redis](https://redis.io/)) has been extracted from the playbook and now lives in its [own repository](https://gitlab.com/etke.cc/roles/redis). This makes it possible to easily use it in other Ansible playbooks. + +You need to **update you roles** (`just roles` or `make roles`) regardless of whether you're enabling Ntfy or not. If you're making use of Ntfy via this playbook, you will need to update variable references in your `vars.yml` file (`matrix_redis_` -> `redis_`). + ## The matrix-ntfy role lives independently now **TLDR**: the `matrix-ntfy` role is now included from another repository. Some variables have been renamed. All functionality remains intact. diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 32b99629c..1cd84c6b5 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -340,7 +340,7 @@ devture_systemd_service_manager_services_list_auto: | + ([{'name': 'matrix-prometheus-nginxlog-exporter.service', 'priority': 3900, 'groups': ['matrix', 'monitoring', 'prometheus', 'prometheus-exporters']}] if matrix_prometheus_nginxlog_exporter_enabled else []) + - ([{'name': 'matrix-redis', 'priority': 750, 'groups': ['matrix', 'redis']}] if matrix_redis_enabled else []) + ([{'name': (redis_identifier + '.service'), 'priority': 750, 'groups': ['matrix', 'redis']}] if redis_enabled else []) + ([{'name': 'matrix-registration.service', 'priority': 4000, 'groups': ['matrix', 'registration']}] if matrix_registration_enabled else []) + @@ -2923,15 +2923,22 @@ ntfy_visitor_request_limit_exempt_hosts_hostnames_auto: | ###################################################################### # -# matrix-redis +# etke/redis # ###################################################################### -matrix_redis_enabled: "{{ matrix_synapse_workers_enabled }}" +redis_enabled: "{{ matrix_synapse_workers_enabled }}" + +redis_identifier: matrix-redis + +redis_uid: "{{ matrix_user_uid }}" +redis_gid: "{{ matrix_user_gid }}" + +redis_base_path: "{{ matrix_base_data_path }}/redis" ###################################################################### # -# /matrix-redis +# /etle/redis # ###################################################################### @@ -3077,7 +3084,12 @@ matrix_synapse_container_metrics_api_host_bind_port: "{{ (matrix_playbook_servic # # For exposing the Synapse Manhole port (plain HTTP) to the local host. matrix_synapse_container_manhole_api_host_bind_port: "{{ (matrix_playbook_service_host_bind_interface_prefix ~ '9000') if matrix_synapse_metrics_enabled and matrix_playbook_service_host_bind_interface_prefix else '' }}" -# + +matrix_synapse_container_additional_networks: | + {{ + ([redis_container_network] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == redis_identifier else []) + }} + # For exposing the Synapse worker (and metrics) ports to the local host. matrix_synapse_workers_container_host_bind_address: "{{ matrix_playbook_service_host_bind_interface_prefix[0:-1] if (matrix_synapse_workers_enabled and matrix_playbook_service_host_bind_interface_prefix) else '' }}" @@ -3134,6 +3146,8 @@ matrix_synapse_systemd_required_services_list: | + ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else []) + + ([redis_identifier ~ '.service'] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == redis_identifier else []) + + (['matrix-goofys.service'] if matrix_s3_media_store_enabled else []) }} @@ -3145,9 +3159,9 @@ matrix_synapse_systemd_wanted_services_list: | }} # Synapse workers (used for parallel load-scaling) need Redis for IPC. -matrix_synapse_redis_enabled: "{{ matrix_redis_enabled }}" -matrix_synapse_redis_host: "{{ 'matrix-redis' if matrix_redis_enabled else '' }}" -matrix_synapse_redis_password: "{{ matrix_redis_connection_password if matrix_redis_enabled else '' }}" +matrix_synapse_redis_enabled: "{{ redis_enabled }}" +matrix_synapse_redis_host: "{{ redis_identifier if redis_enabled else '' }}" +matrix_synapse_redis_password: "{{ redis_connection_password if redis_enabled else '' }}" matrix_synapse_container_extra_arguments_auto: "{{ matrix_homeserver_container_extra_arguments_auto }}" matrix_synapse_app_service_config_files_auto: "{{ matrix_homeserver_app_service_config_files_auto }}" diff --git a/playbooks/matrix.yml b/playbooks/matrix.yml index f6a83ccbc..6980a6f9b 100755 --- a/playbooks/matrix.yml +++ b/playbooks/matrix.yml @@ -43,7 +43,7 @@ - role: galaxy/com.devture.ansible.role.postgres - - custom/matrix-redis + - galaxy/redis - custom/matrix-corporal - custom/matrix-bridge-appservice-discord - custom/matrix-bridge-appservice-slack diff --git a/requirements.yml b/requirements.yml index 54c172ec4..cc270a6c5 100644 --- a/requirements.yml +++ b/requirements.yml @@ -45,6 +45,9 @@ - src: git+https://gitlab.com/etke.cc/roles/ntfy.git version: v2.0.0-0 +- src: git+https://gitlab.com/etke.cc/roles/redis.git + version: v7.0.7-0 + - src: git+https://github.com/devture/com.devture.ansible.role.traefik.git version: fb09fd26f877372417d5586f1e79e83f983f0bd6 diff --git a/roles/custom/matrix-redis/defaults/main.yml b/roles/custom/matrix-redis/defaults/main.yml deleted file mode 100644 index 779ec3a30..000000000 --- a/roles/custom/matrix-redis/defaults/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Project source code URL: https://github.com/redis/redis - -matrix_redis_enabled: true - -matrix_redis_connection_password: "" - -matrix_redis_base_path: "{{ matrix_base_data_path }}/redis" -matrix_redis_data_path: "{{ matrix_redis_base_path }}/data" - -matrix_redis_version: 7.0.7-alpine -matrix_redis_docker_image_v6: "{{ matrix_container_global_registry_prefix }}redis:{{ matrix_redis_version }}" -matrix_redis_docker_image_latest: "{{ matrix_redis_docker_image_v6 }}" -matrix_redis_docker_image_to_use: '{{ matrix_redis_docker_image_latest }}' - -matrix_redis_docker_image_force_pull: "{{ matrix_redis_docker_image_to_use.endswith(':latest') }}" - -# A list of extra arguments to pass to the container -matrix_redis_container_extra_arguments: [] - -# Controls whether the matrix-redis container exposes a port (tcp/6379 in the container) -# that can be used to access redis from outside the container -# -# Takes an ":" or "" value (e.g. "127.0.0.1:6379"), or empty string to not expose. -matrix_redis_container_redis_bind_port: "" diff --git a/roles/custom/matrix-redis/tasks/main.yml b/roles/custom/matrix-redis/tasks/main.yml deleted file mode 100644 index 296400059..000000000 --- a/roles/custom/matrix-redis/tasks/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- - -- block: - - when: matrix_redis_enabled | bool - ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_install.yml" - tags: - - setup-all - - setup-redis - - install-all - - install-redis - -- block: - - when: not matrix_redis_enabled | bool - ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" - tags: - - setup-all - - setup-redis diff --git a/roles/custom/matrix-redis/tasks/setup_install.yml b/roles/custom/matrix-redis/tasks/setup_install.yml deleted file mode 100644 index 1384c042c..000000000 --- a/roles/custom/matrix-redis/tasks/setup_install.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -- name: Ensure redis Docker image is pulled - community.docker.docker_image: - name: "{{ matrix_redis_docker_image_to_use }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_redis_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_redis_docker_image_force_pull }}" - register: result - retries: "{{ devture_playbook_help_container_retries_count }}" - delay: "{{ devture_playbook_help_container_retries_delay }}" - until: result is not failed - -- name: Ensure redis paths exist - ansible.builtin.file: - path: "{{ item }}" - state: directory - mode: 0700 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_username }}" - with_items: - - "{{ matrix_redis_base_path }}" - - "{{ matrix_redis_data_path }}" - -# We do this as a separate task, because: -# - we'd like to do it for the data path only, not for the base path (which contains root-owned environment variable files we'd like to leave as-is) -# - we need to do it without `mode`, or we risk making certain `.conf` and other files's executable bit to flip to true -- name: Ensure redis data path ownership is correct - ansible.builtin.file: - path: "{{ matrix_redis_data_path }}" - state: directory - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_username }}" - recurse: true - -- name: Ensure redis environment variables file created - ansible.builtin.template: - src: "{{ role_path }}/templates/{{ item }}.j2" - dest: "{{ matrix_redis_base_path }}/{{ item }}" - mode: 0644 - with_items: - - "redis.conf" - -- name: Ensure matrix-redis.service installed - ansible.builtin.template: - src: "{{ role_path }}/templates/systemd/matrix-redis.service.j2" - dest: "{{ devture_systemd_docker_base_systemd_path }}/matrix-redis.service" - mode: 0644 - register: matrix_redis_systemd_service_result diff --git a/roles/custom/matrix-redis/tasks/setup_uninstall.yml b/roles/custom/matrix-redis/tasks/setup_uninstall.yml deleted file mode 100644 index e2bbc9e44..000000000 --- a/roles/custom/matrix-redis/tasks/setup_uninstall.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Check existence of matrix-redis service - ansible.builtin.stat: - path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-redis.service" - register: matrix_redis_service_stat - -- when: matrix_redis_service_stat.stat.exists | bool - block: - - name: Ensure matrix-redis is stopped - ansible.builtin.service: - name: matrix-redis - state: stopped - enabled: false - daemon_reload: true - - - name: Ensure matrix-redis.service doesn't exist - ansible.builtin.file: - path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-redis.service" - state: absent - -- name: Check existence of matrix-redis local data path - ansible.builtin.stat: - path: "{{ matrix_redis_data_path }}" - register: matrix_redis_data_path_stat - -# We just want to notify the user. Deleting data is too destructive. -- name: Notify if matrix-redis local data remains - ansible.builtin.debug: - msg: "Note: You are not using a local redis instance, but some old data remains from before in `{{ matrix_redis_data_path }}`. Feel free to delete it." - when: matrix_redis_data_path_stat.stat.exists | bool diff --git a/roles/custom/matrix-redis/templates/redis.conf.j2 b/roles/custom/matrix-redis/templates/redis.conf.j2 deleted file mode 100644 index 343713566..000000000 --- a/roles/custom/matrix-redis/templates/redis.conf.j2 +++ /dev/null @@ -1,4 +0,0 @@ -#jinja2: lstrip_blocks: "True" -{% if matrix_redis_connection_password %} -requirepass {{ matrix_redis_connection_password }} -{% endif %} diff --git a/roles/custom/matrix-redis/templates/systemd/matrix-redis.service.j2 b/roles/custom/matrix-redis/templates/systemd/matrix-redis.service.j2 deleted file mode 100644 index 5f6699f83..000000000 --- a/roles/custom/matrix-redis/templates/systemd/matrix-redis.service.j2 +++ /dev/null @@ -1,37 +0,0 @@ -#jinja2: lstrip_blocks: "True" -[Unit] -Description=Matrix Redis server -After=docker.service -Requires=docker.service - -[Service] -Type=simple -ExecStartPre=-/usr/bin/docker stop matrix-redis -ExecStartPre=-/usr/bin/docker rm matrix-redis - -ExecStart=/usr/bin/docker run --rm --name matrix-redis \ - --log-driver=none \ - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ - --cap-drop=ALL \ - --read-only \ - --tmpfs=/tmp:rw,noexec,nosuid,size=100m \ - --network={{ matrix_docker_network }} \ - {% if matrix_redis_container_redis_bind_port %} - -p {{ matrix_redis_container_redis_bind_port }}:6379 \ - {% endif %} - --mount type=bind,src={{ matrix_redis_base_path }}/redis.conf,dst=/usr/local/etc/redis/redis.conf,ro \ - --mount type=bind,src={{ matrix_redis_data_path }},dst=/data \ - {% for arg in matrix_redis_container_extra_arguments %} - {{ arg }} \ - {% endfor %} - {{ matrix_redis_docker_image_to_use }} \ - redis-server /usr/local/etc/redis/redis.conf - -ExecStop=-/usr/bin/docker stop matrix-redis -ExecStop=-/usr/bin/docker rm matrix-redis -Restart=always -RestartSec=30 -SyslogIdentifier=matrix-redis - -[Install] -WantedBy=multi-user.target diff --git a/roles/custom/matrix_playbook_migration/defaults/main.yml b/roles/custom/matrix_playbook_migration/defaults/main.yml index f8ccc88a5..514125ec0 100644 --- a/roles/custom/matrix_playbook_migration/defaults/main.yml +++ b/roles/custom/matrix_playbook_migration/defaults/main.yml @@ -14,3 +14,6 @@ matrix_playbook_migration_matrix_grafana_migration_validation_enabled: true # Controls if (`matrix_ntfy` -> `ntfy`) validation will run. matrix_playbook_migration_matrix_ntfy_migration_validation_enabled: true + +# Controls if (`matrix_redis` -> `redis`) validation will run. +matrix_playbook_migration_matrix_redis_migration_validation_enabled: true diff --git a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml index c17bec69d..7afd21540 100644 --- a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml +++ b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml @@ -125,3 +125,18 @@ Please change your configuration (vars.yml) to rename all variables (`matrix_ntfy_` -> `ntfy_`). We found usage of the following variables: {{ matrix_playbook_migration_ntfy_migration_vars.keys() | join(', ') }} when: "matrix_playbook_migration_ntfy_migration_vars | length > 0" + +- when: matrix_playbook_migration_matrix_redis_migration_validation_enabled | bool + block: + - ansible.builtin.set_fact: + matrix_playbook_migration_redis_migration_vars: |- + {{ vars | dict2items | selectattr('key', 'match', 'matrix_redis_.*') | list | items2dict }} + + - name: (Deprecation) Catch and report matrix_redis variables + ansible.builtin.fail: + msg: >- + The matrix-redis role that used to be part of this playbook has been replaced by https://gitlab.com/etke.cc/roles/redis. + The new role is compatible with the old one, but uses different names for its variables. + Please change your configuration (vars.yml) to rename all variables (`matrix_redis_` -> `redis_`). + We found usage of the following variables: {{ matrix_playbook_migration_redis_migration_vars.keys() | join(', ') }} + when: "matrix_playbook_migration_redis_migration_vars | length > 0"