From 7b52e6ad5ed8aece4d86ce0f3063fabeed544294 Mon Sep 17 00:00:00 2001
From: Stuart Mumford <stuart@cadair.com>
Date: Wed, 30 Jun 2021 20:52:49 +0000
Subject: [PATCH 1/4] Add worker metrics to prometheus exporter

---
 roles/matrix-prometheus/templates/prometheus.yml.j2 | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2
index b3ee3b86..edfa476b 100644
--- a/roles/matrix-prometheus/templates/prometheus.yml.j2
+++ b/roles/matrix-prometheus/templates/prometheus.yml.j2
@@ -30,7 +30,15 @@ scrape_configs:
   - job_name: 'synapse'
     metrics_path: '/_synapse/metrics'
     static_configs:
-    - targets: {{ matrix_prometheus_scraper_synapse_targets|to_json }}
+    - targets:
+    {% for target in matrix_prometheus_scraper_synapse_targets %}
+      - {{ target }}
+    {% endfor %}
+    {% for worker in matrix_synapse_workers_enabled_list|d([]) %}
+    {% if worker.metrics_port != 0 %}
+      - 'matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}'
+    {% endif %}
+    {% endfor %}
   {% endif %}
 
   {% if matrix_prometheus_scraper_node_enabled %}
@@ -43,4 +51,4 @@ scrape_configs:
   - job_name: postgres
     static_configs:
       - targets: {{ matrix_prometheus_scraper_postgres_targets|to_json }}
-  {% endif %}
\ No newline at end of file
+  {% endif %}

From 3d063f6acef368ae213c2987bbc00aeffc575099 Mon Sep 17 00:00:00 2001
From: Stuart Mumford <stuart@cadair.com>
Date: Wed, 30 Jun 2021 21:30:18 +0000
Subject: [PATCH 2/4] make them show as jobs in grafana

---
 .../templates/prometheus.yml.j2               | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2
index edfa476b..2ab7c4e1 100644
--- a/roles/matrix-prometheus/templates/prometheus.yml.j2
+++ b/roles/matrix-prometheus/templates/prometheus.yml.j2
@@ -30,15 +30,20 @@ scrape_configs:
   - job_name: 'synapse'
     metrics_path: '/_synapse/metrics'
     static_configs:
-    - targets:
-    {% for target in matrix_prometheus_scraper_synapse_targets %}
-      - {{ target }}
-    {% endfor %}
-    {% for worker in matrix_synapse_workers_enabled_list|d([]) %}
-    {% if worker.metrics_port != 0 %}
-      - 'matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}'
-    {% endif %}
-    {% endfor %}
+    - targets: {{ matrix_prometheus_scraper_synapse_targets|to_json }}
+      labels:
+        instance: {{ matrix_domain }}
+        job: master
+        index: 1
+  {% for worker in matrix_synapse_workers_enabled_list|d([]) %}
+  {% if worker.metrics_port != 0 %}
+    - targets: ['matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}']
+      labels:
+        instance: {{ matrix_domain }}
+        job: {{ worker.type }}
+        index: {{ worker.instanceId }}
+  {% endif %}
+  {% endfor %}
   {% endif %}
 
   {% if matrix_prometheus_scraper_node_enabled %}

From 09ee5ce52e9240e7aa620fed9093ded12950536b Mon Sep 17 00:00:00 2001
From: Stuart Mumford <stuart@cadair.com>
Date: Wed, 30 Jun 2021 21:32:19 +0000
Subject: [PATCH 3/4] we index from 0 apparently

---
 roles/matrix-prometheus/templates/prometheus.yml.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2
index 2ab7c4e1..21a31ce1 100644
--- a/roles/matrix-prometheus/templates/prometheus.yml.j2
+++ b/roles/matrix-prometheus/templates/prometheus.yml.j2
@@ -34,7 +34,7 @@ scrape_configs:
       labels:
         instance: {{ matrix_domain }}
         job: master
-        index: 1
+        index: 0
   {% for worker in matrix_synapse_workers_enabled_list|d([]) %}
   {% if worker.metrics_port != 0 %}
     - targets: ['matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}']

From 2aa457efcc0a973d062d8240760e7095226199b4 Mon Sep 17 00:00:00 2001
From: Stuart Mumford <stuart@cadair.com>
Date: Fri, 2 Jul 2021 15:41:36 +0000
Subject: [PATCH 4/4] Use a prom variable and not a synapse role variable

---
 group_vars/matrix_servers                           | 1 +
 roles/matrix-prometheus/defaults/main.yml           | 1 +
 roles/matrix-prometheus/templates/prometheus.yml.j2 | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 96b933f3..f52b4f10 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1768,6 +1768,7 @@ matrix_prometheus_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_en
 
 matrix_prometheus_scraper_synapse_enabled: "{{ matrix_synapse_enabled and matrix_synapse_metrics_enabled }}"
 matrix_prometheus_scraper_synapse_targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}']
+matrix_prometheus_scraper_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}"
 matrix_prometheus_scraper_synapse_rules_synapse_tag: "{{ matrix_synapse_docker_image_tag }}"
 
 matrix_prometheus_scraper_node_enabled: "{{ matrix_prometheus_node_exporter_enabled }}"
diff --git a/roles/matrix-prometheus/defaults/main.yml b/roles/matrix-prometheus/defaults/main.yml
index 3cd2599b..79282f91 100644
--- a/roles/matrix-prometheus/defaults/main.yml
+++ b/roles/matrix-prometheus/defaults/main.yml
@@ -34,6 +34,7 @@ matrix_prometheus_scraper_synapse_rules_synapse_tag: "master"
 matrix_prometheus_scraper_synapse_rules_download_url: "https://raw.githubusercontent.com/matrix-org/synapse/{{ matrix_prometheus_scraper_synapse_rules_synapse_tag }}/contrib/prometheus/synapse-v2.rules"
 
 matrix_prometheus_scraper_synapse_targets: []
+matrix_prometheus_scraper_synapse_workers_enabled_list: []
 
 # Tells whether the "node" scraper configuration is enabled.
 # This configuration aims to scrape the current node (this server).
diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2
index 21a31ce1..869b2da8 100644
--- a/roles/matrix-prometheus/templates/prometheus.yml.j2
+++ b/roles/matrix-prometheus/templates/prometheus.yml.j2
@@ -35,7 +35,7 @@ scrape_configs:
         instance: {{ matrix_domain }}
         job: master
         index: 0
-  {% for worker in matrix_synapse_workers_enabled_list|d([]) %}
+  {% for worker in matrix_prometheus_scraper_synapse_workers_enabled_list %}
   {% if worker.metrics_port != 0 %}
     - targets: ['matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}']
       labels: