kgess-mirror/element-synapse
1
0
Fork 0
mirror of https://github.com/element-hq/synapse.git synced 2026-01-16 23:00:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 18
element-synapse/contrib/grafana/synapse.json
Eric Eastwood 87d93b1ae6
Latest changes from importing/exporting from Grafana 12.3.1 (#19381) 
These are automatic changes from importing/exporting from Grafana
12.3.1.

In order to verify that I'm not sneaking in any changes, you can follow
these steps to get the same output.

Reproduction instructions:

 1. Start [Grafana](https://hub.docker.com/r/grafana/grafana)
    ```
docker run -d --name=grafana --add-host
host.docker.internal:host-gateway -p 3000:3000 grafana/grafana
    ```
1. Visit the Grafana dashboard, http://localhost:3000/ (Credentials:
`admin`/`admin`)
 1. Import the Synapse dashboard: `contrib/grafana/synapse.json`
1. Export the Synapse dashboard. On the dashboard page -> **Export** ->
**Export as code** -> Using the **Classic** model -> Check **Export for
sharing externally** -> Copy
 1. Paste into `contrib/grafana/synapse.json`
 1. `git status`/`git diff` to check if there is any diff

Sanity checked the dashboard itself by importing the dashboard on
https://grafana.matrix.org/ (Grafana 10.4.1 according to
https://grafana.matrix.org/api/health). The process-level metrics won't
work because https://github.com/element-hq/synapse/pull/19337 just
merged and isn't on `matrix.org` yet. Also just generally, this
dashboard works for me locally with the
[load-tests](https://github.com/element-hq/synapse-rust-apps/pull/397)
I've been doing.


### Motivation

There are few fixes I want to make to the Grafana dashboard and it sucks
having to manually translate everything back over because we have
different formatting.

Hopefully after this bulk change, future exports will have exactly what
we want to change.
2026-01-16 11:36:49 -06:00

7706 lines
229 KiB
JSON
Raw Permalink Blame History

{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "12.3.1"
},
{
"type": "panel",
"id": "heatmap",
"name": "Heatmap",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"enable": false,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"limit": 100,
"name": "Annotations & Alerts",
"showIn": 0,
"type": "dashboard"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"enable": true,
"expr": "(\n changes(process_start_time_seconds{job=\"synapse\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) * on (instance, job, index) group_left(version) synapse_build_info{job=\"synapse\"}",
"hide": false,
"iconColor": "purple",
"name": "deploys",
"titleFormat": "Deployed {{version}}"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [
{
"asDropdown": false,
"icon": "external link",
"includeVars": true,
"keepTime": true,
"tags": [
"matrix"
],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 73,
"panels": [],
"title": "Overview",
"type": "row"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 1
},
"id": 189,
"options": {
"calculate": false,
"calculation": {},
"cellGap": -1,
"cellRadius": 0,
"cellValues": {},
"color": {
"exponent": 0.5,
"fill": "#b4ff00",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Inferno",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": false
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"reverse": false,
"unit": "s"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le)",
"format": "heatmap",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "Event Send Time (excluding errors, all workers)",
"type": "heatmap"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 35,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 0,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": 0
},
{
"color": "orange",
"value": 1
},
{
"color": "red",
"value": 2
}
]
},
"unit": "s"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Avg"
},
"properties": [
{
"id": "custom.fillOpacity",
"value": 0
},
{
"id": "custom.lineWidth",
"value": 3
}
]
},
{
"matcher": {
"id": "byName",
"options": "99%"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#C4162A",
"mode": "fixed"
}
},
{
"id": "custom.fillBelowTo",
"value": "90%"
}
]
},
{
"matcher": {
"id": "byName",
"options": "90%"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#FF7383",
"mode": "fixed"
}
},
{
"id": "custom.fillBelowTo",
"value": "75%"
}
]
},
{
"matcher": {
"id": "byName",
"options": "75%"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#FFEE52",
"mode": "fixed"
}
},
{
"id": "custom.fillBelowTo",
"value": "50%"
}
]
},
{
"matcher": {
"id": "byName",
"options": "50%"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#73BF69",
"mode": "fixed"
}
},
{
"id": "custom.fillBelowTo",
"value": "25%"
}
]
},
{
"matcher": {
"id": "byName",
"options": "25%"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#1F60C4",
"mode": "fixed"
}
},
{
"id": "custom.fillBelowTo",
"value": "5%"
}
]
},
{
"matcher": {
"id": "byName",
"options": "5%"
},
"properties": [
{
"id": "custom.lineWidth",
"value": 0
}
]
},
{
"matcher": {
"id": "byName",
"options": "Average"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "rgb(255, 255, 255)",
"mode": "fixed"
}
},
{
"id": "custom.drawStyle",
"value": "line"
},
{
"id": "custom.lineWidth",
"value": 3
}
]
},
{
"matcher": {
"id": "byName",
"options": "Local events being persisted"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#96d98D",
"mode": "fixed"
}
},
{
"id": "custom.showPoints",
"value": "always"
},
{
"id": "unit",
"value": "hertz"
},
{
"id": "custom.axisPlacement",
"value": "right"
}
]
},
{
"matcher": {
"id": "byName",
"options": "All events being persisted"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#B877D9",
"mode": "fixed"
}
},
{
"id": "custom.showPoints",
"value": "always"
},
{
"id": "unit",
"value": "hertz"
},
{
"id": "custom.axisPlacement",
"value": "right"
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 1
},
"id": 152,
"options": {
"alertThreshold": true,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "99%",
"refId": "D"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "50%",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))",
"legendFormat": "25%",
"refId": "F"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))",
"legendFormat": "5%",
"refId": "G"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size]))",
"legendFormat": "Average",
"range": true,
"refId": "H"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size]))",
"hide": false,
"instant": false,
"legendFormat": "Local events being persisted",
"refId": "E"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(synapse_storage_events_persisted_events_total{server_name=\"$server_name\"}[$bucket_size]))",
"hide": false,
"instant": false,
"legendFormat": "All events being persisted",
"refId": "I"
}
],
"title": "Event Send Time Quantiles (excluding errors, all workers)",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 3,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line+area"
}
},
"links": [],
"mappings": [],
"max": 1.5,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": 0
},
{
"color": "red",
"value": 1
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 10
},
"id": 75,
"options": {
"alertThreshold": true,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(process_cpu_seconds_total{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} ",
"refId": "A"
}
],
"title": "CPU usage",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 3,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 10
},
"id": 198,
"options": {
"alertThreshold": true,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "process_resident_memory_bytes{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}} {{index}}",
"refId": "A",
"step": 20,
"target": ""
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(process_resident_memory_bytes{job=~\"$job\",index=~\"$index\"}) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"hide": true,
"interval": "",
"legendFormat": "total",
"refId": "B"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMax": 1,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 10,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 19
},
"id": 245,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "(\n synapse_build_info{job=\"synapse\"} * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) - 1",
"legendFormat": "version {{version}}",
"range": true,
"refId": "deployed_synapse_versions"
}
],
"title": "Deployed Synapse versions over time",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "/max$/"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#890F02",
"mode": "fixed"
}
},
{
"id": "custom.fillOpacity",
"value": 0
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 19
},
"id": 37,
"options": {
"alertThreshold": true,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "process_open_fds{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}}",
"refId": "A",
"step": 20
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "process_max_fds{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"hide": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} max",
"refId": "B",
"step": 20
}
],
"title": "Open FDs",
"type": "timeseries"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 26
},
"id": 54,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 27
},
"id": 5,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(process_cpu_system_seconds_total{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} system ",
"metric": "",
"range": true,
"refId": "B",
"step": 20
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(process_cpu_user_seconds_total{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} user",
"refId": "A",
"step": 20
}
],
"title": "CPU",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 27
},
"id": 105,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",job=~\"$job\"}[$bucket_size])) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"hide": false,
"interval": "",
"legendFormat": "{{job}}-{{index}} 99.9%",
"refId": "E"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",job=~\"$job\"}[$bucket_size])) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} 99%",
"refId": "A",
"step": 20
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",job=~\"$job\"}[$bucket_size])) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} 95%",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",job=~\"$job\"}[$bucket_size])) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} 90%",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "(\n\trate(python_twisted_reactor_tick_time_sum{index=~\"$index\",job=~\"$job\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n\tsynapse_server_name_info{server_name=\"$server_name\"}\n) / (\n\trate(python_twisted_reactor_tick_time_count{index=~\"$index\",job=~\"$job\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n\tsynapse_server_name_info{server_name=\"$server_name\"}\n)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} mean",
"refId": "D"
}
],
"title": "Reactor tick quantiles",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 34
},
"id": 34,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "process_resident_memory_bytes{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}} {{index}}",
"refId": "A",
"step": 20,
"target": ""
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (server_name) (\n process_resident_memory_bytes{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n)",
"interval": "",
"legendFormat": "total",
"refId": "B"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 34
},
"id": 49,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "scrape_duration_seconds{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}}",
"refId": "A",
"step": 20
}
],
"title": "Prometheus scrape time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 41
},
"id": 53,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "min_over_time(up{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "(\n synapse_build_info{job=\"synapse\"} * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) - 1",
"hide": false,
"legendFormat": "version {{version}}",
"range": true,
"refId": "deployed_synapse_versions"
}
],
"title": "Up",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 41
},
"id": 120,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_http_server_response_ru_utime_seconds{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"hide": false,
"instant": false,
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_background_process_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "B"
}
],
"title": "Stacked CPU usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 48
},
"id": 136,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"legendFormat": "{{job}}-{{index}} {{method}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"legendFormat": "{{job}}-{{index}} {{method}} (federation)",
"range": true,
"refId": "B"
}
],
"title": "Outgoing HTTP request rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "active threads",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 48
},
"id": 207,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "synapse_threadpool_working_threads{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "A"
}
],
"title": "Threadpool activity",
"type": "timeseries"
}
],
"title": "Process info",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"id": 56,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 28
},
"id": 85,
"options": {
"calculate": false,
"calculation": {},
"cellGap": -1,
"cellRadius": 0,
"cellValues": {},
"color": {
"exponent": 0.5,
"fill": "#b4ff00",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Inferno",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": false
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"reverse": false,
"unit": "s"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\"}[$bucket_size])) by (le)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "Event Send Time (Including errors, across all workers)",
"type": "heatmap"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 28
},
"id": 33,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_storage_events_persisted_events_total{server_name=\"$server_name\"}[$bucket_size])) without (job,index)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "",
"refId": "A",
"step": 20,
"target": ""
}
],
"title": "Events Persisted (all workers)",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 37
},
"id": 40,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_storage_events_persisted_events_sep_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{origin_type}}",
"refId": "D"
}
],
"title": "Events/s Local vs Remote",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 37
},
"id": 46,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by(type) (rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"instant": false,
"intervalFactor": 2,
"legendFormat": "{{type}}",
"refId": "A",
"step": 20
}
],
"title": "Events/s by Type",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 44
},
"id": 45,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",server_name=\"$server_name\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{origin_entity}} ({{origin_type}})",
"range": true,
"refId": "A",
"step": 20
}
],
"title": "Memberships/s by Origin",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 51
},
"id": 118,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} 99%",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} 95%",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} 90%",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} 50%",
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} mean",
"range": true,
"refId": "E"
}
],
"title": "Event send time quantiles by worker",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 30,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s/s"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 51
},
"id": 222,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": false,
"expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{server_name=\"$server_name\"}[1m]))",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"legendFormat": "DB time",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": false,
"expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{server_name=\"$server_name\"}[1m]))",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"legendFormat": "CPU time",
"refId": "C"
}
],
"title": "Stateres worst-case",
"type": "timeseries"
}
],
"title": "Event persistence",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 28
},
"id": 57,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 29
},
"id": 4,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_http_server_requests_received_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
"refId": "A",
"step": 20
}
],
"title": "Request Count by arrival time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 29
},
"id": 32,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_http_server_requests_received_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{server_name=\"$server_name\",job=~\"$job\",method!=\"OPTIONS\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",
"refId": "A",
"step": 20,
"target": ""
}
],
"title": "Top 10 Request Counts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 37
},
"id": 139,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
"refId": "A",
"step": 20
}
],
"title": "Total CPU Usage by Endpoint",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 37
},
"id": 52,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
"refId": "A",
"step": 20
}
],
"title": "Average CPU Usage by Endpoint",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 45
},
"id": 7,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
"refId": "A",
"step": 20
}
],
"title": "DB Usage by endpoint",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 45
},
"id": 47,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
"refId": "A",
"step": 20
}
],
"title": "Non-sync avg response time",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 53
},
"id": 103,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "topk(10,synapse_http_server_in_flight_requests_count{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"})",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))",
"interval": "",
"legendFormat": "Total",
"refId": "B"
}
],
"title": "Requests in flight",
"type": "timeseries"
}
],
"title": "Requests",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 29
},
"id": 97,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 30
},
"id": 99,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_background_process_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "A"
}
],
"title": "CPU usage by background jobs",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 30
},
"id": 101,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_background_process_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "A"
}
],
"title": "DB usage by background jobs (including scheduling time)",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 39
},
"id": 138,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}",
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "A"
}
],
"title": "Background jobs in flight",
"type": "timeseries"
}
],
"title": "Background jobs",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 30
},
"id": 81,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 31
},
"id": 79,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_federation_client_sent_transactions_total{server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "successful txn rate",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",server_name=\"$server_name\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{server_name=\"$server_name\"}[$bucket_size]))",
"legendFormat": "failed txn rate",
"refId": "B"
}
],
"title": "Outgoing federation transaction rate",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 31
},
"id": 83,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_federation_server_received_pdus_total{server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "pdus",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_federation_server_received_edus_total{server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "edus",
"refId": "B"
}
],
"title": "Incoming PDU/EDU rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 40
},
"id": 109,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "pdus",
"range": true,
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_federation_client_sent_edus_total{server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "edus",
"refId": "B"
}
],
"title": "Outgoing PDU/EDU rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 40
},
"id": 111,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_federation_client_sent_edus_by_type_total{server_name=\"$server_name\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{type}}",
"range": true,
"refId": "A"
}
],
"title": "Outgoing EDUs by type",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Triangular growth may indicate a problem with federation sending from the remote host --- but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMax": 60,
"axisSoftMin": 0,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 60
}
]
},
"unit": "m"
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"libera.chat "
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": true,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 49
},
"id": 243,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"exemplar": false,
"expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_received_pdu_time[10m]))) / 60",
"instant": false,
"legendFormat": "{{origin_server_name}} ",
"range": true,
"refId": "A"
}
],
"title": "Age of last PDU received from nominated hosts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Triangular growth may indicate a problem with federation senders on the monitored instance---but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMax": 60,
"axisSoftMin": 0,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 60
}
]
},
"unit": "m"
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"libera.chat"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": true,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 49
},
"id": 241,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"exemplar": false,
"expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_sent_pdu_time[10m]))) / 60",
"instant": false,
"legendFormat": "{{destination_server_name}}",
"range": true,
"refId": "A"
}
],
"title": "Age of last PDU sent to nominated hosts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "The number of events in the in-memory queues ",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 57
},
"id": 142,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "synapse_federation_transaction_queue_pending_pdus{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
"legendFormat": "pending PDUs {{job}}-{{index}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_federation_transaction_queue_pending_edus{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
"legendFormat": "pending EDUs {{job}}-{{index}}",
"refId": "B"
}
],
"title": "In-memory federation transmission queues",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "Number of events queued up on the master process for processing by the federation sender",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 57
},
"id": 140,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_federation_send_queue_presence_changed_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "presence changed",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_federation_send_queue_presence_map_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "presence map",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_federation_send_queue_presence_destinations_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "presence destinations",
"refId": "E"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_federation_send_queue_keyed_edu_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "keyed edus",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_federation_send_queue_edus_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "other edus",
"refId": "D"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_federation_send_queue_pos_time_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "stream positions",
"refId": "F"
}
],
"title": "Outgoing EDU queues on master",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 66
},
"id": 166,
"options": {
"calculate": false,
"calculation": {},
"cellGap": -1,
"cellValues": {
"decimals": 2
},
"color": {
"exponent": 0.5,
"fill": "#b4ff00",
"min": 0,
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Inferno",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": false
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"decimals": 0,
"reverse": false,
"unit": "s"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{server_name=\"$server_name\",name=\"federation_sender\"}[$bucket_size])) by (le)",
"format": "heatmap",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ le }}",
"refId": "A"
}
],
"title": "Federation send PDU lag",
"type": "heatmap"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 66
},
"id": 162,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "99%",
"refId": "D"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "50%",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"interval": "",
"legendFormat": "25%",
"refId": "F"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"interval": "",
"legendFormat": "5%",
"refId": "G"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))",
"interval": "",
"legendFormat": "Average",
"refId": "H"
}
],
"title": "Federation send PDU lag quantiles",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 75
},
"id": 164,
"options": {
"calculate": false,
"calculation": {},
"cellGap": -1,
"cellValues": {
"decimals": 2
},
"color": {
"exponent": 0.5,
"fill": "#b4ff00",
"min": 0,
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Inferno",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": false
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"decimals": 0,
"reverse": false,
"unit": "s"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{server_name=\"$server_name\"}[$bucket_size])) by (le)",
"format": "heatmap",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ le }}",
"refId": "A"
}
],
"title": "Handle inbound PDU time",
"type": "heatmap"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 75
},
"id": 203,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}}",
"range": true,
"refId": "A",
"step": 4
}
],
"title": "Age of oldest event in staging area",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 84
},
"id": 202,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}}",
"range": true,
"refId": "A",
"step": 4
}
],
"title": "Number of events in federation staging area",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 84
},
"id": 205,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_federation_soft_failed_events_total{server_name=\"$server_name\"}[$bucket_size]))",
"interval": "",
"legendFormat": "soft-failed events",
"refId": "A"
}
],
"title": "Soft-failed event rate",
"type": "timeseries"
}
],
"title": "Federation",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 31
},
"id": 227,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 154
},
"id": 239,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(increase(synapse_rate_limit_reject_total{server_name=\"$server_name\"}[$bucket_size]))",
"refId": "A"
}
],
"title": "Number of rate limit rejected requests",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 154
},
"id": 235,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(increase(synapse_rate_limit_sleep_total{server_name=\"$server_name\"}[$bucket_size]))",
"refId": "A"
}
],
"title": "Number of requests being slept by the rate limiter",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Why is the data zero (0)? https://github.com/matrix-org/synapse/pull/13541#discussion_r951926322",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 162
},
"id": 237,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{server_name=\"$server_name\"}[$bucket_size]))",
"refId": "A"
}
],
"title": "Number of hosts being rejected by the rate limiter",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "https://github.com/matrix-org/synapse/pull/13541",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 162
},
"id": 233,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{server_name=\"$server_name\"}[$bucket_size]))",
"refId": "A"
}
],
"title": "Number of hosts being slept by the rate limiter",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 170
},
"id": 229,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
"legendFormat": ">99%",
"range": true,
"refId": "E"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "99%",
"range": true,
"refId": "D"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "50%",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"legendFormat": "25%",
"refId": "F"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))",
"legendFormat": "5%",
"refId": "G"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))",
"legendFormat": "Average",
"refId": "H"
}
],
"title": "Rate limit queue wait time Quantiles (all workers)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
}
]
},
"unit": "hertz"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Default reject threshold (50 requests within a second)"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
},
{
"id": "custom.lineWidth",
"value": 2
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 170
},
"id": 231,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_rate_limit_sleep_total{server_name=\"$server_name\"}[$bucket_size]))",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "50",
"hide": false,
"legendFormat": "Default reject threshold (50 requests within a second)",
"range": true,
"refId": "B"
}
],
"title": "Rate of requests being slept by the rate limiter",
"type": "timeseries"
}
],
"title": "Federation rate limiter",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 32
},
"id": 60,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "hertz"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 155
},
"id": 51,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_http_httppusher_http_pushes_processed_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "processed {{job}}-{{index}}",
"range": true,
"refId": "A",
"step": 20
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_http_httppusher_http_pushes_failed_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "failed {{job}}-{{index}}",
"range": true,
"refId": "B",
"step": 20
}
],
"title": "HTTP Push rate",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 155
},
"id": 134,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", server_name=\"$server_name\"})",
"legendFormat": "{{kind}} {{app_id}}",
"refId": "A"
}
],
"title": "Active pusher instances by app",
"type": "timeseries"
}
],
"title": "Pushes",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 33
},
"id": 219,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "How many entries in current state that we are iterating over while calculating push rules.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 33
},
"id": 209,
"options": {
"alertThreshold": true
},
"pluginVersion": "8.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{index}}",
"metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total",
"refId": "A",
"step": 2
}
],
"title": "Iterations over State",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Rate that the cached push rules for a room get invalidated due to underlying push rules being changed. ",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 33
},
"id": 211,
"options": {
"alertThreshold": true
},
"pluginVersion": "8.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{index}}",
"metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
"refId": "A",
"step": 2
}
],
"title": "Push Rule Invalidations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "How often the \"delta optimisation\" happens.\n\nThe delta optimisation is when we update the push rules for a room incrementally after a state change where we know the delta between the old state and the new state.\n\nThis can't happen if we don't the delta or we're calculating push rules from scratch.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 40
},
"id": 213,
"options": {
"alertThreshold": true
},
"pluginVersion": "8.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",server_name=\"$server_name\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "Hit Rate",
"metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
"refId": "A",
"step": 2
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "Number of calls",
"refId": "B",
"step": 2
}
],
"title": "Delta Optimisation",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "How often we have the correct cached push rules for a room.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 40
},
"id": 215,
"options": {
"alertThreshold": true
},
"pluginVersion": "8.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",server_name=\"$server_name\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "Hit Rate",
"metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
"refId": "A",
"step": 2
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "Number of calls",
"refId": "B",
"step": 2
}
],
"title": "How often we reuse existing calculated push rules",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "How often we have existing cached push rules for the room. \n\nNote that these might be outdated and need to be recalculated if the state has changed.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 47
},
"id": 217,
"options": {
"alertThreshold": true
},
"pluginVersion": "8.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",server_name=\"$server_name\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "Hit Rate",
"metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
"refId": "A",
"step": 2
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "Number of calls",
"refId": "B",
"step": 2
}
],
"title": "How often we have the RulesForRoom cached",
"type": "timeseries"
}
],
"title": "Push Rule Cache",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 34
},
"id": 58,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 35
},
"id": 48,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_storage_schedule_time_sum{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}}",
"refId": "A",
"step": 20
}
],
"title": "Avg time waiting for db conn",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 35
},
"id": 104,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
"legendFormat": "{{job}} {{index}} 99%",
"refId": "A",
"step": 20
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}} {{index}} 95%",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}} {{index}} 90%",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_storage_schedule_time_sum{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}} {{index}} mean",
"refId": "D"
}
],
"title": "Db scheduling time quantiles",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 42
},
"id": 10,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "topk(10, rate(synapse_storage_transaction_time_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{desc}}",
"refId": "A",
"step": 20
}
],
"title": "Top DB transactions by txn rate",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 42
},
"id": 11,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_storage_transaction_time_sum_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{desc}}",
"refId": "A",
"step": 20
}
],
"title": "DB transactions by total txn time",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 49
},
"id": 180,
"options": {
"alertThreshold": false
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_storage_transaction_time_sum_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{desc}}",
"refId": "A",
"step": 20
}
],
"title": "Average DB txn time",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 49
},
"id": 200,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",server_name=\"$server_name\",job=\"$job\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "99%",
"refId": "D"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",server_name=\"$server_name\",job=\"$job\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",server_name=\"$server_name\",job=\"$job\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",server_name=\"$server_name\",job=\"$job\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "50%",
"refId": "B"
}
],
"title": "Time waiting for DB connection quantiles",
"type": "timeseries"
}
],
"title": "Database",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 35
},
"id": 59,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 12,
"x": 0,
"y": 158
},
"id": 12,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A",
"step": 20
}
],
"title": "Total CPU Usage by Block",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 12,
"x": 12,
"y": 158
},
"id": 26,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A",
"step": 20
}
],
"title": "Average CPU Time per Block",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 12,
"x": 0,
"y": 171
},
"id": 13,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A",
"step": 20
}
],
"title": "Total DB Usage by Block",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 12,
"x": 12,
"y": 171
},
"id": 27,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A",
"step": 20
}
],
"title": "Average Database Transaction time, by Block",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 12,
"x": 0,
"y": 184
},
"id": 28,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A",
"step": 20
}
],
"title": "Average Transactions per Block",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 12,
"x": 12,
"y": 184
},
"id": 25,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_metrics_block_time_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A",
"step": 20
}
],
"title": "Average Wallclock Time per Block",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 15,
"w": 12,
"x": 0,
"y": 197
},
"id": 154,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_metrics_block_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A"
}
],
"title": "Block count",
"type": "timeseries"
}
],
"title": "Per-block metrics",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 36
},
"id": 61,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 36
},
"id": 1,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{name}} {{job}}-{{index}}",
"refId": "A",
"step": 20
}
],
"title": "Cache Hit Ratio",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 36
},
"id": 8,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_util_caches_cache_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{name}} {{job}}-{{index}}",
"refId": "A",
"step": 20
}
],
"title": "Cache Size",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 46
},
"id": 38,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{name}} {{job}}-{{index}}",
"refId": "A",
"step": 20
}
],
"title": "Cache request rate",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 46
},
"id": 39,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{name}} {{job}}-{{index}}",
"refId": "A",
"step": 20
}
],
"title": "Top 10 cache misses",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 56
},
"id": 65,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_caches_cache_evicted_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{name}} ({{reason}}) {{job}}-{{index}}",
"refId": "A"
}
],
"title": "Cache eviction rate",
"type": "timeseries"
}
],
"title": "Caches",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 37
},
"id": 148,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {},
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 29
},
"id": 146,
"options": {
"alertThreshold": true
},
"pluginVersion": "7.3.7",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_util_caches_response_cache_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
"legendFormat": "{{name}} {{job}}-{{index}}",
"refId": "A"
}
],
"title": "Response cache size",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {},
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 29
},
"id": 150,
"options": {
"alertThreshold": true
},
"pluginVersion": "7.3.7",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_util_caches_response_cache_hits{server_name=\"$server_name\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{server_name=\"$server_name\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{name}} {{job}}-{{index}}",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "",
"interval": "",
"legendFormat": "",
"refId": "B"
}
],
"title": "Response cache hit rate",
"type": "timeseries"
}
],
"title": "Response caches",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 38
},
"id": 62,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {},
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 30
},
"id": 91,
"options": {
"alertThreshold": true
},
"pluginVersion": "7.3.7",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(python_gc_time_sum{job=~\"$job\",index=~\"$index\"}[10m]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"instant": false,
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} gen {{gen}}",
"refId": "A"
}
],
"title": "Total GC time by bucket (10m smoothing)",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {},
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 30
},
"id": 21,
"options": {
"alertThreshold": true
},
"pluginVersion": "7.3.7",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "(\n rate(python_gc_time_sum{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) / (\n rate(python_gc_time_count{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{index}} gen {{gen}} ",
"refId": "A",
"step": 20,
"target": ""
}
],
"title": "Average GC Time Per Collection",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
"fieldConfig": {
"defaults": {
"custom": {},
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 39
},
"id": 89,
"options": {
"alertThreshold": true
},
"pluginVersion": "7.3.7",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "python_gc_counts{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} gen {{gen}}",
"refId": "A"
}
],
"title": "Allocation counts",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {},
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 39
},
"id": 93,
"options": {
"alertThreshold": true
},
"pluginVersion": "7.3.7",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "(\n rate(python_gc_unreachable_total{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) / (\n rate(python_gc_time_count{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} gen {{gen}}",
"refId": "A"
}
],
"title": "Object counts per collection",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {},
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 48
},
"id": 95,
"options": {
"alertThreshold": true
},
"pluginVersion": "7.3.7",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(python_gc_time_count{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} gen {{gen}}",
"refId": "A"
}
],
"title": "GC frequency",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 48
},
"id": 87,
"options": {},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "GC durations",
"type": "heatmap"
}
],
"title": "GC",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 39
},
"id": 63,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 162
},
"id": 43,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{command}}",
"refId": "A",
"step": 20
}
],
"title": "Rate of outgoing commands",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "hertz"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 162
},
"id": 41,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{stream_name}}",
"refId": "A",
"step": 20
}
],
"title": "Rate of outgoing RDATA commands, by stream",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "hertz"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 169
},
"id": 42,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{command}}",
"refId": "A",
"step": 20
}
],
"title": "Rate of incoming commands (including echoes)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMin": 1,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "hertz"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 169
},
"id": 220,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{stream_name}}",
"refId": "A",
"step": 20
}
],
"title": "Rate of incoming RDATA commands (excluding echoes), by stream",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 176
},
"id": 144,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_replication_tcp_command_queue{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
"legendFormat": "{{stream_name}} {{job}}-{{index}}",
"refId": "A"
}
],
"title": "Queued incoming RDATA commands, by stream",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 176
},
"id": 115,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{reason_type}}",
"refId": "A"
}
],
"title": "Replication connection close reasons",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 183
},
"id": 113,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{stream_name}}",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}}",
"refId": "B"
}
],
"title": "Replication connections",
"type": "timeseries"
}
],
"title": "Replication",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 40
},
"id": 69,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 163
},
"id": 67,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "max(synapse_event_persisted_position{server_name=\"$server_name\"}) - on () group_right() synapse_event_processing_positions{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "A"
}
],
"title": "Event processing lag",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 163
},
"id": 71,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "time()*1000-synapse_event_processing_last_ts{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "B"
}
],
"title": "Age of last processed event",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 172
},
"id": 121,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "deriv(synapse_event_processing_last_ts{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "B"
}
],
"title": "Event processing catchup rate",
"type": "timeseries"
}
],
"title": "Event processing loop positions",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 41
},
"id": 126,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 42
},
"id": 122,
"options": {
"calculate": false,
"calculation": {},
"cellGap": 1,
"cellValues": {},
"color": {
"exponent": 0.5,
"fill": "#B877D9",
"min": 0,
"mode": "opacity",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"decimals": 0,
"reverse": false,
"unit": "short"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_forward_extremities_bucket{server_name=\"$server_name\"} and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "Number of rooms, by number of forward extremities in room",
"type": "heatmap"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 42
},
"id": 124,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_forward_extremities_bucket{server_name=\"$server_name\"} > 0",
"format": "heatmap",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "Room counts, by number of extremities",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 50
},
"id": 127,
"options": {
"calculate": false,
"calculation": {},
"cellGap": 1,
"cellValues": {},
"color": {
"exponent": 0.5,
"fill": "#5794F2",
"min": 0,
"mode": "opacity",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"decimals": 0,
"reverse": false,
"unit": "short"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "Events persisted, by number of forward extremities in room (heatmap)",
"type": "heatmap"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 50
},
"id": 128,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "50%",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "99%",
"refId": "D"
}
],
"title": "Events persisted, by number of forward extremities in room (quantiles)",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 58
},
"id": 129,
"options": {
"calculate": false,
"calculation": {},
"cellGap": 1,
"cellValues": {},
"color": {
"exponent": 0.5,
"fill": "#FF9830",
"min": 0,
"mode": "opacity",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"decimals": 0,
"reverse": false,
"unit": "short"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "Events persisted, by number of stale forward extremities in room (heatmap)",
"type": "heatmap"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 58
},
"id": 130,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "50%",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "99%",
"refId": "D"
}
],
"title": "Events persisted, by number of stale forward extremities in room (quantiles)",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 66
},
"id": 131,
"options": {
"calculate": false,
"calculation": {},
"cellGap": 1,
"cellValues": {},
"color": {
"exponent": 0.5,
"fill": "#73BF69",
"min": 0,
"mode": "opacity",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"decimals": 0,
"reverse": false,
"unit": "short"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "heatmap",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "Number of state resolution performed, by number of state groups involved (heatmap)",
"type": "heatmap"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 66
},
"id": 132,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "50%",
"range": true,
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "99%",
"refId": "D"
}
],
"title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 74
},
"id": 179,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"interval": "",
"legendFormat": "State res ",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"interval": "",
"legendFormat": "Potential to prune",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"interval": "",
"legendFormat": "Pruned",
"refId": "C"
}
],
"title": "Stale extremity dropping",
"type": "timeseries"
}
],
"title": "Extremities",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 42
},
"id": 158,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 43
},
"id": 156,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max(synapse_admin_mau_max{server_name=\"$server_name\"})",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "Max",
"range": true,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max(synapse_admin_mau_current{server_name=\"$server_name\"})",
"hide": false,
"legendFormat": "Current",
"range": true,
"refId": "C"
}
],
"title": "MAU Limits",
"type": "timeseries"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 43
},
"id": 160,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "synapse_admin_mau_current_mau_by_service{server_name=\"$server_name\"}",
"interval": "",
"legendFormat": "{{ app_service }}",
"refId": "A"
}
],
"title": "MAU by Appservice",
"type": "timeseries"
}
],
"title": "MAU",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 43
},
"id": 177,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 44
},
"id": 173,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
"legendFormat": "{{stream}} {{index}}",
"metric": "synapse_notifier",
"range": true,
"refId": "A",
"step": 2
}
],
"title": "Notifier Streams Woken",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 44
},
"id": 175,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",server_name=\"$server_name\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{type}} {{index}}",
"range": true,
"refId": "A",
"step": 2
}
],
"title": "Presence Stream Fetch Type Rates",
"type": "timeseries"
}
],
"title": "Notifier",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 44
},
"id": 170,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 45
},
"id": 168,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_appservice_api_sent_events_total{server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{service}}",
"range": true,
"refId": "A"
}
],
"title": "Sent Events rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 45
},
"id": 171,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_appservice_api_sent_transactions_total{server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{exported_service }} {{ service }}",
"range": true,
"refId": "A"
}
],
"title": "Transactions rate",
"type": "timeseries"
}
],
"title": "Appservices",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 45
},
"id": 188,
"panels": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 46
},
"id": 182,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "Notified",
"refId": "A"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "Remote ping",
"refId": "B"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "Total updates",
"refId": "C"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "Remote updates",
"refId": "D"
},
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "Bump active time",
"refId": "E"
}
],
"title": "Presence",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 46
},
"id": 184,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{from}} -> {{to}}",
"range": true,
"refId": "A"
}
],
"title": "Presence state transitions",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 54
},
"id": 186,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{reason}}",
"range": true,
"refId": "A"
}
],
"title": "Presence notify reason",
"type": "timeseries"
}
],
"title": "Presence",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 46
},
"id": 197,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "hertz"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 47
},
"id": 191,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_external_cache_set{job=~\"$job\", server_name=\"$server_name\", index=~\"$index\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
"range": true,
"refId": "A"
}
],
"title": "External Cache Set Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 47
},
"id": 193,
"options": {
"alertThreshold": true
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", server_name=\"$server_name\", index=~\"$index\"}[$bucket_size]))",
"interval": "",
"legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
"range": true,
"refId": "A"
}
],
"title": "External Cache Get Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 55
},
"id": 195,
"options": {
"calculate": false,
"calculation": {},
"cellGap": -1,
"cellValues": {
"decimals": 2
},
"color": {
"exponent": 0.5,
"fill": "#b4ff00",
"min": 0,
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Inferno",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": false
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": true
},
"yAxis": {
"axisPlacement": "left",
"decimals": 0,
"reverse": false,
"unit": "s"
}
},
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\",job=~\"$job\"}[$bucket_size])) by (le)",
"format": "heatmap",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"title": "External Cache Response Time",
"type": "heatmap"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
}
]
},
"unit": "hertz"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 55
},
"id": 223,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.4",
"targets": [
{
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(synapse_external_cache_get{job=~\"$job\", server_name=\"$server_name\", index=~\"$index\", hit=\"False\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
"range": true,
"refId": "A"
}
],
"title": "External Cache Miss Rate",
"type": "timeseries"
}
],
"title": "External Cache",
"type": "row"
}
],
"preload": false,
"refresh": "",
"schemaVersion": 42,
"tags": [
"matrix"
],
"templating": {
"list": [
{
"current": {
"text": "",
"value": "${DS_PROMETHEUS}",
"selected": true
},
"includeAll": false,
"label": "Datasource",
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"type": "datasource"
},
{
"auto": true,
"auto_count": 100,
"auto_min": "30s",
"current": {
"text": "$__auto",
"value": "$__auto"
},
"label": "Bucket Size",
"name": "bucket_size",
"options": [
{
"selected": false,
"text": "30s",
"value": "30s"
},
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "2m",
"value": "2m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
}
],
"query": "30s,1m,2m,5m,10m,15m",
"refresh": 2,
"type": "interval"
},
{
"current": {},
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"definition": "",
"includeAll": false,
"name": "server_name",
"options": [],
"query": {
"query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, server_name)",
"refId": "Prometheus-instance-Variable-Query"
},
"refresh": 2,
"regex": "",
"sort": 1,
"type": "query"
},
{
"current": {},
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"definition": "",
"includeAll": true,
"label": "Job",
"multi": true,
"name": "job",
"options": [],
"query": {
"query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, job)",
"refId": "Prometheus-job-Variable-Query"
},
"refresh": 2,
"regex": "",
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {},
"datasource": {
"uid": "${DS_PROMETHEUS}"
},
"definition": "",
"includeAll": true,
"multi": true,
"name": "index",
"options": [],
"query": {
"query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, index)",
"refId": "Prometheus-index-Variable-Query"
},
"refresh": 2,
"regex": "",
"sort": 3,
"type": "query"
}
]
},
"time": {
"from": "now-3h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Synapse",
"uid": "000000012",
"version": 1,
"weekStart": ""
}
Reference in a new issue View git blame Copy permalink