pds.dad
This repository has no description
13 kB
271 lines
1{
2 "annotations": { "list": [] },
3 "editable": true,
4 "fiscalYearStartMonth": 0,
5 "graphTooltip": 1,
6 "links": [],
7 "panels": [
8 {
9 "id": 13,
10 "type": "stat",
11 "title": "Repos left to sync",
12 "description": "Repos not yet mirrored: pending + resyncing + desynchronized + error. Parked/unavailable states (takendown, suspended, deactivated, deleted) are excluded since they can't be fetched until the account comes back. Drains to 0 as the first sync completes.",
13 "datasource": { "type": "prometheus", "uid": "prometheus" },
14 "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 },
15 "fieldConfig": {
16 "defaults": {
17 "color": { "mode": "thresholds" },
18 "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 0 } ] }
19 },
20 "overrides": []
21 },
22 "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "textMode": "auto" },
23 "targets": [
24 { "expr": "sum(replica_repos_total{state=~\"pending|resyncing|desynchronized|error\"})", "legendFormat": "left to sync", "refId": "A" }
25 ]
26 },
27 {
28 "id": 14,
29 "type": "gauge",
30 "title": "First sync progress",
31 "description": "Fraction of syncable repos that are now active. Denominator excludes parked/deleted states so a PDS with permanently-unavailable accounts can still reach 100%.",
32 "datasource": { "type": "prometheus", "uid": "prometheus" },
33 "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 },
34 "fieldConfig": {
35 "defaults": {
36 "color": { "mode": "thresholds" },
37 "min": 0,
38 "max": 1,
39 "unit": "percentunit",
40 "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 0.5 }, { "color": "yellow", "value": 0.9 }, { "color": "green", "value": 0.99 } ] }
41 },
42 "overrides": []
43 },
44 "options": { "showThresholdLabels": false, "showThresholdMarkers": true },
45 "targets": [
46 { "expr": "sum(replica_repos_total{state=\"active\"}) / sum(replica_repos_total{state=~\"pending|resyncing|active|desynchronized|error\"})", "legendFormat": "progress", "refId": "A" }
47 ]
48 },
49 {
50 "id": 15,
51 "type": "stat",
52 "title": "Estimated time remaining",
53 "description": "Rough ETA based on the resync completion rate over the last hour. Inaccurate while throughput is stalled, rate-limited, or only a few workers are active. Treat as a guideline, not a deadline.",
54 "datasource": { "type": "prometheus", "uid": "prometheus" },
55 "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 },
56 "fieldConfig": {
57 "defaults": {
58 "color": { "mode": "thresholds" },
59 "unit": "s",
60 "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 86400 }, { "color": "orange", "value": 604800 }, { "color": "red", "value": 2592000 } ] }
61 },
62 "overrides": []
63 },
64 "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto" },
65 "targets": [
66 { "expr": "sum(replica_repos_total{state=~\"pending|resyncing|desynchronized|error\"}) / clamp_min(sum(rate(replica_resyncs_completed_total[1h])), 0.00001)", "legendFormat": "eta", "refId": "A" }
67 ]
68 },
69 {
70 "id": 1,
71 "type": "timeseries",
72 "title": "Repos by state",
73 "datasource": { "type": "prometheus", "uid": "prometheus" },
74 "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 },
75 "fieldConfig": {
76 "defaults": {
77 "custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" }, "showPoints": "never" },
78 "min": 0
79 },
80 "overrides": []
81 },
82 "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi" } },
83 "targets": [
84 { "expr": "sum by (state) (replica_repos_total)", "legendFormat": "{{state}}", "refId": "A" }
85 ]
86 },
87 {
88 "id": 2,
89 "type": "timeseries",
90 "title": "Firehose events/s by type",
91 "datasource": { "type": "prometheus", "uid": "prometheus" },
92 "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 },
93 "fieldConfig": {
94 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
95 "overrides": []
96 },
97 "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi" } },
98 "targets": [
99 { "expr": "sum by (type) (rate(replica_firehose_events_received_total[$__rate_interval]))", "legendFormat": "{{type}}", "refId": "A" }
100 ]
101 },
102 {
103 "id": 3,
104 "type": "timeseries",
105 "title": "Firehose cursor lag (seq)",
106 "description": "Gap between the newest event seen in memory and the last cursor persisted to the DB.",
107 "datasource": { "type": "prometheus", "uid": "prometheus" },
108 "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 },
109 "fieldConfig": {
110 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
111 "overrides": []
112 },
113 "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
114 "targets": [
115 { "expr": "replica_firehose_last_seq - replica_firehose_cursor_persisted_seq", "legendFormat": "cursor lag", "refId": "A" }
116 ]
117 },
118 {
119 "id": 4,
120 "type": "timeseries",
121 "title": "CAR disk usage",
122 "datasource": { "type": "prometheus", "uid": "prometheus" },
123 "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 },
124 "fieldConfig": {
125 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0, "unit": "bytes" },
126 "overrides": []
127 },
128 "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
129 "targets": [
130 { "expr": "replica_repos_disk_bytes", "legendFormat": "disk bytes", "refId": "A" }
131 ]
132 },
133 {
134 "id": 5,
135 "type": "timeseries",
136 "title": "Firehose commits appended/s",
137 "datasource": { "type": "prometheus", "uid": "prometheus" },
138 "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 },
139 "fieldConfig": {
140 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
141 "overrides": []
142 },
143 "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
144 "targets": [
145 { "expr": "rate(replica_firehose_commits_appended_total[$__rate_interval])", "legendFormat": "commits/s", "refId": "A" },
146 { "expr": "rate(replica_firehose_blocks_appended_total[$__rate_interval])", "legendFormat": "blocks/s", "refId": "B" }
147 ]
148 },
149 {
150 "id": 6,
151 "type": "timeseries",
152 "title": "Firehose commit bytes/s",
153 "datasource": { "type": "prometheus", "uid": "prometheus" },
154 "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 },
155 "fieldConfig": {
156 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0, "unit": "Bps" },
157 "overrides": []
158 },
159 "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
160 "targets": [
161 { "expr": "rate(replica_firehose_commit_bytes_total[$__rate_interval])", "legendFormat": "commit bytes/s", "refId": "A" }
162 ]
163 },
164 {
165 "id": 7,
166 "type": "timeseries",
167 "title": "Resync outcomes/s",
168 "datasource": { "type": "prometheus", "uid": "prometheus" },
169 "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 },
170 "fieldConfig": {
171 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
172 "overrides": []
173 },
174 "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi" } },
175 "targets": [
176 { "expr": "rate(replica_resyncs_completed_total[$__rate_interval])", "legendFormat": "completed", "refId": "A" },
177 { "expr": "rate(replica_resyncs_failed_total[$__rate_interval])", "legendFormat": "failed", "refId": "B" },
178 { "expr": "rate(replica_resyncs_rate_limited_total[$__rate_interval])", "legendFormat": "rate limited", "refId": "C" }
179 ]
180 },
181 {
182 "id": 8,
183 "type": "timeseries",
184 "title": "Resync duration p50 / p95",
185 "datasource": { "type": "prometheus", "uid": "prometheus" },
186 "gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 },
187 "fieldConfig": {
188 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "never" }, "min": 0, "unit": "s" },
189 "overrides": []
190 },
191 "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
192 "targets": [
193 { "expr": "histogram_quantile(0.50, rate(replica_resync_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p50", "refId": "A" },
194 { "expr": "histogram_quantile(0.95, rate(replica_resync_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p95", "refId": "B" }
195 ]
196 },
197 {
198 "id": 9,
199 "type": "stat",
200 "title": "Desyncs flagged",
201 "description": "Firehose + scrubber verification failures. Any non-zero value deserves attention.",
202 "datasource": { "type": "prometheus", "uid": "prometheus" },
203 "gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 },
204 "fieldConfig": {
205 "defaults": {
206 "color": { "mode": "thresholds" },
207 "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }
208 },
209 "overrides": []
210 },
211 "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "textMode": "auto" },
212 "targets": [
213 { "expr": "replica_firehose_desync_total", "legendFormat": "firehose desyncs", "refId": "A" },
214 { "expr": "replica_scrub_repos_flagged_total", "legendFormat": "scrub flagged", "refId": "B" }
215 ]
216 },
217 {
218 "id": 10,
219 "type": "timeseries",
220 "title": "Scrub repos checked/s",
221 "datasource": { "type": "prometheus", "uid": "prometheus" },
222 "gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 },
223 "fieldConfig": {
224 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
225 "overrides": []
226 },
227 "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
228 "targets": [
229 { "expr": "rate(replica_scrub_repos_checked_total[$__rate_interval])", "legendFormat": "checked/s", "refId": "A" }
230 ]
231 },
232 {
233 "id": 11,
234 "type": "timeseries",
235 "title": "Repo state transitions/s",
236 "datasource": { "type": "prometheus", "uid": "prometheus" },
237 "gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 },
238 "fieldConfig": {
239 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" }, "showPoints": "never" }, "min": 0 },
240 "overrides": []
241 },
242 "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi" } },
243 "targets": [
244 { "expr": "sum by (to) (rate(replica_repos_state_transitions_total[$__rate_interval]))", "legendFormat": "→ {{to}}", "refId": "A" }
245 ]
246 },
247 {
248 "id": 12,
249 "type": "timeseries",
250 "title": "Firehose reconnects",
251 "datasource": { "type": "prometheus", "uid": "prometheus" },
252 "gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 },
253 "fieldConfig": {
254 "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
255 "overrides": []
256 },
257 "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
258 "targets": [
259 { "expr": "replica_firehose_reconnects_total", "legendFormat": "reconnects", "refId": "A" }
260 ]
261 }
262 ],
263 "refresh": "30s",
264 "schemaVersion": 39,
265 "tags": ["replica", "atproto", "pds"],
266 "time": { "from": "now-6h", "to": "now" },
267 "timezone": "browser",
268 "title": "PDS Replica",
269 "uid": "replica-overview",
270 "version": 2
271}