pds.dad
This repository has no description
Docker and dashboards
author
Bailey Townsend
date
(Jul 19, 2026, 2:27 PM -0500)
commit
79b06e45
79b06e45d95be8fdaf7bbdc8220183bcfba6deca
parent
1b53b4b2
1b53b4b2198f78a2a10b786e34aa51e42135a63f
+427
+9
.dockerignore
+9
.dockerignore
+24
.env.example
+24
.env.example
···
1
+
# Copy to .env and fill in. docker compose reads .env automatically.
2
+
3
+
# --- Replica service (required) ---
4
+
# URL of the PDS to replicate, e.g. https://pds.example.com
5
+
REPLICA_PDS=
6
+
7
+
# --- Replica service (optional overrides; defaults work in Docker) ---
8
+
# REPLICA_DATABASE_URL=/data/pds-replica.db
9
+
# REPLICA_DATA_DIR=/data/repos
10
+
# REPLICA_SCRUB_INTERVAL=24h
11
+
# REPLICA_DELETED_RETENTION=720h
12
+
# REPLICA_METRICS_LISTEN=:9091
13
+
14
+
# --- Host port mappings ---
15
+
REPLICA_METRICS_PORT=9091
16
+
PROMETHEUS_PORT=9090
17
+
GRAFANA_PORT=3000
18
+
19
+
# --- Prometheus ---
20
+
# PROMETHEUS_RETENTION=15d
21
+
22
+
# --- Grafana (admin user; anonymous viewers are enabled in compose) ---
23
+
GF_SECURITY_ADMIN_USER=admin
24
+
GF_SECURITY_ADMIN_PASSWORD=changeme
+37
Dockerfile
+37
Dockerfile
···
1
+
# syntax=docker/dockerfile:1
2
+
3
+
# Build stage: CGO is required for the mattn/go-sqlite3 driver.
4
+
FROM golang:1.26-alpine3.22 AS build
5
+
6
+
RUN apk add --no-cache gcc musl-dev
7
+
8
+
WORKDIR /src
9
+
10
+
# Cache module downloads separately from source changes.
11
+
COPY go.mod go.sum ./
12
+
RUN go mod download
13
+
14
+
COPY . .
15
+
RUN CGO_ENABLED=1 go build -trimpath -ldflags="-s -w" -o /out/replica ./cmd
16
+
17
+
# Runtime stage.
18
+
FROM alpine:3.22
19
+
20
+
RUN apk add --no-cache ca-certificates tzdata wget \
21
+
&& adduser -D -u 10001 replica
22
+
23
+
COPY --from=build /out/replica /usr/local/bin/replica
24
+
25
+
# Single volume holding both the sqlite DB and the CAR data dir
26
+
# (paths configured via REPLICA_DATABASE_URL / REPLICA_DATA_DIR).
27
+
RUN mkdir -p /data && chown replica:replica /data
28
+
VOLUME /data
29
+
30
+
USER replica
31
+
32
+
EXPOSE 9091
33
+
34
+
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
35
+
CMD wget -q -O /dev/null http://localhost:9091/_health || exit 1
36
+
37
+
ENTRYPOINT ["/usr/local/bin/replica", "run"]
+54
docker-compose.yml
+54
docker-compose.yml
···
1
+
name: replica
2
+
3
+
services:
4
+
replica:
5
+
build: .
6
+
restart: unless-stopped
7
+
environment:
8
+
REPLICA_PDS: ${REPLICA_PDS:?REPLICA_PDS must be set in .env}
9
+
REPLICA_DATABASE_URL: ${REPLICA_DATABASE_URL:-/data/pds-replica.db}
10
+
REPLICA_DATA_DIR: ${REPLICA_DATA_DIR:-/data/repos}
11
+
REPLICA_SCRUB_INTERVAL: ${REPLICA_SCRUB_INTERVAL:-24h}
12
+
REPLICA_DELETED_RETENTION: ${REPLICA_DELETED_RETENTION:-720h}
13
+
REPLICA_METRICS_LISTEN: ${REPLICA_METRICS_LISTEN:-:9091}
14
+
volumes:
15
+
- replica-data:/data
16
+
ports:
17
+
- "${REPLICA_METRICS_PORT:-9091}:9091"
18
+
19
+
prometheus:
20
+
image: prom/prometheus:v3.13.1
21
+
restart: unless-stopped
22
+
command:
23
+
- --config.file=/etc/prometheus/prometheus.yml
24
+
- --storage.tsdb.path=/prometheus
25
+
- --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d}
26
+
volumes:
27
+
- ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
28
+
- prometheus-data:/prometheus
29
+
ports:
30
+
- "${PROMETHEUS_PORT:-9090}:9090"
31
+
depends_on:
32
+
- replica
33
+
34
+
grafana:
35
+
image: grafana/grafana:12.4.5
36
+
restart: unless-stopped
37
+
environment:
38
+
GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER:-admin}
39
+
GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD:?GF_SECURITY_ADMIN_PASSWORD must be set in .env}
40
+
GF_AUTH_ANONYMOUS_ENABLED: "true"
41
+
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
42
+
volumes:
43
+
- grafana-data:/var/lib/grafana
44
+
- ./docker/grafana/provisioning:/etc/grafana/provisioning:ro
45
+
- ./docker/grafana/dashboards:/var/lib/grafana/dashboards:ro
46
+
ports:
47
+
- "${GRAFANA_PORT:-3000}:3000"
48
+
depends_on:
49
+
- prometheus
50
+
51
+
volumes:
52
+
replica-data:
53
+
prometheus-data:
54
+
grafana-data:
+271
docker/grafana/dashboards/replica.json
+271
docker/grafana/dashboards/replica.json
···
1
+
{
2
+
"annotations": { "list": [] },
3
+
"editable": true,
4
+
"fiscalYearStartMonth": 0,
5
+
"graphTooltip": 1,
6
+
"links": [],
7
+
"panels": [
8
+
{
9
+
"id": 13,
10
+
"type": "stat",
11
+
"title": "Repos left to sync",
12
+
"description": "Repos not yet mirrored: pending + resyncing + desynchronized + error. Parked/unavailable states (takendown, suspended, deactivated, deleted) are excluded since they can't be fetched until the account comes back. Drains to 0 as the first sync completes.",
13
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
14
+
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 },
15
+
"fieldConfig": {
16
+
"defaults": {
17
+
"color": { "mode": "thresholds" },
18
+
"thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 0 } ] }
19
+
},
20
+
"overrides": []
21
+
},
22
+
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "textMode": "auto" },
23
+
"targets": [
24
+
{ "expr": "sum(replica_repos_total{state=~\"pending|resyncing|desynchronized|error\"})", "legendFormat": "left to sync", "refId": "A" }
25
+
]
26
+
},
27
+
{
28
+
"id": 14,
29
+
"type": "gauge",
30
+
"title": "First sync progress",
31
+
"description": "Fraction of syncable repos that are now active. Denominator excludes parked/deleted states so a PDS with permanently-unavailable accounts can still reach 100%.",
32
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
33
+
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 },
34
+
"fieldConfig": {
35
+
"defaults": {
36
+
"color": { "mode": "thresholds" },
37
+
"min": 0,
38
+
"max": 1,
39
+
"unit": "percentunit",
40
+
"thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 0.5 }, { "color": "yellow", "value": 0.9 }, { "color": "green", "value": 0.99 } ] }
41
+
},
42
+
"overrides": []
43
+
},
44
+
"options": { "showThresholdLabels": false, "showThresholdMarkers": true },
45
+
"targets": [
46
+
{ "expr": "sum(replica_repos_total{state=\"active\"}) / sum(replica_repos_total{state=~\"pending|resyncing|active|desynchronized|error\"})", "legendFormat": "progress", "refId": "A" }
47
+
]
48
+
},
49
+
{
50
+
"id": 15,
51
+
"type": "stat",
52
+
"title": "Estimated time remaining",
53
+
"description": "Rough ETA based on the resync completion rate over the last hour. Inaccurate while throughput is stalled, rate-limited, or only a few workers are active. Treat as a guideline, not a deadline.",
54
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
55
+
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 },
56
+
"fieldConfig": {
57
+
"defaults": {
58
+
"color": { "mode": "thresholds" },
59
+
"unit": "s",
60
+
"thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 86400 }, { "color": "orange", "value": 604800 }, { "color": "red", "value": 2592000 } ] }
61
+
},
62
+
"overrides": []
63
+
},
64
+
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto" },
65
+
"targets": [
66
+
{ "expr": "sum(replica_repos_total{state=~\"pending|resyncing|desynchronized|error\"}) / clamp_min(sum(rate(replica_resyncs_completed_total[1h])), 0.00001)", "legendFormat": "eta", "refId": "A" }
67
+
]
68
+
},
69
+
{
70
+
"id": 1,
71
+
"type": "timeseries",
72
+
"title": "Repos by state",
73
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
74
+
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 },
75
+
"fieldConfig": {
76
+
"defaults": {
77
+
"custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" }, "showPoints": "never" },
78
+
"min": 0
79
+
},
80
+
"overrides": []
81
+
},
82
+
"options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi" } },
83
+
"targets": [
84
+
{ "expr": "sum by (state) (replica_repos_total)", "legendFormat": "{{state}}", "refId": "A" }
85
+
]
86
+
},
87
+
{
88
+
"id": 2,
89
+
"type": "timeseries",
90
+
"title": "Firehose events/s by type",
91
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
92
+
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 },
93
+
"fieldConfig": {
94
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
95
+
"overrides": []
96
+
},
97
+
"options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi" } },
98
+
"targets": [
99
+
{ "expr": "sum by (type) (rate(replica_firehose_events_received_total[$__rate_interval]))", "legendFormat": "{{type}}", "refId": "A" }
100
+
]
101
+
},
102
+
{
103
+
"id": 3,
104
+
"type": "timeseries",
105
+
"title": "Firehose cursor lag (seq)",
106
+
"description": "Gap between the newest event seen in memory and the last cursor persisted to the DB.",
107
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
108
+
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 },
109
+
"fieldConfig": {
110
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
111
+
"overrides": []
112
+
},
113
+
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
114
+
"targets": [
115
+
{ "expr": "replica_firehose_last_seq - replica_firehose_cursor_persisted_seq", "legendFormat": "cursor lag", "refId": "A" }
116
+
]
117
+
},
118
+
{
119
+
"id": 4,
120
+
"type": "timeseries",
121
+
"title": "CAR disk usage",
122
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
123
+
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 },
124
+
"fieldConfig": {
125
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0, "unit": "bytes" },
126
+
"overrides": []
127
+
},
128
+
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
129
+
"targets": [
130
+
{ "expr": "replica_repos_disk_bytes", "legendFormat": "disk bytes", "refId": "A" }
131
+
]
132
+
},
133
+
{
134
+
"id": 5,
135
+
"type": "timeseries",
136
+
"title": "Firehose commits appended/s",
137
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
138
+
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 },
139
+
"fieldConfig": {
140
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
141
+
"overrides": []
142
+
},
143
+
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
144
+
"targets": [
145
+
{ "expr": "rate(replica_firehose_commits_appended_total[$__rate_interval])", "legendFormat": "commits/s", "refId": "A" },
146
+
{ "expr": "rate(replica_firehose_blocks_appended_total[$__rate_interval])", "legendFormat": "blocks/s", "refId": "B" }
147
+
]
148
+
},
149
+
{
150
+
"id": 6,
151
+
"type": "timeseries",
152
+
"title": "Firehose commit bytes/s",
153
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
154
+
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 },
155
+
"fieldConfig": {
156
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0, "unit": "Bps" },
157
+
"overrides": []
158
+
},
159
+
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
160
+
"targets": [
161
+
{ "expr": "rate(replica_firehose_commit_bytes_total[$__rate_interval])", "legendFormat": "commit bytes/s", "refId": "A" }
162
+
]
163
+
},
164
+
{
165
+
"id": 7,
166
+
"type": "timeseries",
167
+
"title": "Resync outcomes/s",
168
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
169
+
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 },
170
+
"fieldConfig": {
171
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
172
+
"overrides": []
173
+
},
174
+
"options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi" } },
175
+
"targets": [
176
+
{ "expr": "rate(replica_resyncs_completed_total[$__rate_interval])", "legendFormat": "completed", "refId": "A" },
177
+
{ "expr": "rate(replica_resyncs_failed_total[$__rate_interval])", "legendFormat": "failed", "refId": "B" },
178
+
{ "expr": "rate(replica_resyncs_rate_limited_total[$__rate_interval])", "legendFormat": "rate limited", "refId": "C" }
179
+
]
180
+
},
181
+
{
182
+
"id": 8,
183
+
"type": "timeseries",
184
+
"title": "Resync duration p50 / p95",
185
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
186
+
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 },
187
+
"fieldConfig": {
188
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "never" }, "min": 0, "unit": "s" },
189
+
"overrides": []
190
+
},
191
+
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
192
+
"targets": [
193
+
{ "expr": "histogram_quantile(0.50, rate(replica_resync_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p50", "refId": "A" },
194
+
{ "expr": "histogram_quantile(0.95, rate(replica_resync_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p95", "refId": "B" }
195
+
]
196
+
},
197
+
{
198
+
"id": 9,
199
+
"type": "stat",
200
+
"title": "Desyncs flagged",
201
+
"description": "Firehose + scrubber verification failures. Any non-zero value deserves attention.",
202
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
203
+
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 },
204
+
"fieldConfig": {
205
+
"defaults": {
206
+
"color": { "mode": "thresholds" },
207
+
"thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }
208
+
},
209
+
"overrides": []
210
+
},
211
+
"options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "textMode": "auto" },
212
+
"targets": [
213
+
{ "expr": "replica_firehose_desync_total", "legendFormat": "firehose desyncs", "refId": "A" },
214
+
{ "expr": "replica_scrub_repos_flagged_total", "legendFormat": "scrub flagged", "refId": "B" }
215
+
]
216
+
},
217
+
{
218
+
"id": 10,
219
+
"type": "timeseries",
220
+
"title": "Scrub repos checked/s",
221
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
222
+
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 },
223
+
"fieldConfig": {
224
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
225
+
"overrides": []
226
+
},
227
+
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
228
+
"targets": [
229
+
{ "expr": "rate(replica_scrub_repos_checked_total[$__rate_interval])", "legendFormat": "checked/s", "refId": "A" }
230
+
]
231
+
},
232
+
{
233
+
"id": 11,
234
+
"type": "timeseries",
235
+
"title": "Repo state transitions/s",
236
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
237
+
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 },
238
+
"fieldConfig": {
239
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" }, "showPoints": "never" }, "min": 0 },
240
+
"overrides": []
241
+
},
242
+
"options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi" } },
243
+
"targets": [
244
+
{ "expr": "sum by (to) (rate(replica_repos_state_transitions_total[$__rate_interval]))", "legendFormat": "→ {{to}}", "refId": "A" }
245
+
]
246
+
},
247
+
{
248
+
"id": 12,
249
+
"type": "timeseries",
250
+
"title": "Firehose reconnects",
251
+
"datasource": { "type": "prometheus", "uid": "prometheus" },
252
+
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 },
253
+
"fieldConfig": {
254
+
"defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "showPoints": "never" }, "min": 0 },
255
+
"overrides": []
256
+
},
257
+
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
258
+
"targets": [
259
+
{ "expr": "replica_firehose_reconnects_total", "legendFormat": "reconnects", "refId": "A" }
260
+
]
261
+
}
262
+
],
263
+
"refresh": "30s",
264
+
"schemaVersion": 39,
265
+
"tags": ["replica", "atproto", "pds"],
266
+
"time": { "from": "now-6h", "to": "now" },
267
+
"timezone": "browser",
268
+
"title": "PDS Replica",
269
+
"uid": "replica-overview",
270
+
"version": 2
271
+
}
+13
docker/grafana/provisioning/dashboards/dashboards.yml
+13
docker/grafana/provisioning/dashboards/dashboards.yml
+9
docker/grafana/provisioning/datasources/prometheus.yml
+9
docker/grafana/provisioning/datasources/prometheus.yml