bas.sh
Archive of https://github.com/w-social-eu/w-social-atproto (see also https://github.com/brw/w-social-atproto-archive)
add dataplane service and AppView CI workflow
author
Denys Zarubin
date
(May 18, 2026, 1:12 PM +0200)
commit
30d177bf
30d177bf15d74d6c23cac583b1a70126a5c7e719
parent
31b2e84e
31b2e84e475426bc51d6a60fca9992d553472287
+402
-5
+222
.github/workflows/build-and-deploy-appview.yaml
+222
.github/workflows/build-and-deploy-appview.yaml
···
1
+
name: Build & Deploy AppView
2
+
3
+
on:
4
+
push:
5
+
branches:
6
+
- develop
7
+
- main
8
+
tags:
9
+
- 'v*'
10
+
paths:
11
+
- 'packages/bsky/**'
12
+
- 'packages/bsync/**'
13
+
- 'packages/sync/**'
14
+
- 'packages/identity/**'
15
+
- 'packages/api/**'
16
+
- 'packages/common/**'
17
+
- 'packages/common-web/**'
18
+
- 'packages/crypto/**'
19
+
- 'packages/did/**'
20
+
- 'packages/repo/**'
21
+
- 'packages/syntax/**'
22
+
- 'packages/lexicon/**'
23
+
- 'packages/lexicon-resolver/**'
24
+
- 'packages/xrpc/**'
25
+
- 'packages/xrpc-server/**'
26
+
- 'services/bsky/**'
27
+
- 'services/bsync/**'
28
+
- 'services/dataplane/**'
29
+
- 'lexicons/**'
30
+
workflow_dispatch:
31
+
inputs:
32
+
environment:
33
+
description: 'Target environment'
34
+
required: false
35
+
default: 'dev'
36
+
type: choice
37
+
options:
38
+
- dev
39
+
- stage
40
+
- prod
41
+
42
+
env:
43
+
REGISTRY: ghcr.io
44
+
BSKY_IMAGE: ghcr.io/w-social-eu/w-social-atproto/bsky
45
+
BSYNC_IMAGE: ghcr.io/w-social-eu/w-social-atproto/bsync
46
+
DATAPLANE_IMAGE: ghcr.io/w-social-eu/w-social-atproto/dataplane
47
+
48
+
jobs:
49
+
config:
50
+
runs-on: ubuntu-latest
51
+
outputs:
52
+
environment: ${{ steps.config.outputs.environment }}
53
+
image_tag: ${{ steps.config.outputs.image_tag }}
54
+
kubeconfig_secret: ${{ steps.config.outputs.kubeconfig_secret }}
55
+
steps:
56
+
- name: Determine environment
57
+
id: config
58
+
run: |
59
+
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
60
+
ENV="${{ github.event.inputs.environment }}"
61
+
elif [[ "${{ github.ref_type }}" == "tag" && "${{ github.ref_name }}" == v* ]]; then
62
+
ENV="prod"
63
+
elif [[ "${{ github.ref_name }}" == "main" ]]; then
64
+
ENV="stage"
65
+
elif [[ "${{ github.ref_name }}" == "develop" || "${{ github.ref_name }}" == feature/* ]]; then
66
+
ENV="dev"
67
+
else
68
+
ENV="dev"
69
+
fi
70
+
SHORT_SHA="${GITHUB_SHA::7}"
71
+
echo "environment=$ENV" >> $GITHUB_OUTPUT
72
+
echo "image_tag=custom-${ENV}-${SHORT_SHA}" >> $GITHUB_OUTPUT
73
+
case "$ENV" in
74
+
dev) echo "kubeconfig_secret=KUBECONFIG_DEV" >> $GITHUB_OUTPUT ;;
75
+
stage) echo "kubeconfig_secret=KUBECONFIG_STAGE" >> $GITHUB_OUTPUT ;;
76
+
prod) echo "kubeconfig_secret=KUBECONFIG_PROD" >> $GITHUB_OUTPUT ;;
77
+
esac
78
+
79
+
build-bsky:
80
+
runs-on: ubuntu-latest
81
+
needs: config
82
+
permissions:
83
+
contents: read
84
+
packages: write
85
+
steps:
86
+
- uses: actions/checkout@v4
87
+
- uses: docker/setup-buildx-action@v3
88
+
- uses: docker/login-action@v3
89
+
with:
90
+
registry: ${{ env.REGISTRY }}
91
+
username: ${{ github.actor }}
92
+
password: ${{ secrets.GITHUB_TOKEN }}
93
+
- name: Build and push bsky
94
+
uses: docker/build-push-action@v5
95
+
with:
96
+
context: .
97
+
file: services/bsky/Dockerfile
98
+
push: true
99
+
tags: |
100
+
${{ env.BSKY_IMAGE }}:${{ needs.config.outputs.image_tag }}
101
+
${{ env.BSKY_IMAGE }}:${{ needs.config.outputs.environment }}-latest
102
+
${{ github.ref_type == 'tag' && format('{0}:{1}', env.BSKY_IMAGE, github.ref_name) || '' }}
103
+
no-cache: true
104
+
105
+
build-bsync:
106
+
runs-on: ubuntu-latest
107
+
needs: config
108
+
permissions:
109
+
contents: read
110
+
packages: write
111
+
steps:
112
+
- uses: actions/checkout@v4
113
+
- uses: docker/setup-buildx-action@v3
114
+
- uses: docker/login-action@v3
115
+
with:
116
+
registry: ${{ env.REGISTRY }}
117
+
username: ${{ github.actor }}
118
+
password: ${{ secrets.GITHUB_TOKEN }}
119
+
- name: Build and push bsync
120
+
uses: docker/build-push-action@v5
121
+
with:
122
+
context: .
123
+
file: services/bsync/Dockerfile
124
+
push: true
125
+
tags: |
126
+
${{ env.BSYNC_IMAGE }}:${{ needs.config.outputs.image_tag }}
127
+
${{ env.BSYNC_IMAGE }}:${{ needs.config.outputs.environment }}-latest
128
+
${{ github.ref_type == 'tag' && format('{0}:{1}', env.BSYNC_IMAGE, github.ref_name) || '' }}
129
+
no-cache: true
130
+
131
+
build-dataplane:
132
+
runs-on: ubuntu-latest
133
+
needs: config
134
+
permissions:
135
+
contents: read
136
+
packages: write
137
+
steps:
138
+
- uses: actions/checkout@v4
139
+
- uses: docker/setup-buildx-action@v3
140
+
- uses: docker/login-action@v3
141
+
with:
142
+
registry: ${{ env.REGISTRY }}
143
+
username: ${{ github.actor }}
144
+
password: ${{ secrets.GITHUB_TOKEN }}
145
+
- name: Build and push dataplane
146
+
uses: docker/build-push-action@v5
147
+
with:
148
+
context: .
149
+
file: services/dataplane/Dockerfile
150
+
push: true
151
+
tags: |
152
+
${{ env.DATAPLANE_IMAGE }}:${{ needs.config.outputs.image_tag }}
153
+
${{ env.DATAPLANE_IMAGE }}:${{ needs.config.outputs.environment }}-latest
154
+
${{ github.ref_type == 'tag' && format('{0}:{1}', env.DATAPLANE_IMAGE, github.ref_name) || '' }}
155
+
no-cache: true
156
+
157
+
deploy:
158
+
runs-on: ubuntu-latest
159
+
needs: [config, build-bsky, build-bsync, build-dataplane]
160
+
steps:
161
+
- name: Checkout appview charts
162
+
uses: actions/checkout@v4
163
+
with:
164
+
repository: w-social-eu/w-social-appview
165
+
token: ${{ secrets.PAT_TOKEN }}
166
+
path: appview
167
+
168
+
- uses: azure/setup-helm@v4
169
+
170
+
- name: Setup kubeconfig
171
+
run: |
172
+
mkdir -p ~/.kube
173
+
echo "${{ secrets[needs.config.outputs.kubeconfig_secret] }}" > ~/.kube/config
174
+
chmod 600 ~/.kube/config
175
+
176
+
- name: Deploy dataplane
177
+
run: |
178
+
ENV="${{ needs.config.outputs.environment }}"
179
+
TAG="${{ needs.config.outputs.image_tag }}"
180
+
helm upgrade --install dataplane appview/helm/charts/dataplane \
181
+
-f appview/helm/charts/dataplane/values.yaml \
182
+
-f appview/helm/charts/dataplane/values-${ENV}.yaml \
183
+
--set image.repository=${{ env.DATAPLANE_IMAGE }} \
184
+
--set image.tag="${TAG}" \
185
+
-n backend \
186
+
--wait --timeout 10m
187
+
188
+
- name: Deploy bsync
189
+
run: |
190
+
ENV="${{ needs.config.outputs.environment }}"
191
+
TAG="${{ needs.config.outputs.image_tag }}"
192
+
helm upgrade --install bsync appview/helm/charts/bsync \
193
+
-f appview/helm/charts/bsync/values.yaml \
194
+
-f appview/helm/charts/bsync/values-${ENV}.yaml \
195
+
--set image.repository=${{ env.BSYNC_IMAGE }} \
196
+
--set image.tag="${TAG}" \
197
+
-n backend \
198
+
--wait --timeout 5m
199
+
200
+
- name: Deploy bsky
201
+
run: |
202
+
ENV="${{ needs.config.outputs.environment }}"
203
+
TAG="${{ needs.config.outputs.image_tag }}"
204
+
helm upgrade --install bsky appview/helm/charts/bsky \
205
+
-f appview/helm/charts/bsky/values.yaml \
206
+
-f appview/helm/charts/bsky/values-${ENV}.yaml \
207
+
--set image.repository=${{ env.BSKY_IMAGE }} \
208
+
--set image.tag="${TAG}" \
209
+
-n backend \
210
+
--wait --timeout 5m
211
+
212
+
- name: Summary
213
+
if: always()
214
+
run: |
215
+
echo "## AppView Build & Deploy" >> $GITHUB_STEP_SUMMARY
216
+
echo "" >> $GITHUB_STEP_SUMMARY
217
+
echo "| Service | Image |" >> $GITHUB_STEP_SUMMARY
218
+
echo "|---------|-------|" >> $GITHUB_STEP_SUMMARY
219
+
echo "| bsky | \`${{ env.BSKY_IMAGE }}:${{ needs.config.outputs.image_tag }}\` |" >> $GITHUB_STEP_SUMMARY
220
+
echo "| bsync | \`${{ env.BSYNC_IMAGE }}:${{ needs.config.outputs.image_tag }}\` |" >> $GITHUB_STEP_SUMMARY
221
+
echo "| dataplane | \`${{ env.DATAPLANE_IMAGE }}:${{ needs.config.outputs.image_tag }}\` |" >> $GITHUB_STEP_SUMMARY
222
+
echo "| Environment | **${{ needs.config.outputs.environment }}** |" >> $GITHUB_STEP_SUMMARY
+15
-5
pnpm-lock.yaml
+15
-5
pnpm-lock.yaml
···
2297
2297
specifier: 3.13.2
2298
2298
version: 3.13.2
2299
2299
2300
+
services/dataplane:
2301
+
dependencies:
2302
+
'@atproto/bsky':
2303
+
specifier: workspace:^
2304
+
version: link:../../packages/bsky
2305
+
dd-trace:
2306
+
specifier: 3.13.2
2307
+
version: 3.13.2
2308
+
2300
2309
services/ozone:
2301
2310
dependencies:
2302
2311
'@atproto/aws':
···
9632
9641
mime-types: 2.1.35
9633
9642
negotiator: 0.6.3
9634
9643
9635
-
/acorn-import-assertions@1.9.0(acorn@8.10.0):
9644
+
/acorn-import-assertions@1.9.0(acorn@8.15.0):
9636
9645
resolution: {integrity: sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==}
9637
9646
peerDependencies:
9638
9647
acorn: ^8
9639
9648
dependencies:
9640
-
acorn: 8.10.0
9649
+
acorn: 8.15.0
9641
9650
dev: false
9642
9651
9643
9652
/acorn-jsx@5.3.2(acorn@8.10.0):
···
9657
9666
resolution: {integrity: sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==}
9658
9667
engines: {node: '>=0.4.0'}
9659
9668
hasBin: true
9669
+
dev: true
9660
9670
9661
9671
/acorn@8.14.1:
9662
9672
resolution: {integrity: sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==}
···
11165
11175
'@datadog/sketches-js': 2.1.0
11166
11176
crypto-randomuuid: 1.0.0
11167
11177
diagnostics_channel: 1.1.0
11168
-
ignore: 5.2.4
11178
+
ignore: 5.3.2
11169
11179
import-in-the-middle: 1.4.2
11170
11180
ipaddr.js: 2.1.0
11171
11181
istanbul-lib-coverage: 3.2.0
···
13644
13654
/import-in-the-middle@1.4.2:
13645
13655
resolution: {integrity: sha512-9WOz1Yh/cvO/p69sxRmhyQwrIGGSp7EIdcb+fFNVi7CzQGQB8U1/1XrKVSbEd/GNOAeM0peJtmi7+qphe7NvAw==}
13646
13656
dependencies:
13647
-
acorn: 8.10.0
13648
-
acorn-import-assertions: 1.9.0(acorn@8.10.0)
13657
+
acorn: 8.15.0
13658
+
acorn-import-assertions: 1.9.0(acorn@8.15.0)
13649
13659
cjs-module-lexer: 1.2.3
13650
13660
module-details-from-path: 1.0.3
13651
13661
dev: false
+81
services/dataplane/Dockerfile
+81
services/dataplane/Dockerfile
···
1
+
FROM node:20.20-alpine AS build
2
+
3
+
RUN corepack enable
4
+
5
+
WORKDIR /app
6
+
7
+
COPY ./package.json ./
8
+
RUN corepack prepare --activate
9
+
10
+
# Move files into the image and install
11
+
COPY ./*.* ./
12
+
COPY ./tsconfig ./tsconfig
13
+
COPY ./lexicons ./lexicons
14
+
COPY ./packages/internal ./packages/internal
15
+
COPY ./packages/lex ./packages/lex
16
+
17
+
# NOTE dataplane's transitive dependencies match bsky's (DataPlaneServer lives in @atproto/bsky)
18
+
COPY ./packages/api ./packages/api
19
+
COPY ./packages/aws ./packages/aws
20
+
COPY ./packages/bsky ./packages/bsky
21
+
COPY ./packages/common ./packages/common
22
+
COPY ./packages/common-web ./packages/common-web
23
+
COPY ./packages/crypto ./packages/crypto
24
+
COPY ./packages/did ./packages/did
25
+
COPY ./packages/identity ./packages/identity
26
+
COPY ./packages/lex-cli ./packages/lex-cli
27
+
COPY ./packages/lexicon ./packages/lexicon
28
+
COPY ./packages/lexicon-resolver ./packages/lexicon-resolver
29
+
COPY ./packages/oauth/jwk ./packages/oauth/jwk
30
+
COPY ./packages/oauth/jwk-jose ./packages/oauth/jwk-jose
31
+
COPY ./packages/oauth/jwk-webcrypto ./packages/oauth/jwk-webcrypto
32
+
COPY ./packages/oauth/oauth-client ./packages/oauth/oauth-client
33
+
COPY ./packages/oauth/oauth-client-browser ./packages/oauth/oauth-client-browser
34
+
COPY ./packages/oauth/oauth-client-browser-example ./packages/oauth/oauth-client-browser-example
35
+
COPY ./packages/oauth/oauth-provider ./packages/oauth/oauth-provider
36
+
COPY ./packages/oauth/oauth-provider-api ./packages/oauth/oauth-provider-api
37
+
COPY ./packages/oauth/oauth-provider-frontend ./packages/oauth/oauth-provider-frontend
38
+
COPY ./packages/oauth/oauth-provider-ui ./packages/oauth/oauth-provider-ui
39
+
COPY ./packages/oauth/oauth-scopes ./packages/oauth/oauth-scopes
40
+
COPY ./packages/oauth/oauth-types ./packages/oauth/oauth-types
41
+
COPY ./packages/pds ./packages/pds
42
+
COPY ./packages/repo ./packages/repo
43
+
COPY ./packages/sync ./packages/sync
44
+
COPY ./packages/syntax ./packages/syntax
45
+
COPY ./packages/ws-client ./packages/ws-client
46
+
COPY ./packages/xrpc ./packages/xrpc
47
+
COPY ./packages/xrpc-server ./packages/xrpc-server
48
+
49
+
COPY ./services/dataplane ./services/dataplane
50
+
51
+
# install all deps
52
+
RUN pnpm install --frozen-lockfile
53
+
# build all packages with external node_modules
54
+
RUN pnpm run --recursive --stream --filter '@atproto/crypto...' build
55
+
RUN pnpm run --recursive --stream --filter '@atproto/bsky...' build
56
+
# clean up
57
+
RUN rm -rf node_modules
58
+
# install only prod deps, hoisted to root node_modules dir
59
+
RUN pnpm install --prod --shamefully-hoist --frozen-lockfile --prefer-offline
60
+
61
+
WORKDIR services/dataplane
62
+
63
+
FROM node:20.20-alpine
64
+
65
+
RUN apk add --update dumb-init runit
66
+
67
+
ENTRYPOINT ["dumb-init", "--"]
68
+
69
+
WORKDIR /app/services/dataplane
70
+
COPY --from=build /app /app
71
+
72
+
EXPOSE 3001
73
+
ENV DATAPLANE_PORT=3001
74
+
ENV NODE_ENV=production
75
+
76
+
USER node
77
+
CMD ["node", "--heapsnapshot-signal=SIGUSR2", "--enable-source-maps", "--max-old-space-size=3072", "index.js"]
78
+
79
+
LABEL org.opencontainers.image.source=https://github.com/w-social-eu/w-social-atproto
80
+
LABEL org.opencontainers.image.description="Bsky DataPlane Service"
81
+
LABEL org.opencontainers.image.licenses=MIT
+75
services/dataplane/index.js
+75
services/dataplane/index.js
···
1
+
/* eslint-env node */
2
+
3
+
'use strict'
4
+
5
+
const dd = require('dd-trace')
6
+
7
+
dd.tracer.init({ logInjection: true })
8
+
9
+
// Tracer code above must come before anything else
10
+
const assert = require('node:assert')
11
+
const { Database, DataPlaneServer, RepoSubscription } = require('@atproto/bsky')
12
+
13
+
const main = async () => {
14
+
const port = parseInt(process.env.DATAPLANE_PORT || '3001', 10)
15
+
const dbUrl = process.env.DATAPLANE_DB_POSTGRES_URL
16
+
const subscriptionEndpoint = process.env.DATAPLANE_SUBSCRIPTION_ENDPOINT
17
+
const plcUrl =
18
+
process.env.DATAPLANE_DID_PLC_URL || 'https://plc.directory'
19
+
const poolSize = parseInt(process.env.DATAPLANE_DB_POOL_SIZE || '10', 10)
20
+
const runMigrations = process.env.DATAPLANE_DB_MIGRATE !== '0'
21
+
22
+
// When true, start from the current tail of the firehose instead of cursor 0.
23
+
// Required for high-volume relays (bsky.network) to avoid OOM on historical backfill.
24
+
// Set DATAPLANE_START_FROM_LATEST=true when connecting to bsky.network.
25
+
const startFromLatest = process.env.DATAPLANE_START_FROM_LATEST === 'true'
26
+
27
+
assert(dbUrl, 'DATAPLANE_DB_POSTGRES_URL is required')
28
+
assert(subscriptionEndpoint, 'DATAPLANE_SUBSCRIPTION_ENDPOINT is required')
29
+
30
+
if (runMigrations) {
31
+
console.log('dataplane: running db migrations')
32
+
const migrationDb = new Database({ url: dbUrl })
33
+
await migrationDb.migrateToLatestOrThrow()
34
+
await migrationDb.close()
35
+
console.log('dataplane: migrations complete')
36
+
}
37
+
38
+
const db = new Database({ url: dbUrl, poolSize })
39
+
40
+
const dataplane = await DataPlaneServer.create(db, port, plcUrl)
41
+
console.log(`dataplane: gRPC server listening on port ${port}`)
42
+
43
+
const sub = new RepoSubscription({
44
+
service: subscriptionEndpoint,
45
+
db,
46
+
idResolver: dataplane.idResolver,
47
+
})
48
+
49
+
if (startFromLatest) {
50
+
// RepoSubscription hardcodes startCursor: 0 which backfills all history.
51
+
// Setting runner.cursor = undefined makes Firehose start from current tail.
52
+
sub.runner.cursor = undefined
53
+
console.log('dataplane: starting from current firehose tail (no historical backfill)')
54
+
}
55
+
56
+
sub.start()
57
+
console.log(`dataplane: subscribed to firehose at ${subscriptionEndpoint}`)
58
+
59
+
const shutdown = async () => {
60
+
console.log('dataplane: shutting down')
61
+
await sub.destroy()
62
+
await dataplane.destroy()
63
+
await db.close()
64
+
console.log('dataplane: stopped')
65
+
process.exit(0)
66
+
}
67
+
68
+
process.on('SIGTERM', shutdown)
69
+
process.on('disconnect', shutdown)
70
+
}
71
+
72
+
main().catch((err) => {
73
+
console.error('dataplane: startup failed', err)
74
+
process.exit(1)
75
+
})