goat.navy
forked from
tranquil.farm/tranquil-pds
Our Personal Data Server from scratch!
fix(firehose): lost events if seq commits out of order
Lewis: May this revision serve well! <lu5a@proton.me>
author
Lewis
committer
Tangled
date
(May 30, 2026, 9:10 PM +0300)
commit
e9dc57d6
e9dc57d6f458aa8692e41e41da99674c28bf6276
parent
4e2525b2
4e2525b245782671fab73bfa81561d60a606b841
change-id
rlxwxzuv
rlxwxzuvnqvuxosnuztsznpwonswvppz
+640
-472
+4
-5
.sqlx/query-0349b2af4d21105029ecc2bf81ad7be9987ec17ac088730596849d7a45558e57.json
.sqlx/query-957bc81c63510ab502d2f8cea75dafcbdf2c8ebb448c86990d5e404ee5280438.json
+4
-5
.sqlx/query-0349b2af4d21105029ecc2bf81ad7be9987ec17ac088730596849d7a45558e57.json
.sqlx/query-957bc81c63510ab502d2f8cea75dafcbdf2c8ebb448c86990d5e404ee5280438.json
···
1
1
{
2
2
"db_name": "PostgreSQL",
3
-
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2",
3
+
"query": "SELECT seq as \"seq!\", did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC",
4
4
"describe": {
5
5
"columns": [
6
6
{
7
7
"ordinal": 0,
8
-
"name": "seq",
8
+
"name": "seq!",
9
9
"type_info": "Int8"
10
10
},
11
11
{
···
86
86
],
87
87
"parameters": {
88
88
"Left": [
89
-
"Int8",
90
89
"Int8"
91
90
]
92
91
},
93
92
"nullable": [
94
-
false,
93
+
true,
95
94
false,
96
95
false,
97
96
false,
···
109
108
true
110
109
]
111
110
},
112
-
"hash": "0349b2af4d21105029ecc2bf81ad7be9987ec17ac088730596849d7a45558e57"
111
+
"hash": "957bc81c63510ab502d2f8cea75dafcbdf2c8ebb448c86990d5e404ee5280438"
113
112
}
+20
.sqlx/query-09142b5a6b91fe4fd795fb6c85af5461e2621f6c3545b6d26e4618d813904b75.json
+20
.sqlx/query-09142b5a6b91fe4fd795fb6c85af5461e2621f6c3545b6d26e4618d813904b75.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "SELECT nextval('firehose_seq') as \"seq!\"",
4
+
"describe": {
5
+
"columns": [
6
+
{
7
+
"ordinal": 0,
8
+
"name": "seq!",
9
+
"type_info": "Int8"
10
+
}
11
+
],
12
+
"parameters": {
13
+
"Left": []
14
+
},
15
+
"nullable": [
16
+
null
17
+
]
18
+
},
19
+
"hash": "09142b5a6b91fe4fd795fb6c85af5461e2621f6c3545b6d26e4618d813904b75"
20
+
}
+15
.sqlx/query-0b895d105363a60040323c5c47bc897905af3da980d0dc1b3b60bbdcdd90ea7a.json
+15
.sqlx/query-0b895d105363a60040323c5c47bc897905af3da980d0dc1b3b60bbdcdd90ea7a.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "UPDATE repo_seq r\n SET seq = $1::bigint + (o.ord - 1)\n FROM unnest($2::bigint[]) WITH ORDINALITY AS o(id, ord)\n WHERE r.id = o.id",
4
+
"describe": {
5
+
"columns": [],
6
+
"parameters": {
7
+
"Left": [
8
+
"Int8",
9
+
"Int8Array"
10
+
]
11
+
},
12
+
"nullable": []
13
+
},
14
+
"hash": "0b895d105363a60040323c5c47bc897905af3da980d0dc1b3b60bbdcdd90ea7a"
15
+
}
+14
.sqlx/query-122d79e006c0e11214e3cf11cfb98bdff6fbf156893b482572532ebcd8979e61.json
+14
.sqlx/query-122d79e006c0e11214e3cf11cfb98bdff6fbf156893b482572532ebcd8979e61.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "\n DELETE FROM repo_seq\n WHERE did = $1\n AND id <> (SELECT id FROM repo_seq WHERE did = $1 ORDER BY id DESC LIMIT 1)\n ",
4
+
"describe": {
5
+
"columns": [],
6
+
"parameters": {
7
+
"Left": [
8
+
"Text"
9
+
]
10
+
},
11
+
"nullable": []
12
+
},
13
+
"hash": "122d79e006c0e11214e3cf11cfb98bdff6fbf156893b482572532ebcd8979e61"
14
+
}
+22
.sqlx/query-3cc7247d2cd7e85a6f21d783dd956f8fecc6f63c994014bb4e91504d498b5fd5.json
+22
.sqlx/query-3cc7247d2cd7e85a6f21d783dd956f8fecc6f63c994014bb4e91504d498b5fd5.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "SELECT pg_try_advisory_xact_lock($1) as \"locked!\"",
4
+
"describe": {
5
+
"columns": [
6
+
{
7
+
"ordinal": 0,
8
+
"name": "locked!",
9
+
"type_info": "Bool"
10
+
}
11
+
],
12
+
"parameters": {
13
+
"Left": [
14
+
"Int8"
15
+
]
16
+
},
17
+
"nullable": [
18
+
null
19
+
]
20
+
},
21
+
"hash": "3cc7247d2cd7e85a6f21d783dd956f8fecc6f63c994014bb4e91504d498b5fd5"
22
+
}
+4
-4
.sqlx/query-4702a94384ce938d3f5384bc2106a48500f8749134e6bfd1c619a9151f5d3772.json
.sqlx/query-7cdf3b8136782a5b2e4aff91907a680566b4610e6cbf9d9396d31f9bbd99c3b8.json
+4
-4
.sqlx/query-4702a94384ce938d3f5384bc2106a48500f8749134e6bfd1c619a9151f5d3772.json
.sqlx/query-7cdf3b8136782a5b2e4aff91907a680566b4610e6cbf9d9396d31f9bbd99c3b8.json
···
1
1
{
2
2
"db_name": "PostgreSQL",
3
-
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq = $1",
3
+
"query": "SELECT seq as \"seq!\", did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq = $1",
4
4
"describe": {
5
5
"columns": [
6
6
{
7
7
"ordinal": 0,
8
-
"name": "seq",
8
+
"name": "seq!",
9
9
"type_info": "Int8"
10
10
},
11
11
{
···
90
90
]
91
91
},
92
92
"nullable": [
93
-
false,
93
+
true,
94
94
false,
95
95
false,
96
96
false,
···
108
108
true
109
109
]
110
110
},
111
-
"hash": "4702a94384ce938d3f5384bc2106a48500f8749134e6bfd1c619a9151f5d3772"
111
+
"hash": "7cdf3b8136782a5b2e4aff91907a680566b4610e6cbf9d9396d31f9bbd99c3b8"
112
112
}
+15
.sqlx/query-4f3ef746cb5fc0916d3df5f92494c538497d3b9c1cfcc6e44c9b80f2193094f6.json
+15
.sqlx/query-4f3ef746cb5fc0916d3df5f92494c538497d3b9c1cfcc6e44c9b80f2193094f6.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "\n INSERT INTO repo_seq (did, event_type, handle)\n VALUES ($1, 'identity', $2)\n ",
4
+
"describe": {
5
+
"columns": [],
6
+
"parameters": {
7
+
"Left": [
8
+
"Text",
9
+
"Text"
10
+
]
11
+
},
12
+
"nullable": []
13
+
},
14
+
"hash": "4f3ef746cb5fc0916d3df5f92494c538497d3b9c1cfcc6e44c9b80f2193094f6"
15
+
}
-23
.sqlx/query-50b2178cd174af1dc132a22048e52a6af492f4d8025ec512dd9c0c9ba62eb202.json
-23
.sqlx/query-50b2178cd174af1dc132a22048e52a6af492f4d8025ec512dd9c0c9ba62eb202.json
···
1
-
{
2
-
"db_name": "PostgreSQL",
3
-
"query": "\n INSERT INTO repo_seq (did, event_type, handle)\n VALUES ($1, 'identity', $2)\n RETURNING seq\n ",
4
-
"describe": {
5
-
"columns": [
6
-
{
7
-
"ordinal": 0,
8
-
"name": "seq",
9
-
"type_info": "Int8"
10
-
}
11
-
],
12
-
"parameters": {
13
-
"Left": [
14
-
"Text",
15
-
"Text"
16
-
]
17
-
},
18
-
"nullable": [
19
-
false
20
-
]
21
-
},
22
-
"hash": "50b2178cd174af1dc132a22048e52a6af492f4d8025ec512dd9c0c9ba62eb202"
23
-
}
-26
.sqlx/query-5996774d097a484f81353d49e4a99b07c30a003377c924add576bfb48ddde29b.json
-26
.sqlx/query-5996774d097a484f81353d49e4a99b07c30a003377c924add576bfb48ddde29b.json
···
1
-
{
2
-
"db_name": "PostgreSQL",
3
-
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, rev, block_cids, block_data)\n VALUES ($1, 'sync', $2, $3, $4, $5)\n RETURNING seq\n ",
4
-
"describe": {
5
-
"columns": [
6
-
{
7
-
"ordinal": 0,
8
-
"name": "seq",
9
-
"type_info": "Int8"
10
-
}
11
-
],
12
-
"parameters": {
13
-
"Left": [
14
-
"Text",
15
-
"Text",
16
-
"Text",
17
-
"ByteaArray",
18
-
"ByteaArray"
19
-
]
20
-
},
21
-
"nullable": [
22
-
false
23
-
]
24
-
},
25
-
"hash": "5996774d097a484f81353d49e4a99b07c30a003377c924add576bfb48ddde29b"
26
-
}
-24
.sqlx/query-623042c0b901e17ff37c5bcc1a4eefe56fc8b996ca8326e75796897d13c9e9ed.json
-24
.sqlx/query-623042c0b901e17ff37c5bcc1a4eefe56fc8b996ca8326e75796897d13c9e9ed.json
···
1
-
{
2
-
"db_name": "PostgreSQL",
3
-
"query": "\n INSERT INTO repo_seq (did, event_type, active, status)\n VALUES ($1, 'account', $2, $3)\n RETURNING seq\n ",
4
-
"describe": {
5
-
"columns": [
6
-
{
7
-
"ordinal": 0,
8
-
"name": "seq",
9
-
"type_info": "Int8"
10
-
}
11
-
],
12
-
"parameters": {
13
-
"Left": [
14
-
"Text",
15
-
"Bool",
16
-
"Text"
17
-
]
18
-
},
19
-
"nullable": [
20
-
false
21
-
]
22
-
},
23
-
"hash": "623042c0b901e17ff37c5bcc1a4eefe56fc8b996ca8326e75796897d13c9e9ed"
24
-
}
+22
.sqlx/query-714a1c9a73e2fed8182d4ab2fe4a2c33bc10a52e27ed95d3a6641477ae42395e.json
+22
.sqlx/query-714a1c9a73e2fed8182d4ab2fe4a2c33bc10a52e27ed95d3a6641477ae42395e.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "SELECT id as \"id!\" FROM repo_seq WHERE seq IS NULL ORDER BY id LIMIT $1",
4
+
"describe": {
5
+
"columns": [
6
+
{
7
+
"ordinal": 0,
8
+
"name": "id!",
9
+
"type_info": "Int8"
10
+
}
11
+
],
12
+
"parameters": {
13
+
"Left": [
14
+
"Int8"
15
+
]
16
+
},
17
+
"nullable": [
18
+
false
19
+
]
20
+
},
21
+
"hash": "714a1c9a73e2fed8182d4ab2fe4a2c33bc10a52e27ed95d3a6641477ae42395e"
22
+
}
+15
.sqlx/query-856e458c1c78f030545dc147f76d1b91fa4cc6e178eb51133c52358855ba3f02.json
+15
.sqlx/query-856e458c1c78f030545dc147f76d1b91fa4cc6e178eb51133c52358855ba3f02.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "DELETE FROM repo_seq WHERE did = $1 AND id <> $2",
4
+
"describe": {
5
+
"columns": [],
6
+
"parameters": {
7
+
"Left": [
8
+
"Text",
9
+
"Int8"
10
+
]
11
+
},
12
+
"nullable": []
13
+
},
14
+
"hash": "856e458c1c78f030545dc147f76d1b91fa4cc6e178eb51133c52358855ba3f02"
15
+
}
+4
-4
.sqlx/query-89c8ce80d8c52b4668c34f63e48b51cacc7e8ff300958a192378f1d72de04f9a.json
.sqlx/query-da2770e44c0ec21756ce4b6b0c57abff4f38cd8c680c5ced7d3b35f3c1f58da7.json
+4
-4
.sqlx/query-89c8ce80d8c52b4668c34f63e48b51cacc7e8ff300958a192378f1d72de04f9a.json
.sqlx/query-da2770e44c0ec21756ce4b6b0c57abff4f38cd8c680c5ced7d3b35f3c1f58da7.json
···
1
1
{
2
2
"db_name": "PostgreSQL",
3
-
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2",
3
+
"query": "SELECT seq as \"seq!\", did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2",
4
4
"describe": {
5
5
"columns": [
6
6
{
7
7
"ordinal": 0,
8
-
"name": "seq",
8
+
"name": "seq!",
9
9
"type_info": "Int8"
10
10
},
11
11
{
···
91
91
]
92
92
},
93
93
"nullable": [
94
-
false,
94
+
true,
95
95
false,
96
96
false,
97
97
false,
···
109
109
true
110
110
]
111
111
},
112
-
"hash": "89c8ce80d8c52b4668c34f63e48b51cacc7e8ff300958a192378f1d72de04f9a"
112
+
"hash": "da2770e44c0ec21756ce4b6b0c57abff4f38cd8c680c5ced7d3b35f3c1f58da7"
113
113
}
+16
.sqlx/query-8ccd42b5b1b75f45e094889ee26af8517de28ca687efa81fd9c4fce598338fe9.json
+16
.sqlx/query-8ccd42b5b1b75f45e094889ee26af8517de28ca687efa81fd9c4fce598338fe9.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "\n INSERT INTO repo_seq (did, event_type, active, status)\n VALUES ($1, 'account', $2, $3)\n ",
4
+
"describe": {
5
+
"columns": [],
6
+
"parameters": {
7
+
"Left": [
8
+
"Text",
9
+
"Bool",
10
+
"Text"
11
+
]
12
+
},
13
+
"nullable": []
14
+
},
15
+
"hash": "8ccd42b5b1b75f45e094889ee26af8517de28ca687efa81fd9c4fce598338fe9"
16
+
}
+4
-12
.sqlx/query-97dd3cfe72b74b4dac686c030a07f6eb56dabb2df72815f3248760b4e3f7950e.json
.sqlx/query-82276a34f473ff8b9790b78915ae3f72873d4b748cdb47bd2b81edc2f7a6e5a8.json
+4
-12
.sqlx/query-97dd3cfe72b74b4dac686c030a07f6eb56dabb2df72815f3248760b4e3f7950e.json
.sqlx/query-82276a34f473ff8b9790b78915ae3f72873d4b748cdb47bd2b81edc2f7a6e5a8.json
···
1
1
{
2
2
"db_name": "PostgreSQL",
3
-
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n RETURNING seq\n ",
3
+
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n ",
4
4
"describe": {
5
-
"columns": [
6
-
{
7
-
"ordinal": 0,
8
-
"name": "seq",
9
-
"type_info": "Int8"
10
-
}
11
-
],
5
+
"columns": [],
12
6
"parameters": {
13
7
"Left": [
14
8
"Text",
···
23
17
"Text"
24
18
]
25
19
},
26
-
"nullable": [
27
-
false
28
-
]
20
+
"nullable": []
29
21
},
30
-
"hash": "97dd3cfe72b74b4dac686c030a07f6eb56dabb2df72815f3248760b4e3f7950e"
22
+
"hash": "82276a34f473ff8b9790b78915ae3f72873d4b748cdb47bd2b81edc2f7a6e5a8"
31
23
}
+18
.sqlx/query-9ba7869f29286cfd6b811ffa1e26bda1d637b98b44be66cdc265b1af4cec2d4f.json
+18
.sqlx/query-9ba7869f29286cfd6b811ffa1e26bda1d637b98b44be66cdc265b1af4cec2d4f.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, rev, block_cids, block_data)\n VALUES ($1, 'sync', $2, $3, $4, $5)\n ",
4
+
"describe": {
5
+
"columns": [],
6
+
"parameters": {
7
+
"Left": [
8
+
"Text",
9
+
"Text",
10
+
"Text",
11
+
"ByteaArray",
12
+
"ByteaArray"
13
+
]
14
+
},
15
+
"nullable": []
16
+
},
17
+
"hash": "9ba7869f29286cfd6b811ffa1e26bda1d637b98b44be66cdc265b1af4cec2d4f"
18
+
}
+22
.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json
+22
.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "SELECT pg_advisory_xact_lock($1)",
4
+
"describe": {
5
+
"columns": [
6
+
{
7
+
"ordinal": 0,
8
+
"name": "pg_advisory_xact_lock",
9
+
"type_info": "Void"
10
+
}
11
+
],
12
+
"parameters": {
13
+
"Left": [
14
+
"Int8"
15
+
]
16
+
},
17
+
"nullable": [
18
+
null
19
+
]
20
+
},
21
+
"hash": "a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247"
22
+
}
+3
-3
.sqlx/query-a0ea3630adb9e411d94d88084df6b05d7fc3ba8c306146fc932b736fe342eceb.json
.sqlx/query-2c3c0740793f29b9bd9e0f81c2372d427dee091e20da8f71ae1e5f2769aa985d.json
+3
-3
.sqlx/query-a0ea3630adb9e411d94d88084df6b05d7fc3ba8c306146fc932b736fe342eceb.json
.sqlx/query-2c3c0740793f29b9bd9e0f81c2372d427dee091e20da8f71ae1e5f2769aa985d.json
···
1
1
{
2
2
"db_name": "PostgreSQL",
3
-
"query": "\n INSERT INTO repo_seq (did, event_type, active, status)\n VALUES ($1, 'account', false, 'deleted')\n RETURNING seq\n ",
3
+
"query": "\n INSERT INTO repo_seq (did, event_type, active, status)\n VALUES ($1, 'account', false, 'deleted')\n RETURNING id\n ",
4
4
"describe": {
5
5
"columns": [
6
6
{
7
7
"ordinal": 0,
8
-
"name": "seq",
8
+
"name": "id",
9
9
"type_info": "Int8"
10
10
}
11
11
],
···
18
18
false
19
19
]
20
20
},
21
-
"hash": "a0ea3630adb9e411d94d88084df6b05d7fc3ba8c306146fc932b736fe342eceb"
21
+
"hash": "2c3c0740793f29b9bd9e0f81c2372d427dee091e20da8f71ae1e5f2769aa985d"
22
22
}
+4
-12
.sqlx/query-a325862f897484bb1a8863fd7a1bc3fd17ab20b6e30498f9900daf80609c413d.json
.sqlx/query-800b93a38eef70bafdf0a1b221994f3c19ab562f84d18265376296a592b4c767.json
+4
-12
.sqlx/query-a325862f897484bb1a8863fd7a1bc3fd17ab20b6e30498f9900daf80609c413d.json
.sqlx/query-800b93a38eef70bafdf0a1b221994f3c19ab562f84d18265376296a592b4c767.json
···
1
1
{
2
2
"db_name": "PostgreSQL",
3
-
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, rev)\n VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7, $8)\n RETURNING seq\n ",
3
+
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, rev)\n VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7, $8)\n ",
4
4
"describe": {
5
-
"columns": [
6
-
{
7
-
"ordinal": 0,
8
-
"name": "seq",
9
-
"type_info": "Int8"
10
-
}
11
-
],
5
+
"columns": [],
12
6
"parameters": {
13
7
"Left": [
14
8
"Text",
···
21
15
"Text"
22
16
]
23
17
},
24
-
"nullable": [
25
-
false
26
-
]
18
+
"nullable": []
27
19
},
28
-
"hash": "a325862f897484bb1a8863fd7a1bc3fd17ab20b6e30498f9900daf80609c413d"
20
+
"hash": "800b93a38eef70bafdf0a1b221994f3c19ab562f84d18265376296a592b4c767"
29
21
}
+4
-4
.sqlx/query-ad784aff9bb5c1f19f90a3b1c3b3694a3e1aa889b0fe3e55cfebb7af8c620256.json
.sqlx/query-dfe011b2d2bafbbbbea508a4977fc34bb1be4cf2612296df23763e6e5841c33e.json
+4
-4
.sqlx/query-ad784aff9bb5c1f19f90a3b1c3b3694a3e1aa889b0fe3e55cfebb7af8c620256.json
.sqlx/query-dfe011b2d2bafbbbbea508a4977fc34bb1be4cf2612296df23763e6e5841c33e.json
···
1
1
{
2
2
"db_name": "PostgreSQL",
3
-
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1 AND seq < $2\n ORDER BY seq ASC",
3
+
"query": "SELECT seq as \"seq!\", did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1 AND seq < $2\n ORDER BY seq ASC",
4
4
"describe": {
5
5
"columns": [
6
6
{
7
7
"ordinal": 0,
8
-
"name": "seq",
8
+
"name": "seq!",
9
9
"type_info": "Int8"
10
10
},
11
11
{
···
91
91
]
92
92
},
93
93
"nullable": [
94
-
false,
94
+
true,
95
95
false,
96
96
false,
97
97
false,
···
109
109
true
110
110
]
111
111
},
112
-
"hash": "ad784aff9bb5c1f19f90a3b1c3b3694a3e1aa889b0fe3e55cfebb7af8c620256"
112
+
"hash": "dfe011b2d2bafbbbbea508a4977fc34bb1be4cf2612296df23763e6e5841c33e"
113
113
}
-15
.sqlx/query-b6d6548acb89d6384cd226f6ed0d66de27fde3af24b4a7a3fce7e098812e38a5.json
-15
.sqlx/query-b6d6548acb89d6384cd226f6ed0d66de27fde3af24b4a7a3fce7e098812e38a5.json
···
1
-
{
2
-
"db_name": "PostgreSQL",
3
-
"query": "DELETE FROM repo_seq WHERE did = $1 AND seq != $2",
4
-
"describe": {
5
-
"columns": [],
6
-
"parameters": {
7
-
"Left": [
8
-
"Text",
9
-
"Int8"
10
-
]
11
-
},
12
-
"nullable": []
13
-
},
14
-
"hash": "b6d6548acb89d6384cd226f6ed0d66de27fde3af24b4a7a3fce7e098812e38a5"
15
-
}
+22
.sqlx/query-bd0407818c8bdae1f1024777951b08cb9c386fe751722e1caf6ad243e271697a.json
+22
.sqlx/query-bd0407818c8bdae1f1024777951b08cb9c386fe751722e1caf6ad243e271697a.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "SELECT setval('firehose_seq', $1)",
4
+
"describe": {
5
+
"columns": [
6
+
{
7
+
"ordinal": 0,
8
+
"name": "setval",
9
+
"type_info": "Int8"
10
+
}
11
+
],
12
+
"parameters": {
13
+
"Left": [
14
+
"Int8"
15
+
]
16
+
},
17
+
"nullable": [
18
+
null
19
+
]
20
+
},
21
+
"hash": "bd0407818c8bdae1f1024777951b08cb9c386fe751722e1caf6ad243e271697a"
22
+
}
+5
-4
.sqlx/query-c47746c256fd8c9d386551f3bd270e9a96fecdb713642d07d60fb4aeb02c8426.json
.sqlx/query-f16d41b41a5388997e6ed408da55056173340ed12b60b1a956de87f73f9d0824.json
+5
-4
.sqlx/query-c47746c256fd8c9d386551f3bd270e9a96fecdb713642d07d60fb4aeb02c8426.json
.sqlx/query-f16d41b41a5388997e6ed408da55056173340ed12b60b1a956de87f73f9d0824.json
···
1
1
{
2
2
"db_name": "PostgreSQL",
3
-
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC",
3
+
"query": "SELECT seq as \"seq!\", did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2",
4
4
"describe": {
5
5
"columns": [
6
6
{
7
7
"ordinal": 0,
8
-
"name": "seq",
8
+
"name": "seq!",
9
9
"type_info": "Int8"
10
10
},
11
11
{
···
86
86
],
87
87
"parameters": {
88
88
"Left": [
89
+
"Int8",
89
90
"Int8"
90
91
]
91
92
},
92
93
"nullable": [
93
-
false,
94
+
true,
94
95
false,
95
96
false,
96
97
false,
···
108
109
true
109
110
]
110
111
},
111
-
"hash": "c47746c256fd8c9d386551f3bd270e9a96fecdb713642d07d60fb4aeb02c8426"
112
+
"hash": "f16d41b41a5388997e6ed408da55056173340ed12b60b1a956de87f73f9d0824"
112
113
}
+12
.sqlx/query-d4eacb9bca838dd18dcfd8ca39b83ef50968bfaaabc213536c049cc222ffe9c8.json
+12
.sqlx/query-d4eacb9bca838dd18dcfd8ca39b83ef50968bfaaabc213536c049cc222ffe9c8.json
+2
-9
crates/tranquil-api/src/identity/plc/submit.rs
+2
-9
crates/tranquil-api/src/identity/plc/submit.rs
···
129
129
.await
130
130
.map_err(ApiError::from)?;
131
131
132
-
match state
132
+
if let Err(e) = state
133
133
.repos
134
134
.repo
135
135
.insert_identity_event(did, Some(&user.handle))
136
136
.await
137
137
{
138
-
Ok(seq) => {
139
-
if let Err(e) = state.repos.repo.notify_update(seq).await {
140
-
warn!("Failed to notify identity event: {:?}", e);
141
-
}
142
-
}
143
-
Err(e) => {
144
-
warn!("Failed to sequence identity event: {:?}", e);
145
-
}
138
+
warn!("Failed to sequence identity event: {:?}", e);
146
139
}
147
140
let _ = state
148
141
.cache
+1
-2
crates/tranquil-api/src/repo/import.rs
+1
-2
crates/tranquil-api/src/repo/import.rs
+12
-17
crates/tranquil-api/src/server/account_status.rs
+12
-17
crates/tranquil-api/src/server/account_status.rs
···
665
665
error!("DB error deleting account: {:?}", e);
666
666
ApiError::InternalError(None)
667
667
})?;
668
-
let account_seq = tranquil_pds::repo_ops::sequence_account_event(
668
+
if let Err(e) = tranquil_pds::repo_ops::sequence_account_event(
669
669
&state,
670
670
did,
671
671
tranquil_db_traits::AccountStatus::Deleted,
672
672
)
673
-
.await;
674
-
match account_seq {
675
-
Ok(seq) => {
676
-
if let Err(e) = state.repos.repo.delete_sequences_except(did, seq).await {
677
-
warn!(
678
-
"Failed to cleanup sequences for deleted account {}: {}",
679
-
did, e
680
-
);
681
-
}
682
-
}
683
-
Err(e) => {
684
-
warn!(
685
-
"Failed to sequence account deletion event for {}: {}",
686
-
did, e
687
-
);
688
-
}
673
+
.await
674
+
{
675
+
warn!(
676
+
"Failed to sequence account deletion event for {}: {}",
677
+
did, e
678
+
);
679
+
} else if let Err(e) = state.repos.repo.purge_did_events_keeping_latest(did).await {
680
+
warn!(
681
+
"Failed to cleanup sequences for deleted account {}: {}",
682
+
did, e
683
+
);
689
684
}
690
685
let _ = state
691
686
.cache
+15
-18
crates/tranquil-db-traits/src/repo.rs
+15
-18
crates/tranquil-db-traits/src/repo.rs
···
319
319
320
320
#[derive(Debug, Clone)]
321
321
pub struct ApplyCommitResult {
322
-
pub seq: i64,
323
322
pub is_account_active: bool,
324
323
}
325
324
···
445
444
446
445
async fn count_user_blocks(&self, user_id: Uuid) -> Result<i64, DbError>;
447
446
448
-
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError>;
447
+
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<(), DbError>;
449
448
450
449
async fn insert_identity_event(
451
450
&self,
452
451
did: &Did,
453
452
handle: Option<&Handle>,
454
-
) -> Result<SequenceNumber, DbError>;
453
+
) -> Result<(), DbError>;
455
454
456
-
async fn insert_account_event(
457
-
&self,
458
-
did: &Did,
459
-
status: AccountStatus,
460
-
) -> Result<SequenceNumber, DbError>;
455
+
async fn insert_account_event(&self, did: &Did, status: AccountStatus) -> Result<(), DbError>;
461
456
462
457
async fn insert_sync_event(
463
458
&self,
···
465
460
commit_cid: &CidLink,
466
461
rev: Option<&str>,
467
462
commit_bytes: &[u8],
468
-
) -> Result<SequenceNumber, DbError>;
463
+
) -> Result<(), DbError>;
469
464
470
465
async fn insert_genesis_commit_event(
471
466
&self,
···
475
470
rev: &str,
476
471
commit_bytes: &[u8],
477
472
mst_root_bytes: &[u8],
478
-
) -> Result<SequenceNumber, DbError>;
479
-
480
-
async fn delete_sequences_except(
481
-
&self,
482
-
did: &Did,
483
-
keep_seq: SequenceNumber,
484
473
) -> Result<(), DbError>;
485
474
475
+
async fn purge_did_events_keeping_latest(&self, did: &Did) -> Result<(), DbError>;
476
+
477
+
async fn assign_pending_sequences(&self) -> Result<u64, DbError> {
478
+
Ok(0)
479
+
}
480
+
481
+
async fn flush_pending_sequences(&self) -> Result<(), DbError> {
482
+
Ok(())
483
+
}
484
+
486
485
async fn prune_events_older_than(&self, cutoff: DateTime<Utc>) -> Result<PruneCount, DbError>;
487
486
488
487
async fn get_max_seq(&self) -> Result<SequenceNumber, DbError>;
···
526
525
async fn get_repo_root_cid_by_user_id(&self, user_id: Uuid)
527
526
-> Result<Option<CidLink>, DbError>;
528
527
529
-
async fn notify_update(&self, seq: SequenceNumber) -> Result<(), DbError>;
530
-
531
528
async fn import_repo_data(
532
529
&self,
533
530
user_id: Uuid,
···
563
560
564
561
#[async_trait]
565
562
pub trait RepoEventReceiver: Send {
566
-
async fn recv(&mut self) -> Option<i64>;
563
+
async fn recv(&mut self) -> Option<()>;
567
564
}
+1
-1
crates/tranquil-db-traits/src/user.rs
+1
-1
crates/tranquil-db-traits/src/user.rs
···
551
551
limit: i64,
552
552
) -> Result<Vec<ScheduledDeletionAccount>, DbError>;
553
553
554
-
async fn delete_account_with_firehose(&self, user_id: Uuid, did: &Did) -> Result<i64, DbError>;
554
+
async fn delete_account_with_firehose(&self, user_id: Uuid, did: &Did) -> Result<(), DbError>;
555
555
556
556
async fn create_password_account(
557
557
&self,
+2
-2
crates/tranquil-db/src/postgres/event_notifier.rs
+2
-2
crates/tranquil-db/src/postgres/event_notifier.rs
···
35
35
36
36
#[async_trait]
37
37
impl RepoEventReceiver for PostgresRepoEventReceiver {
38
-
async fn recv(&mut self) -> Option<i64> {
38
+
async fn recv(&mut self) -> Option<()> {
39
39
match self.listener.recv().await {
40
-
Ok(notification) => notification.payload().parse().ok(),
40
+
Ok(_) => Some(()),
41
41
Err(_) => None,
42
42
}
43
43
}
+145
-88
crates/tranquil-db/src/postgres/repo.rs
+145
-88
crates/tranquil-db/src/postgres/repo.rs
···
111
111
})
112
112
}
113
113
114
+
const SEQUENCER_LOCK_KEY: i64 = 0x0074_7261_6e73_6571;
115
+
const SEQUENCER_BATCH_SIZE: i64 = 1000;
116
+
117
+
async fn notify_repo_pending(pool: &PgPool) {
118
+
if let Err(e) = sqlx::query!("NOTIFY repo_updates").execute(pool).await {
119
+
tracing::warn!(error = %e, "failed to notify repo_updates listeners");
120
+
}
121
+
}
122
+
123
+
async fn assign_one_batch(
124
+
mut tx: sqlx::Transaction<'_, sqlx::Postgres>,
125
+
pool: &PgPool,
126
+
) -> Result<i64, DbError> {
127
+
let pending_ids: Vec<i64> = sqlx::query_scalar!(
128
+
r#"SELECT id as "id!" FROM repo_seq WHERE seq IS NULL ORDER BY id LIMIT $1"#,
129
+
SEQUENCER_BATCH_SIZE
130
+
)
131
+
.fetch_all(&mut *tx)
132
+
.await
133
+
.map_err(map_sqlx_error)?;
134
+
135
+
let count = pending_ids.len() as i64;
136
+
if count == 0 {
137
+
tx.commit().await.map_err(map_sqlx_error)?;
138
+
return Ok(0);
139
+
}
140
+
141
+
let start = sqlx::query_scalar!(r#"SELECT nextval('firehose_seq') as "seq!""#)
142
+
.fetch_one(&mut *tx)
143
+
.await
144
+
.map_err(map_sqlx_error)?;
145
+
if count > 1 {
146
+
sqlx::query!("SELECT setval('firehose_seq', $1)", start + count - 1)
147
+
.fetch_one(&mut *tx)
148
+
.await
149
+
.map_err(map_sqlx_error)?;
150
+
}
151
+
152
+
let assigned = sqlx::query!(
153
+
r#"UPDATE repo_seq r
154
+
SET seq = $1::bigint + (o.ord - 1)
155
+
FROM unnest($2::bigint[]) WITH ORDINALITY AS o(id, ord)
156
+
WHERE r.id = o.id"#,
157
+
start,
158
+
&pending_ids
159
+
)
160
+
.execute(&mut *tx)
161
+
.await
162
+
.map_err(map_sqlx_error)?
163
+
.rows_affected();
164
+
165
+
tx.commit().await.map_err(map_sqlx_error)?;
166
+
if assigned > 0 {
167
+
notify_repo_pending(pool).await;
168
+
}
169
+
Ok(count)
170
+
}
171
+
114
172
pub struct PostgresRepoRepository {
115
173
pool: PgPool,
116
174
}
···
716
774
Ok(rows.into_iter().map(|(cid,)| cid).collect())
717
775
}
718
776
719
-
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError> {
777
+
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<(), DbError> {
720
778
let (block_cids, block_data) = inline_to_paired_blocks(data.blocks.as_deref());
721
-
let seq = sqlx::query_scalar!(
779
+
sqlx::query!(
722
780
r#"
723
781
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)
724
782
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
725
-
RETURNING seq
726
783
"#,
727
784
data.did.as_str(),
728
785
data.event_type.as_str(),
···
735
792
data.prev_data_cid.as_ref().map(|c| c.as_str()),
736
793
data.rev
737
794
)
738
-
.fetch_one(&self.pool)
795
+
.execute(&self.pool)
739
796
.await
740
797
.map_err(map_sqlx_error)?;
741
798
742
-
Ok(seq.into())
799
+
notify_repo_pending(&self.pool).await;
800
+
Ok(())
743
801
}
744
802
745
803
async fn insert_identity_event(
746
804
&self,
747
805
did: &Did,
748
806
handle: Option<&Handle>,
749
-
) -> Result<SequenceNumber, DbError> {
807
+
) -> Result<(), DbError> {
750
808
let handle_str = handle.map(|h| h.as_str());
751
-
let seq = sqlx::query_scalar!(
809
+
sqlx::query!(
752
810
r#"
753
811
INSERT INTO repo_seq (did, event_type, handle)
754
812
VALUES ($1, 'identity', $2)
755
-
RETURNING seq
756
813
"#,
757
814
did.as_str(),
758
815
handle_str
759
816
)
760
-
.fetch_one(&self.pool)
817
+
.execute(&self.pool)
761
818
.await
762
819
.map_err(map_sqlx_error)?;
763
820
764
-
sqlx::query(&format!("NOTIFY repo_updates, '{}'", seq))
765
-
.execute(&self.pool)
766
-
.await
767
-
.map_err(map_sqlx_error)?;
768
-
769
-
Ok(seq.into())
821
+
notify_repo_pending(&self.pool).await;
822
+
Ok(())
770
823
}
771
824
772
-
async fn insert_account_event(
773
-
&self,
774
-
did: &Did,
775
-
status: AccountStatus,
776
-
) -> Result<SequenceNumber, DbError> {
825
+
async fn insert_account_event(&self, did: &Did, status: AccountStatus) -> Result<(), DbError> {
777
826
let active = status.is_active();
778
827
let status_str = status.for_firehose();
779
-
let seq = sqlx::query_scalar!(
828
+
sqlx::query!(
780
829
r#"
781
830
INSERT INTO repo_seq (did, event_type, active, status)
782
831
VALUES ($1, 'account', $2, $3)
783
-
RETURNING seq
784
832
"#,
785
833
did.as_str(),
786
834
active,
787
835
status_str
788
836
)
789
-
.fetch_one(&self.pool)
837
+
.execute(&self.pool)
790
838
.await
791
839
.map_err(map_sqlx_error)?;
792
840
793
-
sqlx::query(&format!("NOTIFY repo_updates, '{}'", seq))
794
-
.execute(&self.pool)
795
-
.await
796
-
.map_err(map_sqlx_error)?;
797
-
798
-
Ok(seq.into())
841
+
notify_repo_pending(&self.pool).await;
842
+
Ok(())
799
843
}
800
844
801
845
async fn insert_sync_event(
···
804
848
commit_cid: &CidLink,
805
849
rev: Option<&str>,
806
850
commit_bytes: &[u8],
807
-
) -> Result<SequenceNumber, DbError> {
851
+
) -> Result<(), DbError> {
808
852
let cid_bytes = commit_cid
809
853
.to_cid()
810
854
.map(|c| c.to_bytes())
811
855
.unwrap_or_default();
812
856
let block_cids: Vec<Vec<u8>> = vec![cid_bytes];
813
857
let block_data: Vec<Vec<u8>> = vec![commit_bytes.to_vec()];
814
-
let seq = sqlx::query_scalar!(
858
+
sqlx::query!(
815
859
r#"
816
860
INSERT INTO repo_seq (did, event_type, commit_cid, rev, block_cids, block_data)
817
861
VALUES ($1, 'sync', $2, $3, $4, $5)
818
-
RETURNING seq
819
862
"#,
820
863
did.as_str(),
821
864
commit_cid.as_str(),
···
823
866
&block_cids as &[Vec<u8>],
824
867
&block_data as &[Vec<u8>]
825
868
)
826
-
.fetch_one(&self.pool)
869
+
.execute(&self.pool)
827
870
.await
828
871
.map_err(map_sqlx_error)?;
829
872
830
-
sqlx::query(&format!("NOTIFY repo_updates, '{}'", seq))
831
-
.execute(&self.pool)
832
-
.await
833
-
.map_err(map_sqlx_error)?;
834
-
835
-
Ok(seq.into())
873
+
notify_repo_pending(&self.pool).await;
874
+
Ok(())
836
875
}
837
876
838
877
async fn insert_genesis_commit_event(
···
843
882
rev: &str,
844
883
commit_bytes: &[u8],
845
884
mst_root_bytes: &[u8],
846
-
) -> Result<SequenceNumber, DbError> {
885
+
) -> Result<(), DbError> {
847
886
let ops = serde_json::json!([]);
848
887
let blobs: Vec<String> = vec![];
849
888
let commit_cid_bytes = commit_cid
···
858
897
let block_data: Vec<Vec<u8>> = vec![commit_bytes.to_vec(), mst_root_bytes.to_vec()];
859
898
let prev_cid: Option<&str> = None;
860
899
861
-
let seq = sqlx::query_scalar!(
900
+
sqlx::query!(
862
901
r#"
863
902
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, rev)
864
903
VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7, $8)
865
-
RETURNING seq
866
904
"#,
867
905
did.as_str(),
868
906
commit_cid.as_str(),
···
873
911
&block_data as &[Vec<u8>],
874
912
rev
875
913
)
876
-
.fetch_one(&self.pool)
914
+
.execute(&self.pool)
877
915
.await
878
916
.map_err(map_sqlx_error)?;
879
917
880
-
sqlx::query(&format!("NOTIFY repo_updates, '{}'", seq))
881
-
.execute(&self.pool)
882
-
.await
883
-
.map_err(map_sqlx_error)?;
884
-
885
-
Ok(seq.into())
918
+
notify_repo_pending(&self.pool).await;
919
+
Ok(())
886
920
}
887
921
888
-
async fn delete_sequences_except(
889
-
&self,
890
-
did: &Did,
891
-
keep_seq: SequenceNumber,
892
-
) -> Result<(), DbError> {
922
+
async fn purge_did_events_keeping_latest(&self, did: &Did) -> Result<(), DbError> {
893
923
sqlx::query!(
894
-
"DELETE FROM repo_seq WHERE did = $1 AND seq != $2",
895
-
did.as_str(),
896
-
keep_seq.as_i64()
924
+
r#"
925
+
DELETE FROM repo_seq
926
+
WHERE did = $1
927
+
AND id <> (SELECT id FROM repo_seq WHERE did = $1 ORDER BY id DESC LIMIT 1)
928
+
"#,
929
+
did.as_str()
897
930
)
898
931
.execute(&self.pool)
899
932
.await
···
902
935
Ok(())
903
936
}
904
937
938
+
async fn assign_pending_sequences(&self) -> Result<u64, DbError> {
939
+
let mut total: u64 = 0;
940
+
loop {
941
+
let mut tx = self.pool.begin().await.map_err(map_sqlx_error)?;
942
+
let acquired = sqlx::query_scalar!(
943
+
r#"SELECT pg_try_advisory_xact_lock($1) as "locked!""#,
944
+
SEQUENCER_LOCK_KEY
945
+
)
946
+
.fetch_one(&mut *tx)
947
+
.await
948
+
.map_err(map_sqlx_error)?;
949
+
if !acquired {
950
+
return Ok(total);
951
+
}
952
+
let count = assign_one_batch(tx, &self.pool).await?;
953
+
total += count as u64;
954
+
if count < SEQUENCER_BATCH_SIZE {
955
+
return Ok(total);
956
+
}
957
+
}
958
+
}
959
+
960
+
async fn flush_pending_sequences(&self) -> Result<(), DbError> {
961
+
loop {
962
+
let mut tx = self.pool.begin().await.map_err(map_sqlx_error)?;
963
+
sqlx::query!("SELECT pg_advisory_xact_lock($1)", SEQUENCER_LOCK_KEY)
964
+
.execute(&mut *tx)
965
+
.await
966
+
.map_err(map_sqlx_error)?;
967
+
let count = assign_one_batch(tx, &self.pool).await?;
968
+
if count < SEQUENCER_BATCH_SIZE {
969
+
return Ok(());
970
+
}
971
+
}
972
+
}
973
+
905
974
async fn prune_events_older_than(&self, cutoff: DateTime<Utc>) -> Result<PruneCount, DbError> {
906
975
let result = sqlx::query!("DELETE FROM repo_seq WHERE created_at < $1", cutoff)
907
976
.execute(&self.pool)
···
965
1034
Some(lim) => {
966
1035
let rows = sqlx::query_as!(
967
1036
SequencedEventRow,
968
-
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
1037
+
r#"SELECT seq as "seq!", did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
969
1038
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
970
1039
FROM repo_seq
971
1040
WHERE seq > $1
···
982
1051
None => {
983
1052
let rows = sqlx::query_as!(
984
1053
SequencedEventRow,
985
-
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
1054
+
r#"SELECT seq as "seq!", did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
986
1055
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
987
1056
FROM repo_seq
988
1057
WHERE seq > $1
···
1004
1073
) -> Result<Vec<SequencedEvent>, DbError> {
1005
1074
let rows = sqlx::query_as!(
1006
1075
SequencedEventRow,
1007
-
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
1076
+
r#"SELECT seq as "seq!", did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
1008
1077
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
1009
1078
FROM repo_seq
1010
1079
WHERE seq > $1 AND seq < $2
···
1024
1093
) -> Result<Option<SequencedEvent>, DbError> {
1025
1094
let row = sqlx::query_as!(
1026
1095
SequencedEventRow,
1027
-
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
1096
+
r#"SELECT seq as "seq!", did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
1028
1097
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
1029
1098
FROM repo_seq
1030
1099
WHERE seq = $1"#,
···
1043
1112
) -> Result<Vec<SequencedEvent>, DbError> {
1044
1113
let rows = sqlx::query_as!(
1045
1114
SequencedEventRow,
1046
-
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
1115
+
r#"SELECT seq as "seq!", did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
1047
1116
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
1048
1117
FROM repo_seq
1049
1118
WHERE seq > $1
···
1102
1171
.await
1103
1172
.map_err(map_sqlx_error)?;
1104
1173
Ok(cid.map(CidLink::from))
1105
-
}
1106
-
1107
-
async fn notify_update(&self, seq: SequenceNumber) -> Result<(), DbError> {
1108
-
sqlx::query(&format!("NOTIFY repo_updates, '{}'", seq.as_i64()))
1109
-
.execute(&self.pool)
1110
-
.await
1111
-
.map_err(map_sqlx_error)?;
1112
-
Ok(())
1113
1174
}
1114
1175
1115
1176
async fn import_repo_data(
···
1402
1463
1403
1464
let event = input.commit_event;
1404
1465
let (event_block_cids, event_block_data) = inline_into_paired_blocks(event.blocks);
1405
-
let seq: i64 = sqlx::query_scalar(
1466
+
sqlx::query!(
1406
1467
r#"
1407
1468
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)
1408
1469
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
1409
-
RETURNING seq
1410
1470
"#,
1471
+
event.did.as_str(),
1472
+
event.event_type.as_str(),
1473
+
event.commit_cid.as_ref().map(|c| c.as_str()),
1474
+
event.prev_cid.as_ref().map(|c| c.as_str()),
1475
+
event.ops,
1476
+
event.blobs.as_deref(),
1477
+
&event_block_cids as &[Vec<u8>],
1478
+
&event_block_data as &[Vec<u8>],
1479
+
event.prev_data_cid.as_ref().map(|c| c.as_str()),
1480
+
event.rev
1411
1481
)
1412
-
.bind(event.did.as_str())
1413
-
.bind(event.event_type.as_str())
1414
-
.bind(event.commit_cid.as_ref().map(|c| c.as_str()))
1415
-
.bind(event.prev_cid.as_ref().map(|c| c.as_str()))
1416
-
.bind(&event.ops)
1417
-
.bind(&event.blobs)
1418
-
.bind(&event_block_cids)
1419
-
.bind(&event_block_data)
1420
-
.bind(event.prev_data_cid.as_ref().map(|c| c.as_str()))
1421
-
.bind(&event.rev)
1422
-
.fetch_one(&mut *tx)
1482
+
.execute(&mut *tx)
1423
1483
.await
1424
1484
.map_err(|e| ApplyCommitError::Database(e.to_string()))?;
1425
1485
1426
-
sqlx::query(&format!("NOTIFY repo_updates, '{}'", seq))
1486
+
sqlx::query!("NOTIFY repo_updates")
1427
1487
.execute(&mut *tx)
1428
1488
.await
1429
1489
.map_err(|e| ApplyCommitError::Database(e.to_string()))?;
···
1432
1492
.await
1433
1493
.map_err(|e| ApplyCommitError::Database(e.to_string()))?;
1434
1494
1435
-
Ok(tranquil_db_traits::ApplyCommitResult {
1436
-
seq,
1437
-
is_account_active,
1438
-
})
1495
+
Ok(tranquil_db_traits::ApplyCommitResult { is_account_active })
1439
1496
}
1440
1497
1441
1498
async fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, DbError> {
+7
-7
crates/tranquil-db/src/postgres/user.rs
+7
-7
crates/tranquil-db/src/postgres/user.rs
···
2291
2291
.collect())
2292
2292
}
2293
2293
2294
-
async fn delete_account_with_firehose(&self, user_id: Uuid, did: &Did) -> Result<i64, DbError> {
2294
+
async fn delete_account_with_firehose(&self, user_id: Uuid, did: &Did) -> Result<(), DbError> {
2295
2295
let mut tx = self.pool.begin().await.map_err(map_sqlx_error)?;
2296
2296
2297
2297
sqlx::query!("DELETE FROM blobs WHERE created_by_user = $1", user_id)
···
2370
2370
.await
2371
2371
.map_err(map_sqlx_error)?;
2372
2372
2373
-
let account_seq: i64 = sqlx::query_scalar!(
2373
+
let event_id: i64 = sqlx::query_scalar!(
2374
2374
r#"
2375
2375
INSERT INTO repo_seq (did, event_type, active, status)
2376
2376
VALUES ($1, 'account', false, 'deleted')
2377
-
RETURNING seq
2377
+
RETURNING id
2378
2378
"#,
2379
2379
did.as_str()
2380
2380
)
···
2383
2383
.map_err(map_sqlx_error)?;
2384
2384
2385
2385
sqlx::query!(
2386
-
"DELETE FROM repo_seq WHERE did = $1 AND seq != $2",
2386
+
"DELETE FROM repo_seq WHERE did = $1 AND id <> $2",
2387
2387
did.as_str(),
2388
-
account_seq
2388
+
event_id
2389
2389
)
2390
2390
.execute(&mut *tx)
2391
2391
.await
···
2393
2393
2394
2394
tx.commit().await.map_err(map_sqlx_error)?;
2395
2395
2396
-
sqlx::query(&format!("NOTIFY repo_updates, '{}'", account_seq))
2396
+
sqlx::query!("NOTIFY repo_updates")
2397
2397
.execute(&self.pool)
2398
2398
.await
2399
2399
.map_err(map_sqlx_error)?;
2400
2400
2401
-
Ok(account_seq)
2401
+
Ok(())
2402
2402
}
2403
2403
2404
2404
async fn create_password_account(
+4
-5
crates/tranquil-pds/src/repo_ops.rs
+4
-5
crates/tranquil-pds/src/repo_ops.rs
···
19
19
use std::sync::Arc;
20
20
use tokio::sync::OwnedMutexGuard;
21
21
use tracing::{error, warn};
22
-
use tranquil_db_traits::SequenceNumber;
23
22
use uuid::Uuid;
24
23
25
24
#[derive(Debug)]
···
776
775
state: &AppState,
777
776
did: &Did,
778
777
handle: Option<&Handle>,
779
-
) -> Result<SequenceNumber, CommitError> {
778
+
) -> Result<(), CommitError> {
780
779
state
781
780
.repos
782
781
.repo
···
788
787
state: &AppState,
789
788
did: &Did,
790
789
status: tranquil_db_traits::AccountStatus,
791
-
) -> Result<SequenceNumber, CommitError> {
790
+
) -> Result<(), CommitError> {
792
791
state
793
792
.repos
794
793
.repo
···
801
800
did: &Did,
802
801
commit_cid: &str,
803
802
rev: Option<&str>,
804
-
) -> Result<SequenceNumber, CommitError> {
803
+
) -> Result<(), CommitError> {
805
804
let cid_link: crate::types::CidLink = commit_cid
806
805
.parse()
807
806
.map_err(|_| CommitError::InvalidCid(commit_cid.to_string()))?;
···
829
828
commit_cid: &Cid,
830
829
mst_root_cid: &Cid,
831
830
rev: &str,
832
-
) -> Result<SequenceNumber, CommitError> {
831
+
) -> Result<(), CommitError> {
833
832
let commit_cid_link = crate::types::CidLink::from(commit_cid);
834
833
let mst_root_cid_link = crate::types::CidLink::from(mst_root_cid);
835
834
let commit_bytes = state
+1
-1
crates/tranquil-pds/src/scheduled.rs
+1
-1
crates/tranquil-pds/src/scheduled.rs
+34
crates/tranquil-pds/tests/common/mod.rs
+34
crates/tranquil-pds/tests/common/mod.rs
···
921
921
.expect("TEST_BLOCK_STORE not initialized")
922
922
}
923
923
924
+
#[allow(dead_code)]
925
+
pub async fn flushed_max_seq(
926
+
repos: &tranquil_db::PostgresRepositories,
927
+
) -> tranquil_db_traits::SequenceNumber {
928
+
repos
929
+
.repo
930
+
.flush_pending_sequences()
931
+
.await
932
+
.expect("flush_pending_sequences");
933
+
repos.repo.get_max_seq().await.expect("get_max_seq")
934
+
}
935
+
936
+
#[allow(dead_code)]
937
+
pub async fn sequenced_event_for_did(
938
+
repos: &tranquil_db::PostgresRepositories,
939
+
baseline: tranquil_db_traits::SequenceNumber,
940
+
did: &tranquil_types::Did,
941
+
) -> tranquil_db_traits::SequencedEvent {
942
+
repos
943
+
.repo
944
+
.flush_pending_sequences()
945
+
.await
946
+
.expect("flush_pending_sequences");
947
+
repos
948
+
.repo
949
+
.get_events_since_seq(baseline, None)
950
+
.await
951
+
.expect("get_events_since_seq")
952
+
.into_iter()
953
+
.filter(|event| &event.did == did)
954
+
.last()
955
+
.unwrap_or_else(|| panic!("event for did {did} not found after flush"))
956
+
}
957
+
924
958
fn extract_verification_code(body_text: &str) -> String {
925
959
let lines: Vec<&str> = body_text.lines().collect();
926
960
lines
+16
-14
crates/tranquil-pds/tests/firehose_inline_blocks.rs
+16
-14
crates/tranquil-pds/tests/firehose_inline_blocks.rs
···
66
66
let repos = get_test_repos().await;
67
67
let typed_did = tranquil_types::Did::new(did.clone()).unwrap();
68
68
69
+
repos
70
+
.repo
71
+
.flush_pending_sequences()
72
+
.await
73
+
.expect("flush_pending_sequences");
69
74
let events = repos
70
75
.repo
71
76
.get_events_since_seq(SequenceNumber::ZERO, None)
···
142
147
let cid_link: CidLink = (&commit_cid).into();
143
148
let rev = "3kabcdefghij2";
144
149
145
-
let seq = repos
150
+
let baseline = repos.repo.get_max_seq().await.expect("get_max_seq");
151
+
repos
146
152
.repo
147
153
.insert_sync_event(&did, &cid_link, Some(rev), &commit_bytes)
148
154
.await
149
155
.expect("insert_sync_event");
150
156
151
-
let event = repos
152
-
.repo
153
-
.get_event_by_seq(seq)
154
-
.await
155
-
.expect("get_event_by_seq")
156
-
.expect("event present");
157
+
let event = sequenced_event_for_did(repos, baseline, &did).await;
157
158
158
159
assert_eq!(event.event_type, RepoEventType::Sync);
159
160
let blocks = event
···
193
194
let mst_link: CidLink = (&mst_root_cid).into();
194
195
let rev = "3kabcdefghij3";
195
196
196
-
let seq = repos
197
+
let baseline = repos.repo.get_max_seq().await.expect("get_max_seq");
198
+
repos
197
199
.repo
198
200
.insert_genesis_commit_event(
199
201
&did,
···
206
208
.await
207
209
.expect("insert_genesis_commit_event");
208
210
209
-
let event = repos
210
-
.repo
211
-
.get_event_by_seq(seq)
212
-
.await
213
-
.expect("get_event_by_seq")
214
-
.expect("event present");
211
+
let event = sequenced_event_for_did(repos, baseline, &did).await;
215
212
216
213
assert_eq!(event.event_type, RepoEventType::Commit);
217
214
let blocks = event
···
329
326
330
327
let repos = get_test_repos().await;
331
328
let typed_did = tranquil_types::Did::new(did.clone()).unwrap();
329
+
repos
330
+
.repo
331
+
.flush_pending_sequences()
332
+
.await
333
+
.expect("flush_pending_sequences");
332
334
let events = repos
333
335
.repo
334
336
.get_events_since_seq(SequenceNumber::ZERO, None)
+1
-1
crates/tranquil-pds/tests/firehose_validation.rs
+1
-1
crates/tranquil-pds/tests/firehose_validation.rs
···
801
801
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
802
802
803
803
let repos = get_test_repos().await;
804
-
let max_seq = repos.repo.get_max_seq().await.unwrap().as_i64();
804
+
let max_seq = flushed_max_seq(repos).await.as_i64();
805
805
let outdated_cursor = (max_seq - 100).max(1);
806
806
let url = format!(
807
807
"ws://127.0.0.1:{}/xrpc/com.atproto.sync.subscribeRepos?cursor={}",
+5
-5
crates/tranquil-pds/tests/mst_firehose_e2e.rs
+5
-5
crates/tranquil-pds/tests/mst_firehose_e2e.rs
···
223
223
let (token, did) = create_account_and_login(&client).await;
224
224
225
225
let repos = get_test_repos().await;
226
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
226
+
let cursor = flushed_max_seq(repos).await.as_i64();
227
227
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
228
228
tokio::time::sleep(Duration::from_millis(100)).await;
229
229
···
298
298
let (token, did) = create_account_and_login(&client).await;
299
299
300
300
let repos = get_test_repos().await;
301
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
301
+
let cursor = flushed_max_seq(repos).await.as_i64();
302
302
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
303
303
tokio::time::sleep(Duration::from_millis(100)).await;
304
304
···
340
340
create_record(&client, &token, &did, &rkey_for(i), "pre").await;
341
341
}
342
342
343
-
let resume_cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
343
+
let resume_cursor = flushed_max_seq(repos).await.as_i64();
344
344
345
345
for i in 5..12 {
346
346
create_record(&client, &token, &did, &rkey_for(i), "post").await;
···
371
371
let client = client();
372
372
let (token, did) = create_account_and_login(&client).await;
373
373
let repos = get_test_repos().await;
374
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
374
+
let cursor = flushed_max_seq(repos).await.as_i64();
375
375
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
376
376
tokio::time::sleep(Duration::from_millis(100)).await;
377
377
···
417
417
let client = client();
418
418
let (token, did) = create_account_and_login(&client).await;
419
419
let repos = get_test_repos().await;
420
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
420
+
let cursor = flushed_max_seq(repos).await.as_i64();
421
421
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
422
422
tokio::time::sleep(Duration::from_millis(100)).await;
423
423
+5
crates/tranquil-pds/tests/mst_inductive_firehose.rs
+5
crates/tranquil-pds/tests/mst_inductive_firehose.rs
···
247
247
async fn our_commit_events(did: &str) -> Vec<SequencedEvent> {
248
248
let repos = get_test_repos().await;
249
249
let typed_did = Did::new(did.to_string()).unwrap();
250
+
repos
251
+
.repo
252
+
.flush_pending_sequences()
253
+
.await
254
+
.expect("flush_pending_sequences");
250
255
let events = repos
251
256
.repo
252
257
.get_events_since_seq(SequenceNumber::ZERO, None)
+7
-2
crates/tranquil-pds/tests/mst_integrity.rs
+7
-2
crates/tranquil-pds/tests/mst_integrity.rs
···
216
216
217
217
let repos = get_test_repos().await;
218
218
let typed_did = Did::new(did.clone()).unwrap();
219
+
repos
220
+
.repo
221
+
.flush_pending_sequences()
222
+
.await
223
+
.expect("flush_pending_sequences");
219
224
let events = repos
220
225
.repo
221
226
.get_events_since_seq(SequenceNumber::ZERO, None)
···
253
258
let (token, did) = create_account_and_login(&client).await;
254
259
255
260
let repos = get_test_repos().await;
256
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
261
+
let cursor = flushed_max_seq(repos).await.as_i64();
257
262
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
258
263
tokio::time::sleep(Duration::from_millis(100)).await;
259
264
···
284
289
let (token, did) = create_account_and_login(&client).await;
285
290
286
291
let repos = get_test_repos().await;
287
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
292
+
let cursor = flushed_max_seq(repos).await.as_i64();
288
293
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
289
294
tokio::time::sleep(Duration::from_millis(100)).await;
290
295
+5
crates/tranquil-pds/tests/mst_property_fuzz.rs
+5
crates/tranquil-pds/tests/mst_property_fuzz.rs
···
267
267
268
268
let repos = get_test_repos().await;
269
269
let typed_did = Did::new(did.clone()).unwrap();
270
+
repos
271
+
.repo
272
+
.flush_pending_sequences()
273
+
.await
274
+
.expect("flush_pending_sequences");
270
275
let events = repos
271
276
.repo
272
277
.get_events_since_seq(SequenceNumber::ZERO, None)
+8
-8
crates/tranquil-pds/tests/repo_lifecycle.rs
+8
-8
crates/tranquil-pds/tests/repo_lifecycle.rs
···
59
59
let (token, did) = create_account_and_login(&client).await;
60
60
61
61
let repos = get_test_repos().await;
62
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
62
+
let cursor = flushed_max_seq(repos).await.as_i64();
63
63
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
64
64
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
65
65
···
134
134
let v1_cid = Cid::from_str(v1_cid_str).unwrap();
135
135
136
136
let repos = get_test_repos().await;
137
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
137
+
let cursor = flushed_max_seq(repos).await.as_i64();
138
138
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
139
139
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
140
140
···
203
203
let rkey = parts[parts.len() - 1];
204
204
205
205
let repos = get_test_repos().await;
206
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
206
+
let cursor = flushed_max_seq(repos).await.as_i64();
207
207
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
208
208
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
209
209
···
246
246
let (token, did) = create_account_and_login(&client).await;
247
247
248
248
let repos = get_test_repos().await;
249
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
249
+
let cursor = flushed_max_seq(repos).await.as_i64();
250
250
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
251
251
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
252
252
···
315
315
let (token, did) = create_account_and_login(&client).await;
316
316
317
317
let repos = get_test_repos().await;
318
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
318
+
let cursor = flushed_max_seq(repos).await.as_i64();
319
319
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
320
320
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
321
321
···
396
396
};
397
397
398
398
let repos = get_test_repos().await;
399
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
399
+
let cursor = flushed_max_seq(repos).await.as_i64();
400
400
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
401
401
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
402
402
···
444
444
let (token, did) = create_account_and_login(&client).await;
445
445
446
446
let repos = get_test_repos().await;
447
-
let baseline_seq = repos.repo.get_max_seq().await.unwrap().as_i64();
447
+
let baseline_seq = flushed_max_seq(repos).await.as_i64();
448
448
449
449
let mut expected_cids: Vec<String> = Vec::with_capacity(5);
450
450
let texts = [
···
496
496
let (bob_token, bob_did) = create_account_and_login(&client).await;
497
497
498
498
let repos = get_test_repos().await;
499
-
let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
499
+
let cursor = flushed_max_seq(repos).await.as_i64();
500
500
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
501
501
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
502
502
+6
-4
crates/tranquil-pds/tests/store_parity.rs
+6
-4
crates/tranquil-pds/tests/store_parity.rs
···
1628
1628
rev: Some("rev0".to_string()),
1629
1629
};
1630
1630
1631
-
let pg_seq = f.pg.repo.insert_commit_event(&event).await.unwrap();
1632
-
let store_seq = f.store.repo.insert_commit_event(&event).await.unwrap();
1633
-
assert!(pg_seq.as_i64() > 0);
1634
-
assert!(store_seq.as_i64() > 0);
1631
+
let baseline = f.pg.repo.get_max_seq().await.unwrap();
1632
+
f.pg.repo.insert_commit_event(&event).await.unwrap();
1633
+
f.store.repo.insert_commit_event(&event).await.unwrap();
1634
+
let pg_seq = common::sequenced_event_for_did(&f.pg, baseline, &did)
1635
+
.await
1636
+
.seq;
1635
1637
1636
1638
let past_cutoff = chrono::Utc::now() - chrono::Duration::hours(24);
1637
1639
let pg_pruned_past =
+2
-2
crates/tranquil-store/src/eventlog/notifier.rs
+2
-2
crates/tranquil-store/src/eventlog/notifier.rs
···
30
30
31
31
#[async_trait]
32
32
impl<S: StorageIO + 'static> RepoEventReceiver for EventLogEventReceiver<S> {
33
-
async fn recv(&mut self) -> Option<i64> {
34
-
self.subscriber.next().await.map(|event| event.seq.as_i64())
33
+
async fn recv(&mut self) -> Option<()> {
34
+
self.subscriber.next().await.map(|_| ())
35
35
}
36
36
}
+13
-32
crates/tranquil-store/src/metastore/client.rs
+13
-32
crates/tranquil-store/src/metastore/client.rs
···
474
474
recv(rx).await
475
475
}
476
476
477
-
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError> {
477
+
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<(), DbError> {
478
478
let (tx, rx) = oneshot::channel();
479
479
self.pool
480
480
.send(MetastoreRequest::Event(EventRequest::InsertCommitEvent {
481
481
data: data.clone(),
482
482
tx,
483
483
}))?;
484
-
recv(rx).await
484
+
recv(rx).await.map(|_: SequenceNumber| ())
485
485
}
486
486
487
487
async fn insert_identity_event(
488
488
&self,
489
489
did: &Did,
490
490
handle: Option<&Handle>,
491
-
) -> Result<SequenceNumber, DbError> {
491
+
) -> Result<(), DbError> {
492
492
let (tx, rx) = oneshot::channel();
493
493
self.pool
494
494
.send(MetastoreRequest::Event(EventRequest::InsertIdentityEvent {
···
496
496
handle: handle.cloned(),
497
497
tx,
498
498
}))?;
499
-
recv(rx).await
499
+
recv(rx).await.map(|_: SequenceNumber| ())
500
500
}
501
501
502
-
async fn insert_account_event(
503
-
&self,
504
-
did: &Did,
505
-
status: AccountStatus,
506
-
) -> Result<SequenceNumber, DbError> {
502
+
async fn insert_account_event(&self, did: &Did, status: AccountStatus) -> Result<(), DbError> {
507
503
let (tx, rx) = oneshot::channel();
508
504
self.pool
509
505
.send(MetastoreRequest::Event(EventRequest::InsertAccountEvent {
···
511
507
status,
512
508
tx,
513
509
}))?;
514
-
recv(rx).await
510
+
recv(rx).await.map(|_: SequenceNumber| ())
515
511
}
516
512
517
513
async fn insert_sync_event(
···
520
516
commit_cid: &CidLink,
521
517
rev: Option<&str>,
522
518
commit_bytes: &[u8],
523
-
) -> Result<SequenceNumber, DbError> {
519
+
) -> Result<(), DbError> {
524
520
let (tx, rx) = oneshot::channel();
525
521
self.pool
526
522
.send(MetastoreRequest::Event(EventRequest::InsertSyncEvent {
···
530
526
commit_bytes: commit_bytes.to_vec(),
531
527
tx,
532
528
}))?;
533
-
recv(rx).await
529
+
recv(rx).await.map(|_: SequenceNumber| ())
534
530
}
535
531
536
532
async fn insert_genesis_commit_event(
···
541
537
rev: &str,
542
538
commit_bytes: &[u8],
543
539
mst_root_bytes: &[u8],
544
-
) -> Result<SequenceNumber, DbError> {
540
+
) -> Result<(), DbError> {
545
541
let (tx, rx) = oneshot::channel();
546
542
self.pool.send(MetastoreRequest::Event(
547
543
EventRequest::InsertGenesisCommitEvent {
···
554
550
tx,
555
551
},
556
552
))?;
557
-
recv(rx).await
553
+
recv(rx).await.map(|_: SequenceNumber| ())
558
554
}
559
555
560
-
async fn delete_sequences_except(
561
-
&self,
562
-
did: &Did,
563
-
keep_seq: SequenceNumber,
564
-
) -> Result<(), DbError> {
556
+
async fn purge_did_events_keeping_latest(&self, did: &Did) -> Result<(), DbError> {
565
557
let (tx, rx) = oneshot::channel();
566
558
self.pool.send(MetastoreRequest::Event(
567
-
EventRequest::DeleteSequencesExcept {
559
+
EventRequest::PurgeDidEventsKeepingLatest {
568
560
did: did.clone(),
569
-
keep_seq,
570
561
tx,
571
562
},
572
563
))?;
···
710
701
self.pool.send(MetastoreRequest::Repo(
711
702
RepoRequest::GetRepoRootCidByUserId { user_id, tx },
712
703
))?;
713
-
recv(rx).await
714
-
}
715
-
716
-
async fn notify_update(&self, seq: SequenceNumber) -> Result<(), DbError> {
717
-
let (tx, rx) = oneshot::channel();
718
-
self.pool
719
-
.send(MetastoreRequest::Event(EventRequest::NotifyUpdate {
720
-
seq,
721
-
tx,
722
-
}))?;
723
704
recv(rx).await
724
705
}
725
706
···
4898
4879
recv(rx).await
4899
4880
}
4900
4881
4901
-
async fn delete_account_with_firehose(&self, user_id: Uuid, did: &Did) -> Result<i64, DbError> {
4882
+
async fn delete_account_with_firehose(&self, user_id: Uuid, did: &Did) -> Result<(), DbError> {
4902
4883
let (tx, rx) = oneshot::channel();
4903
4884
self.pool.send(MetastoreRequest::User(
4904
4885
UserRequest::DeleteAccountWithFirehose {
+6
-10
crates/tranquil-store/src/metastore/commit_ops.rs
+6
-10
crates/tranquil-store/src/metastore/commit_ops.rs
···
245
245
.serialize()
246
246
.map_err(|e| ApplyCommitError::Database(e.to_string()))?;
247
247
248
-
let (seq, deferred) = self
248
+
let (_seq, deferred) = self
249
249
.event_ops
250
250
.append_commit_event_into_batch(
251
251
&mut batch,
···
261
261
self.event_ops.complete_broadcast(deferred);
262
262
263
263
Ok(ApplyCommitResult {
264
-
seq: seq.as_i64(),
265
264
is_account_active: is_active,
266
265
})
267
266
}
···
477
476
use crate::eventlog::{EventLog, EventLogConfig};
478
477
use crate::io::RealIO;
479
478
use crate::metastore::{Metastore, MetastoreConfig};
480
-
use tranquil_db_traits::{CommitEventData, RepoEventType, SequenceNumber};
479
+
use tranquil_db_traits::{CommitEventData, RepoEventType};
481
480
use tranquil_types::{Handle, Nsid, Rkey};
482
481
483
482
struct TestHarness {
···
599
598
};
600
599
601
600
let result = ops.apply_commit(input).unwrap();
602
-
assert!(result.seq > 0);
603
601
assert!(result.is_account_active);
604
602
605
603
let repo = h.metastore.repo_ops().get_repo(user_id).unwrap().unwrap();
···
812
810
},
813
811
};
814
812
815
-
let result = ops.apply_commit(input).unwrap();
816
-
let seq = SequenceNumber::from_raw(result.seq);
813
+
ops.apply_commit(input).unwrap();
814
+
let seq = ops.event_ops.get_max_seq();
817
815
818
816
let event = ops.event_ops.get_event_by_seq(seq).unwrap().unwrap();
819
817
assert_eq!(event.did, did);
···
954
952
},
955
953
};
956
954
957
-
let result = ops.apply_commit(input).unwrap();
958
-
assert!(result.seq > 0);
955
+
ops.apply_commit(input).unwrap();
959
956
}
960
957
961
958
#[test]
···
1144
1141
},
1145
1142
};
1146
1143
1147
-
let result = ops.apply_commit(input).unwrap();
1148
-
assert!(result.seq > 0);
1144
+
ops.apply_commit(input).unwrap();
1149
1145
metastore.persist().unwrap();
1150
1146
}
1151
1147
+27
-4
crates/tranquil-store/src/metastore/event_ops.rs
+27
-4
crates/tranquil-store/src/metastore/event_ops.rs
···
41
41
self.bridge.notifier()
42
42
}
43
43
44
-
pub fn notify_update(&self, _seq: SequenceNumber) -> Result<(), DbError> {
45
-
Ok(())
46
-
}
47
-
48
44
pub fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError> {
49
45
let event = Self::build_commit_event(data);
50
46
self.append_and_index(&event, &data.did, data.rev.as_deref())
···
380
376
batch.commit().map_err(fjall_to_db)?;
381
377
382
378
Ok(())
379
+
}
380
+
381
+
pub fn purge_did_events_keeping_latest(&self, did: &Did) -> Result<(), DbError> {
382
+
let user_hash = UserHash::from_did(did.as_str());
383
+
let prefix = did_events_prefix(user_hash);
384
+
let upper = exclusive_upper_bound(prefix.as_slice())
385
+
.expect("did_events prefix can never be all-0xFF");
386
+
387
+
let latest = self
388
+
.repo_data
389
+
.range(prefix.as_slice()..upper.as_slice())
390
+
.map(|guard| {
391
+
let (key, _) = guard.into_inner().map_err(fjall_to_db)?;
392
+
decode_did_events_seq(key.as_ref())
393
+
})
394
+
.collect::<Result<Vec<u64>, DbError>>()?
395
+
.into_iter()
396
+
.max();
397
+
398
+
match latest {
399
+
Some(seq) => {
400
+
let keep = i64::try_from(seq)
401
+
.map_err(|_| DbError::Query("sequence number out of range".to_owned()))?;
402
+
self.delete_sequences_except(did, SequenceNumber::from_raw(keep))
403
+
}
404
+
None => Ok(()),
405
+
}
383
406
}
384
407
385
408
pub fn read_last_applied_cursor(&self) -> Result<Option<u64>, DbError> {
+7
-16
crates/tranquil-store/src/metastore/handler.rs
+7
-16
crates/tranquil-store/src/metastore/handler.rs
···
362
362
mst_root_bytes: Vec<u8>,
363
363
tx: Tx<SequenceNumber>,
364
364
},
365
-
DeleteSequencesExcept {
365
+
PurgeDidEventsKeepingLatest {
366
366
did: Did,
367
-
keep_seq: SequenceNumber,
368
367
tx: Tx<()>,
369
368
},
370
369
GetMaxSeq {
···
393
392
limit: i64,
394
393
tx: Tx<Vec<SequencedEvent>>,
395
394
},
396
-
NotifyUpdate {
397
-
seq: SequenceNumber,
398
-
tx: Tx<()>,
399
-
},
400
395
}
401
396
402
397
impl EventRequest {
···
409
404
| Self::InsertAccountEvent { did, .. }
410
405
| Self::InsertSyncEvent { did, .. }
411
406
| Self::InsertGenesisCommitEvent { did, .. }
412
-
| Self::DeleteSequencesExcept { did, .. } => {
407
+
| Self::PurgeDidEventsKeepingLatest { did, .. } => {
413
408
Routing::Sharded(UserHash::from_did(did.as_str()).raw())
414
409
}
415
410
Self::GetMaxSeq { .. }
···
417
412
| Self::GetEventsSinceSeq { .. }
418
413
| Self::GetEventsInSeqRange { .. }
419
414
| Self::GetEventBySeq { .. }
420
-
| Self::GetEventsSinceCursor { .. }
421
-
| Self::NotifyUpdate { .. } => Routing::Global,
415
+
| Self::GetEventsSinceCursor { .. } => Routing::Global,
422
416
}
423
417
}
424
418
}
···
1513
1507
DeleteAccountWithFirehose {
1514
1508
user_id: Uuid,
1515
1509
did: Did,
1516
-
tx: Tx<i64>,
1510
+
tx: Tx<()>,
1517
1511
},
1518
1512
CreatePasswordAccount {
1519
1513
input: CreatePasswordAccountInput,
···
2967
2961
);
2968
2962
let _ = tx.send(result);
2969
2963
}
2970
-
EventRequest::DeleteSequencesExcept { did, keep_seq, tx } => {
2971
-
let result = state.event_ops.delete_sequences_except(&did, keep_seq);
2964
+
EventRequest::PurgeDidEventsKeepingLatest { did, tx } => {
2965
+
let result = state.event_ops.purge_did_events_keeping_latest(&did);
2972
2966
let _ = tx.send(result);
2973
2967
}
2974
2968
EventRequest::GetMaxSeq { tx } => {
···
2996
2990
}
2997
2991
EventRequest::GetEventsSinceCursor { cursor, limit, tx } => {
2998
2992
let _ = tx.send(state.event_ops.get_events_since_cursor(cursor, limit));
2999
-
}
3000
-
EventRequest::NotifyUpdate { seq, tx } => {
3001
-
let _ = tx.send(state.event_ops.notify_update(seq));
3002
2993
}
3003
2994
}
3004
2995
}
···
5804
5795
.event_ops
5805
5796
.insert_account_event(&did, AccountStatus::Deleted)
5806
5797
});
5807
-
let _ = tx.send(result.map(|seq| seq.as_i64()));
5798
+
let _ = tx.send(result.map(|_| ()));
5808
5799
}
5809
5800
UserRequest::CreatePasswordAccount { input, tx } => {
5810
5801
let result = user.create_password_account(&input).and_then(|result| {
+2
-11
crates/tranquil-store/tests/sim_metastore.rs
+2
-11
crates/tranquil-store/tests/sim_metastore.rs
···
405
405
),
406
406
};
407
407
408
-
let result = h.apply_commit(input).unwrap_or_else(|e| {
408
+
h.apply_commit(input).unwrap_or_else(|e| {
409
409
panic!("seed={seed} commit {commit_idx}: {e:?}")
410
410
});
411
-
assert!(
412
-
result.seq > 0,
413
-
"seed={seed} commit {commit_idx} seq must be positive"
414
-
);
415
411
416
412
let new_persisted = match commit_idx <= crash_after {
417
413
true => {
···
689
685
receivers.into_iter().for_each(|(idx, rx)| {
690
686
let result = rt.block_on(rx);
691
687
match result {
692
-
Ok(Ok(commit_result)) => {
693
-
assert!(
694
-
commit_result.seq > 0,
695
-
"seed={seed} idx={idx} commit seq must be positive"
696
-
);
697
-
}
688
+
Ok(Ok(_commit_result)) => {}
698
689
Ok(Err(e)) => {
699
690
panic!("seed={seed} idx={idx} commit failed: {e:?}");
700
691
}
+46
-77
crates/tranquil-sync/src/listener.rs
+46
-77
crates/tranquil-sync/src/listener.rs
···
1
1
use std::sync::atomic::{AtomicI64, Ordering};
2
-
use tracing::{debug, error, info, warn};
2
+
use std::time::Duration;
3
+
use tracing::{error, info, warn};
3
4
use tranquil_db_traits::SequenceNumber;
4
5
use tranquil_pds::state::AppState;
5
6
use tranquil_pds::sync::firehose::SequencedEvent;
6
7
7
8
static LAST_BROADCAST_SEQ: AtomicI64 = AtomicI64::new(0);
9
+
10
+
const DRAIN_BATCH_SIZE: i64 = 1000;
11
+
const POLL_INTERVAL: Duration = Duration::from_secs(1);
8
12
9
13
pub async fn start_sequencer_listener(state: AppState) {
10
14
let initial_seq = state
···
23
27
loop {
24
28
if let Err(e) = listen_loop(state.clone()).await {
25
29
error!("Sequencer listener failed: {}. Restarting in 5s...", e);
26
-
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
30
+
tokio::time::sleep(Duration::from_secs(5)).await;
27
31
}
28
32
}
29
33
});
···
37
41
.await
38
42
.map_err(|e| anyhow::anyhow!("Failed to subscribe to events: {:?}", e))?;
39
43
info!("Connected to database and listening for repo updates");
40
-
let catchup_start = SequenceNumber::from_raw(LAST_BROADCAST_SEQ.load(Ordering::SeqCst));
41
-
let events = state
42
-
.repos
43
-
.repo
44
-
.get_events_since_seq(catchup_start, None)
45
-
.await
46
-
.map_err(|e| anyhow::anyhow!("Failed to fetch catchup events: {:?}", e))?;
47
-
if !events.is_empty() {
48
-
info!(
49
-
count = events.len(),
50
-
from_seq = catchup_start.as_i64(),
51
-
"Broadcasting catch-up events"
52
-
);
53
-
events.into_iter().for_each(|event| {
54
-
let seq = event.seq;
55
-
let firehose_event = to_firehose_event(event);
56
-
let _ = state.firehose_tx.send(firehose_event);
57
-
LAST_BROADCAST_SEQ.store(seq.as_i64(), Ordering::SeqCst);
58
-
});
59
-
}
44
+
45
+
let mut last_seq = LAST_BROADCAST_SEQ.load(Ordering::SeqCst);
46
+
sequence_and_broadcast(&state, &mut last_seq).await;
47
+
60
48
loop {
61
-
let Some(seq_id) = receiver.recv().await else {
62
-
return Err(anyhow::anyhow!("Event receiver disconnected"));
63
-
};
64
-
debug!(seq = seq_id, "Received event notification");
65
-
let last_seq = LAST_BROADCAST_SEQ.load(Ordering::SeqCst);
66
-
if seq_id <= last_seq {
67
-
debug!(
68
-
seq = seq_id,
69
-
last = last_seq,
70
-
"Skipping already-broadcast event"
71
-
);
72
-
continue;
73
-
}
74
-
if seq_id > last_seq + 1 {
75
-
let gap_events = state
76
-
.repos
77
-
.repo
78
-
.get_events_in_seq_range(
79
-
SequenceNumber::from_raw(last_seq),
80
-
SequenceNumber::from_raw(seq_id),
81
-
)
82
-
.await
83
-
.unwrap_or_default();
84
-
if !gap_events.is_empty() {
85
-
debug!(count = gap_events.len(), "Filling sequence gap");
86
-
gap_events.into_iter().for_each(|event| {
87
-
let seq = event.seq;
88
-
let firehose_event = to_firehose_event(event);
89
-
let _ = state.firehose_tx.send(firehose_event);
90
-
LAST_BROADCAST_SEQ.store(seq.as_i64(), Ordering::SeqCst);
91
-
});
49
+
tokio::select! {
50
+
received = receiver.recv() => {
51
+
if received.is_none() {
52
+
return Err(anyhow::anyhow!("Event receiver disconnected"));
53
+
}
92
54
}
55
+
_ = tokio::time::sleep(POLL_INTERVAL) => {}
93
56
}
94
-
let event = state
57
+
sequence_and_broadcast(&state, &mut last_seq).await;
58
+
}
59
+
}
60
+
61
+
async fn sequence_and_broadcast(state: &AppState, last_seq: &mut i64) {
62
+
if let Err(e) = state.repos.repo.assign_pending_sequences().await {
63
+
warn!("Failed to assign pending firehose sequences: {:?}", e);
64
+
}
65
+
loop {
66
+
let events = match state
95
67
.repos
96
68
.repo
97
-
.get_event_by_seq(SequenceNumber::from_raw(seq_id))
69
+
.get_events_since_seq(SequenceNumber::from_raw(*last_seq), Some(DRAIN_BATCH_SIZE))
98
70
.await
99
-
.ok()
100
-
.flatten();
101
-
if let Some(event) = event {
102
-
let seq = event.seq;
71
+
{
72
+
Ok(events) => events,
73
+
Err(e) => {
74
+
warn!("Sequencer broadcast query failed: {:?}", e);
75
+
return;
76
+
}
77
+
};
78
+
if events.is_empty() {
79
+
return;
80
+
}
81
+
let batch_len = events.len();
82
+
for event in events {
83
+
let seq = event.seq.as_i64();
103
84
let firehose_event = to_firehose_event(event);
104
-
match state.firehose_tx.send(firehose_event) {
105
-
Ok(receiver_count) => {
106
-
debug!(
107
-
seq = seq_id,
108
-
receivers = receiver_count,
109
-
"Broadcast event to firehose"
110
-
);
111
-
}
112
-
Err(e) => {
113
-
warn!(seq = seq_id, error = %e, "Failed to broadcast event (no receivers?)");
114
-
}
115
-
}
116
-
LAST_BROADCAST_SEQ.store(seq.as_i64(), Ordering::SeqCst);
117
-
} else {
118
-
warn!(
119
-
seq = seq_id,
120
-
"Received notification but could not find row in repo_seq"
121
-
);
85
+
let _ = state.firehose_tx.send(firehose_event);
86
+
*last_seq = seq;
87
+
LAST_BROADCAST_SEQ.store(seq, Ordering::SeqCst);
88
+
}
89
+
if (batch_len as i64) < DRAIN_BATCH_SIZE {
90
+
return;
122
91
}
123
92
}
124
93
}
+15
migrations/20260529_firehose_outbox_sequencing.sql
+15
migrations/20260529_firehose_outbox_sequencing.sql
···
1
+
ALTER TABLE repo_seq ADD COLUMN id BIGSERIAL;
2
+
3
+
ALTER TABLE repo_seq DROP CONSTRAINT repo_seq_pkey;
4
+
ALTER TABLE repo_seq ADD PRIMARY KEY (id);
5
+
6
+
ALTER TABLE repo_seq ALTER COLUMN seq DROP DEFAULT;
7
+
ALTER TABLE repo_seq ALTER COLUMN seq DROP NOT NULL;
8
+
9
+
DROP INDEX IF EXISTS idx_repo_seq_seq;
10
+
ALTER TABLE repo_seq ADD CONSTRAINT repo_seq_seq_key UNIQUE (seq);
11
+
12
+
CREATE SEQUENCE IF NOT EXISTS firehose_seq;
13
+
SELECT setval('firehose_seq', (SELECT COALESCE(MAX(seq), 0) + 1 FROM repo_seq), false);
14
+
15
+
CREATE INDEX idx_repo_seq_unsequenced ON repo_seq (id) WHERE seq IS NULL;