solpbc.org
personal memory agent
1#!/usr/bin/env python3
2# SPDX-License-Identifier: AGPL-3.0-only
3# Copyright (c) 2026 sol pbc
4
5"""Check native-sol parity coverage against current HTTP authorities."""
6
7from __future__ import annotations
8
9import json
10import subprocess
11import sys
12from collections import Counter
13from pathlib import Path
14from typing import Any
15
16try:
17 from scripts.build_native_sol_inventory import (
18 FINAL_HTTP_TOTAL,
19 FINAL_STUB_COUNTS,
20 FINAL_TOP_LEVEL_CHAT_TOTAL,
21 FINAL_TOP_LEVEL_IMPORT_TOTAL,
22 FINAL_TOP_LEVEL_LINK_TOTAL,
23 FINAL_TOP_LEVEL_NOTIFY_TOTAL,
24 REPO_ROOT,
25 discover,
26 )
27except ModuleNotFoundError: # pragma: no cover - direct script execution path.
28 from build_native_sol_inventory import ( # type: ignore[no-redef]
29 FINAL_HTTP_TOTAL,
30 FINAL_STUB_COUNTS,
31 FINAL_TOP_LEVEL_CHAT_TOTAL,
32 FINAL_TOP_LEVEL_IMPORT_TOTAL,
33 FINAL_TOP_LEVEL_LINK_TOTAL,
34 FINAL_TOP_LEVEL_NOTIFY_TOTAL,
35 REPO_ROOT,
36 discover,
37 )
38
39SCHEMA = "native-sol-applicability-v1"
40APPLICABILITY = REPO_ROOT / "core/fixtures/native-sol/applicability.json"
41PARITY_DIR = REPO_ROOT / "core/fixtures/native-sol/parity"
42RUST_MANIFEST = REPO_ROOT / "core/Cargo.toml"
43
44
45def main() -> int:
46 errors = check_coverage()
47 if errors:
48 print("native sol coverage failed:", file=sys.stderr)
49 for error in errors:
50 print(f"- {error}", file=sys.stderr)
51 return 1
52 print("native sol coverage ok")
53 return 0
54
55
56def check_coverage(root: Path = REPO_ROOT) -> list[str]:
57 entries = discover(root)
58 required = {
59 entry.operation_id
60 for entry in entries
61 if entry.surface == "sol-call" and entry.entry_type == "http"
62 }
63 required_stubs = {
64 entry.operation_id
65 for entry in entries
66 if entry.surface == "sol-call" and entry.entry_type in {"moved-stub", "local"}
67 }
68 required_top_level_chat = {
69 entry.operation_id
70 for entry in entries
71 if entry.surface == "sol-chat" and entry.entry_type == "top-level-chat"
72 }
73 required_top_level_import = {
74 entry.operation_id
75 for entry in entries
76 if entry.surface == "sol-import" and entry.entry_type == "top-level-import"
77 }
78 required_top_level_notify = {
79 entry.operation_id
80 for entry in entries
81 if entry.surface == "sol-notify" and entry.entry_type == "top-level-notify"
82 }
83 required_top_level_link = {
84 entry.operation_id
85 for entry in entries
86 if entry.surface == "sol-link" and entry.entry_type == "top-level-link"
87 }
88 required_dispatch = (
89 required
90 | required_stubs
91 | required_top_level_chat
92 | required_top_level_import
93 | required_top_level_link
94 | required_top_level_notify
95 )
96 vectors = load_vectors(PARITY_DIR)
97 resolved = resolve_vectors(PARITY_DIR, vectors)
98 applicability, applicability_errors = load_applicability(APPLICABILITY)
99
100 errors: list[str] = []
101 errors.extend(applicability_errors)
102 if len(required) != FINAL_HTTP_TOTAL:
103 errors.append(
104 f"current HTTP authority count {len(required)} != {FINAL_HTTP_TOTAL}"
105 )
106 stub_counts = Counter(
107 entry.entry_type
108 for entry in entries
109 if entry.surface == "sol-call" and entry.entry_type in FINAL_STUB_COUNTS
110 )
111 for entry_type, expected in sorted(FINAL_STUB_COUNTS.items()):
112 if stub_counts[entry_type] != expected:
113 errors.append(
114 f"current {entry_type} authority count {stub_counts[entry_type]} "
115 f"!= {expected}"
116 )
117 if len(required_top_level_chat) != FINAL_TOP_LEVEL_CHAT_TOTAL:
118 errors.append(
119 f"current top-level chat authority count {len(required_top_level_chat)} "
120 f"!= {FINAL_TOP_LEVEL_CHAT_TOTAL}"
121 )
122 if not applicability_errors:
123 entries = applicability["entries"]
124 keys = set(entries)
125 http_count = applicability.get("http_count")
126 if http_count != FINAL_HTTP_TOTAL:
127 errors.append(
128 f"applicability http_count {http_count!r} != {FINAL_HTTP_TOTAL}"
129 )
130 if http_count != len(entries):
131 errors.append(
132 f"applicability http_count {http_count!r} != entries {len(entries)}"
133 )
134 if http_count != len(required):
135 errors.append(
136 f"applicability http_count {http_count!r} != current HTTP "
137 f"authority count {len(required)}"
138 )
139 errors.extend(compare_sets("applicability keys", required, keys))
140 top_level_entries = applicability.get("top_level_entries", {})
141 if not isinstance(top_level_entries, dict):
142 errors.append("applicability top_level_entries must be an object")
143 top_level_entries = {}
144 import_keys = set(top_level_entries)
145 errors.extend(
146 compare_sets(
147 "top-level import applicability keys",
148 required_top_level_import,
149 import_keys,
150 )
151 )
152
153 if len(required_top_level_import) != FINAL_TOP_LEVEL_IMPORT_TOTAL:
154 errors.append(
155 f"current top-level import authority count {len(required_top_level_import)} "
156 f"!= {FINAL_TOP_LEVEL_IMPORT_TOTAL}"
157 )
158 if len(required_top_level_notify) != FINAL_TOP_LEVEL_NOTIFY_TOTAL:
159 errors.append(
160 f"current top-level notify authority count {len(required_top_level_notify)} "
161 f"!= {FINAL_TOP_LEVEL_NOTIFY_TOTAL}"
162 )
163 if len(required_top_level_link) != FINAL_TOP_LEVEL_LINK_TOTAL:
164 errors.append(
165 f"current top-level link authority count {len(required_top_level_link)} "
166 f"!= {FINAL_TOP_LEVEL_LINK_TOTAL}"
167 )
168 if not required_dispatch:
169 errors.append("native dispatch authority set is empty")
170 resolved_operations = {
171 str(item["operation_id"])
172 for item in resolved.values()
173 if item.get("operation_id") is not None
174 }
175 missing_dispatch = sorted(required_dispatch - resolved_operations)
176 if missing_dispatch:
177 errors.append(
178 "production aggregate dispatch missing required operations "
179 f"{missing_dispatch!r}"
180 )
181
182 buckets = collect_buckets(vectors, resolved, required, {"http"}, errors)
183 for bucket_name in ("request_binding", "success", "failure"):
184 errors.extend(compare_sets(bucket_name, required, buckets[bucket_name]))
185 import_buckets = collect_buckets(
186 vectors,
187 resolved,
188 required_top_level_import,
189 {"top-level-import"},
190 errors,
191 )
192 for bucket_name in ("request_binding", "success", "failure"):
193 errors.extend(
194 compare_sets(
195 f"top-level import {bucket_name}",
196 required_top_level_import,
197 import_buckets[bucket_name],
198 )
199 )
200 notify_buckets = collect_buckets(
201 vectors,
202 resolved,
203 required_top_level_notify,
204 {"top-level-notify"},
205 errors,
206 )
207 for bucket_name in ("notification_binding", "success", "failure"):
208 errors.extend(
209 compare_sets(
210 f"top-level notify {bucket_name}",
211 required_top_level_notify,
212 notify_buckets[bucket_name],
213 )
214 )
215 link_buckets = collect_buckets(
216 vectors,
217 resolved,
218 required_top_level_link,
219 {"top-level-link"},
220 errors,
221 )
222 for bucket_name in ("success", "failure"):
223 errors.extend(
224 compare_sets(
225 f"top-level link {bucket_name}",
226 required_top_level_link,
227 link_buckets[bucket_name],
228 )
229 )
230
231 if not applicability_errors:
232 errors.extend(
233 check_applicability_requirements(
234 applicability["entries"], vectors, resolved, buckets
235 )
236 )
237 errors.extend(
238 check_top_level_import_cases(
239 applicability.get("top_level_entries", {}),
240 vectors,
241 resolved,
242 )
243 )
244 return errors
245
246
247def load_applicability(path: Path) -> tuple[dict[str, Any], list[str]]:
248 try:
249 payload = json.loads(path.read_text(encoding="utf-8"))
250 except FileNotFoundError:
251 return {}, [f"{path.relative_to(REPO_ROOT)} is missing"]
252 except json.JSONDecodeError as error:
253 return {}, [f"{path.relative_to(REPO_ROOT)} is malformed JSON: {error}"]
254 errors: list[str] = []
255 if payload.get("schema") != SCHEMA:
256 errors.append(f"applicability schema must be {SCHEMA!r}")
257 entries = payload.get("entries")
258 if not isinstance(entries, dict):
259 errors.append("applicability entries must be an object")
260 else:
261 for operation_id, entry in entries.items():
262 if not isinstance(operation_id, str) or not operation_id:
263 errors.append("applicability operation ids must be non-empty strings")
264 if not isinstance(entry, dict):
265 errors.append(f"{operation_id}: applicability entry must be an object")
266 continue
267 for key in (
268 "path",
269 "group",
270 "pagination",
271 "mutation",
272 "upload",
273 "env_default",
274 "confirmation",
275 "consent",
276 "dry_run",
277 "multi_request",
278 ):
279 if key not in entry:
280 errors.append(f"{operation_id}: missing applicability field {key}")
281 top_level_entries = payload.get("top_level_entries", {})
282 if top_level_entries is not None and not isinstance(top_level_entries, dict):
283 errors.append("applicability top_level_entries must be an object")
284 return payload, errors
285
286
287def load_vectors(directory: Path) -> dict[str, dict[str, Any]]:
288 vectors: dict[str, dict[str, Any]] = {}
289 for path in sorted(directory.glob("*.jsonl")):
290 for line_number, line in enumerate(
291 path.read_text(encoding="utf-8").splitlines(), 1
292 ):
293 if not line.strip():
294 continue
295 vector = json.loads(line)
296 vector_id = str(vector.get("id") or "")
297 if not vector_id:
298 raise ValueError(
299 f"{path.relative_to(REPO_ROOT)}:{line_number}: missing id"
300 )
301 if vector_id in vectors:
302 raise ValueError(
303 f"{path.relative_to(REPO_ROOT)}:{line_number}: duplicate id {vector_id}"
304 )
305 vector["_fixture"] = str(path.relative_to(REPO_ROOT))
306 vectors[vector_id] = vector
307 return vectors
308
309
310def resolve_vectors(
311 directory: Path, vectors: dict[str, dict[str, Any]]
312) -> dict[str, dict[str, Any]]:
313 paths = sorted(directory.glob("*.jsonl"))
314 completed = subprocess.run(
315 [
316 "cargo",
317 "run",
318 "--quiet",
319 "--manifest-path",
320 str(RUST_MANIFEST),
321 "-p",
322 "solstone-core-sol-client-cli",
323 "--bin",
324 "resolve_parity_leaves",
325 "--",
326 *(str(path) for path in paths),
327 ],
328 check=False,
329 cwd=REPO_ROOT,
330 text=True,
331 stdout=subprocess.PIPE,
332 stderr=subprocess.PIPE,
333 )
334 if completed.returncode != 0:
335 raise RuntimeError(
336 "production parity leaf resolver failed:\n" + completed.stderr
337 )
338 resolved: dict[str, dict[str, Any]] = {}
339 for line in completed.stdout.splitlines():
340 if not line.strip():
341 continue
342 item = json.loads(line)
343 vector_id = str(item["id"])
344 if vector_id not in vectors:
345 raise RuntimeError(f"resolver returned unknown vector id {vector_id}")
346 resolved[vector_id] = item
347 missing = sorted(set(vectors) - set(resolved))
348 if missing:
349 raise RuntimeError(f"resolver omitted vectors {missing!r}")
350 return resolved
351
352
353def collect_buckets(
354 vectors: dict[str, dict[str, Any]],
355 resolved: dict[str, dict[str, Any]],
356 required: set[str],
357 entry_types: set[str],
358 errors: list[str],
359) -> dict[str, set[str]]:
360 buckets: dict[str, set[str]] = {
361 "request_binding": set(),
362 "notification_binding": set(),
363 "success": set(),
364 "failure": set(),
365 }
366 for vector_id, vector in vectors.items():
367 operation_id = resolved[vector_id].get("operation_id")
368 entry_type = resolved[vector_id].get("entry_type")
369 if vector.get("surface", "sol-call") == "sol-call" and operation_id is None:
370 errors.append(
371 f"{vector_id}: argv did not resolve through production dispatch"
372 )
373 continue
374 if entry_type not in entry_types or operation_id not in required:
375 continue
376 expected = vector.get("expected") or {}
377 requests = expected.get("requests") if isinstance(expected, dict) else None
378 notifications = (
379 expected.get("notifications") if isinstance(expected, dict) else None
380 )
381 if pins_request_shape(requests):
382 buckets["request_binding"].add(operation_id)
383 if pins_notification_shape(notifications):
384 buckets["notification_binding"].add(operation_id)
385 if is_failure_vector(vector, expected):
386 buckets["failure"].add(operation_id)
387 elif is_success_vector(
388 expected, vector.get("transport", {}).get("requests", [])
389 ):
390 buckets["success"].add(operation_id)
391 return buckets
392
393
394def pins_notification_shape(notifications: Any) -> bool:
395 return isinstance(notifications, list) and any(
396 isinstance(item, str) and item.endswith("\n") for item in notifications
397 )
398
399
400def pins_request_shape(requests: Any) -> bool:
401 if not isinstance(requests, list):
402 return False
403 return any(
404 isinstance(request, dict)
405 and isinstance(request.get("method"), str)
406 and isinstance(request.get("path"), str)
407 for request in requests
408 )
409
410
411def is_failure_vector(vector: dict[str, Any], expected: Any) -> bool:
412 if isinstance(expected, dict) and expected.get("exit") != 0:
413 return True
414 for request in vector.get("transport", {}).get("requests", []):
415 if not isinstance(request, dict):
416 continue
417 if "fault" in request:
418 return True
419 status = (
420 request.get("response", {}).get("status")
421 if isinstance(request.get("response"), dict)
422 else None
423 )
424 if status is not None and not (200 <= int(status) <= 299):
425 return True
426 return False
427
428
429def is_success_vector(expected: Any, transport_requests: Any) -> bool:
430 if not isinstance(expected, dict) or expected.get("exit") != 0:
431 return False
432 if not isinstance(transport_requests, list):
433 return False
434 for request in transport_requests:
435 if not isinstance(request, dict) or "fault" in request:
436 return False
437 response = request.get("response")
438 if not isinstance(response, dict):
439 return False
440 status = int(response.get("status", 200))
441 if not (200 <= status <= 299):
442 return False
443 return True
444
445
446def compare_sets(label: str, required: set[str], actual: set[str]) -> list[str]:
447 errors: list[str] = []
448 missing = sorted(required - actual)
449 extra = sorted(actual - required)
450 if missing:
451 errors.append(f"{label} missing {missing!r}")
452 if extra:
453 errors.append(f"{label} extra {extra!r}")
454 return errors
455
456
457def check_top_level_import_cases(
458 entries: Any,
459 vectors: dict[str, dict[str, Any]],
460 resolved: dict[str, dict[str, Any]],
461) -> list[str]:
462 if not isinstance(entries, dict):
463 return []
464 errors: list[str] = []
465 for operation_id, entry in sorted(entries.items()):
466 if not isinstance(entry, dict):
467 errors.append(
468 f"{operation_id}: top-level applicability entry must be an object"
469 )
470 continue
471 case_ids = entry.get("case_ids", {})
472 if not isinstance(case_ids, dict):
473 errors.append(f"{operation_id}: top-level case_ids must be an object")
474 continue
475 for case_name, ids in sorted(case_ids.items()):
476 if not isinstance(ids, list) or not ids:
477 errors.append(
478 f"{operation_id}: top-level case {case_name} must be non-empty"
479 )
480 continue
481 for vector_id in ids:
482 if vector_id not in vectors:
483 errors.append(
484 f"{operation_id}: top-level case {case_name} unknown vector {vector_id!r}"
485 )
486 continue
487 mapped = resolved[vector_id].get("operation_id")
488 if mapped != operation_id:
489 errors.append(
490 f"{operation_id}: top-level case {case_name} vector "
491 f"{vector_id!r} maps to {mapped!r}"
492 )
493 return errors
494
495
496def check_applicability_requirements(
497 entries: dict[str, dict[str, Any]],
498 vectors: dict[str, dict[str, Any]],
499 resolved: dict[str, dict[str, Any]],
500 buckets: dict[str, set[str]],
501) -> list[str]:
502 errors: list[str] = []
503 for operation_id, entry in sorted(entries.items()):
504 for dimension in (
505 "pagination",
506 "upload",
507 "env_default",
508 "dry_run",
509 "multi_request",
510 ):
511 payload = entry.get(dimension)
512 if not isinstance(payload, dict) or not payload.get("enabled"):
513 continue
514 errors.extend(
515 check_named_cases(operation_id, dimension, payload, vectors, resolved)
516 )
517 if bool(entry.get("upload", {}).get("enabled")):
518 upload = entry["upload"]
519 operation_vectors = mapped_vectors(vectors, resolved, operation_id)
520 if not any(
521 "UPLOAD" in expected_request_methods(vector)
522 for vector in operation_vectors
523 ):
524 errors.append(f"{operation_id}: upload=true but no UPLOAD vector")
525 errors.extend(
526 check_required_case_names(
527 operation_id,
528 "upload",
529 upload,
530 ["missing_file", "unreadable_file", "rejection_before_mutation"],
531 )
532 )
533 if upload.get("file_count") == "multi" and not any(
534 has_later_upload_failure(vector) for vector in operation_vectors
535 ):
536 errors.append(
537 f"{operation_id}: upload multi has no later-file-failure vector"
538 )
539 if bool(entry.get("multi_request", {}).get("enabled")):
540 multi_request = entry["multi_request"]
541 operation_vectors = mapped_vectors(vectors, resolved, operation_id)
542 if not any(
543 expected_request_count(vector) > 1 for vector in operation_vectors
544 ):
545 errors.append(
546 f"{operation_id}: multi_request=true but no multi-request vector"
547 )
548 if multi_request_can_exceed_one(
549 multi_request.get("request_count")
550 ) and not any(
551 has_later_boundary_failure(vector) for vector in operation_vectors
552 ):
553 errors.append(
554 f"{operation_id}: multi_request has no later-boundary-failure vector"
555 )
556 if bool(entry.get("env_default", {}).get("enabled")):
557 env_default = entry["env_default"]
558 mode = env_default.get("mode", "required")
559 if mode in {"valid_absent", "valid-absent"}:
560 required_cases = ["explicit", "valid_absent"]
561 elif mode == "required":
562 required_cases = ["explicit", "absent"]
563 elif mode in {"required_with_invalid", "required-with-invalid"}:
564 required_cases = ["explicit", "absent", "invalid"]
565 else:
566 errors.append(
567 f"{operation_id}: env_default.mode {mode!r} is unsupported"
568 )
569 required_cases = []
570 errors.extend(
571 check_required_case_names(
572 operation_id, "env_default", env_default, required_cases
573 )
574 )
575 if bool(entry.get("dry_run", {}).get("enabled")):
576 if operation_id not in buckets["success"]:
577 errors.append(f"{operation_id}: dry_run=true but no success coverage")
578 if operation_id not in buckets["failure"]:
579 errors.append(f"{operation_id}: dry_run=true but no failure coverage")
580 return errors
581
582
583def check_required_case_names(
584 operation_id: str,
585 dimension: str,
586 payload: dict[str, Any],
587 required_cases: list[str],
588) -> list[str]:
589 case_ids = payload.get("case_ids", {})
590 if not isinstance(case_ids, dict):
591 return []
592 errors = []
593 for case_name in required_cases:
594 ids = case_ids.get(case_name)
595 if not isinstance(ids, list) or not ids:
596 errors.append(
597 f"{operation_id}: {dimension} missing required case {case_name}"
598 )
599 return errors
600
601
602def check_named_cases(
603 operation_id: str,
604 dimension: str,
605 payload: dict[str, Any],
606 vectors: dict[str, dict[str, Any]],
607 resolved: dict[str, dict[str, Any]],
608) -> list[str]:
609 errors: list[str] = []
610 case_ids = payload.get("case_ids", {})
611 if not isinstance(case_ids, dict):
612 errors.append(f"{operation_id}: {dimension}.case_ids must be an object")
613 return errors
614 for case_name, ids in sorted(case_ids.items()):
615 if not isinstance(ids, list) or not ids:
616 errors.append(
617 f"{operation_id}: {dimension}.case_ids.{case_name} must be non-empty"
618 )
619 continue
620 for vector_id in ids:
621 if vector_id not in vectors:
622 errors.append(
623 f"{operation_id}: {dimension}.case_ids.{case_name} unknown "
624 f"vector {vector_id!r}"
625 )
626 continue
627 mapped = resolved[vector_id].get("operation_id")
628 if mapped != operation_id:
629 errors.append(
630 f"{operation_id}: {dimension}.case_ids.{case_name} vector "
631 f"{vector_id!r} maps to {mapped!r}"
632 )
633 return errors
634
635
636def mapped_vectors(
637 vectors: dict[str, dict[str, Any]],
638 resolved: dict[str, dict[str, Any]],
639 operation_id: str,
640) -> list[dict[str, Any]]:
641 return [
642 vector
643 for vector_id, vector in vectors.items()
644 if resolved[vector_id].get("operation_id") == operation_id
645 ]
646
647
648def expected_request_methods(vector: dict[str, Any]) -> set[str]:
649 requests = vector.get("expected", {}).get("requests", [])
650 methods: set[str] = set()
651 for request in requests if isinstance(requests, list) else []:
652 if isinstance(request, dict) and isinstance(request.get("method"), str):
653 methods.add(request["method"])
654 return methods
655
656
657def expected_request_count(vector: dict[str, Any]) -> int:
658 requests = vector.get("expected", {}).get("requests", [])
659 return len(requests) if isinstance(requests, list) else 0
660
661
662def multi_request_can_exceed_one(value: Any) -> bool:
663 if isinstance(value, int):
664 return value > 1
665 if not isinstance(value, str):
666 return False
667 if value == "single":
668 return False
669 if ".." not in value:
670 try:
671 return int(value) > 1
672 except ValueError:
673 return True
674 _minimum, maximum = value.split("..", 1)
675 if maximum == "n":
676 return True
677 try:
678 return int(maximum) > 1
679 except ValueError:
680 return True
681
682
683def has_later_boundary_failure(vector: dict[str, Any]) -> bool:
684 requests = transport_requests(vector)
685 for index, request in enumerate(requests):
686 if index == 0 or not request_failed(request):
687 continue
688 if all(request_succeeded(previous) for previous in requests[:index]):
689 return True
690 return False
691
692
693def has_later_upload_failure(vector: dict[str, Any]) -> bool:
694 requests = transport_requests(vector)
695 upload_indices = [
696 index
697 for index, request in enumerate(requests)
698 if isinstance(request, dict) and request.get("method") == "UPLOAD"
699 ]
700 for upload_position, request_index in enumerate(upload_indices):
701 if upload_position == 0 or not request_failed(requests[request_index]):
702 continue
703 prior_uploads = (requests[index] for index in upload_indices[:upload_position])
704 if all(request_succeeded(request) for request in prior_uploads):
705 return True
706 return False
707
708
709def transport_requests(vector: dict[str, Any]) -> list[dict[str, Any]]:
710 requests = vector.get("transport", {}).get("requests", [])
711 return [request for request in requests if isinstance(request, dict)]
712
713
714def request_failed(request: dict[str, Any]) -> bool:
715 if "fault" in request:
716 return True
717 status = response_status(request)
718 return status is not None and not (200 <= status <= 299)
719
720
721def request_succeeded(request: dict[str, Any]) -> bool:
722 if "fault" in request:
723 return False
724 status = response_status(request)
725 return status is not None and 200 <= status <= 299
726
727
728def response_status(request: dict[str, Any]) -> int | None:
729 response = request.get("response")
730 if not isinstance(response, dict):
731 return None
732 return int(response.get("status", 200))
733
734
735if __name__ == "__main__":
736 raise SystemExit(main())