solpbc.org
personal memory agent
1# SPDX-License-Identifier: AGPL-3.0-only
2# Copyright (c) 2026 sol pbc
3
4"""Utilities for API response baseline verification."""
5
6from __future__ import annotations
7
8import argparse
9import json
10import os
11import re
12from contextlib import contextmanager
13from difflib import unified_diff
14from pathlib import Path
15from tempfile import TemporaryDirectory
16from typing import Any
17
18from freezegun import freeze_time
19
20try:
21 from tests._baseline_harness import (
22 FROZEN_DATE,
23 FROZEN_TZ_OFFSET,
24 isolated_app_env,
25 make_test_client,
26 prepare_isolated_journal,
27 )
28except ModuleNotFoundError:
29 from _baseline_harness import (
30 FROZEN_DATE,
31 FROZEN_TZ_OFFSET,
32 isolated_app_env,
33 make_test_client,
34 prepare_isolated_journal,
35 )
36
37ENDPOINTS = [
38 # convey/config.py
39 {
40 "app": "config",
41 "name": "convey",
42 "path": "/api/config/convey",
43 "params": {},
44 "status": 200,
45 },
46 # apps/sol/routes.py
47 {
48 "app": "sol",
49 "name": "talents-day",
50 "path": "/app/sol/api/talents/20260304",
51 "params": {"facet": "work"},
52 "status": 200,
53 },
54 {
55 "app": "sol",
56 "name": "run-detail",
57 "path": "/app/sol/api/run/1700000000001",
58 "params": {},
59 "status": 200,
60 },
61 {
62 "app": "sol",
63 "name": "preview",
64 "path": "/app/sol/api/preview/chat",
65 "params": {},
66 "status": 200,
67 },
68 {
69 "app": "sol",
70 "name": "stats-month",
71 "path": "/app/sol/api/stats/202603",
72 "params": {},
73 "status": 200,
74 },
75 {
76 "app": "sol",
77 "name": "badge-count",
78 "path": "/app/sol/api/badge-count",
79 "params": {},
80 "status": 200,
81 "sandbox_only": True, # reads date.today() live + sandbox produces boot-time talent runs
82 },
83 {
84 "app": "sol",
85 "name": "updated-days",
86 "path": "/app/sol/api/updated-days",
87 "params": {},
88 "status": 200,
89 "sandbox_only": True, # live indexer computes differently than Flask test client
90 },
91 # convey/chat.py
92 {
93 "app": "chat",
94 "name": "session",
95 "path": "/api/chat/session",
96 "params": {},
97 "status": 200,
98 },
99 # apps/activities/routes.py
100 {
101 "app": "activities",
102 "name": "stats-month",
103 "path": "/app/activities/api/stats/202603",
104 "params": {},
105 "status": 200,
106 },
107 {
108 "app": "activities",
109 "name": "day-activities",
110 "path": "/app/activities/api/day/20260304/activities",
111 "params": {"facet": "work"},
112 "status": 200,
113 },
114 # apps/entities/routes.py
115 {
116 "app": "entities",
117 "name": "facet-entities",
118 "path": "/app/entities/api/work",
119 "params": {},
120 "status": 200,
121 },
122 {
123 "app": "entities",
124 "name": "entity-detail",
125 "path": "/app/entities/api/work/entity/romeo_montague",
126 "params": {},
127 "status": 200,
128 },
129 {
130 "app": "entities",
131 "name": "entity-types",
132 "path": "/app/entities/api/types",
133 "params": {},
134 "status": 200,
135 },
136 {
137 "app": "entities",
138 "name": "journal-entities",
139 "path": "/app/entities/api/journal",
140 "params": {},
141 "status": 200,
142 },
143 {
144 "app": "entities",
145 "name": "journal-entity-detail",
146 "path": "/app/entities/api/journal/entity/first_test_entity",
147 "params": {},
148 "status": 200,
149 },
150 {
151 "app": "entities",
152 "name": "detected-preview",
153 "path": "/app/entities/api/work/detected/preview",
154 "params": {"name": "Romeo"},
155 "status": 200,
156 },
157 # apps/import/routes.py
158 {
159 "app": "import",
160 "name": "list",
161 "path": "/app/import/api/list",
162 "params": {},
163 "status": 200,
164 },
165 {
166 "app": "import",
167 "name": "import-day",
168 "path": "/app/import/api/20260304",
169 "params": {},
170 "status": 404,
171 },
172 # apps/observer/routes.py
173 {
174 "app": "observer",
175 "name": "list",
176 "path": "/app/observer/api/list",
177 "params": {},
178 "status": 200,
179 },
180 {
181 "app": "observer",
182 "name": "observer-key",
183 "path": "/app/observer/api/example-key/key",
184 "params": {},
185 "status": 404,
186 },
187 {
188 "app": "observer",
189 "name": "ingest-day",
190 "path": "/app/observer/ingest/segments/20260304",
191 "params": {},
192 "status": 401,
193 },
194 # apps/search/routes.py
195 {
196 "app": "search",
197 "name": "search",
198 "path": "/app/search/api/search",
199 "params": {"q": "romeo", "limit": "5", "offset": "0"},
200 "status": 200,
201 "sandbox_only": True,
202 },
203 {
204 "app": "search",
205 "name": "day-results",
206 "path": "/app/search/api/day_results",
207 "params": {"q": "meeting", "day": "20260304", "offset": "0", "limit": "5"},
208 "status": 200,
209 "sandbox_only": True,
210 },
211 # apps/settings/routes.py
212 {
213 "app": "settings",
214 "name": "config",
215 "path": "/app/settings/api/config",
216 "params": {},
217 "status": 200,
218 },
219 {
220 "app": "settings",
221 "name": "transcribe",
222 "path": "/app/settings/api/transcribe",
223 "params": {},
224 "status": 200,
225 },
226 {
227 "app": "thinking",
228 "name": "providers",
229 "path": "/app/thinking/api/providers",
230 "params": {},
231 "status": 200,
232 },
233 {
234 "app": "thinking",
235 "name": "generators",
236 "path": "/app/thinking/api/generators",
237 "params": {},
238 "status": 200,
239 },
240 {
241 "app": "settings",
242 "name": "vision",
243 "path": "/app/settings/api/vision",
244 "params": {},
245 "status": 200,
246 },
247 {
248 "app": "settings",
249 "name": "observe",
250 "path": "/app/settings/api/observe",
251 "params": {},
252 "status": 200,
253 },
254 {
255 "app": "settings",
256 "name": "facet",
257 "path": "/app/settings/api/facet/montague",
258 "params": {},
259 "status": 200,
260 },
261 {
262 "app": "settings",
263 "name": "activities-defaults",
264 "path": "/app/settings/api/activities/defaults",
265 "params": {},
266 "status": 200,
267 },
268 {
269 "app": "settings",
270 "name": "facet-activities",
271 "path": "/app/settings/api/facet/montague/activities",
272 "params": {},
273 "status": 200,
274 },
275 {
276 "app": "settings",
277 "name": "sync",
278 "path": "/app/settings/api/sync",
279 "params": {},
280 "status": 200,
281 },
282 # apps/speakers/routes.py
283 {
284 "app": "speakers",
285 "name": "stats-month",
286 "path": "/app/speakers/api/stats/202603",
287 "params": {},
288 "status": 200,
289 },
290 {
291 "app": "speakers",
292 "name": "segments",
293 "path": "/app/speakers/api/segments/20260304",
294 "params": {},
295 "status": 200,
296 },
297 {
298 "app": "speakers",
299 "name": "speakers-segment",
300 "path": "/app/speakers/api/speakers/20260304/default/090000_300",
301 "params": {},
302 "status": 200,
303 },
304 {
305 "app": "speakers",
306 "name": "review",
307 "path": "/app/speakers/api/review/20260304/default/090000_300/audio",
308 "params": {},
309 "status": 200,
310 },
311 # apps/stats/routes.py
312 {
313 "app": "stats",
314 "name": "stats",
315 "path": "/app/stats/api/stats",
316 "params": {},
317 "status": 200,
318 },
319 # apps/tokens/routes.py
320 {
321 "app": "tokens",
322 "name": "usage",
323 "path": "/app/tokens/api/usage",
324 "params": {"day": "20260304"},
325 "status": 200,
326 },
327 {
328 "app": "tokens",
329 "name": "stats-month",
330 "path": "/app/tokens/api/stats/202603",
331 "params": {},
332 "status": 200,
333 },
334 {
335 "app": "tokens",
336 "name": "daily",
337 "path": "/app/tokens/api/daily",
338 "params": {"days": "14"},
339 "status": 200,
340 },
341 # apps/transcripts/routes.py
342 {
343 "app": "transcripts",
344 "name": "ranges",
345 "path": "/app/transcripts/api/ranges/20260304",
346 "params": {},
347 "status": 200,
348 },
349 {
350 "app": "transcripts",
351 "name": "segments",
352 "path": "/app/transcripts/api/segments/20260304",
353 "params": {},
354 "status": 200,
355 },
356 {
357 "app": "transcripts",
358 "name": "segment-detail",
359 "path": "/app/transcripts/api/segment/20260304/default/090000_300",
360 "params": {},
361 "status": 200,
362 },
363 {
364 "app": "transcripts",
365 "name": "stats-month",
366 "path": "/app/transcripts/api/stats/202603",
367 "params": {},
368 "status": 200,
369 },
370]
371
372
373def normalize(data: Any, journal_path: str) -> Any:
374 """Return a normalized copy of endpoint JSON for deterministic baselines."""
375
376 resolved_journal = str(Path(journal_path).resolve())
377 project_root = str(Path(__file__).resolve().parent.parent)
378
379 # Journal path contains project root, so replace journal first (longer match)
380 path_replacements: list[tuple[str, str]] = []
381 path_replacements.append((resolved_journal, "<JOURNAL>"))
382 # If the fixture journal resolves differently (e.g., symlinks), add that too
383 fixture_journal = str((Path.cwd() / "tests" / "fixtures" / "journal").resolve())
384 if fixture_journal != resolved_journal:
385 path_replacements.append((fixture_journal, "<JOURNAL>"))
386 # Also match the raw (possibly relative) journal_path
387 raw_journal = str(journal_path)
388 if raw_journal != resolved_journal:
389 path_replacements.append((raw_journal, "<JOURNAL>"))
390 # Match the SOLSTONE_JOURNAL env var if set (may be relative)
391 env_journal = os.environ.get("SOLSTONE_JOURNAL", "")
392 if env_journal and env_journal not in (resolved_journal, raw_journal):
393 path_replacements.append((env_journal, "<JOURNAL>"))
394 path_replacements.append((project_root, "<PROJECT>"))
395 # Sort by length descending so longer (more specific) paths match first
396 path_replacements.sort(key=lambda x: len(x[0]), reverse=True)
397
398 def _normalize_string(value: str) -> str:
399 result = value
400 for path, replacement in path_replacements:
401 result = result.replace(path, replacement)
402 # Normalize dynamic timestamp in prompt content
403 result = re.sub(
404 r"^Today is .*", "Today is <TIMESTAMP>", result, flags=re.MULTILINE
405 )
406 return result
407
408 def walk(value: Any, key: str | None = None) -> Any:
409 if isinstance(value, dict):
410 result = {
411 item_key: (
412 0
413 if item_key in {"mtime", "created_at", "file_mtime"}
414 and isinstance(item_value, (int, float))
415 else (
416 "<TIMESTAMP>"
417 if item_key == "generated_at" and isinstance(item_value, str)
418 else (
419 round(item_value, 1)
420 if item_key in {"score", "recency"}
421 and isinstance(item_value, float)
422 else walk(item_value, item_key)
423 )
424 )
425 )
426 for item_key, item_value in value.items()
427 }
428 if key == "provider_status":
429 env_keys = {
430 "anthropic": "ANTHROPIC_API_KEY",
431 "google": "GOOGLE_API_KEY",
432 "openai": "OPENAI_API_KEY",
433 }
434 for _name, status in result.items():
435 if _name in env_keys and isinstance(status, dict):
436 status["configured"] = False
437 status["generate_ready"] = False
438 status["cogitate_ready"] = False
439 status["issues"] = [f"{env_keys[_name]} not set"]
440 continue
441 if _name == "local" and isinstance(status, dict):
442 status["cogitate_ready"] = False
443 status["configured"] = False
444 status["generate_ready"] = False
445 issues = [
446 i
447 for i in status.get("issues", [])
448 if "CLI not found" not in i
449 and "not set" not in i
450 and "not reachable" not in i
451 ]
452 local_issues = [
453 i
454 for i in issues
455 if i
456 in {
457 "binary_missing",
458 "gpu_unavailable",
459 "model_missing",
460 "ram_insufficient",
461 "server_unhealthy",
462 }
463 ]
464 for local_issue in ("binary_missing", "model_missing"):
465 if local_issue not in local_issues:
466 local_issues.append(local_issue)
467 local_issues.append("run `journal install-provider local`")
468 status["issues"] = sorted(local_issues)
469 continue
470 # Normalize env-dependent API key presence
471 if key in ("api_keys", "runtime_env"):
472 for k in result:
473 if isinstance(result[k], bool):
474 result[k] = False
475 if key == "resource" and {
476 "available_memory_gb",
477 "detected",
478 "min_ram_gb",
479 "needs_setup",
480 "notice",
481 "requirement",
482 } <= set(result):
483 for item_key in result:
484 result[item_key] = "<normalized>"
485 return result
486
487 if isinstance(value, list):
488 walked = [walk(item, key) for item in value]
489 # Sort lists of dicts for deterministic comparison
490 if walked and all(isinstance(item, dict) for item in walked):
491 try:
492 walked.sort(key=lambda x: json.dumps(x, sort_keys=True))
493 except TypeError:
494 pass
495 return walked
496
497 if isinstance(value, str):
498 return _normalize_string(str(value))
499
500 return value
501
502 return walk(data)
503
504
505def normalize_for_compare(
506 endpoint: dict[str, Any], data: Any, journal_path: str
507) -> Any:
508 """Normalize payloads for deterministic baseline comparison."""
509
510 normalized = normalize(data, journal_path)
511 if endpoint["app"] == "settings" and endpoint["name"] == "transcribe":
512 if isinstance(normalized, dict):
513 # runtime_label is environment-dependent; tested separately.
514 normalized.pop("runtime_label", None)
515 resource = normalized.get("resource")
516 if isinstance(resource, dict):
517 for key in resource:
518 resource[key] = "<normalized>"
519 return normalized
520
521
522def baseline_path(endpoint: dict[str, str]) -> Path:
523 """Compute baseline file path for an endpoint entry."""
524
525 return Path("tests/baselines/api") / endpoint["app"] / f"{endpoint['name']}.json"
526
527
528def _extract_json(response: Any) -> Any:
529 """Load JSON from either Flask response or requests response."""
530
531 if hasattr(response, "get_json"):
532 payload = response.get_json(silent=True)
533 if payload is None:
534 raise ValueError("response is not JSON")
535 return payload
536
537 try:
538 return response.json()
539 except Exception as exc:
540 raise ValueError("response is not JSON") from exc
541
542
543def fetch_endpoint(client: Any, endpoint: dict[str, Any]) -> tuple[int, Any]:
544 """Call endpoint and return (status_code, parsed_json)."""
545
546 response = client.get(endpoint["path"], query_string=endpoint.get("params", {}))
547 return response.status_code, _extract_json(response)
548
549
550def verify_all(client: Any, journal_path: str) -> list[str]:
551 """Compare all endpoint responses against stored baselines."""
552
553 failures: list[str] = []
554 for endpoint in ENDPOINTS:
555 identifier = f"{endpoint['app']}/{endpoint['name']}"
556 path = baseline_path(endpoint)
557
558 try:
559 status, payload = fetch_endpoint(client, endpoint)
560 except Exception as exc:
561 failures.append(f"{identifier}: failed to fetch endpoint: {exc}")
562 continue
563
564 if status != endpoint["status"]:
565 failures.append(
566 f"{identifier}: expected status {endpoint['status']} got {status}"
567 )
568 continue
569
570 if not path.exists():
571 failures.append(f"{identifier}: baseline file not found: {path}")
572 continue
573
574 actual = normalize_for_compare(endpoint, payload, journal_path)
575 expected = normalize_for_compare(
576 endpoint, json.loads(path.read_text()), journal_path
577 )
578 if actual != expected:
579 actual_dump = json.dumps(
580 actual, indent=2, sort_keys=True, ensure_ascii=False
581 ).splitlines(keepends=True)
582 expected_dump = json.dumps(
583 expected, indent=2, sort_keys=True, ensure_ascii=False
584 ).splitlines(keepends=True)
585 diff = "".join(
586 unified_diff(
587 expected_dump,
588 actual_dump,
589 fromfile=f"{identifier} expected",
590 tofile=f"{identifier} actual",
591 lineterm="",
592 )
593 )
594 failures.append(f"{identifier}:\n{diff}")
595
596 return failures
597
598
599def update_all(
600 client: Any,
601 journal_path: str,
602 *,
603 include_sandbox_only: bool,
604) -> int:
605 """Refresh all endpoint baselines from current responses."""
606
607 updated = 0
608 for endpoint in ENDPOINTS:
609 if endpoint.get("sandbox_only") and not include_sandbox_only:
610 continue
611 identifier = f"{endpoint['app']}/{endpoint['name']}"
612 path = baseline_path(endpoint)
613 path.parent.mkdir(parents=True, exist_ok=True)
614
615 status, payload = fetch_endpoint(client, endpoint)
616 if status != endpoint["status"]:
617 print(
618 f"warn: {identifier} returned {status}, expected {endpoint['status']}, "
619 "still writing normalized payload"
620 )
621
622 normalized = normalize(payload, journal_path)
623 path.write_text(
624 json.dumps(normalized, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
625 )
626 updated += 1
627
628 return updated
629
630
631class _HttpClient:
632 """Minimal requests-like object for endpoint fetching."""
633
634 def __init__(self, base_url: str, password: str | None = None) -> None:
635 self.base_url = base_url.rstrip("/")
636 self._auth = ("", password) if password else None
637
638 def get(self, path: str, query_string: dict[str, Any] | None = None):
639 import requests
640
641 return requests.get(
642 f"{self.base_url}{path}", params=query_string, auth=self._auth
643 )
644
645
646def _resolve_journal_path() -> str:
647 """Resolve journal path from env or sandbox metadata."""
648
649 env_path = Path.cwd() / "tests" / "fixtures" / "journal"
650 journal = Path(os.environ.get("SOLSTONE_JOURNAL", str(env_path)))
651 if journal.is_absolute():
652 return str(journal)
653 return str(Path(journal).resolve())
654
655
656def _resolve_sandbox_journal() -> str | None:
657 marker = Path(".sandbox.journal")
658 if not marker.exists():
659 return None
660 value = marker.read_text().strip()
661 if not value:
662 return None
663 return str(Path(value).resolve())
664
665
666def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
667 parser = argparse.ArgumentParser(description="API baseline verification tool")
668 parser.add_argument(
669 "command",
670 choices=["verify", "update"],
671 help="Whether to verify or regenerate baselines",
672 )
673 parser.add_argument(
674 "--base-url",
675 help="Use HTTP mode against this base URL instead of Flask test client",
676 )
677 parser.add_argument(
678 "--password",
679 help="Password for Basic Auth in HTTP mode",
680 )
681 return parser.parse_args(argv)
682
683
684def _resolve_http_journal() -> str:
685 env_path = os.environ.get("SOLSTONE_JOURNAL")
686 if env_path:
687 return str(Path(env_path).resolve())
688 sandbox_path = _resolve_sandbox_journal()
689 if sandbox_path:
690 return sandbox_path
691 return _resolve_journal_path()
692
693
694@contextmanager
695def client_context(
696 base_url: str | None, password: str | None = None
697) -> tuple[Any, str]:
698 if base_url:
699 yield _HttpClient(base_url, password=password), _resolve_http_journal()
700 return
701
702 with TemporaryDirectory(prefix="api-baseline-journal-") as tmpdir:
703 journal_path = prepare_isolated_journal(Path(tmpdir) / "journal")
704 with freeze_time(FROZEN_DATE, tz_offset=FROZEN_TZ_OFFSET):
705 with isolated_app_env(journal_path):
706 yield make_test_client(journal_path), str(journal_path)
707
708
709def main(argv: list[str] | None = None) -> int:
710 args = parse_args(argv)
711 with client_context(args.base_url, password=args.password) as (
712 client,
713 journal_path,
714 ):
715 if args.command == "verify":
716 failures = verify_all(client, journal_path)
717 if failures:
718 print(f"API baseline verification failed ({len(failures)} endpoints):")
719 for item in failures:
720 print(item)
721 print()
722 print("Run 'make update-api-baselines' to update baselines")
723 return 1
724 print(f"API baseline verification passed for {len(ENDPOINTS)} endpoints.")
725 return 0
726
727 updated = update_all(
728 client,
729 journal_path,
730 include_sandbox_only=bool(args.base_url),
731 )
732 print(f"Updated {updated} baseline files.")
733 return 0
734
735
736if __name__ == "__main__":
737 raise SystemExit(main())