solpbc.org
personal memory agent
1# SPDX-License-Identifier: AGPL-3.0-only
2# Copyright (c) 2026 sol pbc
3
4"""Tests for observe/describe.py category discovery and configuration."""
5
6from pathlib import Path
7
8from solstone.observe import describe as describe_module
9from solstone.observe.describe import _build_redact_instruction
10from solstone.think.prompts import load_prompt
11
12
13def test_categories_discovered():
14 """Test that categories are discovered on import."""
15 CATEGORIES = describe_module.CATEGORIES
16
17 # Should have discovered at least some categories
18 assert isinstance(CATEGORIES, dict)
19 assert len(CATEGORIES) > 0
20
21
22def test_categories_have_required_fields():
23 """Test that all categories have required metadata."""
24 CATEGORIES = describe_module.CATEGORIES
25
26 for category, metadata in CATEGORIES.items():
27 # Every category must have description
28 assert "description" in metadata, f"Category {category} missing 'description'"
29 assert isinstance(metadata["description"], str)
30 assert len(metadata["description"]) > 0
31
32 # Every category should have output field (defaulted if not set)
33 assert "output" in metadata, f"Category {category} missing 'output'"
34 assert metadata["output"] in ("json", "markdown")
35
36 # Every category should have context field (for provider resolution)
37 assert "context" in metadata, f"Category {category} missing 'context'"
38 assert metadata["context"].startswith("observe.describe.")
39
40 # Every category should have an output-token budget
41 assert "max_output_tokens" in metadata
42 assert isinstance(metadata["max_output_tokens"], int)
43 assert metadata["max_output_tokens"] > 0
44
45
46def test_extractable_categories_have_prompts():
47 """Test that extractable categories have valid prompts loaded."""
48 CATEGORIES = describe_module.CATEGORIES
49
50 extractable_count = 0
51 for category, metadata in CATEGORIES.items():
52 if "prompt" in metadata:
53 extractable_count += 1
54 assert isinstance(metadata["prompt"], str)
55 assert len(metadata["prompt"]) > 0, f"Category {category} has empty prompt"
56
57 # Sanity check: we should have at least some extractable categories
58 assert extractable_count > 0, "No extractable categories found"
59
60
61def test_category_max_output_token_defaults_and_overrides():
62 """Test category output-token defaults and explicit overrides."""
63 categories_dir = Path(describe_module.__file__).resolve().parent / "categories"
64 declared_overrides = []
65 defaulted_categories = []
66
67 for md_path in categories_dir.glob("*.md"):
68 category = md_path.stem
69 prompt_content = load_prompt(category, base_dir=categories_dir)
70 metadata = describe_module.CATEGORIES[category]
71
72 if "max_output_tokens" in prompt_content.metadata:
73 declared_overrides.append(category)
74 assert (
75 metadata["max_output_tokens"]
76 == prompt_content.metadata["max_output_tokens"]
77 )
78 else:
79 defaulted_categories.append(category)
80 assert metadata["max_output_tokens"] == 4096
81
82 assert declared_overrides
83 assert defaulted_categories
84
85
86def test_categorization_prompt_built():
87 """Test that categorization prompt is built correctly."""
88 prompt = describe_module.CATEGORIZATION_PROMPT
89
90 # Should contain all category descriptions
91 for category, metadata in describe_module.CATEGORIES.items():
92 assert f"- {category}:" in prompt
93 assert metadata["description"] in prompt
94
95 # Should have the template structure
96 assert "primary" in prompt
97 assert "secondary" in prompt
98 assert "overlap" in prompt
99 assert "Categories (choose one):" in prompt
100
101
102def test_categorization_prompt_alphabetical():
103 """Test that categories in prompt are alphabetically ordered."""
104 prompt = describe_module.CATEGORIZATION_PROMPT
105
106 # Extract category lines from prompt
107 lines = prompt.split("\n")
108 category_lines = [line for line in lines if line.startswith("- ") and ":" in line]
109
110 # Extract category names
111 categories = [line.split(":")[0].replace("- ", "") for line in category_lines]
112
113 # Should be sorted
114 assert categories == sorted(categories)
115
116
117def test_categorization_prompt_is_frozen_literal():
118 """Test that categorization prompt matches the hand-authored literal."""
119 expected = """You have one job: identify the primary foreground and (if present) secondary app categories in this desktop screenshot, and return ONLY this JSON:
120
121{
122 "visual_description":"<1–2 sentences describing what is visible>",
123 "primary": "<largest and most visible app category>",
124 "secondary": "<second most visible app category or 'none'>",
125 "overlap": <boolean, does the primary overlap or cover the secondary, or is it fully standalone and separate>
126}
127
128Rules:
129- For visual_description summarize the **overall desktop view** in **1–2 sentences** for a visually impaired person, first state what kind of content dominates the screen (app UI, photo/video, feed/thread, text document, terminal, or meeting), then summarize layout and window arrangement.
130- For the most visible primary foreground app choose the best category from the list below.
131- Set "secondary" to "none" and "overlap" to true if the primary effectively fills the screen or no distinct second category/window is visible.
132- Set overlap to true if the primary app overlaps, covers, clips, or obscures the secondary in any way.
133- Only set a category for secondary if it is very visible and occupies more than 30% of the screen.
134
135Categories (choose one):
136- browsing: General web browsing, news, shopping, or reference pages without a dominant social feed or media viewer
137- calendar: Calendar and scheduling interfaces: day/week/month views, agenda lists, event details, event creation forms, booking pages, availability grids, and RSVP/scheduling workflows
138- code: Code editors and IDEs
139- gaming: Video games, puzzles, idle games
140- media: Photos, video players, image galleries, or visual media dominating the view, even when displayed inside a browser tab
141- meeting: Video calls/conferencing (Zoom, Meet, Teams, Webex, etc.)
142- messaging: Chat or email apps (Slack, Discord, Messages/iMessage, Gmail, etc.)
143- productivity: Spreadsheets, slides, document editors, task and issue tracking tools, dashboards, other workplace desktop or web apps and professional tools that are not primarily calendar/scheduling views
144- reading: Documents, articles, PDFs, documentation
145- social: Social platforms with feeds, threads, profiles, posts, comments, or timelines (X, Bluesky, Reddit, Instagram, TikTok, LinkedIn, Mastodon, HN)
146- terminal: Command line interfaces, logs, shell
147
148Tie-break rules:
149- If a photo, video, image gallery, or visual media fills most of the screen, choose media even when it is inside a browser.
150- If the dominant surface is a feed, thread, profile, posts, comments, or timeline, choose social rather than browsing.
151- Choose browsing for ordinary web pages, search, news, shopping, or documentation when no social feed or media viewer dominates.
152- Choose calendar for calendar grids, agenda views, event detail/edit forms, availability pickers, booking pages, and scheduling assistants, even when they appear inside a browser or productivity suite.
153- Choose meeting only for an active live call/conference UI; a calendar event for a meeting is calendar, not meeting.
154- Choose messaging/email when the dominant surface is an email or chat conversation, even if it discusses scheduling; choose calendar when an invite/event editor, RSVP pane, availability grid, or booking flow is dominant."""
155 prompt = describe_module.CATEGORIZATION_PROMPT
156
157 assert prompt == expected
158 assert "–" in prompt
159 assert "1-2 sentences" not in prompt
160 assert "�" not in prompt
161 assert "Tie-break rules:" in prompt
162 assert prompt.index("- terminal:") < prompt.index("Tie-break rules:")
163 assert prompt.rstrip().endswith(
164 "choose calendar when an invite/event editor, RSVP pane, availability grid, or booking flow is dominant."
165 )
166
167
168def test_redact_instruction_empty():
169 """Test that empty/missing redact list returns empty string."""
170 assert _build_redact_instruction([]) == ""
171 assert _build_redact_instruction(None) == ""
172
173
174def test_redact_instruction_format():
175 """Test that redact instruction formats rules correctly."""
176 rules = [
177 "use *** instead of any visible passwords",
178 "replace personal email addresses with [redacted]",
179 ]
180 result = _build_redact_instruction(rules)
181
182 # Should contain the header
183 assert "Redaction rules" in result
184 assert "do not generalize" in result
185
186 # Should contain each rule as a bullet
187 for rule in rules:
188 assert f"- {rule}" in result
189
190
191def test_redact_instruction_no_vague_language():
192 """Test that redact instruction doesn't add vague privacy language."""
193 rules = ["replace bank account numbers with ***"]
194 result = _build_redact_instruction(rules)
195
196 # Should not contain vague directives the model could over-apply
197 lower = result.lower()
198 assert "sensitive" not in lower
199 assert "confidential" not in lower
200 assert "personally identifiable" not in lower
201 assert "pii" not in lower