+184 -20

solstone/talent/facet_newsletter.py

View file Reviewed

··· 13 13 from typing import Any 14 14 15 15 from solstone.think.activities import load_activity_records 16 16 + from solstone.think.entities.loading import load_entities 16 17 from solstone.think.facets import get_facet_news 17 18 from solstone.think.indexer.journal import search_journal 18 19 from solstone.think.tools.facets import facet_news, get_facet ··· 23 24 _MAX_ACTIVITY_RECORDS = 12 24 25 _MAX_NARRATIVES_PER_ACTIVITY = 4 25 26 _INDEX_RESULTS_PER_AGENT = 10 27 27 + _MAX_ATTACHED_ENTITY_RECORDS = 12 28 28 + _MAX_DETECTED_ENTITY_RECORDS = 12 26 29 _MAX_ENTITY_RESULTS = 12 27 30 _MAX_TITLE_CHARS = 220 28 31 _MAX_DESCRIPTION_CHARS = 700 ··· 35 38 _MAX_FACET_SUMMARY_CHARS = 3000 36 39 _MAX_PACKET_CHARS = 56000 37 40 38 38 - _TIER_ONE_INDEX_AGENTS = ("event", "meetings", "decisions", "followups") 39 39 - _TIER_TWO_INDEX_AGENTS = ("flow", "span") 41 41 + _TIER_ONE_INDEX_AGENTS = ("flow", "span", "event", "meetings") 42 42 + _TIER_TWO_INDEX_AGENTS = ("decisions", "followups") 40 43 41 44 42 45 def pre_process(config: dict) -> dict | None: ··· 127 130 items.extend(_search_day_evidence(agent, facet=facet, day=day, gaps=gaps)) 128 131 129 132 items.extend(_load_facet_metadata(facet, gaps)) 133 133 + items.extend(_load_facet_entity_context(facet, day, gaps)) 130 134 items.extend(_load_prior_newsletter(facet, day, gaps)) 131 131 - items.extend(_search_facet_entities(facet, gaps)) 135 135 + items.extend(_search_facet_entities(facet, day, gaps)) 132 136 133 137 included, dropped_gaps = _gather_budgeted_items(items) 134 138 gaps.extend(dropped_gaps) 135 139 136 140 counts = _gather_source_counts(included) 141 141 + _add_available_source_counts(counts, items) 137 142 substantive_items = sum(1 for item in included if item["tier"] in (1, 2)) 138 143 counts["substantive_items"] = substantive_items 139 144 ··· 285 290 286 291 tier = 1 if agent in _TIER_ONE_INDEX_AGENTS else 2 287 292 source_order = { 288 288 - "event": 2, 289 289 - "meetings": 3, 290 290 - "decisions": 4, 291 291 - "followups": 5, 292 292 - "flow": 0, 293 293 - "span": 1, 293 293 + "flow": 2, 294 294 + "span": 3, 295 295 + "event": 4, 296 296 + "meetings": 5, 297 297 + "decisions": 6, 298 298 + "followups": 7, 294 299 }.get(agent, 9) 295 300 return [ 296 301 _gather_index_item( ··· 397 402 ] 398 403 399 404 400 400 - def _search_facet_entities(facet: str, gaps: list[str]) -> list[dict[str, Any]]: 405 405 + def _load_facet_entity_context( 406 406 + facet: str, day: str, gaps: list[str] 407 407 + ) -> list[dict[str, Any]]: 408 408 + items: list[dict[str, Any]] = [] 409 409 + 410 410 + try: 411 411 + attached = load_entities(facet) 412 412 + except Exception as exc: 413 413 + logger.warning( 414 414 + "facet newsletter attached entities failed for %s: %s", facet, exc 415 415 + ) 416 416 + gaps.append(f"failed: facet_entities:attached failed for {facet}: {exc}") 417 417 + attached = [] 418 418 + 419 419 + if not attached: 420 420 + gaps.append(f"missing: facet_entities:attached absent for {facet}") 421 421 + else: 422 422 + if len(attached) > _MAX_ATTACHED_ENTITY_RECORDS: 423 423 + gaps.append( 424 424 + "capped: facet_entities:attached limited to " 425 425 + f"{_MAX_ATTACHED_ENTITY_RECORDS}/{len(attached)} items" 426 426 + ) 427 427 + attached = attached[:_MAX_ATTACHED_ENTITY_RECORDS] 428 428 + for index, entity in enumerate(attached): 429 429 + origin = _render_entity_origin( 430 430 + "attached", entity, fallback=f"attached-{index}" 431 431 + ) 432 432 + text, clipped = _render_entity_text(entity, origin, "attached", gaps) 433 433 + items.append( 434 434 + _gather_item( 435 435 + source_class="facet_entities", 436 436 + origin=origin, 437 437 + tier=3, 438 438 + text=text, 439 439 + clipped=clipped, 440 440 + agent="attached", 441 441 + source_label="facet_entities:attached", 442 442 + order_key=(2, 0, index, origin), 443 443 + ) 444 444 + ) 445 445 + 446 446 + try: 447 447 + detected = load_entities(facet, day) 448 448 + except Exception as exc: 449 449 + logger.warning( 450 450 + "facet newsletter detected entities failed for %s %s: %s", 451 451 + facet, 452 452 + day, 453 453 + exc, 454 454 + ) 455 455 + gaps.append(f"failed: facet_entities:detected failed for {facet} {day}: {exc}") 456 456 + detected = [] 457 457 + 458 458 + if not detected: 459 459 + gaps.append(f"missing: facet_entities:detected absent for {facet} {day}") 460 460 + else: 461 461 + if len(detected) > _MAX_DETECTED_ENTITY_RECORDS: 462 462 + gaps.append( 463 463 + "capped: facet_entities:detected limited to " 464 464 + f"{_MAX_DETECTED_ENTITY_RECORDS}/{len(detected)} items" 465 465 + ) 466 466 + detected = detected[:_MAX_DETECTED_ENTITY_RECORDS] 467 467 + for index, entity in enumerate(detected): 468 468 + origin = _render_entity_origin( 469 469 + "detected", entity, fallback=f"detected-{index}" 470 470 + ) 471 471 + text, clipped = _render_entity_text(entity, origin, "detected", gaps) 472 472 + items.append( 473 473 + _gather_item( 474 474 + source_class="facet_entities", 475 475 + origin=origin, 476 476 + tier=3, 477 477 + text=text, 478 478 + clipped=clipped, 479 479 + agent="detected", 480 480 + source_label="facet_entities:detected", 481 481 + order_key=(2, 1, index, origin), 482 482 + ) 483 483 + ) 484 484 + 485 485 + return items 486 486 + 487 487 + 488 488 + def _search_facet_entities( 489 489 + facet: str, day: str, gaps: list[str] 490 490 + ) -> list[dict[str, Any]]: 401 491 try: 402 492 total, results = search_journal( 403 493 "", 404 494 limit=_MAX_ENTITY_RESULTS, 405 495 offset=0, 496 496 + day=day, 406 497 facet=facet, 407 498 agent="entity", 408 499 ) 409 500 except Exception as exc: 410 410 - logger.warning("facet newsletter entity search failed for %s: %s", facet, exc) 411 411 - gaps.append(f"failed: facet_entities failed for {facet}: {exc}") 501 501 + logger.warning( 502 502 + "facet newsletter entity search failed for %s %s: %s", facet, day, exc 503 503 + ) 504 504 + gaps.append(f"failed: facet_entities:indexed failed for {facet} {day}: {exc}") 412 505 return [] 413 506 414 507 if not results: 415 415 - gaps.append(f"missing: facet_entities absent for {facet}") 508 508 + gaps.append(f"missing: facet_entities:indexed absent for {facet} {day}") 416 509 return [] 417 510 418 511 if total > len(results): 419 419 - gaps.append(f"capped: facet_entities limited to {len(results)}/{total} items") 512 512 + gaps.append( 513 513 + f"capped: facet_entities:indexed limited to {len(results)}/{total} items" 514 514 + ) 420 515 421 516 return [ 422 517 _gather_index_item( ··· 426 521 tier=3, 427 522 text_limit=_MAX_ENTITY_TEXT_CHARS, 428 523 gaps=gaps, 429 429 - order_key=(2, index, _render_result_path(result), result.get("id", "")), 524 524 + source_label="facet_entities:indexed", 525 525 + order_key=(2, 2, index, _render_result_path(result), result.get("id", "")), 430 526 ) 431 527 for index, result in enumerate(results) 432 528 ] ··· 441 537 text_limit: int, 442 538 gaps: list[str], 443 539 order_key: tuple, 540 540 + source_label: str | None = None, 444 541 ) -> dict[str, Any]: 445 542 path = _render_result_path(result) 446 543 origin = f"{result.get('id', path)} ({path}; agent={agent})" ··· 462 559 agent=agent, 463 560 path=path, 464 561 result_id=str(result.get("id") or ""), 562 562 + source_label=source_label, 465 563 order_key=order_key, 466 564 ) 467 565 ··· 477 575 agent: str | None = None, 478 576 path: str | None = None, 479 577 result_id: str | None = None, 578 578 + source_label: str | None = None, 480 579 ) -> dict[str, Any]: 481 580 return { 482 581 "source_class": source_class, 582 582 + "source_label": source_label, 483 583 "agent": agent, 484 584 "origin": origin, 485 585 "tier": tier, ··· 497 597 ) -> tuple[list[dict[str, Any]], list[str]]: 498 598 included: list[dict[str, Any]] = [] 499 599 gaps: list[str] = [] 500 500 - remaining = _MAX_PACKET_CHARS 501 600 for item in sorted(items, key=lambda row: (row["tier"], row["order_key"])): 502 502 - length = int(item.get("length") or 0) 503 503 - if length <= remaining: 601 601 + if len(_render_packet([*included, item])) <= _MAX_PACKET_CHARS: 504 602 included.append(item) 505 505 - remaining -= length 506 603 continue 507 604 gaps.append( 508 605 "dropped: " ··· 528 625 "index_result:span": 0, 529 626 "prior_newsletter": 0, 530 627 "facet_metadata": 0, 531 531 - "facet_entities": 0, 628 628 + "facet_entities:attached": 0, 629 629 + "facet_entities:detected": 0, 630 630 + "facet_entities:indexed": 0, 532 631 } 533 632 for item in items: 534 633 tier_key = f"tier{item['tier']}_included" ··· 538 637 return counts 539 638 540 639 640 640 + def _add_available_source_counts( 641 641 + counts: dict[str, int], available_items: list[dict[str, Any]] 642 642 + ) -> None: 643 643 + available = _gather_source_counts(available_items) 644 644 + counts["total_available"] = available["total_included"] 645 645 + counts["tier1_available"] = available["tier1_included"] 646 646 + counts["tier2_available"] = available["tier2_included"] 647 647 + counts["tier3_available"] = available["tier3_included"] 648 648 + for key, value in available.items(): 649 649 + if key == "total_included" or key.startswith("tier"): 650 650 + continue 651 651 + counts[f"{key}_available"] = value 652 652 + counts[f"{key}_included"] = counts.get(key, 0) 653 653 + 654 654 + 541 655 def _render_activity_order(record: dict[str, Any]) -> tuple[int, str, str]: 542 656 try: 543 657 created_at = int(record.get("created_at") or 0) ··· 652 766 653 767 654 768 def _render_source_label(item: dict[str, Any]) -> str: 769 769 + source_label = item.get("source_label") 770 770 + if source_label: 771 771 + return str(source_label) 655 772 source_class = str(item.get("source_class") or "") 656 773 agent = item.get("agent") 657 774 if source_class == "index_result" and agent: 658 775 return f"index_result:{agent}" 776 776 + if source_class == "facet_entities" and agent: 777 777 + entity_kind = "indexed" if agent == "entity" else str(agent) 778 778 + return f"facet_entities:{entity_kind}" 659 779 return source_class 780 780 + 781 781 + 782 782 + def _render_entity_origin(kind: str, entity: dict[str, Any], *, fallback: str) -> str: 783 783 + name = str(entity.get("name") or entity.get("id") or fallback).strip() 784 784 + entity_id = str(entity.get("id") or "").strip() 785 785 + if entity_id and entity_id != name: 786 786 + return f"{kind}:{name} ({entity_id})" 787 787 + return f"{kind}:{name}" 788 788 + 789 789 + 790 790 + def _render_entity_text( 791 791 + entity: dict[str, Any], origin: str, kind: str, gaps: list[str] 792 792 + ) -> tuple[str, bool]: 793 793 + payload = { 794 794 + key: value 795 795 + for key, value in entity.items() 796 796 + if value not in (None, "", [], {}) 797 797 + and key 798 798 + in { 799 799 + "id", 800 800 + "type", 801 801 + "name", 802 802 + "description", 803 803 + "aka", 804 804 + "relationship", 805 805 + "attached_at", 806 806 + "updated_at", 807 807 + "last_seen", 808 808 + "last_active_day", 809 809 + "count", 810 810 + } 811 811 + } 812 812 + text = json.dumps( 813 813 + payload or entity, default=str, ensure_ascii=False, sort_keys=True 814 814 + ) 815 815 + return _render_clipped_text( 816 816 + text, 817 817 + _MAX_ENTITY_TEXT_CHARS, 818 818 + gaps, 819 819 + "facet_entities", 820 820 + origin, 821 821 + "json", 822 822 + agent=kind, 823 823 + ) 660 824 661 825 662 826 def _render_packet(items: list[dict[str, Any]]) -> str:

+74 -2

tests/test_facet_newsletter_pre_hook.py

View file Reviewed

··· 65 65 monkeypatch.setattr(facet_newsletter, "get_facet_news", _empty_newsletter) 66 66 monkeypatch.setattr(facet_newsletter, "get_facet", _facet_summary) 67 67 monkeypatch.setattr(facet_newsletter, "search_journal", _empty_search) 68 68 + monkeypatch.setattr(facet_newsletter, "load_entities", lambda facet, day=None: []) 68 69 monkeypatch.setattr( 69 70 facet_newsletter, 70 71 "load_activity_records", ··· 84 85 encoding="utf-8", 85 86 ) 86 87 88 88 + def fake_search(query, limit=10, offset=0, **kwargs): 89 89 + if kwargs.get("agent") == "span": 90 90 + return 1, [_result(agent="span", text="Indexed span context.")] 91 91 + return 0, [] 92 92 + 93 93 + monkeypatch.setattr(facet_newsletter, "search_journal", fake_search) 94 94 + 87 95 packet = facet_newsletter.pre_process({"facet": FACET, "day": DAY})["template_vars"] 88 96 89 97 assert { ··· 94 102 } <= set(packet) 95 103 assert "Launch review" in packet["source_packet"] 96 104 assert "Narrative launch context." in packet["source_packet"] 105 105 + assert "Indexed span context." in packet["source_packet"] 106 106 + assert "activity_record_available" in packet["source_counts"] 107 107 + assert "activity_record_included" in packet["source_counts"] 97 108 assert isinstance(json.loads(packet["source_gaps"]), list) 98 109 99 110 ··· 147 158 entity_calls = [call for call in calls if call.get("agent") == "entity"] 148 159 assert len(entity_calls) == 1 149 160 assert entity_calls[0].get("facet") == FACET 150 150 - assert "day" not in entity_calls[0] 161 161 + assert entity_calls[0].get("day") == DAY 162 162 + 163 163 + 164 164 + def test_pre_hook_includes_attached_detected_and_indexed_entities( 165 165 + tmp_path, monkeypatch 166 166 + ): 167 167 + _install_common(monkeypatch, tmp_path) 168 168 + monkeypatch.setattr( 169 169 + facet_newsletter, "load_activity_records", lambda facet, day: [] 170 170 + ) 171 171 + 172 172 + def fake_load_entities(facet, day=None): 173 173 + assert facet == FACET 174 174 + if day is None: 175 175 + return [ 176 176 + { 177 177 + "id": "alice", 178 178 + "type": "Person", 179 179 + "name": "Alice", 180 180 + "description": "Product lead.", 181 181 + "last_seen": DAY, 182 182 + } 183 183 + ] 184 184 + assert day == DAY 185 185 + return [ 186 186 + { 187 187 + "id": "bob", 188 188 + "type": "Person", 189 189 + "name": "Bob", 190 190 + "description": "Detected collaborator.", 191 191 + } 192 192 + ] 193 193 + 194 194 + def fake_search(query, limit=10, offset=0, **kwargs): 195 195 + if kwargs.get("agent") == "flow": 196 196 + return 1, [_result(agent="flow", text="Launch flow context.")] 197 197 + if kwargs.get("agent") == "entity": 198 198 + assert kwargs.get("facet") == FACET 199 199 + assert kwargs.get("day") == DAY 200 200 + return 1, [ 201 201 + _result( 202 202 + agent="entity", 203 203 + text="Alice appears in launch planning.", 204 204 + path="entity_search:alice", 205 205 + ) 206 206 + ] 207 207 + return 0, [] 208 208 + 209 209 + monkeypatch.setattr(facet_newsletter, "load_entities", fake_load_entities) 210 210 + monkeypatch.setattr(facet_newsletter, "search_journal", fake_search) 211 211 + 212 212 + packet = facet_newsletter.pre_process({"facet": FACET, "day": DAY})["template_vars"] 213 213 + 214 214 + assert "facet_entities:attached" in packet["source_packet"] 215 215 + assert "facet_entities:detected" in packet["source_packet"] 216 216 + assert "facet_entities:indexed" in packet["source_packet"] 217 217 + assert "Product lead." in packet["source_packet"] 218 218 + assert "Detected collaborator." in packet["source_packet"] 219 219 + assert "Alice appears in launch planning." in packet["source_packet"] 220 220 + assert "facet_entities:attached_available: 1" in packet["source_counts"] 221 221 + assert "facet_entities:detected_available: 1" in packet["source_counts"] 222 222 + assert "facet_entities:indexed_available: 1" in packet["source_counts"] 151 223 152 224 153 225 def test_pre_hook_tier_three_only_skips_no_substantive_sources(tmp_path, monkeypatch): ··· 252 324 253 325 def test_pre_hook_total_budget_drops_lower_tier_items(tmp_path, monkeypatch): 254 326 _install_common(monkeypatch, tmp_path) 255 255 - monkeypatch.setattr(facet_newsletter, "_MAX_PACKET_CHARS", 260) 327 327 + monkeypatch.setattr(facet_newsletter, "_MAX_PACKET_CHARS", 800) 256 328 monkeypatch.setattr( 257 329 facet_newsletter, 258 330 "get_facet",