#!/usr/bin/env python3 """Analyze each island with letta CLI and post summaries back to the API.""" import json import subprocess import sys import time import urllib.parse import urllib.request API_BASE = "https://stigmergic.latha.org" LETTA_BIN = "/home/nandi/.local/npm-global/bin/letta" def fetch_islands(): req = urllib.request.Request( f"{API_BASE}/api/strata/islands", headers={"User-Agent": "stigmergic-analyzer/1.0"}, ) with urllib.request.urlopen(req) as resp: return json.loads(resp.read())["islands"] def build_island_context(island): vertices = island["vertices"] edges = island["edges"] meta = island.get("vertexMeta", {}) titles = [meta.get(v, {}).get("title", v) for v in vertices[:30]] domains = {} for v in vertices: if v.startswith("http"): try: from urllib.parse import urlparse h = urlparse(v).hostname.replace("www.", "") domains[h] = domains.get(h, 0) + 1 except Exception: pass top_domains = sorted(domains.items(), key=lambda x: -x[1])[:5] edge_types = {} for e in edges: t = e.get("connectionType", "unknown") edge_types[t] = edge_types.get(t, 0) + 1 skip_patterns = ( "english", "japanese", "version", "translation", "original", "alternate url", "DID link", ) notes = [ e["note"] for e in edges if e.get("note") and not e["note"].lower().startswith(skip_patterns) ][:5] return { "vertex_count": len(vertices), "edge_count": len(edges), "top_domains": [f"{d[0]} ({d[1]})" for d in top_domains], "edge_types": edge_types, "sample_titles": titles[:15], "notes": notes, } def summarize_with_letta(context): prompt = ( "Write a single concise sentence (max 200 chars) summarizing what this " "cluster of links is about. No preamble, just the summary.\n\n" f"Island data:\n{json.dumps(context, indent=2)}" ) try: result = subprocess.run( [LETTA_BIN, "-p", prompt, "--output-format", "json", "--system", "letta", "--memfs"], capture_output=True, text=True, timeout=60, ) except subprocess.TimeoutExpired: return None if result.returncode != 0: return None try: data = json.loads(result.stdout) if data.get("type") == "result": return data["result"].strip().strip('"').strip()[:300] except Exception: pass return result.stdout.strip()[:300] or None def put_summary(island_id, summary): encoded_id = urllib.parse.quote(island_id, safe="") body = json.dumps({"summary": summary}).encode() req = urllib.request.Request( f"{API_BASE}/api/strata/islands/{encoded_id}/summary", data=body, method="PUT", headers={"Content-Type": "application/json", "User-Agent": "stigmergic-analyzer/1.0"}, ) with urllib.request.urlopen(req) as resp: return resp.status == 200 def main(): islands = fetch_islands() total = len(islands) print(f"Analyzing {total} islands...") for i, island in enumerate(islands): island_id = island["id"] if island.get("summary"): print(f"[{i+1}/{total}] Skip (has summary): {island_id[:60]}...") continue print(f"[{i+1}/{total}] Analyzing: {island_id[:60]}...") context = build_island_context(island) summary = summarize_with_letta(context) if not summary: print(f" Skipping (no summary generated)") continue print(f" Summary: {summary}") put_summary(island_id, summary) print(f" Saved.") time.sleep(1) print("Done.") if __name__ == "__main__": main()