nandi.uk
This repository has no description
3.6 kB
117 lines
1#!/bin/bash
2# Analyze each island with letta CLI and post summaries back to the API.
3# Usage: ./scripts/analyze-islands.sh
4#
5# Requires: letta CLI, CLOUDFLARE_API_TOKEN set
6
7set -euo pipefail
8
9API_BASE="https://stigmergic.latha.org"
10LETTA_BIN="${LETTA_BIN:-/home/nandi/.local/npm-global/bin/letta}"
11
12# Fetch all islands
13islands=$(curl -sf "$API_BASE/api/strata/islands" | python3 -c "
14import json, sys
15data = json.load(sys.stdin)
16for island in data['islands']:
17 print(island['id'])
18")
19
20total=$(echo "$islands" | wc -l)
21current=0
22
23echo "Analyzing $total islands..."
24
25while IFS= read -r island_id; do
26 current=$((current + 1))
27 echo "[$current/$total] Analyzing island: ${island_id:0:60}..."
28
29 # Fetch this island's full data
30 island_json=$(curl -sf "$API_BASE/api/strata/islands" | python3 -c "
31import json, sys
32data = json.load(sys.stdin)
33for island in data['islands']:
34 if island['id'] == sys.argv[1]:
35 # Build a concise context for the LLM
36 vertices = island['vertices']
37 edges = island['edges']
38 meta = island.get('vertexMeta', {})
39
40 titles = [meta.get(v, {}).get('title', v) for v in vertices[:30]]
41 domains = {}
42 for v in vertices:
43 if v.startswith('http'):
44 try:
45 from urllib.parse import urlparse
46 h = urlparse(v).hostname.replace('www.', '')
47 domains[h] = domains.get(h, 0) + 1
48 except: pass
49 top_domains = sorted(domains.items(), key=lambda x: -x[1])[:5]
50
51 edge_types = {}
52 for e in edges:
53 t = e.get('connectionType', 'unknown')
54 edge_types[t] = edge_types.get(t, 0) + 1
55
56 notes = [e['note'] for e in edges if e.get('note')][:5]
57
58 out = {
59 'vertex_count': len(vertices),
60 'edge_count': len(edges),
61 'top_domains': [d[0] + f' ({d[1]})' for d in top_domains],
62 'edge_types': edge_types,
63 'sample_titles': titles[:15],
64 'notes': notes,
65 }
66 print(json.dumps(out))
67 break
68" "$island_id")
69
70 if [ -z "$island_json" ]; then
71 echo " Skipping (no data)"
72 continue
73 fi
74
75 # Ask letta for a one-sentence summary
76 prompt="You are analyzing a knowledge graph island (connected component). Write a single concise sentence (max 200 chars) summarizing what this cluster of links is about. No preamble, just the summary.
77
78Island data:
79$island_json"
80
81 summary=$($LETTA_BIN -p "$prompt" --output-format json --system letta --memfs 2>/dev/null | python3 -c "
82import json, sys
83try:
84 data = json.load(sys.stdin)
85 if data.get('type') == 'result':
86 print(data['result'].strip().strip('\"').strip())
87 else:
88 print(data.get('result', data.get('text', '')), end='')
89except:
90 # Fallback: try reading as plain text
91 sys.stdout.write(sys.stdin.read().strip())
92" 2>/dev/null || echo "")
93
94 if [ -z "$summary" ]; then
95 echo " Skipping (no summary generated)"
96 continue
97 fi
98
99 # Truncate to 300 chars
100 summary=$(echo "$summary" | head -c 300)
101
102 echo " Summary: $summary"
103
104 # PUT the summary back
105 encoded_id=$(python3 -c "import urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=''))" "$island_id")
106 curl -sf -X PUT "$API_BASE/api/strata/islands/$encoded_id/summary" \
107 -H "Content-Type: application/json" \
108 -d "{\"summary\":$(python3 -c "import json; print(json.dumps(sys.argv[1]))" "$summary")}" \
109 > /dev/null
110
111 echo " Saved."
112
113 # Rate limit
114 sleep 2
115done <<< "$islands"
116
117echo "Done. $current islands analyzed."