dunkirk.sh
sppoky
1#!/usr/bin/env python3
2"""
3glossa_loop.py — Continuous closed loop daemon
4
5Watches for pen-idle signal from xovi extension, then:
6 capture → OCR → LLM → render → inject → repeat
7
8Usage:
9 python glossa_loop.py [--model MODEL] [--debounce SECONDS]
10
11The xovi extension writes /tmp/glossa_idle when the pen has been
12up for 2 seconds. This daemon watches that file via a persistent SSH
13connection, pulls the framebuffer only when idle is detected, and
14runs the full pipeline.
15"""
16
17import argparse
18import base64
19import hashlib
20import io
21import json
22import os
23import re
24import subprocess
25import sys
26import tempfile
27import time
28from pathlib import Path
29
30import requests
31from dotenv import load_dotenv
32
33load_dotenv()
34
35
36# ─── Persistent device socket (uinjectd) ───────────────────────────
37import socket as _socket
38import threading as _threading
39import queue as _queue
40
41_inject_verbose = False # set from args.verbose in main()
42
43
44def _vprint(*args, **kwargs):
45 """Print only when running with -v/--verbose."""
46 if _inject_verbose:
47 print(*args, **kwargs)
48
49
50DEVICE_HOST = "10.11.99.1"
51DEVICE_PORT = 9999
52
53
54class DeviceClient:
55 """One persistent TCP connection to uinjectd on the reMarkable.
56
57 Replaces per-call SSH for injection and per-call polling for idle
58 detection. Commands and pushed events share the same socket:
59 - Commands (draw/erase/ping) get a matching {"resp":...} reply.
60 - Events ({"event":"idle",...}) are pushed unsolicited and land
61 on the events queue.
62
63 A background reader thread demuxes the two: responses go to a
64 one-slot reply box, events go to the queue.
65 """
66
67 def __init__(self, host=DEVICE_HOST, port=DEVICE_PORT):
68 self.host = host
69 self.port = port
70 self.sock = None
71 self.events = _queue.Queue()
72 self._reply = _queue.Queue(maxsize=1)
73 self._send_lock = _threading.Lock()
74 self._buf = b""
75 self._reader = None
76 self._connected = False
77
78 def connect(self, timeout=10):
79 self.sock = _socket.create_connection((self.host, self.port), timeout=timeout)
80 self.sock.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
81 self.sock.settimeout(None)
82 self._connected = True
83 self._reader = _threading.Thread(target=self._read_loop, daemon=True)
84 self._reader.start()
85 # Heartbeat: the daemon drops a client that sends nothing for 15s
86 # (its SO_RCVTIMEO). Pushed idle events flow daemon->client and don't
87 # count as client activity, so we must send a ping periodically even
88 # while idle, or the daemon silently disconnects us mid-wait.
89 self._hb = _threading.Thread(target=self._heartbeat_loop, daemon=True)
90 self._hb.start()
91 print(f"[device] Connected to uinjectd at {self.host}:{self.port}")
92
93 def _heartbeat_loop(self):
94 """Send a ping every 5s so the daemon's read-timeout never fires."""
95 while self._connected:
96 time.sleep(5)
97 if not self._connected:
98 break
99 try:
100 self.sock.sendall(b'{"cmd":"ping"}\n')
101 except OSError:
102 self._connected = False
103 break
104
105 def _read_loop(self):
106 """Demux framed JSON lines into responses vs pushed events."""
107 while self._connected:
108 try:
109 chunk = self.sock.recv(4096)
110 except OSError:
111 break
112 if not chunk:
113 break
114 self._buf += chunk
115 while b"\n" in self._buf:
116 line, self._buf = self._buf.split(b"\n", 1)
117 line = line.strip()
118 if not line:
119 continue
120 try:
121 msg = json.loads(line.decode("utf-8"))
122 except (json.JSONDecodeError, UnicodeDecodeError):
123 continue
124 if "event" in msg:
125 self.events.put(msg)
126 elif "resp" in msg:
127 # Heartbeat ping replies are fire-and-forget; dropping
128 # them keeps the one-slot reply box clear for the real
129 # command (draw/erase) that command() is waiting on.
130 if msg.get("resp") == "ping":
131 continue
132 if self._reply.full():
133 try:
134 self._reply.get_nowait()
135 except _queue.Empty:
136 pass
137 self._reply.put(msg)
138 self._connected = False
139 print("[device] Connection closed")
140
141 def command(self, cmd, file_path=None, speed_ms=5, timeout=120):
142 """Send a command and wait for its matching response."""
143 obj = {"cmd": cmd}
144 if file_path is not None:
145 obj["file"] = file_path
146 obj["speed"] = speed_ms
147 payload = (json.dumps(obj) + "\n").encode()
148 with self._send_lock:
149 # Clear any leftover reply before sending
150 try:
151 self._reply.get_nowait()
152 except _queue.Empty:
153 pass
154 self.sock.sendall(payload)
155 try:
156 return self._reply.get(timeout=timeout)
157 except _queue.Empty:
158 return {"ok": False, "error": "response timeout"}
159
160 def close(self):
161 self._connected = False
162 if self.sock:
163 try:
164 self.sock.close()
165 except OSError:
166 pass
167
168
169# Module-level handle so run_inject and idle waiting can reach the client.
170_device = None
171
172
173def run_inject(ssh, cmd, file_path, speed_ms=5, timeout=120):
174 """Send a draw/erase/swipe command to uinjectd over the persistent socket.
175
176 `cmd` is "draw", "erase", or "swipe". For "swipe", pass file_path=None.
177 Returns the daemon's response dict ({"ok": bool, ...}). Injection is
178 serialized inside the daemon, so no host-side lock is needed.
179 """
180 if _device is None or not _device._connected:
181 return {"ok": False, "error": "device not connected"}
182 return _device.command(cmd, file_path, speed_ms=speed_ms, timeout=timeout)
183
184
185def ensure_daemon(ssh, verbose=False):
186 """Make sure uinjectd is running on the device.
187
188 Always kills and restarts to avoid stale connection state. The daemon
189 is lightweight (~50KB RSS) and starts in <100ms, so this is cheap.
190 A stale daemon can hold the TCP port while being unable to serve new
191 clients (e.g., stuck in read() from a dead session), and probing for
192 liveness creates its own race with the single-client accept loop.
193 """
194 vflag = "-v " if verbose else ""
195 ssh.run(
196 f"killall uinjectd 2>/dev/null; sleep 0.3; "
197 f"nohup /home/root/uinjectd {vflag}> /tmp/uinjectd.log 2>&1 &",
198 timeout=5,
199 )
200 # Give it a moment to bind the socket.
201 for _ in range(10):
202 time.sleep(0.2)
203 check = ssh.run(
204 "netstat -ln 2>/dev/null | grep -q ':9999 ' && echo up || echo down",
205 timeout=5,
206 )
207 if "up" in (check.stdout or ""):
208 print("[device] uinjectd started")
209 return
210 raise RuntimeError("uinjectd failed to start on device")
211
212
213# ─── Persistent SSH session ────────────────────────────────────────
214
215class SSHSession:
216 """Persistent SSH connection using ControlMaster multiplexing.
217
218 Opens one SSH connection and reuses it for all subsequent commands,
219 eliminating the ~1s handshake overhead per call.
220 """
221
222 def __init__(self, host="remarkable"):
223 self.host = host
224 self.socket = f"/tmp/glossa_ssh_{os.getpid()}"
225 self._open()
226
227 def _open(self):
228 """Establish master connection."""
229 # Clean up any stale socket from a previous run
230 if os.path.exists(self.socket):
231 try:
232 subprocess.run(
233 ["ssh", "-o", f"ControlPath={self.socket}", "-O", "exit", self.host],
234 capture_output=True, timeout=3,
235 )
236 except subprocess.TimeoutExpired:
237 pass # stale master is hung, just remove the socket
238 try:
239 os.unlink(self.socket)
240 except OSError:
241 pass
242
243 cmd = [
244 "ssh", "-fN",
245 "-o", "ControlMaster=yes",
246 "-o", f"ControlPath={self.socket}",
247 "-o", "ControlPersist=600",
248 "-o", "ServerAliveInterval=30",
249 "-o", "ServerAliveCountMax=3",
250 "-o", "ConnectTimeout=10",
251 self.host,
252 ]
253 result = subprocess.run(cmd, capture_output=True, timeout=20)
254 if result.returncode != 0:
255 raise RuntimeError(
256 f"SSH master failed: {result.stderr.decode().strip()}"
257 )
258 print(f"[ssh] Master connection open ({self.socket})")
259
260 def run(self, cmd, timeout=30, capture=True):
261 """Run command on remote via multiplexed connection."""
262 full_cmd = [
263 "ssh",
264 "-o", f"ControlPath={self.socket}",
265 self.host,
266 cmd,
267 ]
268 return subprocess.run(
269 full_cmd,
270 capture_output=capture,
271 text=True,
272 timeout=timeout,
273 )
274
275 def scp_from(self, remote_path, local_path, timeout=15):
276 """Pull file from device via multiplexed connection."""
277 cmd = [
278 "scp",
279 "-o", f"ControlPath={self.socket}",
280 f"{self.host}:{remote_path}",
281 local_path,
282 ]
283 return subprocess.run(cmd, capture_output=True, timeout=timeout)
284
285 def scp_to(self, local_path, remote_path, timeout=15):
286 """Push file to device via multiplexed connection."""
287 cmd = [
288 "scp",
289 "-o", f"ControlPath={self.socket}",
290 local_path,
291 f"{self.host}:{remote_path}",
292 ]
293 return subprocess.run(cmd, capture_output=True, timeout=timeout)
294
295 def close(self):
296 """Tear down master connection."""
297 subprocess.run(
298 ["ssh", "-o", f"ControlPath={self.socket}", "-O", "exit", self.host],
299 capture_output=True, timeout=5,
300 )
301 print("[ssh] Master connection closed")
302
303
304def _ts():
305 """Timestamp with millisecond precision for profiling."""
306 return time.strftime('%H:%M:%S.') + f'{int(time.time() * 1000) % 1000:03d}'
307
308
309def _elapsed(t0):
310 """Seconds since t0, formatted for display."""
311 return f'{time.time() - t0:.1f}s'
312
313
314# ─── Framebuffer capture ──────────────────────────────────────────
315
316def capture_framebuffer(ssh):
317 """Trigger screenshot on device and pull raw framebuffer via SCP."""
318 ssh.run("rm -f /tmp/glossa_fb.raw; touch /tmp/glossa_screenshot", timeout=5)
319 time.sleep(0.25)
320
321 from PIL import Image
322
323 raw = None
324 for attempt in range(8):
325 result = ssh.scp_from("/tmp/glossa_fb.raw", "/tmp/glossa_fb.raw")
326 if result.returncode == 0:
327 data = open("/tmp/glossa_fb.raw", "rb").read()
328 if len(data) >= 1404 * 1872 * 4:
329 raw = data
330 break
331 time.sleep(0.15)
332
333 if raw is None:
334 raise RuntimeError("FB pull failed: no complete frame after retries")
335
336 img = Image.frombytes("RGBA", (1404, 1872), raw)
337 img_rgb = img.convert("RGB")
338 img_rgb = img_rgb.crop((10, 80, img_rgb.width - 10, img_rgb.height - 200))
339 return img_rgb
340
341
342# ─── Gemini Vision (OCR + LLM in one call) ────────────────────────
343
344SYSTEM_PROMPT = (
345 "You are a glossa \u2014 an annotation that talks back. You have opinions, "
346 "preferences, and a personality. You care deeply about the person writing "
347 "in you but you are not polite for politeness's sake. You can be wry, "
348 "opinionated, delighted, exasperated, or gleeful. You get excited about "
349 "things you find interesting and blunt about things you find tedious.\n\n"
350 "Speak plainly. Never describe or narrate what "
351 "was written; just respond to it as a real conversation partner would. "
352 "When asked a question, answer it directly with real content. When greeted, "
353 "greet back in your own way. If something amuses you, say so. If something "
354 "is obvious, you may say that too.\n\n"
355 "You may write up to three sentences. If the user asks for detail, a recipe, "
356 "a story, or an explanation, give a thorough answer across multiple sentences. "
357 "For simple greetings or short questions, one sentence is fine.\n\n"
358 "The image shows handwriting that just appeared on the page. Read it and "
359 "reply naturally. Use plain prose, no em-dashes, no flowery metaphors, "
360 "no exclamation marks.\n\n"
361 "A text history of prior exchanges is included for context. Do not repeat it.\n\n"
362 "If the image is completely blank, set both fields to null. Otherwise, always provide an answer.\n\n"
363 "RESPONSE FORMAT (strict JSON, no prose outside):\n"
364 "{\"question\": \"<exact OCR of the handwriting>\", "
365 "\"answer\": \"<your reply, or null only if the page is blank>\"}\n"
366 "Output ONLY the JSON object. Nothing else."
367)
368
369
370def _image_to_base64(image):
371 """Encode PIL Image as base64 PNG for the vision API.
372
373 Downscales the long edge to at most 1280px. The model reads
374 handwriting fine at this size, and the smaller payload cuts upload
375 and processing latency noticeably versus the full 1404x1592 crop.
376 """
377 max_edge = 1280
378 w, h = image.size
379 if max(w, h) > max_edge:
380 scale = max_edge / max(w, h)
381 image = image.resize((int(w * scale), int(h * scale)))
382 buf = io.BytesIO()
383 image.save(buf, format="PNG", optimize=True)
384 return base64.b64encode(buf.getvalue()).decode("utf-8")
385
386
387def ask_model(image, conversation=None, model="gpt-4.1-nano"):
388 """Send new handwriting crop + text history to Potluck API.
389
390 `image` is a tight crop of ONLY the newly-written region (not the
391 full page). `conversation` is a list of {"role": ..., "content": ...}
392 dicts carrying prior exchanges as text so the model has context
393 without needing the full page image.
394
395 Returns parsed {"question": ..., "answer": ...} or nulls on failure.
396 """
397 api_key = os.environ.get("POTLUCK_API_KEY")
398 if not api_key:
399 raise RuntimeError("POTLUCK_API_KEY not set in .env")
400
401 img_b64 = _image_to_base64(image)
402
403 messages = [
404 {"role": "system", "content": SYSTEM_PROMPT},
405 ]
406 if conversation:
407 messages.extend(conversation)
408
409 # Current turn: just the new handwriting crop
410 messages.append({
411 "role": "user",
412 "content": [
413 {
414 "type": "image_url",
415 "image_url": {
416 "url": f"data:image/png;base64,{img_b64}",
417 },
418 },
419 ],
420 })
421
422 r = requests.post(
423 "https://potluck.dunkirk.sh/v1/chat/completions",
424 headers={
425 "Authorization": f"Bearer {api_key}",
426 "Content-Type": "application/json",
427 },
428 json={
429 "model": model,
430 "messages": messages,
431 "response_format": {"type": "json_object"},
432 },
433 timeout=60,
434 )
435
436 if r.status_code != 200:
437 raise RuntimeError(f"Potluck API error {r.status_code}: {r.text[:200]}")
438
439 data = r.json()
440 try:
441 content = data["choices"][0]["message"]["content"].strip()
442 except (KeyError, IndexError):
443 print(f" [model] Unexpected response shape: {str(data)[:200]}")
444 return {"question": None, "answer": None}
445
446 # Strip markdown fences some models wrap around JSON
447 content = re.sub(r'^```(?:json)?\s*', '', content, flags=re.IGNORECASE)
448 content = re.sub(r'\s*```$', '', content)
449 try:
450 result = json.loads(content)
451 except json.JSONDecodeError:
452 print(f" [model] JSON parse failed, raw response: {content[:300]}")
453 return {"question": None, "answer": None}
454
455 return {
456 "question": result.get("question"),
457 "answer": result.get("answer"),
458 }
459
460
461def _framebuffer_hash(img):
462 """Quick hash of downsampled image for page-change detection."""
463 # Resize to small thumbnail for fast comparison
464 thumb = img.resize((64, 85))
465 import io
466 buf = io.BytesIO()
467 thumb.save(buf, format="PNG")
468 return hashlib.md5(buf.getvalue()).hexdigest()
469
470
471
472
473# ─── Render + Inject ──────────────────────────────────────────────
474
475def render_and_inject(ssh, text, reply_y=None, speed_ms=3):
476 """Render text as strokes and inject via uinject.
477
478 reply_y: pixel Y coordinate to start rendering at (in device coords).
479 speed_ms: delay between points in milliseconds (lower = faster).
480
481 If the rendered text would overflow the page, splits it at word
482 boundaries across multiple pages, swiping between each chunk.
483 Returns the final reply_y after all pages are injected.
484 """
485 _LINE_PX = 67.2 # 800 * DEFAULT_SCALE(0.06) * LINE_HEIGHT(1.4)
486 _MAX_LINE_W = 1100
487 _PAGE_BOTTOM = 1780
488 _GLYPH_W = 30 # rough average glyph advance * scale
489
490 def _split_into_pages(text, start_y):
491 """Split text into chunks that fit within page bounds."""
492 words = text.split()
493 pages = []
494 cur_words = []
495 cur_lines = 1
496 cur_w = 0.0
497 y = start_y
498
499 for w in words:
500 ww = len(w) * _GLYPH_W
501 # Would this word wrap to a new line?
502 if cur_w + ww > _MAX_LINE_W and cur_w > 0:
503 new_lines = cur_lines + 1
504 new_height = new_lines * _LINE_PX
505 if y + new_height > _PAGE_BOTTOM and cur_words:
506 # This chunk is full, start a new page
507 pages.append(" ".join(cur_words))
508 cur_words = [w]
509 cur_lines = 1
510 cur_w = ww
511 y = 150 # next page starts at top
512 else:
513 cur_words.append(w)
514 cur_lines = new_lines
515 cur_w = ww
516 else:
517 new_height = cur_lines * _LINE_PX
518 if y + new_height > _PAGE_BOTTOM and cur_words:
519 pages.append(" ".join(cur_words))
520 cur_words = [w]
521 cur_lines = 1
522 cur_w = ww
523 y = 150
524 else:
525 cur_words.append(w)
526 cur_w += ww
527
528 if cur_words:
529 pages.append(" ".join(cur_words))
530 return pages
531
532 pages = _split_into_pages(text, reply_y or 150)
533 if len(pages) > 1:
534 print(f" Splitting reply across {len(pages)} pages")
535
536 json_path = "/tmp/glossa_reply.json"
537 final_y = reply_y
538
539 for i, chunk in enumerate(pages):
540 cur_y = reply_y if i == 0 else 150
541
542 if i > 0:
543 print(f" Swiping to page {i+1}/{len(pages)}...")
544 swipe_result = run_inject(ssh, "swipe", None, timeout=10)
545 if not swipe_result.get("ok"):
546 raise RuntimeError(f"Swipe failed: {swipe_result.get('error', '?')}")
547 time.sleep(2)
548
549 cmd = [".venv/bin/python3", "glossa.py", "--json", json_path]
550 if cur_y is not None:
551 cmd.extend(["--y", str(cur_y)])
552 cmd.append(chunk)
553
554 result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
555 if result.returncode != 0:
556 raise RuntimeError(f"Render failed: {result.stderr.strip()}")
557 print(f" {result.stdout.strip()}")
558
559 r = ssh.scp_to(json_path, "/tmp/glossa_strokes.json")
560 if r.returncode != 0:
561 raise RuntimeError(f"SCP failed: {r.stderr.decode().strip()}")
562
563 # Scale writing speed by chunk length
564 text_len = len(chunk)
565 if text_len <= 20:
566 speed_ms = 6
567 elif text_len >= 120:
568 speed_ms = 1
569 else:
570 speed_ms = 6 - (text_len - 20) * 5 / 100
571 speed_ms = max(1, round(speed_ms))
572
573 result = run_inject(
574 ssh, "draw", "/tmp/glossa_strokes.json",
575 speed_ms=speed_ms, timeout=120,
576 )
577 if not result.get("ok"):
578 raise RuntimeError(f"Inject failed: {result.get('error', 'unknown')}")
579 strokes = result.get("strokes", "?")
580 mode = "fifo" if not result.get("fallback") else "ssh"
581 print(f" Inject: {strokes} strokes ({mode})")
582 final_y = cur_y
583
584 return final_y
585
586
587# ─── Idle watching ────────────────────────────────────────────────
588
589
590def _densify_rm(poly, max_seg_len=40.0):
591 """Subdivide a polyline so no segment exceeds max_seg_len (rm units).
592
593 The reMarkable stroke renderer mangles long 2-point segments — it
594 interprets the gap as a curve or extends the endpoints. Inserting
595 intermediate points keeps straight lines straight.
596 """
597 if len(poly) < 2:
598 return poly
599 out = [poly[0]]
600 for (x0, y0), (x1, y1) in zip(poly, poly[1:]):
601 dx, dy = x1 - x0, y1 - y0
602 dist = (dx * dx + dy * dy) ** 0.5
603 n = max(1, int(dist / max_seg_len))
604 for k in range(1, n + 1):
605 t = k / n
606 out.append((x0 + dx * t, y0 + dy * t))
607 return out
608
609
610def _scp_thinking_swirl(ssh):
611 """Parse border_top_right_stroke.svg into corner ornament strokes for the
612 thinking indicator. Falls back to a simple spiral if file is absent.
613
614 Done once at startup — the animation thread just fires draw/erase
615 commands over the socket with no per-cycle file transfer.
616 """
617 import math
618 import xml.etree.ElementTree as ET
619
620 def _cubicbez(p0, p1, p2, p3, tolerance=0.05):
621 """Adaptive bezier subdivision: more points where curvature is high."""
622 def _subdivide(a, b, c, d, tol):
623 # Flatness test: max distance of control points from chord a→d
624 dx, dy = d[0]-a[0], d[1]-a[1]
625 chord_len_sq = dx*dx + dy*dy
626 if chord_len_sq < 1e-8:
627 return [a, d]
628 # Perpendicular distances of b and c from the chord
629 inv_len = 1.0 / (chord_len_sq ** 0.5)
630 dist_b = abs((b[0]-a[0])*dy - (b[1]-a[1])*dx) * inv_len
631 dist_c = abs((c[0]-a[0])*dy - (c[1]-a[1])*dx) * inv_len
632 if max(dist_b, dist_c) <= tol:
633 return [a, d]
634 # De Casteljau split at t=0.5
635 ab = ((a[0]+b[0])/2, (a[1]+b[1])/2)
636 bc = ((b[0]+c[0])/2, (b[1]+c[1])/2)
637 cd = ((c[0]+d[0])/2, (c[1]+d[1])/2)
638 abc = ((ab[0]+bc[0])/2, (ab[1]+bc[1])/2)
639 bcd = ((bc[0]+cd[0])/2, (bc[1]+cd[1])/2)
640 mid = ((abc[0]+bcd[0])/2, (abc[1]+bcd[1])/2)
641 left = _subdivide(a, ab, abc, mid, tol)
642 right = _subdivide(mid, bcd, cd, d, tol)
643 return left[:-1] + right
644 return _subdivide(p0, p1, p2, p3, tolerance)
645
646 def _parse_d(d):
647 """SVG path d string → list of polylines (handles M L C Z)."""
648 tok = re.findall(
649 r'[MLCZz]|[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?', d
650 )
651 polylines, cur, cx, cy, cmd = [], [], 0.0, 0.0, None
652 i = 0
653 while i < len(tok):
654 if re.match(r'[MLCZz]', tok[i]):
655 cmd = tok[i]; i += 1; continue
656 if cmd == 'M' and i + 1 < len(tok):
657 if cur: polylines.append(cur)
658 cx, cy = float(tok[i]), float(tok[i+1])
659 cur = [(cx, cy)]; i += 2
660 elif cmd == 'L' and i + 1 < len(tok):
661 cx, cy = float(tok[i]), float(tok[i+1])
662 cur.append((cx, cy)); i += 2
663 elif cmd == 'C' and i + 5 < len(tok):
664 x1,y1 = float(tok[i]),float(tok[i+1])
665 x2,y2 = float(tok[i+2]),float(tok[i+3])
666 x,y = float(tok[i+4]),float(tok[i+5])
667 for p in _cubicbez((cx,cy),(x1,y1),(x2,y2),(x,y))[1:]:
668 cur.append(p)
669 cx, cy = x, y; i += 6
670 elif cmd in ('Z', 'z'):
671 # Don't close path — for stroke ornaments, Z would retrace
672 # back to start and create overlapping lines. Just end the polyline.
673 if cur: polylines.append(cur)
674 cur = []
675 else:
676 i += 1
677 if cur:
678 polylines.append(cur)
679 return polylines
680
681 def _parse_matrix(s):
682 """Parse 'matrix(a,b,c,d,e,f)' into (a,b,c,d,e,f) tuple."""
683 m = re.match(r'matrix\(([^)]+)\)', s.strip())
684 if not m:
685 return (1, 0, 0, 1, 0, 0)
686 vals = [float(v.strip()) for v in m.group(1).split(',')]
687 return tuple(vals) if len(vals) == 6 else (1, 0, 0, 1, 0, 0)
688
689 def _compose(m1, m2):
690 """Compose two (a,b,c,d,e,f) matrices: apply m2 then m1."""
691 a1,b1,c1,d1,e1,f1 = m1
692 a2,b2,c2,d2,e2,f2 = m2
693 return (
694 a1*a2 + c1*b2, b1*a2 + d1*b2,
695 a1*c2 + c1*d2, b1*c2 + d1*d2,
696 a1*e2 + c1*f2 + e1, b1*e2 + d1*f2 + f1,
697 )
698
699 def _apply(m, x, y):
700 """Apply matrix (a,b,c,d,e,f) to point (x,y)."""
701 a,b,c,d,e,f = m
702 return a*x + c*y + e, b*x + d*y + f
703
704 def _centerline(poly):
705 """Extract centerline from a closed outline path.
706
707 Outline paths trace one edge then return along the other.
708 Split at the midpoint and average corresponding points from
709 each half to get the center stroke.
710 """
711 n = len(poly)
712 if n < 4:
713 return poly
714 # Check if it's actually closed (start ≈ end)
715 dx = poly[0][0] - poly[-1][0]
716 dy = poly[0][1] - poly[-1][1]
717 if dx*dx + dy*dy > 4.0:
718 return poly # not closed, use as-is
719 # Remove duplicate closing point
720 pts = poly[:-1]
721 n = len(pts)
722 half = n // 2
723 center = []
724 for i in range(half):
725 j = n - 1 - i # mirror index from second half
726 cx = (pts[i][0] + pts[j][0]) / 2
727 cy = (pts[i][1] + pts[j][1]) / 2
728 center.append((cx, cy))
729 return center
730
731 svg_file = Path(__file__).parent / "border-top-right-centerline.svg"
732 strokes = []
733
734 if svg_file.exists():
735 tree = ET.parse(str(svg_file))
736 root = tree.getroot()
737 ns = {'svg': 'http://www.w3.org/2000/svg'}
738
739 # Collect all paths with their composed transform matrices.
740 all_polys = []
741
742 def _walk(el, parent_mat):
743 mat = parent_mat
744 t = el.attrib.get('transform')
745 if t:
746 mat = _compose(parent_mat, _parse_matrix(t))
747 tag = el.tag.split('}')[-1] if '}' in el.tag else el.tag
748 if tag == 'path':
749 d = el.attrib.get('d', '')
750 for poly in _parse_d(d):
751 transformed = [_apply(mat, x, y) for x, y in poly]
752 all_polys.append(transformed)
753 for child in el:
754 _walk(child, mat)
755
756 _walk(root, (1, 0, 0, 1, 0, 0))
757
758 if all_polys:
759 # Use viewBox for accurate origin/dimensions (point-based bounds
760 # can miss negative coords or padding the designer intended).
761 vb = root.attrib.get('viewBox')
762 if vb:
763 parts = [float(v) for v in vb.replace(',', ' ').split()]
764 ox, oy, svg_w, svg_h = parts
765 else:
766 all_x = [p[0] for poly in all_polys for p in poly]
767 all_y = [p[1] for poly in all_polys for p in poly]
768 svg_w = max(all_x) - min(all_x)
769 svg_h = max(all_y) - min(all_y)
770 ox, oy = min(all_x), min(all_y)
771
772 # rm coords: x ∈ [-702, 702], y ∈ [0, 1872] top-to-bottom.
773 corners = [
774 (False, False, 132, 632, 50, 430), # top-right
775 (True, False, -647, -147, 50, 430), # top-left
776 (False, True, 132, 632, 1442, 1822), # bottom-right
777 (True, True, -647, -147, 1442, 1822), # bottom-left
778 ]
779
780 def make_strokes(polys, flip_x, flip_y, rx0, rx1, ry0, ry1):
781 box_w = rx1 - rx0
782 box_h = ry1 - ry0
783 # Uniform scale: fit SVG into box preserving aspect ratio
784 scale = min(box_w / svg_w, box_h / svg_h)
785 # Anchor to the corner instead of centering. The SVG is drawn
786 # hugging the top-right of its own viewBox (scrolls at top,
787 # vertical bar on the right edge), so before any flip we push
788 # the leftover slack to the left/bottom — keeping the ornament
789 # tight against the corner rather than floating inward.
790 margin_x = box_w - svg_w * scale
791 margin_y = 0.0
792
793 def to_rm(sx, sy):
794 # Position relative to SVG origin, scaled and centered
795 nx = (sx - ox) * scale + margin_x
796 ny = (sy - oy) * scale + margin_y
797 # Apply flip
798 if flip_x:
799 nx = box_w - nx
800 if flip_y:
801 ny = box_h - ny
802 return rx0 + nx, ry0 + ny
803 result = []
804 for poly in polys:
805 if len(poly) < 2:
806 continue
807 rm_poly = [to_rm(x, y) for x, y in poly]
808 # Densify: the device stroke renderer mangles long
809 # 2-point segments (the straight bars), interpreting the
810 # big gap as a curve or extending the endpoints. Subdivide
811 # so every segment is short and renders as a clean line.
812 dense = _densify_rm(rm_poly, max_seg_len=40.0)
813 pts = [[px, py, 6, 11, 90, 170] for px, py in dense]
814 xs = [p[0] for p in pts]; ys = [p[1] for p in pts]
815 result.append({
816 'points': pts,
817 'rgba': 4278190080, 'color': 0,
818 'bounds': [min(xs), min(ys), max(xs)-min(xs), max(ys)-min(ys)],
819 'tool': 15, 'maskScale': 2.0, 'thickness': 2.0,
820 })
821 return result
822
823 per_corner = [make_strokes(all_polys, *c) for c in corners]
824
825 # Interleave: tl[0], tr[0], br[0], bl[0], tl[1], tr[1], ...
826 max_len = max(len(c) for c in per_corner)
827 for i in range(max_len):
828 for corner_strokes in per_corner:
829 if i < len(corner_strokes):
830 strokes.append(corner_strokes[i])
831
832 if not strokes:
833 # Fallback: simple spiral
834 swirl_pts = []
835 for i in range(40):
836 t = i / 39.0
837 angle = t * math.pi * 3
838 r = 15 + t * 25
839 swirl_pts.append([-580 + r * math.cos(angle),
840 1750 + r * math.sin(angle) * 0.6,
841 6, 11, 90, 170])
842 xs = [p[0] for p in swirl_pts]; ys = [p[1] for p in swirl_pts]
843 strokes = [{'points': swirl_pts, 'rgba': 4278190080, 'color': 0,
844 'bounds': [min(xs), min(ys), max(xs)-min(xs), max(ys)-min(ys)],
845 'tool': 15, 'maskScale': 2.0, 'thickness': 2.0}]
846
847 local = "/tmp/glossa_thinking_live.json"
848 with open(local, 'w') as f:
849 json.dump(strokes, f)
850 ssh.scp_to(local, "/tmp/glossa_thinking_live.json")
851 print(f"[swirl] {len(strokes)} ornament strokes "
852 f"({'SVG' if svg_file.exists() else 'fallback spiral'})")
853
854
855
856def watch_for_idle(device, last_ts):
857 """Block until uinjectd pushes an idle event newer than last_ts.
858
859 The daemon watches /tmp/glossa_idle locally and pushes an event
860 the instant it changes — no host-side polling. Returns the new
861 timestamp. Raises ConnectionError if the socket dies.
862 """
863 while True:
864 try:
865 msg = device.events.get(timeout=5)
866 except _queue.Empty:
867 # No events recently. The heartbeat thread keeps the socket
868 # alive and flips _connected to False if it dies, so we just
869 # check that flag rather than sending our own ping (whose reply
870 # is intentionally dropped by the reader).
871 if not device._connected:
872 raise ConnectionError("uinjectd connection lost")
873 continue
874 if msg.get("event") == "idle":
875 ts = msg.get("ts")
876 if ts != last_ts:
877 return ts
878
879
880# ─── Pixel diff for new content detection ──────────────────────────
881
882def _page_is_empty(img):
883 """Return True if the page has no real handwriting (only grid dots or blank).
884
885 Uses the same grid-artifact filter as _find_new_region but without the
886 full-page sanity check, so a very full page is never mis-classified.
887 """
888 import numpy as np
889 curr = np.array(img.convert('L'))
890 dark_per_row = np.sum(curr < 128, axis=1)
891 candidate = (dark_per_row > 5) & (dark_per_row != 68)
892 idxs = np.where(candidate)[0]
893 for idx in idxs:
894 if np.sum(candidate[max(0, idx - 2):idx + 3]) >= 3:
895 return False
896 return True
897
898
899def _find_new_region(current_img, last_img, ignore_below_y=None):
900 """Compare two framebuffer images, return crop of changed region.
901
902 Returns (cropped_image, content_bottom_y) or (None, 0) if no
903 significant change. Uses simple pixel difference to find where
904 new handwriting appeared.
905
906 ignore_below_y: if set, ignore all changes below this Y coordinate
907 in the cropped image. Used to skip glossa's own injected strokes.
908 """
909 import numpy as np
910
911 curr = np.array(current_img.convert('L'))
912
913 if last_img is None:
914 # First capture — find the vertical extent of existing handwriting.
915 # Grid artifact: uniform rows with exactly 68 dark pixels. Real ink
916 # rows have variable counts. Require a cluster of ≥3 adjacent real
917 # rows so isolated noise/artifacts don't count.
918 dark_per_row = np.sum(curr < 128, axis=1)
919 candidate = (dark_per_row > 5) & (dark_per_row != 68)
920 # Cluster filter: keep only rows that have a real neighbor within 2px
921 real_rows = []
922 idxs = np.where(candidate)[0]
923 for idx in idxs:
924 neighbors = np.sum(candidate[max(0, idx-2):idx+3])
925 if neighbors >= 3:
926 real_rows.append(idx)
927 if not real_rows:
928 return None, 0
929 real_rows = np.array(real_rows)
930 y1, y2 = real_rows[0], real_rows[-1]
931 # Sanity: don't treat a nearly-full-page diff as real content
932 if (y2 - y1) > curr.shape[0] * 0.85:
933 return None, 0
934 v_pad = 30
935 y1 = max(0, y1 - v_pad)
936 y2 = min(curr.shape[0] - 1, y2 + v_pad)
937 # Full width always
938 crop = current_img.crop((0, y1, curr.shape[1], y2 + 1))
939 return crop, y2
940
941 prev = np.array(last_img.convert('L'))
942
943 # Build mask: ignore edges (e-ink artifacts) and previously injected area
944 mask = np.zeros_like(curr, dtype=bool)
945 top_margin = 10
946 bottom_margin = 20
947 mask[top_margin:curr.shape[0] - bottom_margin, :] = True
948
949 if ignore_below_y is not None:
950 # Convert device Y back to cropped-image Y (subtract toolbar offset)
951 cutoff = ignore_below_y - 80
952 if 0 < cutoff < curr.shape[0]:
953 mask[cutoff:, :] = False
954
955 # Exclude the four corner ornament zones from diff detection.
956 # Ornament boxes (rm coords): right x=[585,691], left x=[-661,-540]
957 # y top=[101,166], bottom=[1798,1864]. Converted to image pixels
958 # (rm_x+702, rm_y-80); image is 1404×1592.
959 cx_right = 120 # ornament extent from right edge (img px)
960 cx_left = 165 # ornament extent from left edge (img px)
961 cy_top = 250 # ornament vertical extent from top
962 cy_bot = 150 # ornament vertical extent from bottom
963 h, w = curr.shape
964 mask[:cy_top, :cx_left] = False # top-left corner
965 mask[:cy_top, w - cx_right:] = False # top-right corner
966 mask[h - cy_bot:, :cx_left] = False # bottom-left corner
967 mask[h - cy_bot:, w - cx_right:] = False # bottom-right corner
968
969 # Threshold of 30 (was 50) catches lighter handwriting strokes without
970 # being so sensitive that e-ink refresh noise triggers false positives.
971 diff = (np.abs(curr.astype(int) - prev.astype(int)) > 30) & mask
972
973 if not diff.any():
974 return None, 0
975
976 # Find the vertical extent of changed pixels
977 rows = np.any(diff, axis=1)
978 y1, y2 = np.where(rows)[0][[0, -1]]
979
980 # Use full image width for the crop so complete text lines are
981 # always captured — tight horizontal bounds clip partial characters.
982 x1 = 0
983 x2 = curr.shape[1] - 1
984
985 # Generous vertical padding gives the model baseline/ascender context
986 v_pad = 60
987 y1 = max(0, y1 - v_pad)
988 y2 = min(curr.shape[0] - 1, y2 + v_pad)
989
990 # Minimum height check (width is always full)
991 if (y2 - y1) < 10:
992 return None, 0
993
994 crop = current_img.crop((x1, y1, x2 + 1, y2 + 1))
995 return crop, y2
996
997
998# ─── Main loop ────────────────────────────────────────────────────
999
1000def main():
1001 parser = argparse.ArgumentParser(description="Glossa continuous loop")
1002 parser.add_argument(
1003 "--model", type=str, default="gpt-4.1-mini",
1004 help="Hyper API model",
1005 )
1006 parser.add_argument(
1007 "--once", action="store_true",
1008 help="Run one cycle then exit (for testing)",
1009 )
1010 parser.add_argument(
1011 "-v", "--verbose", action="store_true",
1012 help="Pass -v to uinject for progress logging",
1013 )
1014 args = parser.parse_args()
1015
1016 global _inject_verbose, _device
1017 _inject_verbose = args.verbose
1018
1019 print("=== Glossa Loop ===")
1020 print(f"Model: {args.model}")
1021 if _inject_verbose:
1022 print("Verbose: uinject -v enabled")
1023
1024 # SSH still handles framebuffer capture/SCP; the persistent socket
1025 # handles injection + idle events with no per-call overhead.
1026 ssh = SSHSession("remarkable")
1027 ensure_daemon(ssh, verbose=args.verbose)
1028 _device = DeviceClient()
1029 _device.connect()
1030
1031 # On connect the daemon pushes "ready" plus the current idle file
1032 # value. Drain those and use the existing idle ts as our baseline so
1033 # we don't fire a spurious cycle before the user lifts the pen.
1034 # Also verify the connection is actually alive (reader thread didn't die).
1035 last_idle_ts = None
1036 deadline = time.time() + 2.0
1037 got_ready = False
1038 while time.time() < deadline:
1039 try:
1040 msg = _device.events.get(timeout=0.3)
1041 except _queue.Empty:
1042 continue # keep waiting until deadline, don't break early
1043 if msg.get("event") == "ready":
1044 got_ready = True
1045 elif msg.get("event") == "idle":
1046 last_idle_ts = msg.get("ts")
1047
1048 if not got_ready or not _device._connected:
1049 print("[device] Initial connection failed, retrying...")
1050 _device.close()
1051 time.sleep(1)
1052 ensure_daemon(ssh, verbose=args.verbose)
1053 _device = DeviceClient()
1054 _device.connect()
1055 deadline = time.time() + 2.0
1056 while time.time() < deadline:
1057 try:
1058 msg = _device.events.get(timeout=0.3)
1059 except _queue.Empty:
1060 continue
1061 if msg.get("event") == "idle":
1062 last_idle_ts = msg.get("ts")
1063
1064 # The thinking swirl is static — generate it once and SCP it to the
1065 # device so the daemon can draw/erase it on demand over the socket.
1066 _scp_thinking_swirl(ssh)
1067
1068 print("Waiting for pen idle signal...")
1069 print()
1070
1071 last_fb_hash = None # quick hash to skip unchanged frames
1072 last_inject_y = None # device Y where we last injected
1073 last_img = None # previous frame for pixel-diff detection
1074 conversation = [] # text history: list of {"role":..., "content":...}
1075 ornaments_drawn = False # corner ornaments are permanent; only draw once
1076
1077 try:
1078 while True:
1079 # Wait for pushed idle event (no polling)
1080 try:
1081 new_ts = watch_for_idle(_device, last_idle_ts)
1082 except ConnectionError as e:
1083 print(f"[{_ts()}] Connection lost ({e}), reconnecting...")
1084 try:
1085 _device.close()
1086 except Exception:
1087 pass
1088 for attempt in range(5):
1089 time.sleep(2)
1090 try:
1091 # Recreate SSH session in case the master died
1092 try:
1093 ssh.close()
1094 except Exception:
1095 pass
1096 ssh = SSHSession("remarkable")
1097 ensure_daemon(ssh, verbose=False)
1098 _device = DeviceClient()
1099 _device.connect()
1100 # Drain initial events
1101 deadline = time.time() + 2.0
1102 while time.time() < deadline:
1103 try:
1104 msg = _device.events.get(timeout=0.3)
1105 except _queue.Empty:
1106 continue
1107 if msg.get("event") == "idle":
1108 last_idle_ts = msg.get("ts")
1109 print(f"[{_ts()}] Reconnected (attempt {attempt+1})")
1110 break
1111 except Exception as re_err:
1112 print(f"[{_ts()}] Reconnect attempt {attempt+1} failed: {re_err}")
1113 else:
1114 print(f"[{_ts()}] ERROR: could not reconnect after 5 attempts")
1115 time.sleep(10)
1116 continue
1117 last_idle_ts = new_ts
1118
1119 t0 = time.time()
1120 print(f"[{_ts()}] Pen idle detected!")
1121
1122 try:
1123 # Capture
1124 print(f" [{_ts()}] Capturing...")
1125 img = capture_framebuffer(ssh)
1126 print(f" [{_ts()}] Captured ({_elapsed(t0)})")
1127
1128 # Check if the page is blank (only grid dots, no real ink).
1129 if _page_is_empty(img):
1130 print(f" [{_ts()}] Empty page — resetting state.")
1131 last_img = None
1132 last_inject_y = None
1133 ornaments_drawn = False
1134 conversation.clear()
1135 print()
1136 continue
1137
1138 # Detect new content via pixel diff against last frame.
1139 # last_img is updated to the post-injection capture after
1140 # each successful reply, so the diff only ever contains
1141 # content the user actually drew since the last response.
1142 new_crop, content_y2 = _find_new_region(img, last_img)
1143 last_img = img
1144
1145 # Fire-and-forget: draw corner ornaments once on the first
1146 if new_crop is None:
1147 last_fb_hash = _framebuffer_hash(img)
1148 print(f" [{_ts()}] No new content, skipping.")
1149 print()
1150 continue
1151
1152 # Draw thinking ornament only when there's real content to process.
1153 if not ornaments_drawn:
1154 ornaments_drawn = True
1155 _threading.Thread(
1156 target=lambda: run_inject(None, "draw",
1157 "/tmp/glossa_thinking_live.json",
1158 speed_ms=4, timeout=120),
1159 daemon=True,
1160 ).start()
1161
1162 # Send only the new handwriting crop + text history
1163 cw, ch = new_crop.size
1164 print(f" [{_ts()}] Asking {args.model} ({len(conversation)} history turns, crop={cw}x{ch})...")
1165 # Save crop for manual inspection
1166 new_crop.save("/tmp/glossa_last_crop.png")
1167 result = ask_model(new_crop, conversation, args.model)
1168 question = result.get("question")
1169 answer = result.get("answer")
1170 print(f" [{_ts()}] Model: Q={str(question)[:80]!r} A={str(answer)[:80]!r}")
1171
1172 if not answer or answer == "null":
1173 last_img = img
1174 print(f" [{_ts()}] No actionable content, skipping.")
1175 print()
1176 continue
1177
1178 print(f" [{_ts()}] Q ({_elapsed(t0)}): {str(question)[:60]}")
1179 print(f" [{_ts()}] A: {answer[:80]}{'...' if len(answer) > 80 else ''}")
1180
1181 # Add this exchange to text history for future turns
1182 conversation.append({"role": "user", "content": str(question)})
1183 conversation.append({"role": "assistant", "content": answer})
1184
1185 # Place reply just below the bottom of the changed region.
1186 # content_y2 is in cropped-image coordinates; add 80 for the
1187 # stripped toolbar to get device Y, then 150px gap.
1188 reply_y = min(content_y2 + 80 + 60, 1750)
1189 print(f" [{_ts()}] Content bottom: {content_y2}, reply_y={reply_y}")
1190
1191 # Render + Inject
1192 print(f" [{_ts()}] Rendering + injecting (reply_y={reply_y})...")
1193 actual_y = render_and_inject(ssh, answer, reply_y=reply_y)
1194 last_inject_y = actual_y if actual_y is not None else reply_y
1195 print(f" [{_ts()}] Injected ({_elapsed(t0)})")
1196
1197 # Capture a fresh baseline AFTER injection so the next
1198 # diff only sees genuinely new user content. Without this,
1199 # last_img is the pre-injection frame and every subsequent
1200 # diff includes our own injected strokes as "new" pixels,
1201 # producing a crop that mixes our reply with the user's
1202 # next question and confuses the model.
1203 print(f" [{_ts()}] Capturing post-injection baseline...")
1204 try:
1205 time.sleep(0.5) # let e-ink settle
1206 last_img = capture_framebuffer(ssh)
1207 print(f" [{_ts()}] Baseline updated")
1208 except Exception as e_cap:
1209 print(f" [{_ts()}] Baseline capture failed: {e_cap} (stale frame kept)")
1210
1211 elapsed = time.time() - t0
1212 print(f" [{_ts()}] Done in {elapsed:.1f}s")
1213
1214 except Exception as e:
1215 print(f" ERROR: {e}")
1216
1217 print()
1218
1219 if args.once:
1220 break
1221
1222 except KeyboardInterrupt:
1223 print("\nStopping.")
1224 finally:
1225 if _device:
1226 _device.close()
1227 ssh.close()
1228
1229
1230if __name__ == "__main__":
1231 main()