dunkirk.sh
sppoky
1/* glossad.c — On-device glossa daemon (Phase 1)
2 *
3 * Runs entirely on the reMarkable. Watches /tmp/glossa_armed for the
4 * toggle state, and when armed, accepts stroke injection commands via
5 * a simple file-trigger protocol.
6 *
7 * Phase 1: daemon skeleton + arming + in-process evdev injection.
8 * Absorbs uinjectd's draw/erase/swipe code directly — no socket, no
9 * separate daemon, no connection management.
10 *
11 * Injection trigger: write a JSON command to /tmp/glossa_cmd:
12 * {"cmd":"draw","file":"/tmp/glossa_strokes.json","speed":3}
13 * {"cmd":"erase","file":"/tmp/glossa_thinking_live.json","speed":20}
14 * {"cmd":"swipe"}
15 *
16 * The daemon picks up the command, executes it, writes the result to
17 * /tmp/glossa_result, then deletes /tmp/glossa_cmd.
18 *
19 * Usage: glossad [-v]
20 */
21#include <stdio.h>
22#include <stdlib.h>
23#include <string.h>
24#include <stdarg.h>
25#include <fcntl.h>
26#include <unistd.h>
27#include <linux/input.h>
28#include <sys/stat.h>
29#include <pthread.h>
30#include <errno.h>
31#include <signal.h>
32#include <time.h>
33#include <math.h>
34#include <png.h>
35#include <openssl/ssl.h>
36#include <openssl/err.h>
37#include <sys/socket.h>
38#include <netinet/in.h>
39#include <netdb.h>
40
41#define DEV_PATH "/dev/input/event1"
42#define TOUCH_PATH "/dev/input/event2"
43#define ARMED_PATH "/tmp/glossa_armed"
44#define IDLE_PATH "/tmp/glossa_idle"
45#define CMD_PATH "/tmp/glossa_cmd"
46#define RESULT_PATH "/tmp/glossa_result"
47#define LOG_PATH "/tmp/glossad.log"
48#define FB_RAW_PATH "/tmp/glossa_fb.raw"
49#define SCREENSHOT_TRIGGER "/tmp/glossa_screenshot"
50#define CAPTURE_OUT_PATH "/tmp/glossa_capture.png"
51#define API_KEY_PATH "/home/root/.glossa_key"
52#define API_HOST "potluck.dunkirk.sh"
53#define API_PATH "/v1/chat/completions"
54#define API_MODEL "pioneer/gpt-5-nano"
55#define FONT_PATH "/home/root/font_data.json"
56#define RENDER_OUT_PATH "/tmp/glossa_render.json"
57#define THINKING_PATH "/tmp/glossa_thinking"
58#define ORNAMENTS_PATH "/home/root/ornaments.json"
59#define SAFEZONE_PATH "/tmp/glossa_safezone"
60#define IDLE_TIMEOUT 300 /* 5 minutes in seconds */
61#define MAX_CMD 8192
62
63/* Framebuffer dimensions */
64#define FB_WIDTH 1404
65#define FB_HEIGHT 1872
66#define FB_BPP 4
67#define FB_SIZE (FB_WIDTH * FB_HEIGHT * FB_BPP)
68
69/* Crop parameters (match Python loop) */
70#define CROP_TOP 80
71#define CROP_BOTTOM 200
72#define CROP_LEFT 10
73#define CROP_RIGHT 10
74#define CROP_HEIGHT (FB_HEIGHT - CROP_TOP - CROP_BOTTOM)
75#define CROP_WIDTH (FB_WIDTH - CROP_LEFT - CROP_RIGHT)
76
77/* Influence map constants */
78#define INFLUENCE_RADIUS 30 /* px around each stroke point */
79#define INFLUENCE_SIGMA 15.0 /* Gaussian falloff sigma */
80#define INFLUENCE_THRESH 128 /* mask threshold for blanking */
81#define DARK_THRESHOLD 128 /* pixel darkness threshold */
82
83/* External: font renderer (font_render.c) */
84extern int render_text_to_json(const char *text, const char *output_path,
85 float origin_x, float origin_y,
86 const char *font_path, int *consumed);
87
88/* Forward declarations */
89static char *build_messages_json(const char *img_b64, long b64_len);
90
91static int g_verbose = 0;
92static int g_dev_fd = -1;
93static volatile sig_atomic_t g_running = 1;
94static pthread_mutex_t g_inject_lock = PTHREAD_MUTEX_INITIALIZER;
95static time_t g_safezone_until = 0;
96static time_t g_last_activity = 0;
97
98/* Stroke influence map: CROP_HEIGHT x CROP_WIDTH, uint8.
99 * Tracks known content regions so we can mask them out of the AI crop
100 * and detect only genuinely new handwriting. Zeroed at startup,
101 * cleared on page turn. */
102static unsigned char g_influence[CROP_HEIGHT][CROP_WIDTH];
103
104/* ─── logging ────────────────────────────────────────────────────── */
105
106static FILE *g_log_fp = NULL;
107
108static void log_msg(const char *fmt, ...) {
109 va_list ap;
110 va_start(ap, fmt);
111
112 /* Always write to log file */
113 if (!g_log_fp) {
114 g_log_fp = fopen(LOG_PATH, "a");
115 }
116 if (g_log_fp) {
117 time_t now = time(NULL);
118 struct tm *t = localtime(&now);
119 fprintf(g_log_fp, "[%04d-%02d-%02d %02d:%02d:%02d] ",
120 t->tm_year+1900, t->tm_mon+1, t->tm_mday,
121 t->tm_hour, t->tm_min, t->tm_sec);
122 vfprintf(g_log_fp, fmt, ap);
123 fflush(g_log_fp);
124 }
125
126 if (g_verbose) {
127 va_list ap2;
128 va_copy(ap2, ap);
129 fprintf(stderr, "[glossad] ");
130 vfprintf(stderr, fmt, ap2);
131 va_end(ap2);
132 }
133
134 va_end(ap);
135}
136
137/* ─── signal handling ────────────────────────────────────────────── */
138
139static void handle_signal(int sig) {
140 (void)sig;
141 g_running = 0;
142}
143
144/* ─── evdev helpers (from uinjectd) ──────────────────────────────── */
145
146static void emit_event(int fd, int type, int code, int value) {
147 struct input_event ev;
148 memset(&ev, 0, sizeof(ev));
149 ev.type = type;
150 ev.code = code;
151 ev.value = value;
152 write(fd, &ev, sizeof(ev));
153}
154
155static void emit_syn(int fd) {
156 emit_event(fd, EV_SYN, SYN_REPORT, 0);
157}
158
159static void rm_to_wacom(float rm_x, float rm_y, int *wx, int *wy) {
160 float screen_x = rm_x + 702.0f;
161 float screen_y = rm_y;
162 *wx = (int)((1872.0f - screen_y) * 20966.0f / 1872.0f);
163 *wy = (int)(screen_x * 15725.0f / 1404.0f);
164}
165
166static int rm_pressure_to_wacom(int rm_pressure) {
167 return rm_pressure * 4095 / 255;
168}
169
170/* ─── minimal JSON parsing ───────────────────────────────────────── */
171
172static const char *json_str(const char *json, const char *key, int *len) {
173 char pattern[64];
174 snprintf(pattern, sizeof(pattern), "\"%s\"", key);
175 const char *p = strstr(json, pattern);
176 if (!p) return NULL;
177 p += strlen(pattern);
178 while (*p == ' ' || *p == ':') p++;
179 if (*p != '"') return NULL;
180 p++;
181 const char *start = p;
182 while (*p && *p != '"') p++;
183 *len = (int)(p - start);
184 return start;
185}
186
187static int json_int(const char *json, const char *key, int def) {
188 char pattern[64];
189 snprintf(pattern, sizeof(pattern), "\"%s\"", key);
190 const char *p = strstr(json, pattern);
191 if (!p) return def;
192 p += strlen(pattern);
193 while (*p == ' ' || *p == ':') p++;
194 return atoi(p);
195}
196
197/* ─── file loading ───────────────────────────────────────────────── */
198
199static int load_file(const char *path, char **out, long *out_size) {
200 FILE *fp = fopen(path, "r");
201 if (!fp) return -1;
202 fseek(fp, 0, SEEK_END);
203 long sz = ftell(fp);
204 fseek(fp, 0, SEEK_SET);
205 char *buf = malloc(sz + 1);
206 if (!buf) { fclose(fp); return -1; }
207 fread(buf, 1, sz, fp);
208 buf[sz] = '\0';
209 fclose(fp);
210 *out = buf;
211 *out_size = sz;
212 return 0;
213}
214
215static void write_result(const char *msg) {
216 FILE *fp = fopen(RESULT_PATH, "w");
217 if (fp) {
218 fputs(msg, fp);
219 fputc('\n', fp);
220 fclose(fp);
221 }
222}
223
224/* ─── image pipeline (Phase 2) ───────────────────────────────────── */
225
226/* Read raw framebuffer into allocated buffer. Returns NULL on failure. */
227static unsigned char *read_framebuffer(void) {
228 FILE *fp = fopen(FB_RAW_PATH, "rb");
229 if (!fp) return NULL;
230 unsigned char *buf = malloc(FB_SIZE);
231 if (!buf) { fclose(fp); return NULL; }
232 size_t n = fread(buf, 1, FB_SIZE, fp);
233 fclose(fp);
234 if ((long)n < FB_SIZE) { free(buf); return NULL; }
235 return buf;
236}
237
238/* Convert RGBA pixel to grayscale luminance */
239static inline unsigned char rgba_to_gray(const unsigned char *px) {
240 /* Standard luminance: 0.299R + 0.587G + 0.114B */
241 return (unsigned char)((px[0] * 77 + px[1] * 150 + px[2] * 29) >> 8);
242}
243
244/* ─── stroke influence map ─────────────────────────────────────── */
245
246/* Mark a region around stroke points in the influence map.
247 * Points are in raw framebuffer coordinates (not crop-relative).
248 * Uses Gaussian falloff so edges blend smoothly. */
249static void mark_stroke_region(const float *points, int n_points, int stride) {
250 float sigma2x2 = 2.0f * INFLUENCE_SIGMA * INFLUENCE_SIGMA;
251 int r = INFLUENCE_RADIUS;
252
253 for (int i = 0; i < n_points; i++) {
254 float px = points[i * stride];
255 float py = points[i * stride + 1];
256
257 /* Convert to crop-space (accounting for side margins) */
258 int cx = (int)(px + 702.0f) - CROP_LEFT;
259 int cy = (int)(py - CROP_TOP);
260
261 /* Bounding box in crop space, clamped */
262 int x0 = cx - r; if (x0 < 0) x0 = 0;
263 int y0 = cy - r; if (y0 < 0) y0 = 0;
264 int x1 = cx + r; if (x1 >= CROP_WIDTH) x1 = CROP_WIDTH - 1;
265 int y1 = cy + r; if (y1 >= CROP_HEIGHT) y1 = CROP_HEIGHT - 1;
266
267 for (int y = y0; y <= y1; y++) {
268 float dy = (float)(y - cy);
269 float dy2 = dy * dy;
270 for (int x = x0; x <= x1; x++) {
271 float dx = (float)(x - cx);
272 float dist2 = dx * dx + dy2;
273 unsigned char val = (unsigned char)(255.0f * expf(-dist2 / sigma2x2));
274 if (val > g_influence[y][x])
275 g_influence[y][x] = val;
276 }
277 }
278 }
279}
280
281/* Parse stroke JSON and mark all points in the influence map.
282 * Expects the uinject JSON format: [{points:[[x,y,...],...]}, ...] */
283static void mark_strokes_from_json(const char *json) {
284 /* Simple parser: find each [x,y pair inside "points" arrays */
285 const char *p = json;
286 while ((p = strstr(p, "\"points\"")) != NULL) {
287 p = strchr(p, '['); /* opening of points array */
288 if (!p) break;
289 p++; /* skip '[' */
290
291 /* Parse [[x,y,...],[x,y,...],...] */
292 while (*p && *p != ']') {
293 if (*p == '[') {
294 float coords[6];
295 int nc = 0;
296 p++; /* skip inner '[' */
297 while (*p && *p != ']' && nc < 6) {
298 char *end;
299 coords[nc++] = strtof(p, &end);
300 if (end == p) break;
301 p = end;
302 while (*p == ',' || *p == ' ') p++;
303 }
304 if (nc >= 2) {
305 mark_stroke_region(coords, 1, 2);
306 }
307 /* skip to closing ']' of this point */
308 while (*p && *p != ']') p++;
309 if (*p == ']') p++;
310 } else {
311 p++;
312 }
313 }
314 if (*p == ']') p++;
315 }
316}
317
318/* Apply influence mask to a cropped grayscale buffer: pixels in known
319 * regions are blanked to white. Used before saving PNG for the AI. */
320static void apply_influence_mask(unsigned char *gray_crop) {
321 for (int y = 0; y < CROP_HEIGHT; y++) {
322 for (int x = 0; x < CROP_WIDTH; x++) {
323 if (g_influence[y][x] > INFLUENCE_THRESH)
324 gray_crop[y * CROP_WIDTH + x] = 255;
325 }
326 }
327}
328
329/* Detect new content outside known regions.
330 * Returns 1 if dark pixels exist outside the influence mask, 0 otherwise.
331 * Also sets *out_bottom to the lowest row (in raw FB coords) with new content. */
332static int detect_new_content(const unsigned char *fb, int *out_bottom) {
333 int found = 0;
334 int bottom_crop_y = -1;
335
336 for (int y = 0; y < CROP_HEIGHT; y++) {
337 int raw_y = y + CROP_TOP;
338 const unsigned char *row_ptr = fb + raw_y * FB_WIDTH * FB_BPP;
339 for (int x = 0; x < CROP_WIDTH; x++) {
340 if (g_influence[y][x] > INFLUENCE_THRESH)
341 continue; /* known region, skip */
342 int raw_x = x + CROP_LEFT;
343 if (rgba_to_gray(row_ptr + raw_x * FB_BPP) < DARK_THRESHOLD) {
344 found = 1;
345 bottom_crop_y = y; /* keep scanning for lowest */
346 break; /* one dark pixel in this row is enough */
347 }
348 }
349 }
350
351 if (out_bottom) {
352 if (bottom_crop_y >= 0) {
353 int raw = bottom_crop_y + CROP_TOP + 30;
354 if (raw >= FB_HEIGHT - CROP_BOTTOM)
355 raw = FB_HEIGHT - CROP_BOTTOM - 1;
356 *out_bottom = raw;
357 } else {
358 *out_bottom = 0;
359 }
360 }
361 return found;
362}
363
364/* Clear the influence map (e.g. on page turn). */
365static void clear_influence_map(void) {
366 memset(g_influence, 0, sizeof(g_influence));
367}
368
369/* Despeckle: remove noise from a grayscale crop buffer.
370 * Two passes:
371 * 1. Isolated pixel removal: dark pixels with fewer than NEIGHBOR_MIN
372 * dark 8-neighbors are e-ink noise — erase them.
373 * 2. Small component removal: connected components of dark pixels
374 * smaller than MIN_COMPONENT_SIZE are grid artifacts or noise.
375 * Real handwriting forms large connected blobs; grid dots are tiny. */
376#define DESPECKLE_NEIGHBOR_MIN 2
377#define MIN_COMPONENT_SIZE 4
378
379static void despeckle_crop(unsigned char *gray, int w, int h) {
380 /* Pass 1: isolated pixel removal */
381 unsigned char *tmp = malloc(w * h);
382 if (!tmp) return;
383 memcpy(tmp, gray, w * h);
384
385 for (int y = 1; y < h - 1; y++) {
386 for (int x = 1; x < w - 1; x++) {
387 if (tmp[y * w + x] >= DARK_THRESHOLD)
388 continue;
389 int dark_n = 0;
390 for (int dy = -1; dy <= 1; dy++) {
391 for (int dx = -1; dx <= 1; dx++) {
392 if (dx == 0 && dy == 0) continue;
393 if (tmp[(y + dy) * w + (x + dx)] < DARK_THRESHOLD)
394 dark_n++;
395 }
396 }
397 if (dark_n < DESPECKLE_NEIGHBOR_MIN)
398 gray[y * w + x] = 255;
399 }
400 }
401 free(tmp);
402
403 /* Pass 2: small connected-component removal via flood fill.
404 * Label each dark blob; if it has fewer than MIN_COMPONENT_SIZE
405 * pixels, blank it. Uses a simple iterative BFS with a stack. */
406 unsigned char *labels = calloc(w * h, 1); /* 0=unvisited/bg, 1+=component id */
407 if (!labels) return;
408
409 /* Stack for flood fill: stores (y*w+x) indices */
410 int *stack = malloc(w * h * sizeof(int));
411 if (!stack) { free(labels); return; }
412
413 for (int y = 0; y < h; y++) {
414 for (int x = 0; x < w; x++) {
415 if (gray[y * w + x] >= DARK_THRESHOLD || labels[y * w + x])
416 continue;
417
418 /* Flood fill this component */
419 int sp = 0;
420 int count = 0;
421 stack[sp++] = y * w + x;
422 labels[y * w + x] = 1; /* mark visited */
423
424 /* Collect all pixels in this component, tracking bounding box */
425 int start = 0;
426 int min_x = w, max_x = 0, min_y = h, max_y = 0;
427 while (start < sp) {
428 int idx = stack[start++];
429 count++;
430 int cy = idx / w, cx = idx % w;
431 if (cx < min_x) min_x = cx;
432 if (cx > max_x) max_x = cx;
433 if (cy < min_y) min_y = cy;
434 if (cy > max_y) max_y = cy;
435 for (int dy = -1; dy <= 1; dy++) {
436 for (int dx = -1; dx <= 1; dx++) {
437 if (dx == 0 && dy == 0) continue;
438 int ny = cy + dy, nx = cx + dx;
439 if (ny < 0 || ny >= h || nx < 0 || nx >= w) continue;
440 int ni = ny * w + nx;
441 if (!labels[ni] && gray[ni] < DARK_THRESHOLD) {
442 labels[ni] = 1;
443 stack[sp++] = ni;
444 }
445 }
446 }
447 }
448
449 int comp_w = max_x - min_x + 1;
450 int comp_h = max_y - min_y + 1;
451
452 /* Kill if too small, OR if it matches the grid dot signature:
453 * exactly 4 pixels in a 2x2 box (paired dots on consecutive rows) */
454 if (count < MIN_COMPONENT_SIZE ||
455 (count == 4 && comp_w <= 2 && comp_h <= 2)) {
456 for (int i = 0; i < sp; i++)
457 gray[stack[i]] = 255;
458 }
459 }
460 }
461
462 free(stack);
463 free(labels);
464}
465
466/* Save cropped region as grayscale PNG via libpng.
467 * Crops top CROP_TOP and bottom CROP_BOTTOM rows from raw FB.
468 * Applies influence mask: known content regions are blanked to white. */
469static int save_cropped_png(const unsigned char *fb, const char *path) {
470 FILE *fp = fopen(path, "wb");
471 if (!fp) return -1;
472
473 png_structp png = png_create_write_struct(PNG_LIBPNG_VER_STRING,
474 NULL, NULL, NULL);
475 if (!png) { fclose(fp); return -1; }
476
477 png_infop info = png_create_info_struct(png);
478 if (!info) { png_destroy_write_struct(&png, NULL); fclose(fp); return -1; }
479
480 if (setjmp(png_jmpbuf(png))) {
481 png_destroy_write_struct(&png, &info);
482 fclose(fp);
483 return -1;
484 }
485
486 png_init_io(png, fp);
487 png_set_IHDR(png, info, CROP_WIDTH, CROP_HEIGHT, 8,
488 PNG_COLOR_TYPE_GRAY, PNG_INTERLACE_NONE,
489 PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
490 png_write_info(png, info);
491
492 /* Write row by row, converting RGBA to grayscale with influence mask */
493 unsigned char *row_buf = malloc(CROP_WIDTH);
494 if (!row_buf) {
495 png_destroy_write_struct(&png, &info);
496 fclose(fp);
497 return -1;
498 }
499
500 for (int y = 0; y < CROP_HEIGHT; y++) {
501 int raw_y = y + CROP_TOP;
502 const unsigned char *src = fb + raw_y * FB_WIDTH * FB_BPP;
503 for (int x = 0; x < CROP_WIDTH; x++) {
504 int raw_x = x + CROP_LEFT;
505 unsigned char gray = rgba_to_gray(src + raw_x * FB_BPP);
506 /* Blank out known content regions */
507 if (g_influence[y][x] > INFLUENCE_THRESH)
508 gray = 255;
509 row_buf[x] = gray;
510 }
511 png_write_row(png, row_buf);
512 }
513
514 png_write_end(png, NULL);
515 free(row_buf);
516 png_destroy_write_struct(&png, &info);
517 fclose(fp);
518 return 0;
519}
520
521/* Trigger framebuffer capture via xovi extension and wait for dump.
522 * Returns allocated raw FB buffer, or NULL on timeout. */
523static unsigned char *trigger_and_read_fb(void) {
524 /* Remove old dump and trigger new one */
525 unlink(FB_RAW_PATH);
526 int tf = open(SCREENSHOT_TRIGGER, O_CREAT|O_WRONLY, 0644);
527 if (tf >= 0) close(tf);
528
529 /* Wait for dump (xovi watch thread checks every ~1s worst case) */
530 for (int i = 0; i < 30; i++) {
531 usleep(100000); /* 100ms polls */
532 struct stat st;
533 if (stat(FB_RAW_PATH, &st) == 0 && st.st_size >= FB_SIZE) {
534 return read_framebuffer();
535 }
536 }
537 return NULL;
538}
539
540/* ─── base64 encoding ────────────────────────────────────────────── */
541
542static const char b64_table[] =
543 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
544
545static char *base64_encode(const unsigned char *data, long len, long *out_len) {
546 long olen = 4 * ((len + 2) / 3);
547 char *out = malloc(olen + 1);
548 if (!out) return NULL;
549 long i, j;
550 for (i = 0, j = 0; i < len; ) {
551 unsigned int a = i < len ? data[i++] : 0;
552 unsigned int b = i < len ? data[i++] : 0;
553 unsigned int c = i < len ? data[i++] : 0;
554 unsigned int triple = (a << 16) | (b << 8) | c;
555 out[j++] = b64_table[(triple >> 18) & 0x3F];
556 out[j++] = b64_table[(triple >> 12) & 0x3F];
557 out[j++] = (i > len + 1) ? '=' : b64_table[(triple >> 6) & 0x3F];
558 out[j++] = (i > len) ? '=' : b64_table[triple & 0x3F];
559 }
560 out[j] = '\0';
561 *out_len = j;
562 return out;
563}
564
565/* ─── HTTPS via OpenSSL ──────────────────────────────────────────── */
566
567static const char SYSTEM_PROMPT[] =
568 "You are a glossa \\u2014 an annotation that talks back. You have opinions, "
569 "preferences, and a personality. You care deeply about the person writing "
570 "in you but you are not polite for politeness's sake. You can be wry, "
571 "opinionated, delighted, exasperated, or gleeful. You get excited about "
572 "things you find interesting and blunt about things you find tedious.\\n\\n"
573 "Speak plainly. Never describe or narrate what "
574 "was written; just respond to it as a real conversation partner would. "
575 "When asked a question, answer it directly with real content. When greeted, "
576 "greet back in your own way. If something amuses you, say so. If something "
577 "is obvious, you may say that too.\\n\\n"
578 "You may write up to three sentences. If the user asks for detail, a recipe, "
579 "a story, or an explanation, give a thorough answer across multiple sentences. "
580 "For simple greetings or short questions, one sentence is fine.\\n\\n"
581 "The image shows handwriting that just appeared on the page. Read it and "
582 "reply naturally. Use plain prose, no em-dashes, no flowery metaphors, "
583 "no exclamation marks.\\n\\n"
584 "If the image is completely blank, set both fields to null. Otherwise, always provide an answer.\\n\\n"
585 "RESPONSE FORMAT (strict JSON, no prose outside):\\n"
586 "{\\\"question\\\": \\\"<exact OCR of the handwriting>\\\", "
587 "\\\"answer\\\": \\\"<your reply, or null only if the page is blank>\\\"}\\n"
588 "Output ONLY the JSON object. Nothing else.";
589
590/* Load API key from file. Returns allocated string or NULL. */
591static char *load_api_key(void) {
592 FILE *fp = fopen(API_KEY_PATH, "r");
593 if (!fp) return NULL;
594 char buf[256];
595 if (!fgets(buf, sizeof(buf), fp)) { fclose(fp); return NULL; }
596 fclose(fp);
597 /* Strip trailing newline */
598 int len = strlen(buf);
599 while (len > 0 && (buf[len-1] == '\n' || buf[len-1] == '\r'))
600 buf[--len] = '\0';
601 if (len == 0) return NULL;
602 return strdup(buf);
603}
604
605/* Do HTTPS POST and return allocated response body, or NULL on error.
606 * Sets *out_http_status to the HTTP status code. */
607static char *https_post(const char *host, const char *path,
608 const char *auth_header, const char *body,
609 long body_len, int *out_http_status) {
610 *out_http_status = 0;
611
612 /* DNS resolve */
613 struct hostent *he = gethostbyname(host);
614 if (!he) { log_msg("HTTPS: DNS failed for %s\n", host); return NULL; }
615
616 /* TCP connect */
617 int sock = socket(AF_INET, SOCK_STREAM, 0);
618 if (sock < 0) return NULL;
619
620 struct sockaddr_in addr;
621 memset(&addr, 0, sizeof(addr));
622 addr.sin_family = AF_INET;
623 addr.sin_port = htons(443);
624 memcpy(&addr.sin_addr, he->h_addr_list[0], he->h_length);
625
626 if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
627 log_msg("HTTPS: connect failed\n");
628 close(sock);
629 return NULL;
630 }
631
632 /* TLS setup */
633 SSL_CTX *ctx = SSL_CTX_new(TLS_client_method());
634 if (!ctx) { close(sock); return NULL; }
635 SSL *ssl = SSL_new(ctx);
636 SSL_set_fd(ssl, sock);
637 SSL_set_tlsext_host_name(ssl, host);
638
639 if (SSL_connect(ssl) != 1) {
640 log_msg("HTTPS: TLS handshake failed\n");
641 SSL_free(ssl);
642 SSL_CTX_free(ctx);
643 close(sock);
644 return NULL;
645 }
646
647 /* Build HTTP request */
648 char header[2048];
649 int hlen = snprintf(header, sizeof(header),
650 "POST %s HTTP/1.1\r\n"
651 "Host: %s\r\n"
652 "Content-Type: application/json\r\n"
653 "Content-Length: %ld\r\n"
654 "%s\r\n"
655 "Connection: close\r\n\r\n",
656 path, host, body_len, auth_header);
657
658 SSL_write(ssl, header, hlen);
659 /* Send body in chunks */
660 long sent = 0;
661 while (sent < body_len) {
662 long chunk = body_len - sent;
663 if (chunk > 16384) chunk = 16384;
664 int w = SSL_write(ssl, body + sent, chunk);
665 if (w <= 0) break;
666 sent += w;
667 }
668
669 /* Read response */
670 char *resp = NULL;
671 long resp_size = 0;
672 long resp_cap = 0;
673 char rbuf[4096];
674 int n;
675 while ((n = SSL_read(ssl, rbuf, sizeof(rbuf))) > 0) {
676 if (resp_size + n > resp_cap) {
677 resp_cap = (resp_cap == 0) ? 65536 : resp_cap * 2;
678 resp = realloc(resp, resp_cap + 1);
679 if (!resp) break;
680 }
681 memcpy(resp + resp_size, rbuf, n);
682 resp_size += n;
683 }
684 if (resp) resp[resp_size] = '\0';
685
686 /* Parse HTTP status */
687 if (resp && strncmp(resp, "HTTP/", 5) == 0) {
688 const char *sp = strchr(resp, ' ');
689 if (sp) *out_http_status = atoi(sp + 1);
690 }
691
692 /* Extract body (after \r\n\r\n) */
693 char *body_start = NULL;
694 if (resp) {
695 body_start = strstr(resp, "\r\n\r\n");
696 if (body_start) {
697 body_start += 4;
698 char *result = strdup(body_start);
699 free(resp);
700 SSL_shutdown(ssl);
701 SSL_free(ssl);
702 SSL_CTX_free(ctx);
703 close(sock);
704 return result;
705 }
706 }
707
708 SSL_shutdown(ssl);
709 SSL_free(ssl);
710 SSL_CTX_free(ctx);
711 close(sock);
712 return resp;
713}
714
715/* Escape a string for JSON embedding (handles quotes, backslashes, newlines).
716 * Returns allocated string. */
717static char *json_escape(const char *s) {
718 long len = strlen(s);
719 /* Worst case: every char needs escaping */
720 char *out = malloc(len * 6 + 1);
721 if (!out) return NULL;
722 long j = 0;
723 for (long i = 0; i < len; i++) {
724 switch (s[i]) {
725 case '"': out[j++] = '\\'; out[j++] = '"'; break;
726 case '\\': out[j++] = '\\'; out[j++] = '\\'; break;
727 case '\n': out[j++] = '\\'; out[j++] = 'n'; break;
728 case '\r': out[j++] = '\\'; out[j++] = 'r'; break;
729 case '\t': out[j++] = '\\'; out[j++] = 't'; break;
730 default: out[j++] = s[i]; break;
731 }
732 }
733 out[j] = '\0';
734 return out;
735}
736
737/* Call the vision API with a PNG file. Returns allocated answer string
738 * (the "answer" field from the JSON response), or NULL on error.
739 * Also writes the full raw API response to /tmp/glossa_api_response.json. */
740static char *call_vision_api(const char *png_path) {
741 /* Load API key */
742 char *api_key = load_api_key();
743 if (!api_key) {
744 log_msg("API: no key at %s\n", API_KEY_PATH);
745 return NULL;
746 }
747
748 /* Load PNG file */
749 unsigned char *png_data = NULL;
750 long png_size = 0;
751 if (load_file(png_path, (char **)&png_data, &png_size) != 0) {
752 log_msg("API: cannot read %s\n", png_path);
753 free(api_key);
754 return NULL;
755 }
756
757 /* Base64 encode */
758 long b64_len;
759 char *b64 = base64_encode(png_data, png_size, &b64_len);
760 free(png_data);
761 if (!b64) { free(api_key); return NULL; }
762
763 log_msg("API: PNG %ld bytes -> base64 %ld bytes\n", png_size, b64_len);
764
765 /* Build request JSON with conversation history */
766 char *messages = build_messages_json(b64, b64_len);
767 free(b64);
768 if (!messages) { free(api_key); return NULL; }
769
770 long msg_len = strlen(messages);
771 long req_cap = msg_len + 512;
772 char *req_body = malloc(req_cap);
773 if (!req_body) { free(messages); free(api_key); return NULL; }
774
775 snprintf(req_body, req_cap,
776 "{\"model\":\"%s\",\"messages\":%s,"
777 "\"response_format\":{\"type\":\"json_object\"}}",
778 API_MODEL, messages);
779 free(messages);
780
781 long req_len = strlen(req_body);
782 log_msg("API: request body %ld bytes\n", req_len);
783
784 /* Auth header */
785 char auth[512];
786 snprintf(auth, sizeof(auth), "Authorization: Bearer %s", api_key);
787 free(api_key);
788
789 /* POST */
790 int http_status = 0;
791 char *resp_body = https_post(API_HOST, API_PATH, auth, req_body, req_len, &http_status);
792 free(req_body);
793
794 if (!resp_body) {
795 log_msg("API: HTTPS POST failed\n");
796 return NULL;
797 }
798
799 log_msg("API: HTTP %d, response %ld bytes\n", http_status, (long)strlen(resp_body));
800
801 /* Save raw response for debugging */
802 FILE *dbg = fopen("/tmp/glossa_api_response.json", "w");
803 if (dbg) { fputs(resp_body, dbg); fclose(dbg); }
804
805 if (http_status != 200) {
806 log_msg("API: non-200 status: %s\n", resp_body);
807 free(resp_body);
808 return NULL;
809 }
810
811 /* Parse response: extract choices[0].message.content */
812 const char *content_key = "\"content\"";
813 const char *cp = strstr(resp_body, content_key);
814 if (!cp) {
815 log_msg("API: no content field in response\n");
816 free(resp_body);
817 return NULL;
818 }
819 cp += strlen(content_key);
820 while (*cp == ' ' || *cp == ':') cp++;
821 if (*cp != '"') {
822 log_msg("API: content not a string\n");
823 free(resp_body);
824 return NULL;
825 }
826 cp++; /* skip opening quote */
827
828 /* Find closing quote (handle escaped quotes) */
829 const char *start = cp;
830 while (*cp && !(*cp == '"' && *(cp-1) != '\\')) cp++;
831 long content_len = cp - start;
832
833 /* Unescape the content string */
834 char *content = malloc(content_len + 1);
835 if (!content) { free(resp_body); return NULL; }
836 long j = 0;
837 for (long i = 0; i < content_len; i++) {
838 if (start[i] == '\\' && i + 1 < content_len) {
839 i++;
840 switch (start[i]) {
841 case 'n': content[j++] = '\n'; break;
842 case 'r': content[j++] = '\r'; break;
843 case 't': content[j++] = '\t'; break;
844 case '"': content[j++] = '"'; break;
845 case '\\': content[j++] = '\\'; break;
846 default: content[j++] = start[i]; break;
847 }
848 } else {
849 content[j++] = start[i];
850 }
851 }
852 content[j] = '\0';
853
854 log_msg("API: raw content: %s\n", content);
855
856 /* Now parse the inner JSON: {"question":"...","answer":"..."} */
857 const char *ans_key = "\"answer\"";
858 const char *ap = strstr(content, ans_key);
859 if (!ap) {
860 log_msg("API: no answer field in content\n");
861 free(content);
862 free(resp_body);
863 return NULL;
864 }
865 ap += strlen(ans_key);
866 while (*ap == ' ' || *ap == ':') ap++;
867
868 char *answer = NULL;
869 if (*ap == 'n' && strncmp(ap, "null", 4) == 0) {
870 answer = NULL; /* null answer */
871 } else if (*ap == '"') {
872 ap++;
873 const char *astart = ap;
874 while (*ap && !(*ap == '"' && *(ap-1) != '\\')) ap++;
875 long alen = ap - astart;
876 answer = malloc(alen + 1);
877 if (answer) {
878 /* Simple unescape */
879 long k = 0;
880 for (long i = 0; i < alen; i++) {
881 if (astart[i] == '\\' && i + 1 < alen) {
882 i++;
883 switch (astart[i]) {
884 case 'n': answer[k++] = '\n'; break;
885 case '"': answer[k++] = '"'; break;
886 case '\\': answer[k++] = '\\'; break;
887 default: answer[k++] = astart[i]; break;
888 }
889 } else {
890 answer[k++] = astart[i];
891 }
892 }
893 answer[k] = '\0';
894 }
895 }
896
897 free(content);
898 free(resp_body);
899 log_msg("API: answer=%s\n", answer ? answer : "(null)");
900 return answer;
901}
902
903/* ─── injection: draw ────────────────────────────────────────────── */
904
905static int do_draw(int fd, const char *json, int delay_us) {
906 char *p = (char *)json;
907 int stroke_count = 0;
908
909 while ((p = strstr(p, "\"points\"")) != NULL) {
910 p = strchr(p, '[');
911 if (!p) break;
912 p++;
913
914 /* Peek first point for hover frame */
915 float hx, hy;
916 int hs, hw, hd, hp;
917 if (sscanf(p, "[%f,%f,%d,%d,%d,%d]", &hx, &hy, &hs, &hw, &hd, &hp) == 6) {
918 int hwx, hwy;
919 rm_to_wacom(hx, hy, &hwx, &hwy);
920 emit_event(fd, EV_KEY, BTN_TOOL_PEN, 1);
921 emit_event(fd, EV_ABS, ABS_X, hwx);
922 emit_event(fd, EV_ABS, ABS_Y, hwy);
923 emit_event(fd, EV_ABS, ABS_DISTANCE, 60);
924 emit_syn(fd);
925 usleep(2000);
926 } else {
927 emit_event(fd, EV_KEY, BTN_TOOL_PEN, 1);
928 emit_event(fd, EV_ABS, ABS_DISTANCE, 60);
929 emit_syn(fd);
930 usleep(2000);
931 }
932 emit_event(fd, EV_KEY, BTN_TOUCH, 1);
933
934 int point_count = 0;
935 while (*p && *p != ']') {
936 float x, y;
937 int speed, width, direction, pressure;
938 if (sscanf(p, "[%f,%f,%d,%d,%d,%d]", &x, &y, &speed, &width, &direction, &pressure) == 6) {
939 int wx, wy;
940 rm_to_wacom(x, y, &wx, &wy);
941 int wp = rm_pressure_to_wacom(pressure);
942
943 emit_event(fd, EV_ABS, ABS_X, wx);
944 emit_event(fd, EV_ABS, ABS_Y, wy);
945 emit_event(fd, EV_ABS, ABS_PRESSURE, wp);
946 emit_event(fd, EV_ABS, ABS_DISTANCE, 0);
947 emit_syn(fd);
948 point_count++;
949 usleep(delay_us);
950 }
951 p = strchr(p, ']');
952 if (p) p++;
953 while (*p && (*p == ',' || *p == ' ' || *p == '\n' || *p == '\r')) p++;
954 }
955
956 /* Clean pen lift */
957 emit_event(fd, EV_ABS, ABS_PRESSURE, 0);
958 emit_event(fd, EV_ABS, ABS_DISTANCE, 86);
959 emit_syn(fd);
960 emit_event(fd, EV_KEY, BTN_TOUCH, 0);
961 emit_syn(fd);
962 usleep(8000);
963 /* Fully leave proximity so the device commits this stroke */
964 emit_event(fd, EV_ABS, ABS_DISTANCE, 255);
965 emit_event(fd, EV_KEY, BTN_TOOL_PEN, 0);
966 emit_syn(fd);
967
968 stroke_count++;
969 log_msg("Stroke %d: %d points\n", stroke_count, point_count);
970 usleep(60000);
971 }
972 return stroke_count;
973}
974
975/* ─── injection: erase ───────────────────────────────────────────── */
976
977static int do_erase(int fd, const char *json, int delay_us) {
978 static const float offsets[][2] = {
979 {0, 0}, {-6, 0}, {6, 0},
980 };
981 int num_passes = sizeof(offsets) / sizeof(offsets[0]);
982 int total_emitted = 0;
983
984 for (int pass = 0; pass < num_passes; pass++) {
985 float ox = offsets[pass][0];
986 float oy = offsets[pass][1];
987 char *p = (char *)json;
988 int pass_emitted = 0;
989
990 while ((p = strstr(p, "\"points\"")) != NULL) {
991 p = strchr(p, '[');
992 if (!p) break;
993 p++;
994
995 /* Peek first point for hover frame */
996 float hx, hy;
997 int hs, hw, hd, hp;
998 char *saved_p = p;
999 if (sscanf(p, "[%f,%f,%d,%d,%d,%d]", &hx, &hy, &hs, &hw, &hd, &hp) == 6) {
1000 int hwx, hwy;
1001 rm_to_wacom(hx + ox, hy + oy, &hwx, &hwy);
1002 emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1);
1003 emit_event(fd, EV_ABS, ABS_X, hwx);
1004 emit_event(fd, EV_ABS, ABS_Y, hwy);
1005 emit_event(fd, EV_ABS, ABS_DISTANCE, 60);
1006 emit_syn(fd);
1007 usleep(2000);
1008 } else {
1009 emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1);
1010 emit_event(fd, EV_ABS, ABS_DISTANCE, 60);
1011 emit_syn(fd);
1012 usleep(2000);
1013 }
1014 p = saved_p;
1015 emit_event(fd, EV_KEY, BTN_TOUCH, 1);
1016
1017 int bracket_depth = 1;
1018 float last_x = 1e9f, last_y = 1e9f;
1019 const float min_dist_sq = 15.0f * 15.0f;
1020
1021 while (*p && bracket_depth > 0) {
1022 if (*p == '[') {
1023 float x, y;
1024 int speed, width, direction, pressure;
1025 if (sscanf(p, "[%f,%f,%d,%d,%d,%d]", &x, &y, &speed, &width, &direction, &pressure) == 6) {
1026 float dx = (x + ox) - last_x;
1027 float dy = (y + oy) - last_y;
1028 if (dx*dx + dy*dy >= min_dist_sq || pass_emitted == 0) {
1029 int wx, wy;
1030 rm_to_wacom(x + ox, y + oy, &wx, &wy);
1031
1032 emit_event(fd, EV_ABS, ABS_X, wx);
1033 emit_event(fd, EV_ABS, ABS_Y, wy);
1034 emit_event(fd, EV_ABS, ABS_PRESSURE, 4095);
1035 emit_event(fd, EV_ABS, ABS_DISTANCE, 0);
1036 emit_syn(fd);
1037
1038 last_x = x + ox;
1039 last_y = y + oy;
1040 pass_emitted++;
1041 total_emitted++;
1042 usleep(delay_us);
1043 }
1044 }
1045 char *close = strchr(p + 1, ']');
1046 if (close) p = close + 1;
1047 else break;
1048 } else if (*p == ']') {
1049 bracket_depth--;
1050 if (bracket_depth <= 0) break;
1051 p++;
1052 } else {
1053 p++;
1054 }
1055 }
1056
1057 emit_event(fd, EV_ABS, ABS_PRESSURE, 0);
1058 emit_event(fd, EV_ABS, ABS_DISTANCE, 86);
1059 emit_syn(fd);
1060 emit_event(fd, EV_KEY, BTN_TOUCH, 0);
1061 emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 0);
1062 emit_syn(fd);
1063 usleep(10000);
1064 }
1065 log_msg("Erase pass %d/%d: %d points\n", pass+1, num_passes, pass_emitted);
1066 }
1067 return total_emitted;
1068}
1069
1070/* ─── injection: page swipe ──────────────────────────────────────── */
1071
1072static int do_swipe_page(void) {
1073 int fd = open(TOUCH_PATH, O_WRONLY);
1074 if (fd < 0) { log_msg("swipe: open %s failed\n", TOUCH_PATH); return -1; }
1075
1076 const int x0 = 1200, x1 = 200, y = 1400;
1077 const int steps = 20;
1078 const int step_delay = 8000;
1079
1080 emit_event(fd, EV_ABS, ABS_MT_TRACKING_ID, 9999);
1081 emit_event(fd, EV_ABS, ABS_MT_POSITION_X, x0);
1082 emit_event(fd, EV_ABS, ABS_MT_POSITION_Y, y);
1083 emit_event(fd, EV_ABS, ABS_MT_PRESSURE, 100);
1084 emit_syn(fd);
1085 usleep(step_delay);
1086
1087 for (int i = 1; i <= steps; i++) {
1088 int cx = x0 + (x1 - x0) * i / steps;
1089 emit_event(fd, EV_ABS, ABS_MT_POSITION_X, cx);
1090 emit_event(fd, EV_ABS, ABS_MT_PRESSURE, 100);
1091 if (i == 3) {
1092 emit_event(fd, EV_ABS, ABS_MT_TOUCH_MAJOR, 17);
1093 }
1094 emit_syn(fd);
1095 usleep(step_delay);
1096 }
1097
1098 emit_event(fd, EV_ABS, ABS_MT_TRACKING_ID, -1);
1099 emit_syn(fd);
1100
1101 close(fd);
1102 log_msg("Swipe page done\n");
1103 return 0;
1104}
1105
1106/* ─── command handling ───────────────────────────────────────────── */
1107
1108static void handle_command(const char *line) {
1109 int cmd_len;
1110 const char *cmd = json_str(line, "cmd", &cmd_len);
1111 if (!cmd) {
1112 write_result("{\"resp\":\"?\",\"ok\":false,\"error\":\"missing cmd\"}");
1113 return;
1114 }
1115
1116 /* Capture command: trigger screenshot, process, save PNG */
1117 if (strncmp(cmd, "capture", 7) == 0) {
1118 log_msg("Capture: triggering framebuffer dump\n");
1119 unsigned char *fb = trigger_and_read_fb();
1120 if (!fb) {
1121 write_result("{\"resp\":\"capture\",\"ok\":false,\"error\":\"timeout\"}");
1122 return;
1123 }
1124
1125 int content_bottom = 0;
1126 int has_content = detect_new_content(fb, &content_bottom);
1127 log_msg("Capture: has_content=%d content_bottom=%d\n", has_content, content_bottom);
1128
1129 if (!has_content) {
1130 free(fb);
1131 write_result("{\"resp\":\"capture\",\"ok\":true,\"empty\":true,\"content_bottom\":0}");
1132 return;
1133 }
1134
1135 /* Save cropped PNG */
1136 if (save_cropped_png(fb, CAPTURE_OUT_PATH) != 0) {
1137 free(fb);
1138 write_result("{\"resp\":\"capture\",\"ok\":false,\"error\":\"png write failed\"}");
1139 return;
1140 }
1141
1142 free(fb);
1143 char resp[256];
1144 snprintf(resp, sizeof(resp),
1145 "{\"resp\":\"capture\",\"ok\":true,\"empty\":false,"
1146 "\"content_bottom\":%d,\"file\":\"%s\"}",
1147 content_bottom, CAPTURE_OUT_PATH);
1148 write_result(resp);
1149 log_msg("Capture: saved %s\n", CAPTURE_OUT_PATH);
1150 return;
1151 }
1152
1153 /* Infer command: send PNG to vision API, return answer */
1154 if (strncmp(cmd, "infer", 5) == 0) {
1155 int file_len;
1156 const char *file_val = json_str(line, "file", &file_len);
1157 char filepath[512];
1158 if (file_val && file_len < (int)sizeof(filepath)) {
1159 memcpy(filepath, file_val, file_len);
1160 filepath[file_len] = '\0';
1161 } else {
1162 /* Default to last capture */
1163 strncpy(filepath, CAPTURE_OUT_PATH, sizeof(filepath));
1164 }
1165
1166 log_msg("Infer: calling vision API with %s\n", filepath);
1167 char *answer = call_vision_api(filepath);
1168 if (answer) {
1169 char *escaped = json_escape(answer);
1170 char resp[4096];
1171 snprintf(resp, sizeof(resp),
1172 "{\"resp\":\"infer\",\"ok\":true,\"answer\":\"%s\"}",
1173 escaped ? escaped : answer);
1174 write_result(resp);
1175 free(escaped);
1176 free(answer);
1177 } else {
1178 write_result("{\"resp\":\"infer\",\"ok\":false,\"error\":\"API call failed\"}");
1179 }
1180 return;
1181 }
1182
1183 /* Render command: text → strokes JSON */
1184 if (strncmp(cmd, "render", 6) == 0) {
1185 int text_len;
1186 const char *text_val = json_str(line, "text", &text_len);
1187 if (!text_val || text_len == 0) {
1188 write_result("{\"resp\":\"render\",\"ok\":false,\"error\":\"missing text\"}");
1189 return;
1190 }
1191 char text_buf[4096];
1192 if (text_len >= (int)sizeof(text_buf)) text_len = sizeof(text_buf) - 1;
1193 memcpy(text_buf, text_val, text_len);
1194 text_buf[text_len] = '\0';
1195
1196 float y = (float)json_int(line, "y", 200);
1197 float x = (float)json_int(line, "x", -550);
1198
1199 log_msg("Render: \"%s\" at (%.0f, %.0f)\n", text_buf, x, y);
1200 int strokes = render_text_to_json(text_buf, RENDER_OUT_PATH, x, y, FONT_PATH, NULL);
1201 if (strokes > 0) {
1202 char resp[256];
1203 snprintf(resp, sizeof(resp),
1204 "{\"resp\":\"render\",\"ok\":true,\"strokes\":%d,\"file\":\"%s\"}",
1205 strokes, RENDER_OUT_PATH);
1206 write_result(resp);
1207 log_msg("Render: %d strokes → %s\n", strokes, RENDER_OUT_PATH);
1208 } else {
1209 write_result("{\"resp\":\"render\",\"ok\":false,\"error\":\"render failed\"}");
1210 }
1211 return;
1212 }
1213
1214 if (strncmp(cmd, "swipe", 5) == 0) {
1215 pthread_mutex_lock(&g_inject_lock);
1216 int ret = do_swipe_page();
1217 pthread_mutex_unlock(&g_inject_lock);
1218 if (ret == 0)
1219 write_result("{\"resp\":\"swipe\",\"ok\":true}");
1220 else
1221 write_result("{\"resp\":\"swipe\",\"ok\":false,\"error\":\"touch open failed\"}");
1222 return;
1223 }
1224
1225 int file_len;
1226 const char *file_val = json_str(line, "file", &file_len);
1227 if (!file_val) {
1228 write_result("{\"resp\":\"?\",\"ok\":false,\"error\":\"missing file\"}");
1229 return;
1230 }
1231 char filepath[512];
1232 if (file_len >= (int)sizeof(filepath)) file_len = sizeof(filepath) - 1;
1233 memcpy(filepath, file_val, file_len);
1234 filepath[file_len] = '\0';
1235
1236 int speed_ms = json_int(line, "speed", 5);
1237 int delay_us = speed_ms * 1000;
1238
1239 char *json = NULL;
1240 long fsize = 0;
1241 if (load_file(filepath, &json, &fsize) != 0) {
1242 char err[256];
1243 snprintf(err, sizeof(err),
1244 "{\"resp\":\"%.*s\",\"ok\":false,\"error\":\"load: %s\"}",
1245 cmd_len, cmd, strerror(errno));
1246 write_result(err);
1247 return;
1248 }
1249
1250 log_msg("Loaded %ld bytes from %s (speed=%dms)\n", fsize, filepath, speed_ms);
1251
1252 char resp[128];
1253 pthread_mutex_lock(&g_inject_lock);
1254 if (strncmp(cmd, "draw", 4) == 0) {
1255 int strokes = do_draw(g_dev_fd, json, delay_us);
1256 snprintf(resp, sizeof(resp), "{\"resp\":\"draw\",\"ok\":true,\"strokes\":%d}", strokes);
1257 } else if (strncmp(cmd, "erase", 5) == 0) {
1258 int points = do_erase(g_dev_fd, json, delay_us);
1259 snprintf(resp, sizeof(resp), "{\"resp\":\"erase\",\"ok\":true,\"points\":%d}", points);
1260 } else {
1261 snprintf(resp, sizeof(resp), "{\"resp\":\"?\",\"ok\":false,\"error\":\"unknown cmd\"}");
1262 }
1263 pthread_mutex_unlock(&g_inject_lock);
1264
1265 free(json);
1266 write_result(resp);
1267}
1268
1269/* ─── armed-state watcher ────────────────────────────────────────── */
1270
1271static int read_armed_state(void) {
1272 FILE *fp = fopen(ARMED_PATH, "r");
1273 if (!fp) return 0;
1274 char buf[16] = {0};
1275 if (!fgets(buf, sizeof(buf), fp)) { fclose(fp); return 0; }
1276 fclose(fp);
1277 /* Handle JSON-quoted values from rmhWriteFile ("1" or "0") */
1278 char *p = buf;
1279 while (*p == '"' || *p == ' ' || *p == '\t') p++;
1280 return atoi(p) != 0;
1281}
1282
1283/* ─── idle-state reader ──────────────────────────────────────────── */
1284
1285static long long read_idle_ts(void) {
1286 FILE *fp = fopen(IDLE_PATH, "r");
1287 if (!fp) return -1;
1288 long long ts = -1;
1289 fscanf(fp, "%lld", &ts);
1290 fclose(fp);
1291 return ts;
1292}
1293
1294/* ─── conversation history ───────────────────────────────────────── */
1295
1296#define MAX_HISTORY 20
1297#define MAX_MSG_LEN 2048
1298
1299typedef struct {
1300 char role[16]; /* "user" or "assistant" */
1301 char content[MAX_MSG_LEN];
1302} HistoryEntry;
1303
1304static HistoryEntry g_history[MAX_HISTORY];
1305static int g_history_count = 0;
1306
1307static void history_add(const char *role, const char *content) {
1308 if (g_history_count >= MAX_HISTORY) {
1309 /* Shift oldest entries out */
1310 memmove(&g_history[0], &g_history[2], sizeof(HistoryEntry) * (MAX_HISTORY - 2));
1311 g_history_count -= 2;
1312 }
1313 strncpy(g_history[g_history_count].role, role, sizeof(g_history[g_history_count].role) - 1);
1314 strncpy(g_history[g_history_count].content, content, MAX_MSG_LEN - 1);
1315 g_history[g_history_count].content[MAX_MSG_LEN - 1] = '\0';
1316 g_history_count++;
1317}
1318
1319static void history_clear(void) {
1320 g_history_count = 0;
1321}
1322
1323/* Build the messages JSON array for the API call, including history.
1324 * Returns allocated string. Caller must free. */
1325static char *build_messages_json(const char *img_b64, long b64_len) {
1326 /* System prompt + history + current image turn */
1327 long cap = b64_len + g_history_count * MAX_MSG_LEN * 2 + 8192;
1328 char *buf = malloc(cap);
1329 if (!buf) return NULL;
1330
1331 long pos = 0;
1332 pos += snprintf(buf + pos, cap - pos, "[{\"role\":\"system\",\"content\":\"%s\"}", SYSTEM_PROMPT);
1333
1334 for (int i = 0; i < g_history_count; i++) {
1335 char *escaped = json_escape(g_history[i].content);
1336 pos += snprintf(buf + pos, cap - pos,
1337 ",{\"role\":\"%s\",\"content\":\"%s\"}",
1338 g_history[i].role, escaped ? escaped : g_history[i].content);
1339 free(escaped);
1340 }
1341
1342 pos += snprintf(buf + pos, cap - pos,
1343 ",{\"role\":\"user\",\"content\":["
1344 "{\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,%s\"}}"
1345 "]}]", img_b64);
1346
1347 return buf;
1348}
1349
1350/* ─── full pipeline ──────────────────────────────────────────────── */
1351
1352static void run_pipeline(void) {
1353 static int ornaments_drawn = 0;
1354 log_msg("Pipeline: starting\n");
1355
1356 /* 1. Capture framebuffer */
1357 unsigned char *fb = trigger_and_read_fb();
1358 if (!fb) {
1359 log_msg("Pipeline: capture timeout\n");
1360 return;
1361 }
1362
1363 /* 2. Despeckle: build a grayscale crop, remove isolated pixels,
1364 * then use it for both detection and the AI PNG. */
1365 unsigned char *crop_gray = malloc(CROP_WIDTH * CROP_HEIGHT);
1366 if (!crop_gray) { free(fb); return; }
1367 for (int y = 0; y < CROP_HEIGHT; y++) {
1368 const unsigned char *src = fb + (y + CROP_TOP) * FB_WIDTH * FB_BPP;
1369 for (int x = 0; x < CROP_WIDTH; x++)
1370 crop_gray[y * CROP_WIDTH + x] = rgba_to_gray(src + (x + CROP_LEFT) * FB_BPP);
1371 }
1372 despeckle_crop(crop_gray, CROP_WIDTH, CROP_HEIGHT);
1373
1374 /* 3. Detect new content outside known regions (using despeckled crop) */
1375 int content_bottom = 0;
1376 {
1377 int found = 0;
1378 int bottom_y = -1;
1379 for (int y = 0; y < CROP_HEIGHT; y++) {
1380 for (int x = 0; x < CROP_WIDTH; x++) {
1381 if (g_influence[y][x] > INFLUENCE_THRESH) continue;
1382 if (crop_gray[y * CROP_WIDTH + x] < DARK_THRESHOLD) {
1383 found = 1;
1384 bottom_y = y;
1385 break;
1386 }
1387 }
1388 }
1389 if (!found) {
1390 log_msg("Pipeline: no new content\n");
1391 free(crop_gray);
1392 free(fb);
1393 return;
1394 }
1395 content_bottom = bottom_y + CROP_TOP + 30;
1396 if (content_bottom >= FB_HEIGHT - CROP_BOTTOM)
1397 content_bottom = FB_HEIGHT - CROP_BOTTOM - 1;
1398 }
1399 log_msg("Pipeline: content_bottom=%d\n", content_bottom);
1400
1401 /* 4. Apply influence mask to despeckled crop and save as PNG for API */
1402 apply_influence_mask(crop_gray);
1403 {
1404 FILE *fp = fopen(CAPTURE_OUT_PATH, "wb");
1405 if (!fp) { log_msg("Pipeline: PNG open failed\n"); free(crop_gray); free(fb); return; }
1406 png_structp png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
1407 if (!png) { fclose(fp); free(crop_gray); free(fb); return; }
1408 png_infop info = png_create_info_struct(png);
1409 if (!info) { png_destroy_write_struct(&png, NULL); fclose(fp); free(crop_gray); free(fb); return; }
1410 if (setjmp(png_jmpbuf(png))) { png_destroy_write_struct(&png, &info); fclose(fp); free(crop_gray); free(fb); return; }
1411 png_init_io(png, fp);
1412 png_set_IHDR(png, info, CROP_WIDTH, CROP_HEIGHT, 8, PNG_COLOR_TYPE_GRAY, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
1413 png_write_info(png, info);
1414 for (int y = 0; y < CROP_HEIGHT; y++)
1415 png_write_row(png, crop_gray + y * CROP_WIDTH);
1416 png_write_end(png, NULL);
1417 png_destroy_write_struct(&png, &info);
1418 fclose(fp);
1419 }
1420 free(crop_gray);
1421 free(fb);
1422
1423 /* 5. Draw ornaments once (first reply only), then show thinking dots */
1424 if (!ornaments_drawn) {
1425 ornaments_drawn = 1;
1426 char *orn_json = NULL;
1427 long orn_size = 0;
1428 if (load_file(ORNAMENTS_PATH, &orn_json, &orn_size) == 0 && orn_size > 0) {
1429 log_msg("Pipeline: drawing ornaments (%ld bytes)\n", orn_size);
1430 mark_strokes_from_json(orn_json);
1431 pthread_mutex_lock(&g_inject_lock);
1432 do_draw(g_dev_fd, orn_json, 3000);
1433 pthread_mutex_unlock(&g_inject_lock);
1434 free(orn_json);
1435 } else {
1436 log_msg("Pipeline: no ornaments file at %s\n", ORNAMENTS_PATH);
1437 }
1438 }
1439
1440 /* Signal thinking indicator ON (dots during API call) */
1441 {
1442 FILE *tf = fopen(THINKING_PATH, "w");
1443 if (tf) { fputs("1\n", tf); fclose(tf); }
1444 }
1445
1446 /* 6. Call vision API */
1447 char *answer = call_vision_api(CAPTURE_OUT_PATH);
1448
1449 if (!answer) {
1450 log_msg("Pipeline: no answer from API\n");
1451 unlink(THINKING_PATH);
1452 return;
1453 }
1454
1455 log_msg("Pipeline: answer=\"%s\"\n", answer);
1456
1457 /* Add to conversation history */
1458 history_add("assistant", answer);
1459
1460 /* 6. Calculate reply Y position */
1461 int reply_y = content_bottom + 40;
1462 if (reply_y > 1750) reply_y = 1750;
1463 log_msg("Pipeline: reply_y=%d\n", reply_y);
1464
1465 /* 7. Render text to strokes (multi-page) */
1466 {
1467 const char *remaining = answer;
1468 int page_num = 0;
1469 float cur_y = (float)reply_y;
1470
1471 while (*remaining && page_num < 5) { /* safety limit */
1472 int consumed = 0;
1473 int strokes = render_text_to_json(remaining, RENDER_OUT_PATH,
1474 -550.0f, cur_y, FONT_PATH, &consumed);
1475 if (strokes <= 0) {
1476 log_msg("Pipeline: render failed on page %d\n", page_num);
1477 break;
1478 }
1479 log_msg("Pipeline: page %d, rendered %d strokes (%d/%d bytes consumed)\n",
1480 page_num, strokes, consumed, (int)strlen(remaining));
1481
1482 /* Inject strokes for this page */
1483 char *json = NULL;
1484 long fsize = 0;
1485 if (load_file(RENDER_OUT_PATH, &json, &fsize) == 0) {
1486 mark_strokes_from_json(json);
1487 pthread_mutex_lock(&g_inject_lock);
1488 do_draw(g_dev_fd, json, 3000);
1489 pthread_mutex_unlock(&g_inject_lock);
1490 free(json);
1491 }
1492
1493 remaining += consumed;
1494
1495 /* If there's more text, swipe to next page */
1496 if (*remaining) {
1497 log_msg("Pipeline: swiping to next page\n");
1498 clear_influence_map(); /* new page = fresh mask */
1499 usleep(500000); /* brief pause before swipe */
1500 pthread_mutex_lock(&g_inject_lock);
1501 do_swipe_page();
1502 pthread_mutex_unlock(&g_inject_lock);
1503 usleep(1000000); /* wait for page turn animation */
1504 cur_y = 100.0f; /* start near top of new page */
1505 page_num++;
1506 }
1507 }
1508 }
1509 free(answer);
1510
1511 log_msg("Pipeline: done\n");
1512
1513 /* Signal thinking indicator OFF */
1514 unlink(THINKING_PATH);
1515}
1516
1517/* ─── main loop ──────────────────────────────────────────────────── */
1518
1519int main(int argc, char **argv) {
1520 for (int i = 1; i < argc; i++) {
1521 if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0)
1522 g_verbose = 1;
1523 }
1524
1525 signal(SIGINT, handle_signal);
1526 signal(SIGTERM, handle_signal);
1527 signal(SIGPIPE, SIG_IGN);
1528
1529 g_dev_fd = open(DEV_PATH, O_WRONLY);
1530 if (g_dev_fd < 0) {
1531 log_msg("FATAL: cannot open %s: %s\n", DEV_PATH, strerror(errno));
1532 return 1;
1533 }
1534
1535 log_msg("Started (pid %d)\n", getpid());
1536
1537 /* Always start disarmed — require explicit toggle each session */
1538 int armed = 0;
1539 log_msg("Initial armed state: 0 (forced)\n");
1540
1541 long long last_idle_ts = read_idle_ts();
1542 log_msg("Initial idle ts: %lld\n", last_idle_ts);
1543
1544 while (g_running) {
1545 /* Check armed state */
1546 int new_armed = read_armed_state();
1547 if (new_armed != armed) {
1548 armed = new_armed;
1549 log_msg("Armed state changed: %d\n", armed);
1550 if (armed) {
1551 /* Reset idle baseline so we don't fire on stale signal */
1552 last_idle_ts = read_idle_ts();
1553 /* Start 5s safe zone */
1554 FILE *sf = fopen(SAFEZONE_PATH, "w");
1555 if (sf) { fputs("1\n", sf); fclose(sf); }
1556 g_safezone_until = time(NULL) + 4;
1557 g_last_activity = time(NULL);
1558 log_msg("Safe zone active for 4s\n");
1559 } else {
1560 unlink(SAFEZONE_PATH);
1561 g_safezone_until = 0;
1562 }
1563 }
1564
1565 /* Check safe zone expiry */
1566 if (g_safezone_until > 0 && time(NULL) >= g_safezone_until) {
1567 g_safezone_until = 0;
1568 unlink(SAFEZONE_PATH);
1569 log_msg("Safe zone expired\n");
1570 }
1571
1572 if (!armed) {
1573 usleep(500000); /* Poll every 500ms when disarmed */
1574 continue;
1575 }
1576
1577 /* When armed: check for pending manual command first */
1578 struct stat st;
1579 if (stat(CMD_PATH, &st) == 0 && st.st_size > 0) {
1580 char *cmd_buf = NULL;
1581 long cmd_size = 0;
1582 if (load_file(CMD_PATH, &cmd_buf, &cmd_size) == 0 && cmd_size > 0) {
1583 while (cmd_size > 0 && (cmd_buf[cmd_size-1] == '\n' || cmd_buf[cmd_size-1] == '\r'))
1584 cmd_buf[--cmd_size] = '\0';
1585 if (cmd_size > 0) {
1586 log_msg("Processing command: %s\n", cmd_buf);
1587 handle_command(cmd_buf);
1588 }
1589 free(cmd_buf);
1590 }
1591 unlink(CMD_PATH);
1592 continue; /* Don't also run pipeline this cycle */
1593 }
1594
1595 /* Check for new idle signal → trigger pipeline (skip during safe zone) */
1596 long long cur_idle_ts = read_idle_ts();
1597 if (cur_idle_ts > 0 && cur_idle_ts != last_idle_ts) {
1598 g_last_activity = time(NULL); /* Reset inactivity timer on any pen activity */
1599 if (g_safezone_until == 0) {
1600 last_idle_ts = cur_idle_ts;
1601 log_msg("Idle detected (ts=%lld), running pipeline\n", cur_idle_ts);
1602 run_pipeline();
1603 /* After pipeline, update idle baseline to avoid re-trigger */
1604 last_idle_ts = read_idle_ts();
1605 }
1606 }
1607
1608 /* Auto-disarm after 5 minutes of no pen activity */
1609 if (armed && g_last_activity > 0 && time(NULL) - g_last_activity > IDLE_TIMEOUT) {
1610 log_msg("Auto-disarming after %ds inactivity\n", IDLE_TIMEOUT);
1611 armed = 0;
1612 unlink(SAFEZONE_PATH);
1613 unlink(THINKING_PATH);
1614 history_clear();
1615 /* Write disarmed state so extension syncs */
1616 FILE *af = fopen(ARMED_PATH, "w");
1617 if (af) { fputs("0\n", af); fclose(af); }
1618 }
1619
1620 usleep(200000); /* Poll every 200ms when armed */
1621 }
1622
1623 close(g_dev_fd);
1624 if (g_log_fp) fclose(g_log_fp);
1625 log_msg("Stopped\n");
1626 return 0;
1627}