nandi.uk / character
This repository has no description
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

Add

author
nandi date (May 26, 2026, 5:54 PM -0700) commit a56edcb6 parent 33c95550
+1935 -29
+110
bun.lock
··· 1 + { 2 + "lockfileVersion": 1, 3 + "configVersion": 1, 4 + "workspaces": { 5 + "": { 6 + "name": "character", 7 + "dependencies": { 8 + "@google/genai": "^1.18.0", 9 + "obs-websocket-js": "^5.0.6", 10 + }, 11 + }, 12 + }, 13 + "packages": { 14 + "@google/genai": ["@google/genai@1.52.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q=="], 15 + 16 + "@msgpack/msgpack": ["@msgpack/msgpack@2.8.0", "", {}, "sha512-h9u4u/jiIRKbq25PM+zymTyW6bhTzELvOoUd+AvYriWOAKpLGnIamaET3pnHYoI5iYphAHBI4ayx0MehR+VVPQ=="], 17 + 18 + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], 19 + 20 + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], 21 + 22 + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="], 23 + 24 + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="], 25 + 26 + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="], 27 + 28 + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], 29 + 30 + "@protobufjs/inquire": ["@protobufjs/inquire@1.1.2", "", {}, "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw=="], 31 + 32 + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], 33 + 34 + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], 35 + 36 + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="], 37 + 38 + "@types/node": ["@types/node@25.9.1", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg=="], 39 + 40 + "@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="], 41 + 42 + "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], 43 + 44 + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], 45 + 46 + "bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="], 47 + 48 + "buffer-equal-constant-time": ["buffer-equal-constant-time@1.0.1", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="], 49 + 50 + "crypto-js": ["crypto-js@4.2.0", "", {}, "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q=="], 51 + 52 + "data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="], 53 + 54 + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], 55 + 56 + "ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="], 57 + 58 + "eventemitter3": ["eventemitter3@5.0.4", "", {}, "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw=="], 59 + 60 + "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], 61 + 62 + "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="], 63 + 64 + "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="], 65 + 66 + "gaxios": ["gaxios@7.1.4", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2" } }, "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA=="], 67 + 68 + "gcp-metadata": ["gcp-metadata@8.1.2", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="], 69 + 70 + "google-auth-library": ["google-auth-library@10.6.2", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.1.4", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw=="], 71 + 72 + "google-logging-utils": ["google-logging-utils@1.1.3", "", {}, "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA=="], 73 + 74 + "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], 75 + 76 + "isomorphic-ws": ["isomorphic-ws@5.0.0", "", { "peerDependencies": { "ws": "*" } }, "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw=="], 77 + 78 + "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="], 79 + 80 + "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], 81 + 82 + "jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], 83 + 84 + "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], 85 + 86 + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], 87 + 88 + "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], 89 + 90 + "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], 91 + 92 + "obs-websocket-js": ["obs-websocket-js@5.0.8", "", { "dependencies": { "@msgpack/msgpack": "^2.7.1", "crypto-js": "^4.1.1", "debug": "^4.3.2", "eventemitter3": "^5.0.1", "isomorphic-ws": "^5.0.0", "type-fest": "^3.11.0", "ws": "^8.13.0" } }, "sha512-QDnQJMr5wuCoYugK02ggZ1/cvESs4KJDEK+UhGg0Ry35jnY8tK1Xr3KoPjKyoD6sd8G+WdtIdzuY+yUyPtogQQ=="], 93 + 94 + "p-retry": ["p-retry@4.6.2", "", { "dependencies": { "@types/retry": "0.12.0", "retry": "^0.13.1" } }, "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ=="], 95 + 96 + "protobufjs": ["protobufjs@7.6.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-4K0myLaWL5EteuSAro91EGFgcfVgxb64Jx+7oDAY6GOkXD4M69yuSEljNcInGVCA5sOPxmZ/EqDLj2x0Q0+Ygg=="], 97 + 98 + "retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="], 99 + 100 + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], 101 + 102 + "type-fest": ["type-fest@3.13.1", "", {}, "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g=="], 103 + 104 + "undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], 105 + 106 + "web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="], 107 + 108 + "ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="], 109 + } 110 + }
+46 -1
devenv.nix
··· 1 1 { pkgs, ... }: 2 2 { 3 3 env = { 4 - LD_LIBRARY_PATH = "${pkgs.stdenv.cc.cc.lib}/lib:${pkgs.lib.makeLibraryPath [ pkgs.libxkbcommon pkgs.wayland pkgs.libGL pkgs.vulkan-loader pkgs.alsa-lib pkgs.pulseaudio pkgs.udev ]}:$LD_LIBRARY_PATH"; 4 + LD_LIBRARY_PATH = "${pkgs.stdenv.cc.cc.lib}/lib:${pkgs.lib.makeLibraryPath [ pkgs.libxkbcommon pkgs.wayland pkgs.libGL pkgs.vulkan-loader pkgs.alsa-lib pkgs.pulseaudio pkgs.udev pkgs.libinput pkgs.libudev-zero ]}:$LD_LIBRARY_PATH"; 5 5 DRI_PRIME = "1"; 6 + XDG_SESSION_TYPE = "wayland"; 7 + QT_QPA_PLATFORM = "wayland;xcb"; 8 + SDL_VIDEODRIVER = "wayland,x11"; 6 9 }; 7 10 11 + scripts.run-minimal-haru.exec = '' 12 + set -euo pipefail 13 + 14 + export XDG_RUNTIME_DIR="''${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" 15 + 16 + if [ -z "''${WAYLAND_DISPLAY:-}" ]; then 17 + for sock in "$XDG_RUNTIME_DIR"/wayland-*; do 18 + if [ -S "$sock" ]; then 19 + export WAYLAND_DISPLAY="$(basename "$sock")" 20 + break 21 + fi 22 + done 23 + fi 24 + 25 + if [ -n "''${WAYLAND_DISPLAY:-}" ] && [ -S "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" ]; then 26 + echo "Launching minimal Haru on Wayland: $XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" 27 + cd /home/nandi/code/character/minimal-haru 28 + exec godot4 --path . 29 + fi 30 + 31 + echo "No Wayland display detected; falling back to headless mode" 32 + cd /home/nandi/code/character/minimal-haru 33 + exec godot4 --headless --path . 34 + ''; 35 + 8 36 languages.javascript = { 9 37 enable = true; 10 38 bun.enable = true; ··· 17 45 curl 18 46 godot_4 19 47 obs-studio 48 + libinput 49 + libudev-zero 20 50 python311 21 51 python311Packages.virtualenv 22 52 python313 ··· 34 64 echo "character dev env" 35 65 echo "- bun available" 36 66 echo "- ffmpeg available" 67 + export XDG_RUNTIME_DIR="''${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" 68 + if [ -z "''${WAYLAND_DISPLAY:-}" ]; then 69 + for sock in "$XDG_RUNTIME_DIR"/wayland-*; do 70 + if [ -S "$sock" ]; then 71 + export WAYLAND_DISPLAY="$(basename "$sock")" 72 + break 73 + fi 74 + done 75 + fi 76 + if [ -n "''${WAYLAND_DISPLAY:-}" ] && [ -S "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" ]; then 77 + echo "- wayland display: $WAYLAND_DISPLAY" 78 + else 79 + echo "- wayland display not detected in this shell" 80 + fi 81 + echo "- run minimal Haru with: run-minimal-haru" 37 82 ''; 38 83 }
+69
docs/gemini-live-websocket-notes.md
··· 1 + # Gemini Live WebSocket notes 2 + 3 + These notes capture the verified protocol target for the next implementation step. 4 + 5 + ## Verified target 6 + 7 + Google AI documentation indicates Gemini Live supports: 8 + 9 + - direct WebSocket sessions 10 + - stateful sessions 11 + - text and audio modalities 12 + - JavaScript SDK usage via `@google/genai` 13 + - raw WebSocket usage 14 + 15 + ## Verified endpoint shape 16 + 17 + Raw WebSocket endpoint: 18 + 19 + ```text 20 + wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=YOUR_API_KEY 21 + ``` 22 + 23 + ## Verified setup flow 24 + 25 + The first client message must configure the session. 26 + 27 + The documentation describes a first setup/config message and then subsequent turn content. 28 + 29 + ## Verified JS SDK direction 30 + 31 + Docs show JavaScript examples using: 32 + 33 + ```ts 34 + import { GoogleGenAI, Modality } from '@google/genai'; 35 + ``` 36 + 37 + And a live connect flow shaped like: 38 + 39 + ```ts 40 + const session = await ai.live.connect({ 41 + model, 42 + callbacks, 43 + config, 44 + }); 45 + ``` 46 + 47 + ## Verified model direction 48 + 49 + Examples reference live-capable models such as: 50 + 51 + - `gemini-3.1-flash-live-preview` 52 + 53 + ## Implication for this repo 54 + 55 + The next implementation step should prefer: 56 + 57 + 1. adding `@google/genai` 58 + 2. implementing `GeminiLiveProvider` with `ai.live.connect(...)` 59 + 3. capturing streamed text/audio output into the local runtime event model 60 + 61 + ## Notes for runtime wiring 62 + 63 + Desired mapping in this repo: 64 + 65 + - incoming chat -> `GeminiLiveProvider.generateReply` 66 + - Gemini streamed text -> reply text 67 + - Gemini audio output -> OBS-capturable sink 68 + - Gemini affect/prosody -> avatar action cues for OpenVT bridge 69 +
+220
docs/gemini-openvt-obs-streamplace-plan.md
··· 1 + # Gemini Live + OpenVT + OBS + stream.place integration plan 2 + 3 + ## Objective 4 + 5 + Connect these pieces into one loop: 6 + 7 + 1. stream.place chat arrives 8 + 2. chat triggers an agent response 9 + 3. Gemini Live API generates the live reply / speech stream 10 + 4. OpenVT shows Haru as the avatar renderer 11 + 5. OBS captures OpenVT and audio 12 + 6. OBS streams to stream.place 13 + 14 + ## Current repo state 15 + 16 + ### Already present 17 + 18 + - OpenVT local setup notes and launcher scripts 19 + - working Haru/OpenVT path in `vendor/open-vt` 20 + - simple chat runtime scaffold in `src/index.ts` 21 + - local chat simulation in `scripts/simulate-chat.ts` 22 + - architecture notes mentioning: 23 + - stream.place chat 24 + - LLM reply generation 25 + - TTS 26 + - avatar actions 27 + - WHIP/RTMP uplink 28 + 29 + ### Not yet present 30 + 31 + - real stream.place chat adapter 32 + - Gemini Live API client 33 + - agent orchestration layer 34 + - OpenVT control bridge from external runtime 35 + - OBS automation / obs-websocket control 36 + - actual stream.place ingest automation 37 + 38 + ## Minimal end-to-end architecture 39 + 40 + ```text 41 + stream.place chat 42 + -> chat adapter 43 + -> agent runtime 44 + -> Gemini Live session 45 + -> text reply 46 + -> streaming audio reply 47 + -> optional emotion tags / cues 48 + -> avatar control bridge 49 + -> OpenVT expression / motion triggers 50 + -> OBS scene 51 + -> captures OpenVT window or virtual camera 52 + -> mixes Gemini audio output / mic 53 + -> stream.place ingest (WHIP or RTMP) 54 + ``` 55 + 56 + ## Recommended practical split 57 + 58 + ### Process A: bot runtime 59 + 60 + Responsible for: 61 + 62 + - reading chat from stream.place 63 + - deciding whether to respond 64 + - calling the agent 65 + - opening / maintaining Gemini Live session 66 + - emitting: 67 + - reply text 68 + - audio stream or audio file 69 + - avatar cues like `emotion:happy`, `gesture:nod`, `talk:start` 70 + 71 + ### Process B: OpenVT renderer 72 + 73 + Responsible for: 74 + 75 + - rendering Haru 76 + - microphone / lip sync path 77 + - avatar expressions / motions 78 + - outputting window / virtual camera feed 79 + 80 + ### Process C: OBS production 81 + 82 + Responsible for: 83 + 84 + - capturing OpenVT window or loopback webcam 85 + - ingesting Gemini audio output 86 + - mixing optional mic/music 87 + - sending program output to stream.place 88 + 89 + ## Critical integration gaps 90 + 91 + ### 1. Gemini Live client 92 + 93 + Need a client that can: 94 + 95 + - send text/chat turns 96 + - receive streaming text/audio 97 + - optionally maintain conversational session state 98 + 99 + Likely implementation location: 100 + 101 + - `src/gemini-live.ts` 102 + 103 + ### 2. Chat adapter for stream.place 104 + 105 + Need a real adapter replacing the current scaffold. 106 + 107 + Likely implementation location: 108 + 109 + - `src/streamplace-chat.ts` 110 + 111 + Expected responsibilities: 112 + 113 + - subscribe to chat events (likely Jetstream / ATProto feed path) 114 + - normalize events into local `ChatMessage` 115 + - send replies back if desired 116 + 117 + ### 3. OpenVT control bridge 118 + 119 + OpenVT currently supports internal blueprint / tracker-driven motion, but this repo does not yet expose an external API for: 120 + 121 + - toggling expressions 122 + - triggering gestures / animations 123 + - setting mouth/talk state from external runtime 124 + 125 + Likely easiest bridge options: 126 + 127 + 1. small local websocket server inside OpenVT, or 128 + 2. local IPC / file / TCP bridge from external runtime into OpenVT tracker/action nodes 129 + 130 + Likely implementation target: 131 + 132 + - add a lightweight control node under `vendor/open-vt` or use a separate control process 133 + 134 + ### 4. OBS control 135 + 136 + OBS can be kept manual at first, but for automation the likely path is: 137 + 138 + - `obs-websocket` 139 + 140 + It would allow: 141 + 142 + - scene switching 143 + - source muting/unmuting 144 + - start/stop streaming 145 + 146 + Likely implementation location: 147 + 148 + - `src/obs-control.ts` 149 + 150 + ### 5. stream.place ingest 151 + 152 + Docs in this repo already assume: 153 + 154 + - WHIP if available 155 + - otherwise RTMP 156 + 157 + Likely first step: 158 + 159 + - manual OBS config for stream.place ingest 160 + 161 + ## Smallest viable path 162 + 163 + To minimize scope, do this in order: 164 + 165 + ### Phase 1: fake chat -> fake reply -> OpenVT + OBS manual 166 + 167 + - use local simulated chat 168 + - use deterministic local text reply 169 + - use existing OpenVT Haru setup 170 + - use OBS manual window capture 171 + - no Gemini yet 172 + 173 + ### Phase 2: Gemini Live reply + audio 174 + 175 + - replace fake reply with Gemini Live 176 + - route Gemini audio to system audio or a known sink OBS can capture 177 + - keep OBS manual 178 + 179 + ### Phase 3: external avatar control 180 + 181 + - map reply affect / punctuation / tags to: 182 + - expression 183 + - gesture 184 + - talk state 185 + - send those cues into OpenVT 186 + 187 + ### Phase 4: real stream.place chat + real stream.place ingest 188 + 189 + - replace simulated chat with stream.place adapter 190 + - optionally post replies back to chat 191 + 192 + ## Exact current blocker 193 + 194 + The repo does not yet contain any real implementation for: 195 + 196 + - Gemini Live API calls 197 + - stream.place chat subscription 198 + - OpenVT external control bridge 199 + - OBS websocket integration 200 + 201 + So the task is not “wire existing modules together”; it is “build the missing connectors around the current OpenVT renderer path.” 202 + 203 + ## Recommended next implementation step 204 + 205 + Build the first real connector: 206 + 207 + 1. `src/gemini-live.ts` 208 + 2. `src/runtime.ts` orchestrator 209 + 3. extend `scripts/simulate-chat.ts` to call the runtime 210 + 211 + This gives: 212 + 213 + - simulated chat 214 + - agent-triggered response 215 + - Gemini Live text/audio integration 216 + 217 + Then the next connector should be: 218 + 219 + - OpenVT control bridge for avatar cues 220 +
+72
docs/live-stack-runbook.md
··· 1 + # Live stack runbook 2 + 3 + ## Objective 4 + 5 + Bring up the current integrated stack for: 6 + 7 + - Jetstream/stream.place-style chat intake 8 + - Gemini Live response generation 9 + - OpenVT avatar control 10 + - OBS scene/text control 11 + 12 + ## Required environment 13 + 14 + Set these before starting the live loop: 15 + 16 + ```bash 17 + export GEMINI_API_KEY=... 18 + export OBS_WS_URL=ws://127.0.0.1:4455 19 + export OBS_SCENE_NAME="Your Scene" 20 + export OBS_TEXT_INPUT_NAME="Your Text Source" 21 + # optional 22 + export OBS_WS_PASSWORD=... 23 + ``` 24 + 25 + ## Required running services 26 + 27 + 1. OpenVT running with runtime control listener enabled 28 + 2. OBS running with obs-websocket enabled on port 4455 29 + 30 + ## Preflight check 31 + 32 + ```bash 33 + bun run check:live-stack 34 + ``` 35 + 36 + Expected good state: 37 + 38 + - `GEMINI_API_KEY: set` 39 + - `OBS websocket: open` 40 + - `OpenVT runtime control: open` 41 + 42 + ## Start the loop 43 + 44 + ```bash 45 + ./scripts/run-live-stack.sh 46 + ``` 47 + 48 + or: 49 + 50 + ```bash 51 + bun run src/index.ts --live 52 + ``` 53 + 54 + ## Current behavior 55 + 56 + When chat arrives: 57 + 58 + 1. immediate OpenVT cues are sent: 59 + - `talk:start` 60 + - default `emotion:curious` 61 + 2. runtime generates a reply 62 + 3. reply text is sent to: 63 + - OpenVT text control 64 + - OBS text input (if configured) 65 + 4. remaining avatar actions are sent to OpenVT 66 + 67 + ## What is still manual 68 + 69 + - actual OBS source/scene setup 70 + - stream.place ingest configuration in OBS 71 + - real Gemini credential provisioning 72 +
+86
docs/openvt-control-bridge-plan.md
··· 1 + # OpenVT control bridge plan 2 + 3 + ## Objective 4 + 5 + Take runtime outputs from this repo and make them actionable inside OpenVT. 6 + 7 + Current runtime outputs already include: 8 + 9 + - reply text 10 + - avatar actions such as: 11 + - `talk:start` 12 + - `talk:stop` 13 + - `emotion:*` 14 + - `gesture:*` 15 + 16 + ## Verified useful OpenVT primitives 17 + 18 + ### 1. TCP listener pattern already exists in OpenVT 19 + 20 + OpenVT includes a reusable TCP server pattern in: 21 + 22 + - `vendor/open-vt/lib/tracking/net/tcp_tracker.gd` 23 + 24 + This means a local control bridge can likely be implemented in the same style as tracker ingestion. 25 + 26 + ### 2. JSON decoding already exists in multiple trackers 27 + 28 + Examples: 29 + 30 + - `vendor/open-vt/lib/tracking/camera/vts/vts_tracker.gd` 31 + - `vendor/open-vt/lib/tracking/camera/vts_blendshapes/vts_tracker.gd` 32 + 33 + So a JSON-over-TCP control protocol is a natural fit. 34 + 35 + ### 3. Stage/model control entry points exist 36 + 37 + Verified scene/controller anchors: 38 + 39 + - `vendor/open-vt/studio/stage/stage.gd` 40 + - `vendor/open-vt/lib/model/vt_model.gd` 41 + 42 + OpenVT already exposes model lifecycle and tracking update paths; a bridge can target the active model and/or stage. 43 + 44 + ## Recommended control protocol 45 + 46 + Use a simple local TCP JSON protocol. 47 + 48 + ### Why this path 49 + 50 + - matches existing OpenVT tracker infrastructure 51 + - avoids inventing an HTTP stack inside Godot 52 + - easy to emit from Bun/Node 53 + - good enough for localhost control from the runtime 54 + 55 + ## Suggested messages 56 + 57 + ```json 58 + { "type": "talk", "state": "start" } 59 + { "type": "talk", "state": "stop" } 60 + { "type": "emotion", "value": "happy" } 61 + { "type": "gesture", "value": "nod" } 62 + { "type": "text", "value": "Hello chat!" } 63 + ``` 64 + 65 + ## Minimal OpenVT-side implementation plan 66 + 67 + 1. Add a new control node/script under OpenVT, for example: 68 + - `vendor/open-vt/lib/control/runtime_control.gd` 69 + 2. Reuse the TCP server pattern from `tcp_tracker.gd` 70 + 3. Parse JSON messages 71 + 4. Map actions to existing model/expression/animation hooks 72 + 73 + ## Minimal repo-side implementation plan 74 + 75 + 1. Add a Bun-side TCP client sender 76 + 2. On each runtime output action, emit one JSON message to localhost 77 + 3. Keep OBS separate from this bridge; OBS can continue to capture OpenVT visually 78 + 79 + ## Immediate next step 80 + 81 + Implement the repo-side sender first: 82 + 83 + - `src/openvt-control.ts` 84 + 85 + That preserves the interface even before the OpenVT side is fully patched. 86 +
+92
docs/streamplace-ingest-checklist.md
··· 1 + # stream.place ingest checklist 2 + 3 + ## Objective 4 + 5 + Verify the last leg of the stack: 6 + 7 + - OBS program output reaches stream.place 8 + 9 + ## Current repo status 10 + 11 + This repo currently provides: 12 + 13 + - chat intake/runtime 14 + - Gemini Live seam 15 + - OpenVT control seam 16 + - OBS websocket seam 17 + 18 + It does **not** yet automate stream.place ingest creation or credentials. 19 + 20 + ## Manual checklist 21 + 22 + ### 1. Configure OBS output 23 + 24 + In OBS, set stream output using the stream.place-supported ingest path: 25 + 26 + - WHIP if your stream.place setup provides WHIP 27 + - otherwise RTMP if your stream.place setup provides RTMP 28 + 29 + Required values are expected to come from stream.place, for example: 30 + 31 + - server / endpoint URL 32 + - stream key or bearer/auth material 33 + 34 + ### 2. Verify OBS scene contents 35 + 36 + Minimum recommended scene: 37 + 38 + - OpenVT capture source 39 + - window capture, or 40 + - virtual camera capture path if preferred 41 + - text source for replies 42 + - audio source for Gemini/system output 43 + 44 + ### 3. Verify OBS websocket is enabled 45 + 46 + Expected local control port in this repo: 47 + 48 + - `ws://127.0.0.1:4455` 49 + 50 + ### 4. Verify with repo preflight 51 + 52 + Run: 53 + 54 + ```bash 55 + bun run check:live-stack 56 + ``` 57 + 58 + ### 5. Start runtime 59 + 60 + Run: 61 + 62 + ```bash 63 + bun run run:live-stack 64 + ``` 65 + 66 + ### 6. Confirm end-to-end behavior 67 + 68 + Look for all of the following: 69 + 70 + - incoming chat event from Jetstream 71 + - immediate avatar action in OpenVT 72 + - reply text updated in OBS text source 73 + - Gemini-generated reply content 74 + - OBS program stream active to stream.place 75 + 76 + ## Missing automation 77 + 78 + The following is still not implemented in code: 79 + 80 + - automatic creation of stream.place ingest sessions 81 + - automatic writing of OBS stream settings from stream.place credentials 82 + - automatic verification that stream.place is receiving program output 83 + 84 + ## Best next implementation if needed 85 + 86 + Add a small config layer for stream.place ingest values, for example: 87 + 88 + - `STREAMPLACE_INGEST_URL` 89 + - `STREAMPLACE_STREAM_KEY` 90 + 91 + and document how those feed OBS configuration. 92 +
minimal-haru/addons/gd_cubism/bin/linux/x86_64/libLive2DCubismCore.so

This is a binary file and will not be displayed.

minimal-haru/addons/gd_cubism/bin/linux/x86_64/libgd_cubism.linux.debug.x86_64.so

This is a binary file and will not be displayed.

minimal-haru/addons/gd_cubism/bin/linux/x86_64/libgd_cubism.linux.release.x86_64.so

This is a binary file and will not be displayed.

+9
minimal-haru/addons/gd_cubism/gd_cubism.gdextension
··· 1 + [configuration] 2 + 3 + entry_symbol = "gd_cubism_library_init" 4 + compatibility_minimum = "4.4" 5 + 6 + [libraries] 7 + 8 + linux.debug.x86_64 = "res://addons/gd_cubism/bin/linux/x86_64/libgd_cubism.linux.debug.x86_64.so" 9 + linux.release.x86_64 = "res://addons/gd_cubism/bin/linux/x86_64/libgd_cubism.linux.release.x86_64.so"
+10
minimal-haru/addons/gd_cubism/res/shader/2d_cubism_mask.gdshader
··· 1 + // SPDX-License-Identifier: MIT 2 + // SPDX-FileCopyrightText: 2023 MizunagiKB <mizukb@live.jp> 3 + // GDCubism shader: Mask 4 + shader_type canvas_item; 5 + render_mode blend_mix, unshaded; 6 + 7 + void fragment() { 8 + vec4 color_tex = texture(TEXTURE, UV); 9 + COLOR = vec4(0,0,0,color_tex.a); 10 + }
+76
minimal-haru/addons/gd_cubism/res/shader/2d_cubism_norm_add.gdshader
··· 1 + // SPDX-License-Identifier: MIT 2 + // SPDX-FileCopyrightText: 2023 MizunagiKB <mizukb@live.jp> 3 + // GDCubism shader: Mix 4 + shader_type canvas_item; 5 + render_mode blend_premul_alpha, unshaded; 6 + 7 + instance uniform vec4 color_base: source_color; 8 + instance uniform vec4 color_screen: source_color; 9 + instance uniform vec4 color_multiply: source_color; 10 + instance uniform bool color_override; 11 + 12 + // masking 13 + instance uniform bool has_mask = false; 14 + instance uniform bool invert; 15 + uniform sampler2D tex_mask : filter_linear_mipmap, repeat_disable; 16 + uniform float mask_scale = 1.0; 17 + uniform vec2 canvas_size = vec2(2,2); 18 + uniform vec2 mesh_offset = vec2(0,0); 19 + 20 + varying vec2 MASK_UV; 21 + varying vec4 modulate; 22 + 23 + bool discard_mask() { 24 + if(has_mask) { 25 + if (MASK_UV.x > 1.0 || MASK_UV.y > 1.0 || MASK_UV.x < 0.0 || MASK_UV.y < 0.0) { 26 + return true; 27 + } 28 + } 29 + return false; 30 + } 31 + 32 + vec2 mask_uv(vec2 vtx) { 33 + if (!has_mask) { 34 + return vec2(0,0); 35 + } 36 + vec2 mask_size = vec2(textureSize(tex_mask, 0)) / mask_scale; 37 + vec2 mask_vtx = vtx - mesh_offset; 38 + return mask_vtx / mask_size; 39 + } 40 + 41 + vec4 sample_mask(vec4 color_tex, vec4 base) { 42 + vec4 color_for_mask = color_tex * base; 43 + color_for_mask.rgb = color_for_mask.rgb * color_for_mask.a; 44 + vec4 clip_mask = texture(tex_mask, MASK_UV) * vec4(0,0,0,1); 45 + float mask_val = clip_mask.r + clip_mask.g + clip_mask.b + clip_mask.a; 46 + if (invert) { 47 + mask_val = 1.0 - mask_val; 48 + } 49 + return color_for_mask * mask_val; 50 + } 51 + 52 + void vertex() { 53 + MASK_UV = mask_uv(VERTEX.xy); 54 + modulate = COLOR; 55 + } 56 + 57 + void fragment() { 58 + if (discard_mask()) { 59 + discard; 60 + } 61 + 62 + vec4 color_tex = texture(TEXTURE, UV); 63 + color_tex.rgba = color_tex.rgba * color_multiply.rgba; 64 + 65 + // premul alpha 66 + color_tex.rgb = (color_tex.rgb + color_screen.rgb) - (color_tex.rgb * color_screen.rgb); 67 + 68 + if (has_mask) { 69 + vec4 color_for_mask = sample_mask(color_tex, color_base) * modulate; 70 + COLOR = vec4(color_for_mask.rgb, 0.0); 71 + } else { 72 + // premul alpha 73 + vec4 color = color_tex * color_base * modulate; 74 + COLOR = vec4(color.rgb * color.a, 0.0); 75 + } 76 + }
+76
minimal-haru/addons/gd_cubism/res/shader/2d_cubism_norm_mix.gdshader
··· 1 + // SPDX-License-Identifier: MIT 2 + // SPDX-FileCopyrightText: 2023 MizunagiKB <mizukb@live.jp> 3 + // GDCubism shader: Mix 4 + shader_type canvas_item; 5 + render_mode blend_premul_alpha, unshaded; 6 + 7 + instance uniform vec4 color_base: source_color; 8 + instance uniform vec4 color_screen: source_color; 9 + instance uniform vec4 color_multiply: source_color; 10 + instance uniform bool color_override; 11 + 12 + // masking 13 + instance uniform bool has_mask = false; 14 + instance uniform bool invert; 15 + uniform sampler2D tex_mask : filter_linear_mipmap, repeat_disable; 16 + uniform float mask_scale = 1.0; 17 + uniform vec2 canvas_size = vec2(2,2); 18 + uniform vec2 mesh_offset = vec2(0,0); 19 + 20 + varying vec2 MASK_UV; 21 + varying vec4 modulate; 22 + 23 + bool discard_mask() { 24 + if(has_mask) { 25 + if (MASK_UV.x > 1.0 || MASK_UV.y > 1.0 || MASK_UV.x < 0.0 || MASK_UV.y < 0.0) { 26 + return true; 27 + } 28 + } 29 + return false; 30 + } 31 + 32 + vec2 mask_uv(vec2 vtx) { 33 + if (!has_mask) { 34 + return vec2(0,0); 35 + } 36 + vec2 mask_size = vec2(textureSize(tex_mask, 0)) / mask_scale; 37 + vec2 mask_vtx = vtx - mesh_offset; 38 + return mask_vtx / mask_size; 39 + } 40 + 41 + vec4 sample_mask(vec4 color_tex, vec4 base) { 42 + vec4 color_for_mask = color_tex * base; 43 + color_for_mask.rgb = color_for_mask.rgb * color_for_mask.a; 44 + vec4 clip_mask = texture(tex_mask, MASK_UV) * vec4(0,0,0,1); 45 + float mask_val = clip_mask.r + clip_mask.g + clip_mask.b + clip_mask.a; 46 + if (invert) { 47 + mask_val = 1.0 - mask_val; 48 + } 49 + return color_for_mask * mask_val; 50 + } 51 + 52 + void vertex() { 53 + MASK_UV = mask_uv(VERTEX.xy); 54 + modulate = COLOR; 55 + } 56 + 57 + void fragment() { 58 + if (discard_mask()) { 59 + discard; 60 + } 61 + 62 + vec4 color_tex = texture(TEXTURE, UV); 63 + color_tex.rgba = color_tex.rgba * color_multiply.rgba; 64 + 65 + // premul alpha 66 + color_tex.rgb = (color_tex.rgb + color_screen.rgb) - (color_tex.rgb * color_screen.rgb); 67 + 68 + if (has_mask) { 69 + vec4 color_for_mask = sample_mask(color_tex, color_base); 70 + COLOR = color_for_mask * modulate; 71 + } else { 72 + // premul alpha 73 + vec4 color = color_tex * color_base * modulate; 74 + COLOR = vec4(color.rgb * color.a, color.a); 75 + } 76 + }
+81
minimal-haru/addons/gd_cubism/res/shader/2d_cubism_norm_mul.gdshader
··· 1 + // SPDX-License-Identifier: MIT 2 + // SPDX-FileCopyrightText: 2023 MizunagiKB <mizukb@live.jp> 3 + // GDCubism shader: Mix 4 + shader_type canvas_item; 5 + render_mode blend_mul, unshaded; 6 + 7 + instance uniform vec4 color_base: source_color; 8 + instance uniform vec4 color_screen: source_color; 9 + instance uniform vec4 color_multiply: source_color; 10 + instance uniform bool color_override; 11 + 12 + // masking 13 + instance uniform bool has_mask = false; 14 + instance uniform bool invert; 15 + uniform sampler2D tex_mask : filter_linear_mipmap, repeat_disable; 16 + uniform float mask_scale = 1.0; 17 + uniform vec2 canvas_size = vec2(2,2); 18 + uniform vec2 mesh_offset = vec2(0,0); 19 + 20 + varying vec2 MASK_UV; 21 + varying vec4 modulate; 22 + 23 + bool discard_mask() { 24 + if(has_mask) { 25 + if (MASK_UV.x > 1.0 || MASK_UV.y > 1.0 || MASK_UV.x < 0.0 || MASK_UV.y < 0.0) { 26 + return true; 27 + } 28 + } 29 + return false; 30 + } 31 + 32 + vec2 mask_uv(vec2 vtx) { 33 + if (!has_mask) { 34 + return vec2(0,0); 35 + } 36 + vec2 mask_size = vec2(textureSize(tex_mask, 0)) / mask_scale; 37 + vec2 mask_vtx = vtx - mesh_offset; 38 + return mask_vtx / mask_size; 39 + } 40 + 41 + vec4 sample_mask(vec4 color_tex, vec4 base) { 42 + vec4 color_for_mask = color_tex * base; 43 + color_for_mask.rgb = color_for_mask.rgb * color_for_mask.a; 44 + vec4 clip_mask = texture(tex_mask, MASK_UV) * vec4(0,0,0,1); 45 + float mask_val = clip_mask.r + clip_mask.g + clip_mask.b + clip_mask.a; 46 + if (invert) { 47 + mask_val = 1.0 - mask_val; 48 + } 49 + return color_for_mask * mask_val; 50 + } 51 + 52 + void vertex() { 53 + MASK_UV = mask_uv(VERTEX.xy); 54 + modulate = COLOR; 55 + } 56 + 57 + void fragment() { 58 + if (discard_mask()) { 59 + discard; 60 + } 61 + 62 + vec4 color_tex = texture(TEXTURE, UV); 63 + color_tex.rgba = color_tex.rgba * color_multiply.rgba; 64 + 65 + // premul alpha 66 + color_tex.rgb = color_tex.rgb + color_screen.rgb - (color_tex.rgb * color_screen.rgb); 67 + 68 + if (has_mask) { 69 + vec4 color_for_mask = sample_mask(color_tex, color_base); 70 + COLOR = vec4( 71 + color_for_mask.r + (1.0 - color_for_mask.a), 72 + color_for_mask.g + (1.0 - color_for_mask.a), 73 + color_for_mask.b + (1.0 - color_for_mask.a), 74 + 1.0 75 + ) * modulate; 76 + } else { 77 + // premul alpha 78 + vec4 color = color_tex * color_base * modulate; 79 + COLOR = vec4(color.rgb * color.a, color.a); 80 + } 81 + }
+47
minimal-haru/addons/gd_cubism/res/shader/cubism_mask.gdshaderinc
··· 1 + // masking 2 + instance uniform bool has_mask = false; 3 + instance uniform bool invert; 4 + uniform sampler2D tex_mask : filter_linear_mipmap, repeat_disable; 5 + uniform float mask_scale = 1.0; 6 + uniform vec2 canvas_size = vec2(2,2); 7 + uniform vec2 mesh_offset = vec2(0,0); 8 + 9 + varying vec2 MASK_UV; 10 + varying vec4 modulate; 11 + 12 + bool discard_mask() { 13 + if(has_mask) { 14 + if (MASK_UV.x > 1.0 || MASK_UV.y > 1.0 || MASK_UV.x < 0.0 || MASK_UV.y < 0.0) { 15 + return true; 16 + } 17 + } 18 + return false; 19 + } 20 + 21 + vec2 mask_uv(vec2 vtx) { 22 + if (!has_mask) { 23 + return vec2(0,0); 24 + } 25 + 26 + vec2 mask_size = vec2(textureSize(tex_mask, 0)) / mask_scale; 27 + vec2 mask_vtx = vtx - mesh_offset; 28 + return mask_vtx / mask_size; 29 + } 30 + 31 + vec4 sample_mask(vec4 color_tex, vec4 base) { 32 + vec4 color_for_mask = color_tex * base; 33 + color_for_mask.rgb = color_for_mask.rgb * color_for_mask.a; 34 + 35 + vec4 clip_mask = texture(tex_mask, MASK_UV) * vec4(0,0,0,1); 36 + 37 + float mask_val = clip_mask.r + clip_mask.g + clip_mask.b + clip_mask.a; 38 + if (invert) { 39 + mask_val = 1.0 - mask_val; 40 + } 41 + return color_for_mask * mask_val; 42 + } 43 + 44 + void vertex() { 45 + MASK_UV = mask_uv(VERTEX.xy); 46 + modulate = COLOR; 47 + }
+42
minimal-haru/main.gd
··· 1 + extends Node2D 2 + 3 + const MODEL_PATH = "/home/nandi/.local/share/godot/app_userdata/open-vt/Live2DModels/Haru/Haru.model3.json" 4 + const SHADERS = [ 5 + preload("res://addons/gd_cubism/res/shader/2d_cubism_norm_add.gdshader"), 6 + preload("res://addons/gd_cubism/res/shader/2d_cubism_norm_mix.gdshader"), 7 + preload("res://addons/gd_cubism/res/shader/2d_cubism_norm_mul.gdshader"), 8 + preload("res://addons/gd_cubism/res/shader/2d_cubism_mask.gdshader"), 9 + ] 10 + 11 + func _ready() -> void: 12 + var loader = ClassDB.instantiate("GDCubismModelLoader") 13 + if loader == null: 14 + push_error("minimal-haru: GDCubismModelLoader unavailable") 15 + get_tree().quit(1) 16 + return 17 + 18 + var model = loader.call("load_model", MODEL_PATH, SHADERS, true) 19 + if model == null: 20 + push_error("minimal-haru: failed to load Haru model") 21 + get_tree().quit(1) 22 + return 23 + 24 + model.set("mask_viewport_size", 2048) 25 + model.set_process(true) 26 + model.position = Vector2(640, 360) 27 + model.scale = Vector2.ONE * 0.12 28 + add_child(model) 29 + 30 + var physics = ClassDB.instantiate("GDCubismEffectPhysics") 31 + if physics != null: 32 + physics.name = "Physics" 33 + model.add_child(physics) 34 + 35 + await get_tree().process_frame 36 + 37 + print("MINIMAL_HARU loaded=true") 38 + print("MINIMAL_HARU initialized=%s" % JSON.stringify(model.call("is_initialized"))) 39 + print("MINIMAL_HARU mesh_count=%s" % JSON.stringify(model.call("get_meshes").size())) 40 + print("MINIMAL_HARU position=%s" % JSON.stringify({"x": model.position.x, "y": model.position.y})) 41 + print("MINIMAL_HARU scale=%s" % JSON.stringify({"x": model.scale.x, "y": model.scale.y})) 42 + print("MINIMAL_HARU ready_for_interactive_use=true")
+6
minimal-haru/main.tscn
··· 1 + [gd_scene load_steps=2 format=3] 2 + 3 + [ext_resource type="Script" path="res://main.gd" id="1"] 4 + 5 + [node name="Main" type="Node2D"] 6 + script = ExtResource("1")
+17
minimal-haru/project.godot
··· 1 + config_version=5 2 + 3 + [application] 4 + 5 + config/name="minimal-haru" 6 + run/main_scene="res://main.tscn" 7 + config/features=PackedStringArray("4.6", "Forward Plus") 8 + 9 + [autoload] 10 + 11 + CubismBootstrap="*res://addons/gd_cubism/gd_cubism.gdextension" 12 + 13 + [display] 14 + 15 + window/size/viewport_width=1280 16 + window/size/viewport_height=720 17 + window/stretch/aspect="expand"
+1
miniview/.godot/extension_list.cfg
··· 1 + res://addons/gd_cubism/gd_cubism.gdextension
+1 -2
miniview/project.godot
··· 6 6 [application] 7 7 8 8 config/name="mini-cubism-viewer" 9 - run/main_scene="res://addons/gd_cubism/example/viewer.tscn" 9 + run/main_scene="res://viewer_openvt.tscn" 10 10 config/features=PackedStringArray("4.3", "Forward Plus") 11 - config/icon="res://addons/gd_cubism/example/icon.svg" 12 11 13 12 [display] 14 13
+31
miniview/viewer_min.gd
··· 1 + extends Node2D 2 + 3 + var cubism_model 4 + 5 + func _ready() -> void: 6 + var klass = ClassDB.instantiate("GDCubismUserModel") 7 + if klass == null: 8 + push_error("viewer_min: GDCubismUserModel class unavailable") 9 + return 10 + 11 + cubism_model = klass 12 + add_child(cubism_model) 13 + 14 + var model_path = "res://addons/gd_cubism/example/res/live2d/Haru/Haru.model3.json" 15 + cubism_model.set("assets", model_path) 16 + 17 + await get_tree().process_frame 18 + await get_tree().process_frame 19 + 20 + print("MINIVIEW_HARU assets_assigned=true") 21 + 22 + var canvas_info = cubism_model.get_canvas_info() 23 + var viewport_size = get_viewport_rect().size 24 + if not canvas_info.is_empty(): 25 + var size_px = canvas_info.size_in_pixels 26 + var scale = viewport_size.y / max(size_px.x, size_px.y) 27 + cubism_model.position = viewport_size / 2.0 28 + cubism_model.scale = Vector2(scale, scale) 29 + 30 + print("MINIVIEW_HARU loaded=true") 31 + print("MINIVIEW_HARU assets=%s" % model_path)
+6
miniview/viewer_min.tscn
··· 1 + [gd_scene load_steps=2 format=3] 2 + 3 + [ext_resource type="Script" path="res://viewer_min.gd" id="1"] 4 + 5 + [node name="ViewerMin" type="Node2D"] 6 + script = ExtResource("1")
+45
miniview/viewer_openvt.gd
··· 1 + extends Node2D 2 + 3 + func _ready() -> void: 4 + var tracking = Node.new() 5 + tracking.name = "TrackerSystem" 6 + tracking.add_to_group("system:tracking") 7 + tracking.set_script(preload("res://../vendor/open-vt/lib/tracking/tracking_system.gd")) 8 + add_child(tracking) 9 + 10 + var manager = Node.new() 11 + manager.name = "ModelManager" 12 + manager.set_script(preload("res://../vendor/open-vt/lib/model_manager.gd")) 13 + add_child(manager) 14 + 15 + await get_tree().process_frame 16 + await get_tree().process_frame 17 + 18 + var stage = preload("res://../vendor/open-vt/minimal_stage.tscn").instantiate() 19 + add_child(stage) 20 + await get_tree().process_frame 21 + await get_tree().process_frame 22 + 23 + var mm = get_tree().get_first_node_in_group("system:model") 24 + if mm == null: 25 + push_error("viewer_openvt: model manager unavailable") 26 + return 27 + 28 + var model = mm.make_model("user://Live2DModels/Haru") 29 + if model == null: 30 + push_error("viewer_openvt: failed to make Haru model") 31 + return 32 + 33 + stage.spawn_model(model) 34 + await model.loaded 35 + 36 + if model.is_queued_for_deletion(): 37 + push_error("viewer_openvt: Haru model queued for deletion") 38 + return 39 + 40 + model.position = Vector2(640, 520) 41 + model.scale = Vector2(0.32, 0.32) 42 + 43 + print("MINIVIEW_OPENVT loaded=true") 44 + print("MINIVIEW_OPENVT mesh_count=%s" % JSON.stringify(model.get_meshes().size())) 45 +
+6
miniview/viewer_openvt.tscn
··· 1 + [gd_scene load_steps=2 format=3] 2 + 3 + [ext_resource type="Script" path="res://viewer_openvt.gd" id="1"] 4 + 5 + [node name="ViewerOpenVT" type="Node2D"] 6 + script = ExtResource("1")
+11
package.json
··· 3 3 "version": "0.0.1", 4 4 "private": true, 5 5 "type": "module", 6 + "dependencies": { 7 + "@google/genai": "^1.18.0", 8 + "obs-websocket-js": "^5.0.6" 9 + }, 6 10 "scripts": { 7 11 "dev": "bun run src/index.ts", 12 + "live": "bun run src/index.ts --live", 13 + "check:live-stack": "bun run scripts/check-live-stack.ts", 14 + "configure:obs-websocket": "bun run scripts/configure-obs-websocket.ts", 15 + "configure:obs-scene": "bun run scripts/configure-obs-scene.ts", 16 + "configure:obs-stream": "bun run scripts/configure-obs-stream.ts", 17 + "run:live-stack": "bash ./scripts/run-live-stack.sh", 18 + "verify:live-response": "bun run scripts/verify-live-response.ts", 8 19 "simulate": "bun run scripts/simulate-chat.ts", 9 20 "check": "bun run src/index.ts --help" 10 21 }
+49
scripts/check-live-stack.ts
··· 1 + const requiredEnv = [ 2 + 'GEMINI_API_KEY', 3 + 'OBS_WS_URL', 4 + 'OBS_WS_PASSWORD', 5 + 'OBS_SCENE_NAME', 6 + 'OBS_TEXT_INPUT_NAME', 7 + ]; 8 + 9 + function printEnvStatus() { 10 + console.log('Environment checks:'); 11 + for (const key of requiredEnv) { 12 + const value = process.env[key]; 13 + console.log(`- ${key}: ${value ? 'set' : 'missing'}`); 14 + } 15 + } 16 + 17 + async function probeTcp(name: string, host: string, port: number) { 18 + try { 19 + const socket = await Bun.connect({ 20 + hostname: host, 21 + port, 22 + socket: { 23 + open(socket) { 24 + socket.end(); 25 + }, 26 + data() {}, 27 + close() {}, 28 + error() {}, 29 + }, 30 + }); 31 + socket.unref?.(); 32 + console.log(`- ${name}: open (${host}:${port})`); 33 + } catch { 34 + console.log(`- ${name}: closed (${host}:${port})`); 35 + } 36 + } 37 + 38 + console.log('Live stack readiness'); 39 + printEnvStatus(); 40 + console.log('Port checks:'); 41 + await probeTcp('OBS websocket', '127.0.0.1', 4455); 42 + await probeTcp('OpenVT runtime control', '127.0.0.1', 25733); 43 + 44 + console.log('\nSuggested next actions:'); 45 + console.log('1. Export GEMINI_API_KEY'); 46 + console.log('2. Start OBS with obs-websocket enabled on port 4455'); 47 + console.log('3. Set OBS_SCENE_NAME and OBS_TEXT_INPUT_NAME'); 48 + console.log('4. Start OpenVT so runtime control listens on 25733'); 49 + console.log('5. Run: bun run src/index.ts --live');
+73
scripts/configure-obs-scene.ts
··· 1 + import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; 2 + import { dirname } from 'node:path'; 3 + 4 + const scenePath = '/home/nandi/.config/obs-studio/basic/scenes/Untitled.json'; 5 + const textSourceName = process.env.OBS_TEXT_INPUT_NAME ?? 'Agent Reply'; 6 + const sceneName = process.env.OBS_SCENE_NAME ?? 'Scene'; 7 + 8 + const scene = JSON.parse(readFileSync(scenePath, 'utf8')) as any; 9 + scene.current_scene = sceneName; 10 + scene.current_program_scene = sceneName; 11 + 12 + const sources = Array.isArray(scene.sources) ? scene.sources : []; 13 + const sceneSource = sources.find((s: any) => s.name === sceneName && s.id === 'scene'); 14 + 15 + const hasTextSource = sources.some((s: any) => s.name === textSourceName); 16 + if (!hasTextSource) { 17 + sources.push({ 18 + prev_ver: 536936450, 19 + name: textSourceName, 20 + uuid: '11111111-2222-3333-4444-555555555555', 21 + id: 'text_ft2_source', 22 + versioned_id: 'text_ft2_source', 23 + settings: { 24 + text: '', 25 + }, 26 + mixers: 0, 27 + sync: 0, 28 + flags: 0, 29 + volume: 1.0, 30 + balance: 0.5, 31 + enabled: true, 32 + muted: false, 33 + 'push-to-mute': false, 34 + 'push-to-mute-delay': 0, 35 + 'push-to-talk': false, 36 + 'push-to-talk-delay': 0, 37 + hotkeys: {}, 38 + deinterlace_mode: 0, 39 + deinterlace_field_order: 0, 40 + monitoring_type: 0, 41 + private_settings: {}, 42 + }); 43 + } 44 + 45 + if (sceneSource && Array.isArray(sceneSource.settings?.items)) { 46 + const alreadyPlaced = sceneSource.settings.items.some((item: any) => item.name === textSourceName); 47 + if (!alreadyPlaced) { 48 + sceneSource.settings.items.push({ 49 + name: textSourceName, 50 + source_uuid: '11111111-2222-3333-4444-555555555555', 51 + visible: true, 52 + locked: false, 53 + pos: { x: 40.0, y: 40.0 }, 54 + scale: { x: 1.0, y: 1.0 }, 55 + align: 5, 56 + bounds_type: 0, 57 + bounds_align: 0, 58 + crop_left: 0, 59 + crop_top: 0, 60 + crop_right: 0, 61 + crop_bottom: 0, 62 + id: 1, 63 + group_item_backup: false, 64 + }); 65 + } 66 + } 67 + 68 + scene.sources = sources; 69 + mkdirSync(dirname(scenePath), { recursive: true }); 70 + writeFileSync(scenePath, JSON.stringify(scene, null, 4) + '\n'); 71 + 72 + console.log(scenePath); 73 + console.log(JSON.stringify({ sceneName, textSourceName }, null, 2));
+48
scripts/configure-obs-stream.ts
··· 1 + import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; 2 + import { dirname } from 'node:path'; 3 + 4 + const basicIniPath = '/home/nandi/.config/obs-studio/basic/profiles/Untitled/basic.ini'; 5 + 6 + const streamServer = process.env.OBS_STREAM_SERVER ?? process.env.STREAMPLACE_RTMP_URL ?? ''; 7 + const streamKey = process.env.OBS_STREAM_KEY ?? process.env.STREAMPLACE_STREAM_KEY ?? ''; 8 + 9 + if (!streamServer || !streamKey) { 10 + console.error('Missing OBS_STREAM_SERVER/STREAMPLACE_RTMP_URL or OBS_STREAM_KEY/STREAMPLACE_STREAM_KEY'); 11 + process.exit(1); 12 + } 13 + 14 + let content = ''; 15 + try { 16 + content = readFileSync(basicIniPath, 'utf8'); 17 + } catch { 18 + content = '[General]\nName=Untitled\n'; 19 + } 20 + 21 + const streamBlock = [ 22 + '[SimpleOutput]', 23 + 'StreamEncoder=obs_x264', 24 + '[Output]', 25 + 'Mode=Simple', 26 + '[Stream1]', 27 + 'IgnoreRecommended=false', 28 + 'UseAuth=false', 29 + 'bwtest=false', 30 + 'key=' + streamKey, 31 + 'service=Custom', 32 + 'server=' + streamServer, 33 + ].join('\n'); 34 + 35 + content = content.replace(/\n\[SimpleOutput\][\s\S]*$/m, ''); 36 + content = content.replace(/\n\[Output\][\s\S]*$/m, ''); 37 + content = content.replace(/\n\[Stream1\][\s\S]*$/m, ''); 38 + content = content.trimEnd() + '\n\n' + streamBlock + '\n'; 39 + 40 + mkdirSync(dirname(basicIniPath), { recursive: true }); 41 + writeFileSync(basicIniPath, content); 42 + 43 + console.log(basicIniPath); 44 + console.log(JSON.stringify({ 45 + server: streamServer, 46 + key_present: !!streamKey, 47 + }, null, 2)); 48 +
+19
scripts/configure-obs-websocket.ts
··· 1 + import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; 2 + import { dirname } from 'node:path'; 3 + 4 + const configPath = '/home/nandi/.config/obs-studio/plugin_config/obs-websocket/config.json'; 5 + 6 + const current = JSON.parse(readFileSync(configPath, 'utf8')) as Record<string, unknown>; 7 + current.server_enabled = true; 8 + current.server_port = 4455; 9 + 10 + mkdirSync(dirname(configPath), { recursive: true }); 11 + writeFileSync(configPath, JSON.stringify(current, null, 2) + '\n'); 12 + 13 + console.log(configPath); 14 + console.log(JSON.stringify({ 15 + server_enabled: current.server_enabled, 16 + server_port: current.server_port, 17 + auth_required: current.auth_required, 18 + }, null, 2)); 19 +
+7 -1
scripts/miniview-launch.sh
··· 4 4 ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" 5 5 VIEWER_DIR="$ROOT_DIR/miniview" 6 6 OPENVT_DIR="$ROOT_DIR/vendor/open-vt" 7 + GODOT_BIN="${GODOT_BIN:-/nix/store/q2apsqxsljrzy2xpq26z774r5c30lagr-godot-4.6.2-stable/bin/godot4}" 7 8 8 9 export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" 9 10 export WAYLAND_DISPLAY="${WAYLAND_DISPLAY:-wayland-1}" ··· 14 15 fi 15 16 16 17 mkdir -p "$VIEWER_DIR/addons" 18 + mkdir -p "$VIEWER_DIR/.godot" 17 19 18 20 rm -rf "$VIEWER_DIR/addons/gd_cubism" 19 21 cp -R "$OPENVT_DIR/thirdparty/gd_cubism/demo/addons/gd_cubism" "$VIEWER_DIR/addons/gd_cubism" ··· 22 24 cp "$OPENVT_DIR/addons/gd_cubism/bin/linux/x86_64/libgd_cubism.linux.debug.x86_64.so" "$VIEWER_DIR/addons/gd_cubism/bin/linux/x86_64/" 23 25 cp "$OPENVT_DIR/addons/gd_cubism/bin/linux/x86_64/libLive2DCubismCore.so" "$VIEWER_DIR/addons/gd_cubism/bin/linux/x86_64/" 24 26 27 + cat > "$VIEWER_DIR/.godot/extension_list.cfg" <<'EOF' 28 + res://addons/gd_cubism/gd_cubism.gdextension 29 + EOF 30 + 25 31 mkdir -p "$VIEWER_DIR/addons/gd_cubism/example/res/live2d/Haru" 26 32 rm -rf "$VIEWER_DIR/addons/gd_cubism/example/res/live2d/Haru" 27 33 cp -R /home/nandi/.local/share/godot/app_userdata/open-vt/Live2DModels/Haru "$VIEWER_DIR/addons/gd_cubism/example/res/live2d/Haru" 28 34 29 - exec godot4 --path "$VIEWER_DIR" 35 + exec "$GODOT_BIN" --path "$VIEWER_DIR"
+37
scripts/run-live-stack.sh
··· 1 + #!/usr/bin/env bash 2 + set -euo pipefail 3 + 4 + ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" 5 + SWARM_DIR="$(cd "$ROOT_DIR/../swarm" && pwd)" 6 + 7 + cd "$ROOT_DIR" 8 + 9 + if [[ -z "${GEMINI_API_KEY:-}" ]]; then 10 + echo "== Loading GEMINI_API_KEY from swarm keyvault ==" 11 + GEMINI_API_KEY="$(cd "$SWARM_DIR" && bun run keys get GEMINI_API_KEY | grep -E '^(AIza|AIza[[:alnum:]_-]+)$' | tail -n 1)" 12 + export GEMINI_API_KEY 13 + fi 14 + 15 + OBS_WS_CONFIG="/home/nandi/.config/obs-studio/plugin_config/obs-websocket/config.json" 16 + if [[ -z "${OBS_WS_PASSWORD:-}" && -f "$OBS_WS_CONFIG" ]]; then 17 + echo "== Loading OBS websocket password from OBS config ==" 18 + OBS_WS_PASSWORD="$(python3 - <<'PY' 19 + import json 20 + path = '/home/nandi/.config/obs-studio/plugin_config/obs-websocket/config.json' 21 + with open(path, 'r', encoding='utf-8') as f: 22 + data = json.load(f) 23 + print(data.get('server_password', '')) 24 + PY 25 + )" 26 + export OBS_WS_PASSWORD 27 + fi 28 + 29 + echo "== Live stack preflight ==" 30 + bun run scripts/check-live-stack.ts 31 + 32 + echo 33 + echo "== Starting live loop ==" 34 + echo "This will subscribe to Jetstream chat and attempt Gemini/OpenVT/OBS integration using current env vars." 35 + echo 36 + 37 + exec bun run src/index.ts --live
+1 -1
scripts/simulate-chat.ts
··· 6 6 timestamp: new Date().toISOString() 7 7 }; 8 8 9 - const result = handleMessage(sample); 9 + const result = await handleMessage(sample); 10 10 console.log(JSON.stringify({ input: sample, output: result }, null, 2));
+12
scripts/test-gemini-live.ts
··· 1 + import { GeminiLiveProvider } from '../src/gemini-live'; 2 + 3 + const provider = new GeminiLiveProvider(); 4 + 5 + const reply = await provider.generateReply({ 6 + author: 'viewer1', 7 + text: 'Say hello in one short sentence.', 8 + timestamp: new Date().toISOString(), 9 + }); 10 + 11 + console.log(JSON.stringify({ reply }, null, 2)); 12 +
+16
scripts/test-obs-control.ts
··· 1 + import { OBSControlClient } from '../src/obs-control'; 2 + 3 + const client = new OBSControlClient(); 4 + 5 + console.log(JSON.stringify({ 6 + url: process.env.OBS_WS_URL ?? 'ws://127.0.0.1:4455', 7 + sceneName: process.env.OBS_SCENE_NAME ?? null, 8 + textInputName: process.env.OBS_TEXT_INPUT_NAME ?? null, 9 + }, null, 2)); 10 + 11 + if (process.env.OBS_TEST_CONNECT === '1') { 12 + await client.connect(); 13 + console.log('obs connected'); 14 + await client.disconnect(); 15 + } 16 +
+23
scripts/test-streamplace-chat.ts
··· 1 + import { StreamplaceChatAdapter } from '../src/streamplace-chat'; 2 + 3 + const adapter = new StreamplaceChatAdapter({ 4 + wantedCollections: [process.env.STREAMPLACE_CHAT_NSID ?? 'place.stream.chat.message'], 5 + }); 6 + 7 + console.log(adapter.createUrl()); 8 + 9 + const sample = { 10 + did: 'did:plc:example123', 11 + time_us: 1710000000000000, 12 + kind: 'commit', 13 + commit: { 14 + operation: 'create', 15 + collection: process.env.STREAMPLACE_CHAT_NSID ?? 'place.stream.chat.message', 16 + record: { 17 + message: 'hello from stream.place chat', 18 + createdAt: '2026-05-26T21:23:00.000Z', 19 + }, 20 + }, 21 + }; 22 + 23 + console.log(JSON.stringify(adapter.extractChatEvent(JSON.stringify(sample)), null, 2));
+36
scripts/verify-live-response.ts
··· 1 + import { GeminiLiveProvider } from '../src/gemini-live'; 2 + import { OBSControlClient } from '../src/obs-control'; 3 + import { OpenVTControlClient } from '../src/openvt-control'; 4 + import { CharacterRuntime } from '../src/runtime'; 5 + 6 + const runtime = new CharacterRuntime(new GeminiLiveProvider()); 7 + const openvt = new OpenVTControlClient(); 8 + const obs = new OBSControlClient({ 9 + sceneName: process.env.OBS_SCENE_NAME ?? 'Scene', 10 + textInputName: process.env.OBS_TEXT_INPUT_NAME ?? 'Agent Reply', 11 + }); 12 + 13 + const message = { 14 + author: process.env.TEST_CHAT_AUTHOR ?? 'streamplace-viewer', 15 + text: process.env.TEST_CHAT_TEXT ?? 'hey haru what do you think about stream.place?', 16 + timestamp: new Date().toISOString(), 17 + }; 18 + 19 + console.log(JSON.stringify({ phase: 'input', message }, null, 2)); 20 + 21 + await openvt.sendTalkState('start'); 22 + await openvt.sendEmotion('curious'); 23 + 24 + const result = await runtime.handleMessage(message); 25 + console.log(JSON.stringify({ phase: 'output', result }, null, 2)); 26 + 27 + await openvt.sendText(result.reply); 28 + await obs.setText(result.reply); 29 + await obs.setProgramScene(); 30 + await openvt.sendResponse({ 31 + ...result, 32 + actions: result.actions.filter((action) => action.type !== 'talk:start'), 33 + }); 34 + 35 + console.log(JSON.stringify({ phase: 'delivered' }, null, 2)); 36 +
+97
src/gemini-live.ts
··· 1 + import type { ChatMessage, ResponseProvider } from './runtime'; 2 + import { GoogleGenAI, Modality, type LiveServerMessage } from '@google/genai'; 3 + 4 + type GeminiLiveOptions = { 5 + apiKey?: string; 6 + model?: string; 7 + systemPrompt?: string; 8 + }; 9 + 10 + export class GeminiLiveProvider implements ResponseProvider { 11 + private readonly apiKey?: string; 12 + private readonly model: string; 13 + private readonly systemPrompt: string; 14 + private client?: GoogleGenAI; 15 + 16 + constructor(options: GeminiLiveOptions = {}) { 17 + this.apiKey = options.apiKey ?? process.env.GEMINI_API_KEY; 18 + this.model = options.model ?? 'gemini-2.0-flash-live-001'; 19 + this.systemPrompt = options.systemPrompt ?? 'You are a concise, friendly streaming VTuber agent replying to live chat.'; 20 + } 21 + 22 + async generateReply(message: ChatMessage): Promise<string> { 23 + if (!this.apiKey) { 24 + throw new Error('GEMINI_API_KEY is not set.'); 25 + } 26 + 27 + this.client ??= new GoogleGenAI({ apiKey: this.apiKey }); 28 + 29 + const textChunks: string[] = []; 30 + const queue: LiveServerMessage[] = []; 31 + 32 + const session = await this.client.live.connect({ 33 + model: this.model.startsWith('models/') ? this.model : `models/${this.model}`, 34 + config: { 35 + responseModalities: [Modality.TEXT], 36 + systemInstruction: this.systemPrompt, 37 + }, 38 + callbacks: { 39 + onmessage: (msg) => { 40 + queue.push(msg); 41 + 42 + const anyMsg = msg as any; 43 + const serverContent = anyMsg.serverContent; 44 + const directText = anyMsg.text; 45 + if (typeof directText === 'string' && directText.trim()) { 46 + textChunks.push(directText); 47 + } 48 + 49 + const modelTurn = serverContent?.modelTurn; 50 + const parts = modelTurn?.parts ?? []; 51 + for (const part of parts) { 52 + if (typeof part?.text === 'string' && part.text.trim()) { 53 + textChunks.push(part.text); 54 + } 55 + } 56 + 57 + const outputTranscription = serverContent?.outputTranscription?.text; 58 + if (typeof outputTranscription === 'string' && outputTranscription.trim()) { 59 + textChunks.push(outputTranscription); 60 + } 61 + }, 62 + onerror: (e) => { 63 + throw e; 64 + }, 65 + }, 66 + }); 67 + 68 + try { 69 + await session.sendClientContent({ 70 + turns: `${message.author}: ${message.text}`, 71 + } as any); 72 + 73 + const started = Date.now(); 74 + while (Date.now() - started < 15000) { 75 + const done = queue.find((msg: any) => msg.serverContent?.turnComplete); 76 + if (done) break; 77 + await new Promise((resolve) => setTimeout(resolve, 50)); 78 + } 79 + 80 + const reply = textChunks.join(' ').replace(/\s+/g, ' ').trim(); 81 + if (!reply) { 82 + return `[gemini-live empty:${this.model}] ${message.author}: ${message.text}`; 83 + } 84 + return reply; 85 + } finally { 86 + session.close(); 87 + } 88 + } 89 + 90 + describeSession() { 91 + return { 92 + model: this.model, 93 + systemPrompt: this.systemPrompt, 94 + transport: 'to-be-implemented' 95 + }; 96 + } 97 + }
+11 -24
src/index.ts
··· 1 - type ChatMessage = { 2 - author: string; 3 - text: string; 4 - timestamp: string; 5 - }; 6 - 7 - type AvatarAction = 8 - | { type: 'talk:start' } 9 - | { type: 'talk:stop' } 10 - | { type: 'emotion'; value: 'neutral' | 'happy' | 'excited' | 'curious' } 11 - | { type: 'gesture'; value: 'nod' | 'wave' | 'tilt' }; 1 + import { CharacterRuntime, StubResponseProvider, type ChatMessage } from './runtime'; 2 + import { runLiveLoop } from './live-loop'; 12 3 13 4 function usage() { 14 5 console.log(`character\n\nUsage:\n bun run src/index.ts\n bun run scripts/simulate-chat.ts`); 15 6 } 16 7 17 - function planAvatarActions(reply: string): AvatarAction[] { 18 - const lower = reply.toLowerCase(); 19 - return [ 20 - { type: 'talk:start' }, 21 - { type: 'emotion', value: lower.includes('!') ? 'excited' : lower.includes('?') ? 'curious' : 'happy' }, 22 - { type: 'gesture', value: lower.includes('?') ? 'tilt' : 'nod' }, 23 - { type: 'talk:stop' } 24 - ]; 25 - } 8 + const runtime = new CharacterRuntime(new StubResponseProvider()); 26 9 27 10 export function handleMessage(message: ChatMessage) { 28 - const reply = `Hey ${message.author} — I heard: ${message.text}`; 29 - const actions = planAvatarActions(reply); 30 - return { reply, actions }; 11 + return runtime.handleMessage(message); 31 12 } 32 13 33 14 if (process.argv.includes('--help')) { ··· 35 16 process.exit(0); 36 17 } 37 18 19 + if (process.argv.includes('--live')) { 20 + await runLiveLoop(); 21 + process.stdin.resume(); 22 + process.exit(0); 23 + } 24 + 38 25 console.log('character runtime scaffold ready'); 39 - console.log('next: wire stream.place chat, TTS, avatar renderer, and stream uplink'); 26 + console.log('next: wire Gemini Live, stream.place chat, avatar renderer, and stream uplink');
+59
src/live-loop.ts
··· 1 + import { GeminiLiveProvider } from './gemini-live'; 2 + import { OBSControlClient } from './obs-control'; 3 + import { OpenVTControlClient } from './openvt-control'; 4 + import { CharacterRuntime, StubResponseProvider, type ResponseProvider } from './runtime'; 5 + import { StreamplaceChatAdapter } from './streamplace-chat'; 6 + 7 + function buildProvider(): ResponseProvider { 8 + if (process.env.GEMINI_API_KEY) { 9 + return new GeminiLiveProvider(); 10 + } 11 + return new StubResponseProvider(); 12 + } 13 + 14 + export async function runLiveLoop() { 15 + const runtime = new CharacterRuntime(buildProvider()); 16 + const adapter = new StreamplaceChatAdapter(); 17 + const openvt = new OpenVTControlClient(); 18 + const obs = new OBSControlClient(); 19 + 20 + console.log(`[live-loop] subscribing ${adapter.createUrl()}`); 21 + console.log(`[live-loop] provider=${process.env.GEMINI_API_KEY ? 'gemini-live' : 'stub'}`); 22 + console.log(`[live-loop] collections=${(process.env.STREAMPLACE_CHAT_NSID ?? 'app.bsky.feed.post')}`); 23 + 24 + const ws = adapter.subscribe(async ({ message }) => { 25 + try { 26 + await openvt.sendTalkState('start'); 27 + await openvt.sendEmotion('curious'); 28 + } catch (error) { 29 + console.warn('[live-loop] openvt pre-response control send failed', error); 30 + } 31 + 32 + const result = await runtime.handleMessage(message); 33 + console.log(JSON.stringify({ input: message, output: result }, null, 2)); 34 + 35 + try { 36 + await openvt.sendText(result.reply); 37 + await obs.setText(result.reply); 38 + await obs.setProgramScene(); 39 + await openvt.sendResponse({ 40 + ...result, 41 + actions: result.actions.filter((action) => action.type !== 'talk:start'), 42 + }); 43 + } catch (error) { 44 + console.warn('[live-loop] openvt control send failed', error); 45 + } 46 + }); 47 + 48 + ws.onopen = () => { 49 + console.log('[live-loop] websocket open'); 50 + }; 51 + 52 + ws.onerror = (event) => { 53 + console.error('[live-loop] websocket error', event); 54 + }; 55 + 56 + ws.onclose = (event) => { 57 + console.log(`[live-loop] websocket close code=${event.code} reason=${event.reason}`); 58 + }; 59 + }
+75
src/obs-control.ts
··· 1 + import OBSWebSocket from 'obs-websocket-js'; 2 + 3 + export type OBSControlOptions = { 4 + url?: string; 5 + password?: string; 6 + sceneName?: string; 7 + textInputName?: string; 8 + }; 9 + 10 + export class OBSControlClient { 11 + private readonly obs = new OBSWebSocket(); 12 + private readonly url: string; 13 + private readonly password?: string; 14 + private readonly sceneName?: string; 15 + private readonly textInputName?: string; 16 + private connected = false; 17 + 18 + constructor(options: OBSControlOptions = {}) { 19 + this.url = options.url ?? process.env.OBS_WS_URL ?? 'ws://127.0.0.1:4455'; 20 + this.password = options.password ?? process.env.OBS_WS_PASSWORD; 21 + this.sceneName = options.sceneName ?? process.env.OBS_SCENE_NAME; 22 + this.textInputName = options.textInputName ?? process.env.OBS_TEXT_INPUT_NAME; 23 + } 24 + 25 + async connect() { 26 + if (this.connected) return; 27 + await this.obs.connect(this.url, this.password, { rpcVersion: 1 }); 28 + this.connected = true; 29 + } 30 + 31 + async setProgramScene(sceneName = this.sceneName) { 32 + if (!sceneName) return; 33 + await this.connect(); 34 + await this.obs.call('SetCurrentProgramScene', { sceneName }); 35 + } 36 + 37 + async ensureTextInput(inputName = this.textInputName, sceneName = this.sceneName) { 38 + if (!inputName || !sceneName) return; 39 + await this.connect(); 40 + 41 + try { 42 + await this.obs.call('GetInputSettings', { inputName }); 43 + return; 44 + } catch { 45 + await this.obs.call('CreateInput', { 46 + sceneName, 47 + inputName, 48 + inputKind: 'text_ft2_source', 49 + inputSettings: { 50 + text: '', 51 + }, 52 + sceneItemEnabled: true, 53 + }); 54 + } 55 + } 56 + 57 + async setText(text: string, inputName = this.textInputName) { 58 + if (!inputName) return; 59 + await this.connect(); 60 + await this.ensureTextInput(inputName, this.sceneName); 61 + await this.obs.call('SetInputSettings', { 62 + inputName, 63 + inputSettings: { 64 + text, 65 + }, 66 + overlay: true, 67 + }); 68 + } 69 + 70 + async disconnect() { 71 + if (!this.connected) return; 72 + await this.obs.disconnect(); 73 + this.connected = false; 74 + } 75 + }
+63
src/openvt-control.ts
··· 1 + import type { AgentResponse } from './runtime'; 2 + 3 + export type OpenVTControlOptions = { 4 + host?: string; 5 + port?: number; 6 + }; 7 + 8 + export class OpenVTControlClient { 9 + private readonly host: string; 10 + private readonly port: number; 11 + 12 + constructor(options: OpenVTControlOptions = {}) { 13 + this.host = options.host ?? '127.0.0.1'; 14 + this.port = options.port ?? 25733; 15 + } 16 + 17 + async sendResponse(response: AgentResponse) { 18 + const messages: Array<Record<string, unknown>> = []; 19 + 20 + for (const action of response.actions) { 21 + if (action.type === 'talk:start') messages.push({ type: 'talk', state: 'start' }); 22 + if (action.type === 'talk:stop') messages.push({ type: 'talk', state: 'stop' }); 23 + if (action.type === 'emotion') messages.push({ type: 'emotion', value: action.value }); 24 + if (action.type === 'gesture') messages.push({ type: 'gesture', value: action.value }); 25 + } 26 + 27 + messages.push({ type: 'text', value: response.reply }); 28 + 29 + for (const message of messages) { 30 + await this.sendMessage(message); 31 + } 32 + } 33 + 34 + async sendTalkState(state: 'start' | 'stop') { 35 + await this.sendMessage({ type: 'talk', state }); 36 + } 37 + 38 + async sendEmotion(value: 'neutral' | 'happy' | 'excited' | 'curious') { 39 + await this.sendMessage({ type: 'emotion', value }); 40 + } 41 + 42 + async sendText(value: string) { 43 + await this.sendMessage({ type: 'text', value }); 44 + } 45 + 46 + async sendMessage(message: Record<string, unknown>) { 47 + const socket = await Bun.connect({ 48 + hostname: this.host, 49 + port: this.port, 50 + socket: { 51 + open(socket) { 52 + socket.write(JSON.stringify(message)); 53 + socket.end(); 54 + }, 55 + data() {}, 56 + close() {}, 57 + error() {}, 58 + }, 59 + }); 60 + 61 + socket.unref?.(); 62 + } 63 + }
+58
src/runtime.ts
··· 1 + export type ChatMessage = { 2 + author: string; 3 + text: string; 4 + timestamp: string; 5 + }; 6 + 7 + export type AvatarAction = 8 + | { type: 'talk:start' } 9 + | { type: 'talk:stop' } 10 + | { type: 'emotion'; value: 'neutral' | 'happy' | 'excited' | 'curious' } 11 + | { type: 'gesture'; value: 'nod' | 'wave' | 'tilt' }; 12 + 13 + export type AgentResponse = { 14 + reply: string; 15 + actions: AvatarAction[]; 16 + audio?: { 17 + provider: 'gemini-live' | 'stub'; 18 + text: string; 19 + }; 20 + }; 21 + 22 + export interface ResponseProvider { 23 + generateReply(message: ChatMessage): Promise<string>; 24 + } 25 + 26 + function planAvatarActions(reply: string): AvatarAction[] { 27 + const lower = reply.toLowerCase(); 28 + return [ 29 + { type: 'talk:start' }, 30 + { type: 'emotion', value: lower.includes('!') ? 'excited' : lower.includes('?') ? 'curious' : 'happy' }, 31 + { type: 'gesture', value: lower.includes('?') ? 'tilt' : 'nod' }, 32 + { type: 'talk:stop' } 33 + ]; 34 + } 35 + 36 + export class StubResponseProvider implements ResponseProvider { 37 + async generateReply(message: ChatMessage): Promise<string> { 38 + return `Hey ${message.author} — I heard: ${message.text}`; 39 + } 40 + } 41 + 42 + export class CharacterRuntime { 43 + constructor(private readonly provider: ResponseProvider) {} 44 + 45 + async handleMessage(message: ChatMessage): Promise<AgentResponse> { 46 + const reply = await this.provider.generateReply(message); 47 + const actions = planAvatarActions(reply); 48 + return { 49 + reply, 50 + actions, 51 + audio: { 52 + provider: 'stub', 53 + text: reply 54 + } 55 + }; 56 + } 57 + } 58 +
+91
src/streamplace-chat.ts
··· 1 + import type { ChatMessage } from './runtime'; 2 + 3 + export type StreamplaceChatOptions = { 4 + endpoint?: string; 5 + wantedCollections?: string[]; 6 + wantedDids?: string[]; 7 + WebSocketImpl?: typeof WebSocket; 8 + }; 9 + 10 + export type StreamplaceChatEvent = { 11 + source: 'jetstream'; 12 + raw: unknown; 13 + message: ChatMessage; 14 + }; 15 + 16 + type CommitEnvelope = { 17 + kind?: string; 18 + did?: string; 19 + time_us?: number; 20 + commit?: { 21 + operation?: string; 22 + collection?: string; 23 + record?: { 24 + text?: string; 25 + message?: string; 26 + createdAt?: string; 27 + }; 28 + }; 29 + }; 30 + 31 + const DEFAULT_ENDPOINT = 'wss://jetstream2.us-east.bsky.network/subscribe'; 32 + 33 + export class StreamplaceChatAdapter { 34 + private readonly endpoint: string; 35 + private readonly wantedCollections: string[]; 36 + private readonly wantedDids: string[]; 37 + private readonly WebSocketImpl: typeof WebSocket; 38 + 39 + constructor(options: StreamplaceChatOptions = {}) { 40 + this.endpoint = options.endpoint ?? DEFAULT_ENDPOINT; 41 + this.wantedCollections = options.wantedCollections ?? [process.env.STREAMPLACE_CHAT_NSID ?? 'app.bsky.feed.post']; 42 + this.wantedDids = options.wantedDids ?? []; 43 + this.WebSocketImpl = options.WebSocketImpl ?? WebSocket; 44 + } 45 + 46 + createUrl() { 47 + const url = new URL(this.endpoint); 48 + for (const collection of this.wantedCollections) { 49 + url.searchParams.append('wantedCollections', collection); 50 + } 51 + for (const did of this.wantedDids) { 52 + url.searchParams.append('wantedDids', did); 53 + } 54 + return url.toString(); 55 + } 56 + 57 + extractChatEvent(payload: string): StreamplaceChatEvent | undefined { 58 + const data = JSON.parse(payload) as CommitEnvelope; 59 + if (data.kind !== 'commit') return undefined; 60 + if (data.commit?.operation !== 'create') return undefined; 61 + const collection = data.commit?.collection; 62 + if (!collection || !this.wantedCollections.includes(collection)) return undefined; 63 + 64 + const text = (data.commit.record?.text ?? data.commit.record?.message)?.trim(); 65 + if (!text) return undefined; 66 + 67 + return { 68 + source: 'jetstream', 69 + raw: data, 70 + message: { 71 + author: data.did ?? 'unknown', 72 + text, 73 + timestamp: 74 + data.commit.record?.createdAt ?? 75 + (typeof data.time_us === 'number' ? new Date(Math.floor(data.time_us / 1000)).toISOString() : new Date().toISOString()), 76 + }, 77 + }; 78 + } 79 + 80 + subscribe(onEvent: (event: StreamplaceChatEvent) => void) { 81 + const ws = new this.WebSocketImpl(this.createUrl()); 82 + 83 + ws.onmessage = (event) => { 84 + const text = typeof event.data === 'string' ? event.data : String(event.data); 85 + const parsed = this.extractChatEvent(text); 86 + if (parsed) onEvent(parsed); 87 + }; 88 + 89 + return ws; 90 + } 91 + }