nandi.uk
This repository has no description
Add Gemini live voice loop and tune avatar pose.
Wire Gemini Live audio replies into the avatar player with a feminine voice setting, and keep refining the procedural arm/body pose for a more natural stance.
👾 Generated with [Letta Code](https://letta.com)
Co-Authored-By: Letta Code <noreply@letta.com>
author
nandi
co-author
Letta Code
date
(May 27, 2026, 10:49 AM -0700)
commit
67fd98a1
67fd98a13bd59606c00b38f4a91f694f641d2907
parent
169d34b6
169d34b67dabf2345d6d09cc1e41617de72ced94
+264
-15
+11
README.md
+11
README.md
···
4
4
5
5
The project now includes a default sample avatar at `public/default-avatar.vrm`, so you can run it immediately without finding your own model first.
6
6
7
+
It also includes a Gemini Live text-to-speech loop: enter a prompt, send it to Gemini, receive native audio, and feed that audio back into the avatar player for lip sync.
8
+
7
9
## Run
8
10
9
11
1. Install dependencies:
···
11
13
2. Start dev server:
12
14
bun run dev
13
15
3. Open the local Vite URL.
16
+
17
+
## Gemini setup
18
+
19
+
Set your Gemini API key before starting Vite:
20
+
21
+
```bash
22
+
export VITE_GEMINI_API_KEY=your_key_here
23
+
bun run dev
24
+
```
14
25
15
26
## Usage
16
27
+85
bun.lock
+85
bun.lock
···
5
5
"": {
6
6
"name": "typeface-vrm-lipsync",
7
7
"dependencies": {
8
+
"@google/genai": "^1.21.0",
8
9
"@pixiv/three-vrm": "^3.4.2",
9
10
"three": "^0.179.1",
10
11
},
···
70
71
71
72
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.27.7", "", { "os": "win32", "cpu": "x64" }, "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg=="],
72
73
74
+
"@google/genai": ["@google/genai@1.52.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q=="],
75
+
73
76
"@pixiv/three-vrm": ["@pixiv/three-vrm@3.5.3", "", { "dependencies": { "@pixiv/three-vrm-core": "3.5.3", "@pixiv/three-vrm-materials-hdr-emissive-multiplier": "3.5.3", "@pixiv/three-vrm-materials-mtoon": "3.5.3", "@pixiv/three-vrm-materials-v0compat": "3.5.3", "@pixiv/three-vrm-node-constraint": "3.5.3", "@pixiv/three-vrm-springbone": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-BHvkyedh0RzvSEiwcOf+oOWPAwMGW0xLWQ2CSU+rX/j5E6aII9ool/j97QD67NPlCwBD6lrMRa9RrzEg3xgORA=="],
74
77
75
78
"@pixiv/three-vrm-core": ["@pixiv/three-vrm-core@3.5.3", "", { "dependencies": { "@pixiv/types-vrm-0.0": "3.5.3", "@pixiv/types-vrmc-vrm-1.0": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-iZDOkOxOnvT8vGPE2SgNRee3fQw/u11y4MULE9VPXN/LKSf946aVi+YfVnnMBRdwMfQ2uBNHxUvkIQlRIkXnHQ=="],
···
98
101
99
102
"@pixiv/types-vrmc-vrm-1.0": ["@pixiv/types-vrmc-vrm-1.0@3.5.3", "", {}, "sha512-3HsBZWKXU/CnY2J6uAVuYmM0NPt/e5jroQZH85WYK0EzYmebqTmWKF/fllguNliJQAovdhn8sIA8NbHZoQvbUA=="],
100
103
104
+
"@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="],
105
+
106
+
"@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="],
107
+
108
+
"@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="],
109
+
110
+
"@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="],
111
+
112
+
"@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="],
113
+
114
+
"@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="],
115
+
116
+
"@protobufjs/inquire": ["@protobufjs/inquire@1.1.2", "", {}, "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw=="],
117
+
118
+
"@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="],
119
+
120
+
"@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="],
121
+
122
+
"@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="],
123
+
101
124
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.60.4", "", { "os": "android", "cpu": "arm" }, "sha512-F5QXMSiFebS9hKZj02XhWLLnRpJ3B3AROP0tWbFBSj+6kCbg5m9j5JoHKd4mmSVy5mS/IMQloYgYxCuJC0fxEQ=="],
102
125
103
126
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.60.4", "", { "os": "android", "cpu": "arm64" }, "sha512-GxxTKApUpzRhof7poWvCJHRF51C67u1R7D6DiluBE8wKU1u5GWE8t+v81JvJYtbawoBFX1hLv5Ei4eVjkWokaw=="],
···
152
175
153
176
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
154
177
178
+
"@types/node": ["@types/node@25.9.1", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg=="],
179
+
180
+
"@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="],
181
+
155
182
"@types/stats.js": ["@types/stats.js@0.17.4", "", {}, "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA=="],
156
183
157
184
"@types/three": ["@types/three@0.179.0", "", { "dependencies": { "@dimforge/rapier3d-compat": "~0.12.0", "@tweenjs/tween.js": "~23.1.3", "@types/stats.js": "*", "@types/webxr": "*", "@webgpu/types": "*", "fflate": "~0.8.2", "meshoptimizer": "~0.22.0" } }, "sha512-VgbFG2Pgsm84BqdegZzr7w2aKbQxmgzIu4Dy7/75ygiD/0P68LKmp5ie08KMPNqGTQwIge8s6D1guZf1RnZE0A=="],
···
160
187
161
188
"@webgpu/types": ["@webgpu/types@0.1.70", "", {}, "sha512-LFiNHHKMvmAEvwVew3JLJmTdShhbdwRFSImUshGhE2mGE8ybQzIo63l5uRp+YKnNx+8Qno8Kf6gN+DKMreIJCA=="],
162
189
190
+
"agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
191
+
192
+
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
193
+
194
+
"bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="],
195
+
196
+
"buffer-equal-constant-time": ["buffer-equal-constant-time@1.0.1", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="],
197
+
198
+
"data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="],
199
+
200
+
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
201
+
202
+
"ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="],
203
+
163
204
"esbuild": ["esbuild@0.27.7", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.7", "@esbuild/android-arm": "0.27.7", "@esbuild/android-arm64": "0.27.7", "@esbuild/android-x64": "0.27.7", "@esbuild/darwin-arm64": "0.27.7", "@esbuild/darwin-x64": "0.27.7", "@esbuild/freebsd-arm64": "0.27.7", "@esbuild/freebsd-x64": "0.27.7", "@esbuild/linux-arm": "0.27.7", "@esbuild/linux-arm64": "0.27.7", "@esbuild/linux-ia32": "0.27.7", "@esbuild/linux-loong64": "0.27.7", "@esbuild/linux-mips64el": "0.27.7", "@esbuild/linux-ppc64": "0.27.7", "@esbuild/linux-riscv64": "0.27.7", "@esbuild/linux-s390x": "0.27.7", "@esbuild/linux-x64": "0.27.7", "@esbuild/netbsd-arm64": "0.27.7", "@esbuild/netbsd-x64": "0.27.7", "@esbuild/openbsd-arm64": "0.27.7", "@esbuild/openbsd-x64": "0.27.7", "@esbuild/openharmony-arm64": "0.27.7", "@esbuild/sunos-x64": "0.27.7", "@esbuild/win32-arm64": "0.27.7", "@esbuild/win32-ia32": "0.27.7", "@esbuild/win32-x64": "0.27.7" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w=="],
205
+
206
+
"extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="],
164
207
165
208
"fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
166
209
210
+
"fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="],
211
+
167
212
"fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="],
168
213
214
+
"formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="],
215
+
169
216
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
170
217
218
+
"gaxios": ["gaxios@7.1.4", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2" } }, "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA=="],
219
+
220
+
"gcp-metadata": ["gcp-metadata@8.1.2", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="],
221
+
222
+
"google-auth-library": ["google-auth-library@10.6.2", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.1.4", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw=="],
223
+
224
+
"google-logging-utils": ["google-logging-utils@1.1.3", "", {}, "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA=="],
225
+
226
+
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
227
+
228
+
"json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="],
229
+
230
+
"jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="],
231
+
232
+
"jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="],
233
+
234
+
"long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="],
235
+
171
236
"meshoptimizer": ["meshoptimizer@0.22.0", "", {}, "sha512-IebiK79sqIy+E4EgOr+CAw+Ke8hAspXKzBd0JdgEmPHiAwmvEj2S4h1rfvo+o/BnfEYd/jAOg5IeeIjzlzSnDg=="],
237
+
238
+
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
172
239
173
240
"nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="],
174
241
242
+
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
243
+
244
+
"node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="],
245
+
246
+
"p-retry": ["p-retry@4.6.2", "", { "dependencies": { "@types/retry": "0.12.0", "retry": "^0.13.1" } }, "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ=="],
247
+
175
248
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
176
249
177
250
"picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],
178
251
179
252
"postcss": ["postcss@8.5.15", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A=="],
180
253
254
+
"protobufjs": ["protobufjs@7.6.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-4K0myLaWL5EteuSAro91EGFgcfVgxb64Jx+7oDAY6GOkXD4M69yuSEljNcInGVCA5sOPxmZ/EqDLj2x0Q0+Ygg=="],
255
+
256
+
"retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="],
257
+
181
258
"rollup": ["rollup@4.60.4", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.60.4", "@rollup/rollup-android-arm64": "4.60.4", "@rollup/rollup-darwin-arm64": "4.60.4", "@rollup/rollup-darwin-x64": "4.60.4", "@rollup/rollup-freebsd-arm64": "4.60.4", "@rollup/rollup-freebsd-x64": "4.60.4", "@rollup/rollup-linux-arm-gnueabihf": "4.60.4", "@rollup/rollup-linux-arm-musleabihf": "4.60.4", "@rollup/rollup-linux-arm64-gnu": "4.60.4", "@rollup/rollup-linux-arm64-musl": "4.60.4", "@rollup/rollup-linux-loong64-gnu": "4.60.4", "@rollup/rollup-linux-loong64-musl": "4.60.4", "@rollup/rollup-linux-ppc64-gnu": "4.60.4", "@rollup/rollup-linux-ppc64-musl": "4.60.4", "@rollup/rollup-linux-riscv64-gnu": "4.60.4", "@rollup/rollup-linux-riscv64-musl": "4.60.4", "@rollup/rollup-linux-s390x-gnu": "4.60.4", "@rollup/rollup-linux-x64-gnu": "4.60.4", "@rollup/rollup-linux-x64-musl": "4.60.4", "@rollup/rollup-openbsd-x64": "4.60.4", "@rollup/rollup-openharmony-arm64": "4.60.4", "@rollup/rollup-win32-arm64-msvc": "4.60.4", "@rollup/rollup-win32-ia32-msvc": "4.60.4", "@rollup/rollup-win32-x64-gnu": "4.60.4", "@rollup/rollup-win32-x64-msvc": "4.60.4", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g=="],
259
+
260
+
"safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
182
261
183
262
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
184
263
···
188
267
189
268
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
190
269
270
+
"undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="],
271
+
191
272
"vite": ["vite@7.3.3", "", { "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-/4XH147Ui7OGTjg3HbdWe5arnZQSbfuRzdr9Ec7TQi5I7R+ir0Rlc9GIvD4v0XZurELqA035KVXJXpR61xhiTA=="],
273
+
274
+
"web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="],
275
+
276
+
"ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="],
192
277
}
193
278
}
+1
package.json
+1
package.json
+165
-15
src/main.ts
+165
-15
src/main.ts
···
1
1
import './style.css';
2
2
3
3
import * as THREE from 'three';
4
+
import { GoogleGenAI, Modality, type LiveServerMessage } from '@google/genai';
4
5
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
5
6
import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js';
6
7
import { VRM, VRMExpressionPresetName, VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
···
17
18
<span class="badge">TypeScript + VRM</span>
18
19
<h1>Avatar lip sync MVP</h1>
19
20
<p class="copy">Load a VRM avatar, upload an audio file, and drive mouth movement from the audio amplitude. This is amplitude-based lip sync, so it animates speaking energy rather than phoneme-accurate visemes.</p>
21
+
<p class="copy" style="margin-top: 8px;">Gemini voice is set to <code>Kore</code>, which should read as a feminine voice.</p>
20
22
21
23
<div class="stack">
22
24
<section class="card">
···
37
39
<button id="stopButton" disabled>Stop</button>
38
40
</div>
39
41
<audio id="audio" preload="auto"></audio>
42
+
</section>
43
+
44
+
<section class="card">
45
+
<h2>3. Gemini reply</h2>
46
+
<p class="copy" style="margin: 0 0 12px; font-size: 0.95rem;">Send a text prompt to Gemini Live, receive native audio, and pipe it into the avatar audio player.</p>
47
+
<textarea id="promptInput" rows="4" style="width: 100%; resize: vertical; border-radius: 10px; border: 1px solid rgba(255,255,255,0.12); background: rgba(255,255,255,0.04); color: inherit; padding: 10px;" placeholder="Say something as the avatar..."></textarea>
48
+
<div style="margin-top: 12px; display: flex; gap: 10px; flex-wrap: wrap;">
49
+
<button id="sendPromptButton">Send to Gemini</button>
50
+
</div>
51
+
<p id="geminiStatus" class="footer" style="margin-bottom: 0;">Gemini idle</p>
40
52
</section>
41
53
42
54
<section class="card stats">
43
-
<h2>3. Live values</h2>
55
+
<h2>4. Live values</h2>
44
56
<div class="stat-row"><span>Avatar</span><strong id="avatarStatus">Not loaded</strong></div>
45
57
<div class="stat-row"><span>Audio</span><strong id="audioStatus">Not loaded</strong></div>
46
58
<div class="stat-row"><span>Mouth open</span><strong id="mouthValue">0.00</strong></div>
···
57
69
const vrmInput = requireElement(app.querySelector<HTMLInputElement>('#vrmFile'), 'Missing VRM input');
58
70
const audioInput = requireElement(app.querySelector<HTMLInputElement>('#audioFile'), 'Missing audio input');
59
71
const useSampleAudioButton = requireElement(app.querySelector<HTMLButtonElement>('#useSampleAudioButton'), 'Missing sample audio button');
72
+
const promptInput = requireElement(app.querySelector<HTMLTextAreaElement>('#promptInput'), 'Missing prompt input');
73
+
const sendPromptButton = requireElement(app.querySelector<HTMLButtonElement>('#sendPromptButton'), 'Missing Gemini send button');
74
+
const geminiStatus = requireElement(app.querySelector<HTMLElement>('#geminiStatus'), 'Missing Gemini status');
60
75
const playButton = requireElement(app.querySelector<HTMLButtonElement>('#playButton'), 'Missing play button');
61
76
const stopButton = requireElement(app.querySelector<HTMLButtonElement>('#stopButton'), 'Missing stop button');
62
77
const audioEl = requireElement(app.querySelector<HTMLAudioElement>('#audio'), 'Missing audio element');
···
110
125
let audioObjectUrl: string | null = null;
111
126
let mouthOpenAmount = 0;
112
127
let audioEnergy = 0;
128
+
let geminiResponseParts: Uint8Array[] = [];
113
129
114
130
type PoseBones = {
115
131
neck?: THREE.Object3D | null;
···
126
142
127
143
let poseBones: PoseBones = {};
128
144
145
+
const geminiApiKey = import.meta.env.VITE_GEMINI_API_KEY as string | undefined;
146
+
const geminiModel = 'gemini-3.1-flash-live-preview';
147
+
129
148
function disposeCurrentVrm() {
130
149
if (!currentVrm) return;
131
150
scene.remove(currentVrm.scene);
···
150
169
};
151
170
}
152
171
172
+
function pcm16ToWav(pcmData: Uint8Array, sampleRate = 24000, channels = 1): Blob {
173
+
const bytesPerSample = 2;
174
+
const blockAlign = channels * bytesPerSample;
175
+
const buffer = new ArrayBuffer(44 + pcmData.byteLength);
176
+
const view = new DataView(buffer);
177
+
178
+
const writeString = (offset: number, value: string) => {
179
+
for (let i = 0; i < value.length; i += 1) view.setUint8(offset + i, value.charCodeAt(i));
180
+
};
181
+
182
+
writeString(0, 'RIFF');
183
+
view.setUint32(4, 36 + pcmData.byteLength, true);
184
+
writeString(8, 'WAVE');
185
+
writeString(12, 'fmt ');
186
+
view.setUint32(16, 16, true);
187
+
view.setUint16(20, 1, true);
188
+
view.setUint16(22, channels, true);
189
+
view.setUint32(24, sampleRate, true);
190
+
view.setUint32(28, sampleRate * blockAlign, true);
191
+
view.setUint16(32, blockAlign, true);
192
+
view.setUint16(34, 16, true);
193
+
writeString(36, 'data');
194
+
view.setUint32(40, pcmData.byteLength, true);
195
+
new Uint8Array(buffer, 44).set(pcmData);
196
+
return new Blob([buffer], { type: 'audio/wav' });
197
+
}
198
+
199
+
function concatUint8Arrays(chunks: Uint8Array[]): Uint8Array {
200
+
const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
201
+
const result = new Uint8Array(total);
202
+
let offset = 0;
203
+
for (const chunk of chunks) {
204
+
result.set(chunk, offset);
205
+
offset += chunk.byteLength;
206
+
}
207
+
return result;
208
+
}
209
+
210
+
function base64ToUint8Array(base64: string): Uint8Array {
211
+
const binary = atob(base64);
212
+
const bytes = new Uint8Array(binary.length);
213
+
for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i);
214
+
return bytes;
215
+
}
216
+
217
+
async function playGeneratedSpeech(pcmChunks: Uint8Array[]) {
218
+
if (audioObjectUrl) URL.revokeObjectURL(audioObjectUrl);
219
+
const wavBlob = pcm16ToWav(concatUint8Arrays(pcmChunks));
220
+
audioObjectUrl = URL.createObjectURL(wavBlob);
221
+
audioEl.src = audioObjectUrl;
222
+
audioEl.load();
223
+
audioStatus.textContent = 'Gemini response';
224
+
playButton.disabled = false;
225
+
stopButton.disabled = false;
226
+
ensureAudioGraph();
227
+
if (audioContext?.state === 'suspended') await audioContext.resume();
228
+
await audioEl.play();
229
+
}
230
+
231
+
async function sendPromptToGemini(prompt: string) {
232
+
if (!geminiApiKey) throw new Error('Missing VITE_GEMINI_API_KEY');
233
+
234
+
geminiStatus.textContent = 'Connecting to Gemini…';
235
+
sendPromptButton.disabled = true;
236
+
geminiResponseParts = [];
237
+
238
+
const ai = new GoogleGenAI({ apiKey: geminiApiKey });
239
+
const session = await ai.live.connect({
240
+
model: geminiModel,
241
+
config: {
242
+
responseModalities: [Modality.AUDIO],
243
+
speechConfig: {
244
+
voiceConfig: {
245
+
prebuiltVoiceConfig: {
246
+
voiceName: 'Kore',
247
+
},
248
+
},
249
+
},
250
+
systemInstruction: 'You are a concise speaking avatar. Reply conversationally in under 20 seconds.',
251
+
},
252
+
callbacks: {
253
+
onopen: () => {
254
+
geminiStatus.textContent = 'Gemini connected';
255
+
},
256
+
onmessage: async (message: LiveServerMessage) => {
257
+
const serverContent = message.serverContent;
258
+
const parts = serverContent?.modelTurn?.parts ?? [];
259
+
for (const part of parts) {
260
+
if (part.inlineData?.data) {
261
+
geminiResponseParts.push(base64ToUint8Array(part.inlineData.data));
262
+
}
263
+
}
264
+
265
+
if (serverContent?.turnComplete) {
266
+
geminiStatus.textContent = 'Playing Gemini audio';
267
+
if (geminiResponseParts.length > 0) await playGeneratedSpeech(geminiResponseParts);
268
+
await session.close();
269
+
sendPromptButton.disabled = false;
270
+
geminiStatus.textContent = 'Gemini idle';
271
+
}
272
+
},
273
+
onerror: (error: ErrorEvent) => {
274
+
console.error(error);
275
+
geminiStatus.textContent = 'Gemini error';
276
+
sendPromptButton.disabled = false;
277
+
},
278
+
onclose: () => {
279
+
sendPromptButton.disabled = false;
280
+
},
281
+
},
282
+
});
283
+
284
+
session.sendClientContent({
285
+
turns: [{ role: 'user', parts: [{ text: prompt }] }],
286
+
turnComplete: true,
287
+
});
288
+
}
289
+
153
290
function setMouth(open: number) {
154
291
const clamped = THREE.MathUtils.clamp(open, 0, 1);
155
292
mouthOpenAmount = clamped;
···
254
391
}
255
392
256
393
if (leftUpperArm) {
257
-
leftUpperArm.rotation.z = -0.55 + Math.sin(elapsed * 1.7) * 0.04 + audioEnergy * 0.08;
258
-
leftUpperArm.rotation.x = 0.18 + Math.sin(elapsed * 2.3) * 0.03;
259
-
leftUpperArm.rotation.y = -0.12 - talkSway * 0.4;
394
+
leftUpperArm.rotation.z = 1.12 + Math.sin(elapsed * 1.7) * 0.012 + audioEnergy * 0.02;
395
+
leftUpperArm.rotation.x = 0.03 + Math.sin(elapsed * 2.3) * 0.01;
396
+
leftUpperArm.rotation.y = -0.04 - talkSway * 0.1;
260
397
}
261
398
262
399
if (rightUpperArm) {
263
-
rightUpperArm.rotation.z = 0.55 - Math.sin(elapsed * 1.9) * 0.04 - audioEnergy * 0.08;
264
-
rightUpperArm.rotation.x = 0.18 + Math.sin(elapsed * 2.5) * 0.03;
265
-
rightUpperArm.rotation.y = 0.12 - talkSway * 0.4;
400
+
rightUpperArm.rotation.z = -1.12 - Math.sin(elapsed * 1.9) * 0.012 - audioEnergy * 0.02;
401
+
rightUpperArm.rotation.x = 0.03 + Math.sin(elapsed * 2.5) * 0.01;
402
+
rightUpperArm.rotation.y = 0.04 - talkSway * 0.1;
266
403
}
267
404
268
405
if (leftLowerArm) {
269
-
leftLowerArm.rotation.z = -0.35 - audioEnergy * 0.08;
270
-
leftLowerArm.rotation.x = -0.25 + Math.sin(elapsed * 2.8) * 0.025;
406
+
leftLowerArm.rotation.z = 0;
407
+
leftLowerArm.rotation.x = -0.14 + Math.sin(elapsed * 2.8) * 0.008;
271
408
}
272
409
273
410
if (rightLowerArm) {
274
-
rightLowerArm.rotation.z = 0.35 + audioEnergy * 0.08;
275
-
rightLowerArm.rotation.x = -0.25 + Math.sin(elapsed * 3.0) * 0.025;
411
+
rightLowerArm.rotation.z = 0;
412
+
rightLowerArm.rotation.x = -0.14 + Math.sin(elapsed * 3.0) * 0.008;
276
413
}
277
414
278
415
if (leftHand) {
279
-
leftHand.rotation.y = -0.12 + Math.sin(elapsed * 3.5) * 0.03;
280
-
leftHand.rotation.z = -0.08;
416
+
leftHand.rotation.y = -0.08 + Math.sin(elapsed * 3.5) * 0.02;
417
+
leftHand.rotation.z = -0.03;
281
418
}
282
419
283
420
if (rightHand) {
284
-
rightHand.rotation.y = 0.12 - Math.sin(elapsed * 3.3) * 0.03;
285
-
rightHand.rotation.z = 0.08;
421
+
rightHand.rotation.y = 0.08 - Math.sin(elapsed * 3.3) * 0.02;
422
+
rightHand.rotation.z = 0.03;
286
423
}
287
424
}
288
425
···
380
517
} catch (error) {
381
518
console.error(error);
382
519
audioStatus.textContent = 'Sample load failed';
520
+
}
521
+
});
522
+
523
+
sendPromptButton.addEventListener('click', async () => {
524
+
const prompt = promptInput.value.trim();
525
+
if (!prompt) return;
526
+
527
+
try {
528
+
await sendPromptToGemini(prompt);
529
+
} catch (error) {
530
+
console.error(error);
531
+
geminiStatus.textContent = error instanceof Error ? error.message : 'Gemini request failed';
532
+
sendPromptButton.disabled = false;
383
533
}
384
534
});
385
535
+1
src/vite-env.d.ts
+1
src/vite-env.d.ts
···
1
+
/// <reference types="vite/client" />