nandi.uk
This repository has no description
Initial VRM lip sync avatar demo.
Add a TypeScript web app that loads a VRM avatar, plays speech audio, and drives expressive lip sync and body motion for testing.
👾 Generated with [Letta Code](https://letta.com)
Co-Authored-By: Letta Code <noreply@letta.com>
author
nandi
co-author
Letta Code
date
(May 27, 2026, 10:31 AM -0700)
commit
169d34b6
169d34b67dabf2345d6d09cc1e41617de72ced94
+862
+28
README.md
+28
README.md
···
1
+
# VRM Lip Sync MVP
2
+
3
+
Minimal TypeScript web app that loads a VRM avatar and drives its mouth from uploaded audio using the Web Audio API.
4
+
5
+
The project now includes a default sample avatar at `public/default-avatar.vrm`, so you can run it immediately without finding your own model first.
6
+
7
+
## Run
8
+
9
+
1. Install dependencies:
10
+
bun install
11
+
2. Start dev server:
12
+
bun run dev
13
+
3. Open the local Vite URL.
14
+
15
+
## Usage
16
+
17
+
1. Load a `.vrm` file.
18
+
2. Load an audio file.
19
+
3. Press Play.
20
+
21
+
The avatar uses amplitude-based mouth opening, not phoneme-level viseme detection.
22
+
23
+
## Next upgrades
24
+
25
+
- Add phoneme/viseme inference instead of pure amplitude
26
+
- Add microphone input mode
27
+
- Add drag-and-drop and avatar presets
28
+
- Add eye blink / idle animation
+193
bun.lock
+193
bun.lock
···
1
+
{
2
+
"lockfileVersion": 1,
3
+
"configVersion": 1,
4
+
"workspaces": {
5
+
"": {
6
+
"name": "typeface-vrm-lipsync",
7
+
"dependencies": {
8
+
"@pixiv/three-vrm": "^3.4.2",
9
+
"three": "^0.179.1",
10
+
},
11
+
"devDependencies": {
12
+
"@types/three": "^0.179.0",
13
+
"typescript": "^5.9.2",
14
+
"vite": "^7.1.3",
15
+
},
16
+
},
17
+
},
18
+
"packages": {
19
+
"@dimforge/rapier3d-compat": ["@dimforge/rapier3d-compat@0.12.0", "", {}, "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow=="],
20
+
21
+
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.27.7", "", { "os": "aix", "cpu": "ppc64" }, "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg=="],
22
+
23
+
"@esbuild/android-arm": ["@esbuild/android-arm@0.27.7", "", { "os": "android", "cpu": "arm" }, "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ=="],
24
+
25
+
"@esbuild/android-arm64": ["@esbuild/android-arm64@0.27.7", "", { "os": "android", "cpu": "arm64" }, "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ=="],
26
+
27
+
"@esbuild/android-x64": ["@esbuild/android-x64@0.27.7", "", { "os": "android", "cpu": "x64" }, "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg=="],
28
+
29
+
"@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.27.7", "", { "os": "darwin", "cpu": "arm64" }, "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw=="],
30
+
31
+
"@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.27.7", "", { "os": "darwin", "cpu": "x64" }, "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ=="],
32
+
33
+
"@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.27.7", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w=="],
34
+
35
+
"@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.27.7", "", { "os": "freebsd", "cpu": "x64" }, "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ=="],
36
+
37
+
"@esbuild/linux-arm": ["@esbuild/linux-arm@0.27.7", "", { "os": "linux", "cpu": "arm" }, "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA=="],
38
+
39
+
"@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.27.7", "", { "os": "linux", "cpu": "arm64" }, "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A=="],
40
+
41
+
"@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.27.7", "", { "os": "linux", "cpu": "ia32" }, "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg=="],
42
+
43
+
"@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.27.7", "", { "os": "linux", "cpu": "none" }, "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q=="],
44
+
45
+
"@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.27.7", "", { "os": "linux", "cpu": "none" }, "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw=="],
46
+
47
+
"@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.27.7", "", { "os": "linux", "cpu": "ppc64" }, "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ=="],
48
+
49
+
"@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.27.7", "", { "os": "linux", "cpu": "none" }, "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ=="],
50
+
51
+
"@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.27.7", "", { "os": "linux", "cpu": "s390x" }, "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw=="],
52
+
53
+
"@esbuild/linux-x64": ["@esbuild/linux-x64@0.27.7", "", { "os": "linux", "cpu": "x64" }, "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA=="],
54
+
55
+
"@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.27.7", "", { "os": "none", "cpu": "arm64" }, "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w=="],
56
+
57
+
"@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.27.7", "", { "os": "none", "cpu": "x64" }, "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw=="],
58
+
59
+
"@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.27.7", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A=="],
60
+
61
+
"@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.27.7", "", { "os": "openbsd", "cpu": "x64" }, "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg=="],
62
+
63
+
"@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.27.7", "", { "os": "none", "cpu": "arm64" }, "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw=="],
64
+
65
+
"@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.27.7", "", { "os": "sunos", "cpu": "x64" }, "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA=="],
66
+
67
+
"@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.27.7", "", { "os": "win32", "cpu": "arm64" }, "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA=="],
68
+
69
+
"@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.27.7", "", { "os": "win32", "cpu": "ia32" }, "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw=="],
70
+
71
+
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.27.7", "", { "os": "win32", "cpu": "x64" }, "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg=="],
72
+
73
+
"@pixiv/three-vrm": ["@pixiv/three-vrm@3.5.3", "", { "dependencies": { "@pixiv/three-vrm-core": "3.5.3", "@pixiv/three-vrm-materials-hdr-emissive-multiplier": "3.5.3", "@pixiv/three-vrm-materials-mtoon": "3.5.3", "@pixiv/three-vrm-materials-v0compat": "3.5.3", "@pixiv/three-vrm-node-constraint": "3.5.3", "@pixiv/three-vrm-springbone": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-BHvkyedh0RzvSEiwcOf+oOWPAwMGW0xLWQ2CSU+rX/j5E6aII9ool/j97QD67NPlCwBD6lrMRa9RrzEg3xgORA=="],
74
+
75
+
"@pixiv/three-vrm-core": ["@pixiv/three-vrm-core@3.5.3", "", { "dependencies": { "@pixiv/types-vrm-0.0": "3.5.3", "@pixiv/types-vrmc-vrm-1.0": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-iZDOkOxOnvT8vGPE2SgNRee3fQw/u11y4MULE9VPXN/LKSf946aVi+YfVnnMBRdwMfQ2uBNHxUvkIQlRIkXnHQ=="],
76
+
77
+
"@pixiv/three-vrm-materials-hdr-emissive-multiplier": ["@pixiv/three-vrm-materials-hdr-emissive-multiplier@3.5.3", "", { "dependencies": { "@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-CGPVBzjmrG4yoKkHl7w03/ZJj+RfzQ9LjzI623MYIN9O9YkYd5627ls8ZpkbwN+sgboz7eIx9BkoNfWtLmEVSw=="],
78
+
79
+
"@pixiv/three-vrm-materials-mtoon": ["@pixiv/three-vrm-materials-mtoon@3.5.3", "", { "dependencies": { "@pixiv/types-vrm-0.0": "3.5.3", "@pixiv/types-vrmc-materials-mtoon-1.0": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-lZk2q8mOEzBBy70PP6dMtgaMXn0yXzI4IPja9qU4o0c9EjF0SzkNRXAVBNKqLkqDvInyLi+tYBxhcYpbA5DwcA=="],
80
+
81
+
"@pixiv/three-vrm-materials-v0compat": ["@pixiv/three-vrm-materials-v0compat@3.5.3", "", { "dependencies": { "@pixiv/types-vrm-0.0": "3.5.3", "@pixiv/types-vrmc-materials-mtoon-1.0": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-wU2a38zQasKXaBzm9z0uyp2Ee5+xIJ1FHlVkEcBsZrJ9ri6V0vHkaE5yw5mKaUI6fIHd4GcHlTuWw0Y3pSGh8A=="],
82
+
83
+
"@pixiv/three-vrm-node-constraint": ["@pixiv/three-vrm-node-constraint@3.5.3", "", { "dependencies": { "@pixiv/types-vrmc-node-constraint-1.0": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-rYrM6bNu/IheDtk4xQs+Rn7QgyBp9LmzGyhV48iR+SYXRatztagf/lvxLiB8v70ubVf+uNDEWb5HeVxDMhwjkA=="],
84
+
85
+
"@pixiv/three-vrm-springbone": ["@pixiv/three-vrm-springbone@3.5.3", "", { "dependencies": { "@pixiv/types-vrm-0.0": "3.5.3", "@pixiv/types-vrmc-springbone-1.0": "3.5.3", "@pixiv/types-vrmc-springbone-extended-collider-1.0": "3.5.3" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-/C1PDrJaJt7Hd6qGfVEH00BHCKd/b8ElFj6VBphjdT02Qx8KoVCQH93PMkRcoD3PjqMvvr0FkH5N0cittfkgSA=="],
86
+
87
+
"@pixiv/types-vrm-0.0": ["@pixiv/types-vrm-0.0@3.5.3", "", {}, "sha512-dkXmTqIxKC3A1AMs4Ts3DzlKN7SYab/d5QX9SvAwOUAe4wuafwzIYFzm+q85wNjlgSF4xuNuYIbqQfBcE7Mz0w=="],
88
+
89
+
"@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0": ["@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0@3.5.3", "", {}, "sha512-G+g/Sw+/geG6xu6MYbb0mZFNTgczWXvLCs9CliXxhSKsXYE4SW9AbSlYqg9bPFA0fnNQN8vVQstnhO7FiL0ipg=="],
90
+
91
+
"@pixiv/types-vrmc-materials-mtoon-1.0": ["@pixiv/types-vrmc-materials-mtoon-1.0@3.5.3", "", {}, "sha512-8T6K2j5U7mEs4o5azXFUfXuiddS2gBxPj2Xe9ZVZnguMVsOcNdeiISHXF5CTyfT8UHKbCTfpsn4YgDGpqdXSmw=="],
92
+
93
+
"@pixiv/types-vrmc-node-constraint-1.0": ["@pixiv/types-vrmc-node-constraint-1.0@3.5.3", "", {}, "sha512-72RNVTbP0c4R57SWDT43xhwZE3r6vhUDxD7M34nlX+o2Wis/Kx+R+bnFCf4/proRt+52Xasp3J6l9KiMqCtZ2g=="],
94
+
95
+
"@pixiv/types-vrmc-springbone-1.0": ["@pixiv/types-vrmc-springbone-1.0@3.5.3", "", {}, "sha512-3+Zw++ga16UxHiCTtEbd5HTbtXV+kyQsqvURm8IpVD4XGoSz1e2QJJP+vrbNBzRb7nUKYj4bVOFWGnULSx8sZQ=="],
96
+
97
+
"@pixiv/types-vrmc-springbone-extended-collider-1.0": ["@pixiv/types-vrmc-springbone-extended-collider-1.0@3.5.3", "", {}, "sha512-DaKEfwWsV4xyu0wdWfJ/bPvYpAammj49n0em8JTy52TmOhUm5oTN0ZZ5cmgdL9PVqN9qmWUkOdnK1ck1Id2H3A=="],
98
+
99
+
"@pixiv/types-vrmc-vrm-1.0": ["@pixiv/types-vrmc-vrm-1.0@3.5.3", "", {}, "sha512-3HsBZWKXU/CnY2J6uAVuYmM0NPt/e5jroQZH85WYK0EzYmebqTmWKF/fllguNliJQAovdhn8sIA8NbHZoQvbUA=="],
100
+
101
+
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.60.4", "", { "os": "android", "cpu": "arm" }, "sha512-F5QXMSiFebS9hKZj02XhWLLnRpJ3B3AROP0tWbFBSj+6kCbg5m9j5JoHKd4mmSVy5mS/IMQloYgYxCuJC0fxEQ=="],
102
+
103
+
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.60.4", "", { "os": "android", "cpu": "arm64" }, "sha512-GxxTKApUpzRhof7poWvCJHRF51C67u1R7D6DiluBE8wKU1u5GWE8t+v81JvJYtbawoBFX1hLv5Ei4eVjkWokaw=="],
104
+
105
+
"@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.60.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-tua0TaJxMOB1R0V0RS1jFZ/RpURFDJIOR2A6jWwQeawuFyS4gBW+rntLRaQd0EQ4bd6Vp44Z2rXW+YYDBsj6IA=="],
106
+
107
+
"@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.60.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-CSKq7MsP+5PFIcydhAiR1K0UhEI1A2jWXVKHPCBZ151yOutENwvnPocgVHkivu2kviURtCEB6zUQw0vs8RrhMg=="],
108
+
109
+
"@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.60.4", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-+O8OkVdyvXMtJEciu2wS/pzm1IxntEEQx3z5TAVy4l32G0etZn+RsA48ARRrFm6Ri8fvqPQfgrvNxSjKAbnd3g=="],
110
+
111
+
"@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.60.4", "", { "os": "freebsd", "cpu": "x64" }, "sha512-Iw3oMskH3AfNuhU0MSN7vNbdi4me/NiYo2azqPz/Le16zHSa+3RRmliCMWWQmh4lcndccU40xcJuTYJZxNo/lw=="],
112
+
113
+
"@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.60.4", "", { "os": "linux", "cpu": "arm" }, "sha512-EIPRXTVQpHyF8WOo219AD2yEltPehLTcTMz2fn6JsatLYSzQf00hj3rulF+yauOlF9/FtM2WpkT/hJh/KJFGhA=="],
114
+
115
+
"@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.60.4", "", { "os": "linux", "cpu": "arm" }, "sha512-J3Yh9PzzF1Ovah2At+lHiGQdsYgArxBbXv/zHfSyaiFQEqvNv7DcW98pCrmdjCZBrqBiKrKKe2V+aaSGWuBe/w=="],
116
+
117
+
"@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.60.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-BFDEZMYfUvLn37ONE1yMBojPxnMlTFsdyNoqncT0qFq1mAfllL+ATMMJd8TeuVMiX84s1KbcxcZbXInmcO2mRg=="],
118
+
119
+
"@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.60.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-pc9EYOSlOgdQ2uPl1o9PF6/kLSgaUosia7gOuS8mB69IxJvlclko1MECXysjs5ryez1/5zjYqx3+xYU0TU6R1A=="],
120
+
121
+
"@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.60.4", "", { "os": "linux", "cpu": "none" }, "sha512-NxnomyxYerDh5n4iLrNa+sH+Z+U4BMEE46V2PgQ/hoB909i8gV1M5wPojWg9fk1jWpO3IQnOs20K4wyZuFLEFQ=="],
122
+
123
+
"@rollup/rollup-linux-loong64-musl": ["@rollup/rollup-linux-loong64-musl@4.60.4", "", { "os": "linux", "cpu": "none" }, "sha512-nbJnQ8a3z1mtmrwImCYhc6BGpThAyYVRQxw9uKSKG4wR6aAYno9sVjJ0zaZcW9BPJX1GbrDPf+SvdWjgTuDmnw=="],
124
+
125
+
"@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.60.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-2EU6acNrQLd8tYvo/LXW535wupT3m6fo7HKo6lr7ktQoItxTyOL1ZCR/GfGCuXl2vR+zmfI6eRXkSemafv+iVg=="],
126
+
127
+
"@rollup/rollup-linux-ppc64-musl": ["@rollup/rollup-linux-ppc64-musl@4.60.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WeBtoMuaMxiiIrO2IYP3xs6GMWkJP2C0EoT8beTLkUPmzV1i/UcOSVw1d5r9KBODtHKilG5yFxsGRnBbK3wJ4A=="],
128
+
129
+
"@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.60.4", "", { "os": "linux", "cpu": "none" }, "sha512-FJHFfqpKUI3A10WrWKiFbBZ7yVbGT4q4B5o1qKFFojqpaYoh9LrQgqWCmmcxQzVSXYtyB5bzkXrYzlHTs21MYA=="],
130
+
131
+
"@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.60.4", "", { "os": "linux", "cpu": "none" }, "sha512-mcEl6CUT5IAUmQf1m9FYSmVqCJlpQ8r8eyftFUHG8i9OhY7BkBXSUdnLH5DOf0wCOjcP9v/QO93zpmF1SptCCw=="],
132
+
133
+
"@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.60.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-ynt3JxVd2w2buzoKDWIyiV1pJW93xlQic1THVLXilz429oijRpSHivZAgp65KBu+cMcgf1eVVjdnTLvPxgCuoQ=="],
134
+
135
+
"@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.60.4", "", { "os": "linux", "cpu": "x64" }, "sha512-Boiz5+MsaROEWDf+GGEwF8VMHGhlUoQMtIPjOgA5fv4osupqTVnJteQNKJwUcnUog2G55jYXH7KZFFiJe0TEzQ=="],
136
+
137
+
"@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.60.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+qfSY27qIrFfI/Hom04KYFw3GKZSGU4lXus51wsb5EuySfFlWRwjkKWoE9emgRw/ukoT4Udsj4W/+xxG8VbPKg=="],
138
+
139
+
"@rollup/rollup-openbsd-x64": ["@rollup/rollup-openbsd-x64@4.60.4", "", { "os": "openbsd", "cpu": "x64" }, "sha512-VpTfOPHgVXEBeeR8hZ2O0F3aSso+JDWqTWmTmzcQKted54IAdUVbxE+j/MVxUsKa8L20HJhv3vUezVPoquqWjA=="],
140
+
141
+
"@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.60.4", "", { "os": "none", "cpu": "arm64" }, "sha512-IPOsh5aRYuLv/nkU51X10Bf75Bsf6+gZdx1X+QP5QM6lIJFHHqbHLG0uJn/hWthzo13UAc2umiUorqZy3axoZg=="],
142
+
143
+
"@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.60.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-4QzE9E81OohJ/HKzHhsqU+zcYYojVOXlFMs1DdyMT6qXl/niOH7AVElmmEdUNHHS/oRkc++d5k6Vy85zFs0DEw=="],
144
+
145
+
"@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.60.4", "", { "os": "win32", "cpu": "ia32" }, "sha512-zTPgT1YuHHcd+Tmx7h8aml0FWFVelV5N54oHow9SLj+GfoDy/huQ+UV396N/C7KpMDMiPspRktzM1/0r1usYEA=="],
146
+
147
+
"@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.60.4", "", { "os": "win32", "cpu": "x64" }, "sha512-DRS4G7mi9lJxqEDezIkKCaUIKCrLUUDCUaCsTPCi/rtqaC6D/jjwslMQyiDU50Ka0JKpeXeRBFBAXwArY52vBw=="],
148
+
149
+
"@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.60.4", "", { "os": "win32", "cpu": "x64" }, "sha512-QVTUovf40zgTqlFVrKA1uXMVvU2QWEFWfAH8Wdc48IxLvrJMQVMBRjuQyUpzZCDkakImib9eVazbWlC6ksWtJw=="],
150
+
151
+
"@tweenjs/tween.js": ["@tweenjs/tween.js@23.1.3", "", {}, "sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA=="],
152
+
153
+
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
154
+
155
+
"@types/stats.js": ["@types/stats.js@0.17.4", "", {}, "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA=="],
156
+
157
+
"@types/three": ["@types/three@0.179.0", "", { "dependencies": { "@dimforge/rapier3d-compat": "~0.12.0", "@tweenjs/tween.js": "~23.1.3", "@types/stats.js": "*", "@types/webxr": "*", "@webgpu/types": "*", "fflate": "~0.8.2", "meshoptimizer": "~0.22.0" } }, "sha512-VgbFG2Pgsm84BqdegZzr7w2aKbQxmgzIu4Dy7/75ygiD/0P68LKmp5ie08KMPNqGTQwIge8s6D1guZf1RnZE0A=="],
158
+
159
+
"@types/webxr": ["@types/webxr@0.5.24", "", {}, "sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg=="],
160
+
161
+
"@webgpu/types": ["@webgpu/types@0.1.70", "", {}, "sha512-LFiNHHKMvmAEvwVew3JLJmTdShhbdwRFSImUshGhE2mGE8ybQzIo63l5uRp+YKnNx+8Qno8Kf6gN+DKMreIJCA=="],
162
+
163
+
"esbuild": ["esbuild@0.27.7", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.7", "@esbuild/android-arm": "0.27.7", "@esbuild/android-arm64": "0.27.7", "@esbuild/android-x64": "0.27.7", "@esbuild/darwin-arm64": "0.27.7", "@esbuild/darwin-x64": "0.27.7", "@esbuild/freebsd-arm64": "0.27.7", "@esbuild/freebsd-x64": "0.27.7", "@esbuild/linux-arm": "0.27.7", "@esbuild/linux-arm64": "0.27.7", "@esbuild/linux-ia32": "0.27.7", "@esbuild/linux-loong64": "0.27.7", "@esbuild/linux-mips64el": "0.27.7", "@esbuild/linux-ppc64": "0.27.7", "@esbuild/linux-riscv64": "0.27.7", "@esbuild/linux-s390x": "0.27.7", "@esbuild/linux-x64": "0.27.7", "@esbuild/netbsd-arm64": "0.27.7", "@esbuild/netbsd-x64": "0.27.7", "@esbuild/openbsd-arm64": "0.27.7", "@esbuild/openbsd-x64": "0.27.7", "@esbuild/openharmony-arm64": "0.27.7", "@esbuild/sunos-x64": "0.27.7", "@esbuild/win32-arm64": "0.27.7", "@esbuild/win32-ia32": "0.27.7", "@esbuild/win32-x64": "0.27.7" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w=="],
164
+
165
+
"fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
166
+
167
+
"fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="],
168
+
169
+
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
170
+
171
+
"meshoptimizer": ["meshoptimizer@0.22.0", "", {}, "sha512-IebiK79sqIy+E4EgOr+CAw+Ke8hAspXKzBd0JdgEmPHiAwmvEj2S4h1rfvo+o/BnfEYd/jAOg5IeeIjzlzSnDg=="],
172
+
173
+
"nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="],
174
+
175
+
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
176
+
177
+
"picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],
178
+
179
+
"postcss": ["postcss@8.5.15", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A=="],
180
+
181
+
"rollup": ["rollup@4.60.4", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.60.4", "@rollup/rollup-android-arm64": "4.60.4", "@rollup/rollup-darwin-arm64": "4.60.4", "@rollup/rollup-darwin-x64": "4.60.4", "@rollup/rollup-freebsd-arm64": "4.60.4", "@rollup/rollup-freebsd-x64": "4.60.4", "@rollup/rollup-linux-arm-gnueabihf": "4.60.4", "@rollup/rollup-linux-arm-musleabihf": "4.60.4", "@rollup/rollup-linux-arm64-gnu": "4.60.4", "@rollup/rollup-linux-arm64-musl": "4.60.4", "@rollup/rollup-linux-loong64-gnu": "4.60.4", "@rollup/rollup-linux-loong64-musl": "4.60.4", "@rollup/rollup-linux-ppc64-gnu": "4.60.4", "@rollup/rollup-linux-ppc64-musl": "4.60.4", "@rollup/rollup-linux-riscv64-gnu": "4.60.4", "@rollup/rollup-linux-riscv64-musl": "4.60.4", "@rollup/rollup-linux-s390x-gnu": "4.60.4", "@rollup/rollup-linux-x64-gnu": "4.60.4", "@rollup/rollup-linux-x64-musl": "4.60.4", "@rollup/rollup-openbsd-x64": "4.60.4", "@rollup/rollup-openharmony-arm64": "4.60.4", "@rollup/rollup-win32-arm64-msvc": "4.60.4", "@rollup/rollup-win32-ia32-msvc": "4.60.4", "@rollup/rollup-win32-x64-gnu": "4.60.4", "@rollup/rollup-win32-x64-msvc": "4.60.4", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g=="],
182
+
183
+
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
184
+
185
+
"three": ["three@0.179.1", "", {}, "sha512-5y/elSIQbrvKOISxpwXCR4sQqHtGiOI+MKLc3SsBdDXA2hz3Mdp3X59aUp8DyybMa34aeBwbFTpdoLJaUDEWSw=="],
186
+
187
+
"tinyglobby": ["tinyglobby@0.2.16", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="],
188
+
189
+
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
190
+
191
+
"vite": ["vite@7.3.3", "", { "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-/4XH147Ui7OGTjg3HbdWe5arnZQSbfuRzdr9Ec7TQi5I7R+ir0Rlc9GIvD4v0XZurELqA035KVXJXpR61xhiTA=="],
192
+
}
193
+
}
+12
index.html
+12
index.html
···
1
+
<!doctype html>
2
+
<html lang="en">
3
+
<head>
4
+
<meta charset="UTF-8" />
5
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+
<title>VRM Lip Sync</title>
7
+
<script type="module" src="/src/main.ts"></script>
8
+
</head>
9
+
<body>
10
+
<div id="app"></div>
11
+
</body>
12
+
</html>
+20
package.json
+20
package.json
···
1
+
{
2
+
"name": "typeface-vrm-lipsync",
3
+
"private": true,
4
+
"version": "0.0.0",
5
+
"type": "module",
6
+
"scripts": {
7
+
"dev": "vite",
8
+
"build": "tsc && vite build",
9
+
"preview": "vite preview"
10
+
},
11
+
"devDependencies": {
12
+
"@types/three": "^0.179.0",
13
+
"typescript": "^5.9.2",
14
+
"vite": "^7.1.3"
15
+
},
16
+
"dependencies": {
17
+
"@pixiv/three-vrm": "^3.4.2",
18
+
"three": "^0.179.1"
19
+
}
20
+
}
public/default-avatar.vrm
public/default-avatar.vrm
This is a binary file and will not be displayed.
public/sample-speech.ogg
public/sample-speech.ogg
This is a binary file and will not be displayed.
+433
src/main.ts
+433
src/main.ts
···
1
+
import './style.css';
2
+
3
+
import * as THREE from 'three';
4
+
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
5
+
import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js';
6
+
import { VRM, VRMExpressionPresetName, VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
7
+
8
+
function requireElement<T>(value: T | null, message: string): T {
9
+
if (!value) throw new Error(message);
10
+
return value;
11
+
}
12
+
13
+
const app = requireElement(document.querySelector<HTMLDivElement>('#app'), 'App root not found');
14
+
15
+
app.innerHTML = `
16
+
<aside class="panel">
17
+
<span class="badge">TypeScript + VRM</span>
18
+
<h1>Avatar lip sync MVP</h1>
19
+
<p class="copy">Load a VRM avatar, upload an audio file, and drive mouth movement from the audio amplitude. This is amplitude-based lip sync, so it animates speaking energy rather than phoneme-accurate visemes.</p>
20
+
21
+
<div class="stack">
22
+
<section class="card">
23
+
<h2>1. Avatar</h2>
24
+
<p class="copy" style="margin: 0 0 12px; font-size: 0.95rem;">A sample avatar is loaded by default. Upload a different VRM to replace it.</p>
25
+
<input id="vrmFile" type="file" accept=".vrm,model/gltf-binary" />
26
+
</section>
27
+
28
+
<section class="card">
29
+
<h2>2. Audio</h2>
30
+
<p class="copy" style="margin: 0 0 12px; font-size: 0.95rem;">A public-domain speech sample is bundled by default. Use the button below or upload your own file.</p>
31
+
<div style="margin-bottom: 12px;">
32
+
<button id="useSampleAudioButton">Use sample speech</button>
33
+
</div>
34
+
<input id="audioFile" type="file" accept="audio/*" />
35
+
<div style="margin-top: 12px; display: flex; gap: 10px; flex-wrap: wrap;">
36
+
<button id="playButton" disabled>Play</button>
37
+
<button id="stopButton" disabled>Stop</button>
38
+
</div>
39
+
<audio id="audio" preload="auto"></audio>
40
+
</section>
41
+
42
+
<section class="card stats">
43
+
<h2>3. Live values</h2>
44
+
<div class="stat-row"><span>Avatar</span><strong id="avatarStatus">Not loaded</strong></div>
45
+
<div class="stat-row"><span>Audio</span><strong id="audioStatus">Not loaded</strong></div>
46
+
<div class="stat-row"><span>Mouth open</span><strong id="mouthValue">0.00</strong></div>
47
+
<div class="meter"><div id="mouthMeter"></div></div>
48
+
</section>
49
+
</div>
50
+
51
+
<p class="footer">If your VRM supports expression presets, this app drives <code>aa</code>. Otherwise it falls back to blendshape names like <code>A</code>, <code>aa</code>, or <code>mouthOpen</code>.</p>
52
+
</aside>
53
+
<main class="viewport"></main>
54
+
`;
55
+
56
+
const viewport = requireElement(app.querySelector<HTMLDivElement>('.viewport'), 'Missing viewport');
57
+
const vrmInput = requireElement(app.querySelector<HTMLInputElement>('#vrmFile'), 'Missing VRM input');
58
+
const audioInput = requireElement(app.querySelector<HTMLInputElement>('#audioFile'), 'Missing audio input');
59
+
const useSampleAudioButton = requireElement(app.querySelector<HTMLButtonElement>('#useSampleAudioButton'), 'Missing sample audio button');
60
+
const playButton = requireElement(app.querySelector<HTMLButtonElement>('#playButton'), 'Missing play button');
61
+
const stopButton = requireElement(app.querySelector<HTMLButtonElement>('#stopButton'), 'Missing stop button');
62
+
const audioEl = requireElement(app.querySelector<HTMLAudioElement>('#audio'), 'Missing audio element');
63
+
const avatarStatus = requireElement(app.querySelector<HTMLElement>('#avatarStatus'), 'Missing avatar status');
64
+
const audioStatus = requireElement(app.querySelector<HTMLElement>('#audioStatus'), 'Missing audio status');
65
+
const mouthValue = requireElement(app.querySelector<HTMLElement>('#mouthValue'), 'Missing mouth value');
66
+
const mouthMeter = requireElement(app.querySelector<HTMLElement>('#mouthMeter'), 'Missing mouth meter');
67
+
68
+
const scene = new THREE.Scene();
69
+
scene.background = new THREE.Color(0x0f172a);
70
+
71
+
const camera = new THREE.PerspectiveCamera(35, viewport.clientWidth / viewport.clientHeight, 0.1, 100);
72
+
camera.position.set(0, 1.4, 2.2);
73
+
74
+
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
75
+
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
76
+
renderer.setSize(viewport.clientWidth, viewport.clientHeight);
77
+
renderer.outputColorSpace = THREE.SRGBColorSpace;
78
+
viewport.appendChild(renderer.domElement);
79
+
80
+
const controls = new OrbitControls(camera, renderer.domElement);
81
+
controls.target.set(0, 1.2, 0);
82
+
controls.enableDamping = true;
83
+
controls.minDistance = 1;
84
+
controls.maxDistance = 5;
85
+
controls.update();
86
+
87
+
scene.add(new THREE.HemisphereLight(0xffffff, 0x1e293b, 1.4));
88
+
const dirLight = new THREE.DirectionalLight(0xffffff, 1.4);
89
+
dirLight.position.set(1, 1, 1);
90
+
scene.add(dirLight);
91
+
92
+
const floor = new THREE.Mesh(
93
+
new THREE.CircleGeometry(3, 64),
94
+
new THREE.MeshStandardMaterial({ color: 0x111827, roughness: 0.9, metalness: 0.05 })
95
+
);
96
+
floor.rotation.x = -Math.PI / 2;
97
+
floor.position.y = -0.01;
98
+
scene.add(floor);
99
+
100
+
const clock = new THREE.Clock();
101
+
const loader = new GLTFLoader();
102
+
loader.register((parser) => new VRMLoaderPlugin(parser));
103
+
104
+
let currentVrm: VRM | null = null;
105
+
let audioContext: AudioContext | null = null;
106
+
let analyser: AnalyserNode | null = null;
107
+
let sourceNode: MediaElementAudioSourceNode | null = null;
108
+
let dataArray: Uint8Array<ArrayBuffer> | null = null;
109
+
let vrmObjectUrl: string | null = null;
110
+
let audioObjectUrl: string | null = null;
111
+
let mouthOpenAmount = 0;
112
+
let audioEnergy = 0;
113
+
114
+
type PoseBones = {
115
+
neck?: THREE.Object3D | null;
116
+
spine?: THREE.Object3D | null;
117
+
chest?: THREE.Object3D | null;
118
+
upperChest?: THREE.Object3D | null;
119
+
leftUpperArm?: THREE.Object3D | null;
120
+
rightUpperArm?: THREE.Object3D | null;
121
+
leftLowerArm?: THREE.Object3D | null;
122
+
rightLowerArm?: THREE.Object3D | null;
123
+
leftHand?: THREE.Object3D | null;
124
+
rightHand?: THREE.Object3D | null;
125
+
};
126
+
127
+
let poseBones: PoseBones = {};
128
+
129
+
function disposeCurrentVrm() {
130
+
if (!currentVrm) return;
131
+
scene.remove(currentVrm.scene);
132
+
VRMUtils.deepDispose(currentVrm.scene);
133
+
currentVrm = null;
134
+
poseBones = {};
135
+
}
136
+
137
+
function cachePoseBones(vrm: VRM) {
138
+
const humanoid = vrm.humanoid;
139
+
poseBones = {
140
+
neck: humanoid?.getNormalizedBoneNode('neck'),
141
+
spine: humanoid?.getNormalizedBoneNode('spine'),
142
+
chest: humanoid?.getNormalizedBoneNode('chest'),
143
+
upperChest: humanoid?.getNormalizedBoneNode('upperChest'),
144
+
leftUpperArm: humanoid?.getNormalizedBoneNode('leftUpperArm'),
145
+
rightUpperArm: humanoid?.getNormalizedBoneNode('rightUpperArm'),
146
+
leftLowerArm: humanoid?.getNormalizedBoneNode('leftLowerArm'),
147
+
rightLowerArm: humanoid?.getNormalizedBoneNode('rightLowerArm'),
148
+
leftHand: humanoid?.getNormalizedBoneNode('leftHand'),
149
+
rightHand: humanoid?.getNormalizedBoneNode('rightHand'),
150
+
};
151
+
}
152
+
153
+
function setMouth(open: number) {
154
+
const clamped = THREE.MathUtils.clamp(open, 0, 1);
155
+
mouthOpenAmount = clamped;
156
+
mouthValue.textContent = clamped.toFixed(2);
157
+
mouthMeter.style.width = `${(clamped * 100).toFixed(1)}%`;
158
+
159
+
if (!currentVrm) return;
160
+
161
+
const manager = currentVrm.expressionManager;
162
+
if (!manager) return;
163
+
164
+
manager.setValue(VRMExpressionPresetName.Aa, clamped);
165
+
for (const name of ['A', 'a', 'aa', 'Ah', 'mouthOpen']) {
166
+
try {
167
+
manager.setValue(name as VRMExpressionPresetName, clamped);
168
+
} catch {
169
+
// ignore unsupported expression names
170
+
}
171
+
}
172
+
}
173
+
174
+
function ensureAudioGraph() {
175
+
if (audioContext && analyser && sourceNode && dataArray) return;
176
+
177
+
audioContext = new AudioContext();
178
+
analyser = audioContext.createAnalyser();
179
+
analyser.fftSize = 2048;
180
+
analyser.smoothingTimeConstant = 0.82;
181
+
dataArray = new Uint8Array(new ArrayBuffer(analyser.frequencyBinCount));
182
+
sourceNode = audioContext.createMediaElementSource(audioEl);
183
+
sourceNode.connect(analyser);
184
+
analyser.connect(audioContext.destination);
185
+
}
186
+
187
+
function computeMouthOpen() {
188
+
if (!analyser || !dataArray || audioEl.paused) {
189
+
audioEnergy = THREE.MathUtils.lerp(audioEnergy, 0, 0.18);
190
+
setMouth(0);
191
+
return;
192
+
}
193
+
194
+
analyser.getByteTimeDomainData(dataArray);
195
+
let sum = 0;
196
+
for (let i = 0; i < dataArray.length; i += 1) {
197
+
const centered = (dataArray[i] - 128) / 128;
198
+
sum += centered * centered;
199
+
}
200
+
201
+
const rms = Math.sqrt(sum / dataArray.length);
202
+
const boosted = THREE.MathUtils.clamp((rms - 0.012) * 10.0, 0, 1);
203
+
const exaggerated = THREE.MathUtils.clamp(Math.pow(boosted, 0.7) * 1.25, 0, 1);
204
+
audioEnergy = THREE.MathUtils.lerp(audioEnergy, exaggerated, 0.35);
205
+
const expressiveMouth = THREE.MathUtils.clamp(exaggerated * 1.35 + 0.02, 0, 1);
206
+
setMouth(expressiveMouth);
207
+
}
208
+
209
+
function animatePresence(elapsed: number) {
210
+
if (!currentVrm) return;
211
+
212
+
const idleSway = Math.sin(elapsed * 1.4) * 0.025;
213
+
const idleBob = Math.sin(elapsed * 2.1) * 0.012;
214
+
const talkSway = Math.sin(elapsed * 9.0) * audioEnergy * 0.035;
215
+
const talkNod = Math.sin(elapsed * 7.2) * audioEnergy * 0.03;
216
+
const talkLift = audioEnergy * 0.045;
217
+
218
+
currentVrm.scene.rotation.y = Math.PI + idleSway + talkSway;
219
+
currentVrm.scene.rotation.x = talkNod * 0.35;
220
+
currentVrm.scene.position.y = idleBob + talkLift;
221
+
222
+
const {
223
+
neck,
224
+
spine,
225
+
chest,
226
+
upperChest,
227
+
leftUpperArm,
228
+
rightUpperArm,
229
+
leftLowerArm,
230
+
rightLowerArm,
231
+
leftHand,
232
+
rightHand,
233
+
} = poseBones;
234
+
235
+
if (neck) {
236
+
neck.rotation.y = talkSway * 0.9;
237
+
neck.rotation.x = talkNod * 0.8;
238
+
neck.rotation.z = Math.sin(elapsed * 5.4) * audioEnergy * 0.04;
239
+
}
240
+
241
+
if (spine) {
242
+
spine.rotation.y = idleSway * 0.45 + talkSway * 0.4;
243
+
spine.rotation.x = talkNod * 0.2;
244
+
}
245
+
246
+
if (chest) {
247
+
chest.rotation.x = mouthOpenAmount * 0.05 + audioEnergy * 0.04;
248
+
chest.rotation.z = Math.sin(elapsed * 3.8) * 0.015 + talkSway * 0.3;
249
+
}
250
+
251
+
if (upperChest) {
252
+
upperChest.rotation.y = talkSway * 0.35;
253
+
upperChest.rotation.x = audioEnergy * 0.03;
254
+
}
255
+
256
+
if (leftUpperArm) {
257
+
leftUpperArm.rotation.z = -0.55 + Math.sin(elapsed * 1.7) * 0.04 + audioEnergy * 0.08;
258
+
leftUpperArm.rotation.x = 0.18 + Math.sin(elapsed * 2.3) * 0.03;
259
+
leftUpperArm.rotation.y = -0.12 - talkSway * 0.4;
260
+
}
261
+
262
+
if (rightUpperArm) {
263
+
rightUpperArm.rotation.z = 0.55 - Math.sin(elapsed * 1.9) * 0.04 - audioEnergy * 0.08;
264
+
rightUpperArm.rotation.x = 0.18 + Math.sin(elapsed * 2.5) * 0.03;
265
+
rightUpperArm.rotation.y = 0.12 - talkSway * 0.4;
266
+
}
267
+
268
+
if (leftLowerArm) {
269
+
leftLowerArm.rotation.z = -0.35 - audioEnergy * 0.08;
270
+
leftLowerArm.rotation.x = -0.25 + Math.sin(elapsed * 2.8) * 0.025;
271
+
}
272
+
273
+
if (rightLowerArm) {
274
+
rightLowerArm.rotation.z = 0.35 + audioEnergy * 0.08;
275
+
rightLowerArm.rotation.x = -0.25 + Math.sin(elapsed * 3.0) * 0.025;
276
+
}
277
+
278
+
if (leftHand) {
279
+
leftHand.rotation.y = -0.12 + Math.sin(elapsed * 3.5) * 0.03;
280
+
leftHand.rotation.z = -0.08;
281
+
}
282
+
283
+
if (rightHand) {
284
+
rightHand.rotation.y = 0.12 - Math.sin(elapsed * 3.3) * 0.03;
285
+
rightHand.rotation.z = 0.08;
286
+
}
287
+
}
288
+
289
+
async function loadVrm(file: File) {
290
+
disposeCurrentVrm();
291
+
if (vrmObjectUrl) URL.revokeObjectURL(vrmObjectUrl);
292
+
293
+
vrmObjectUrl = URL.createObjectURL(file);
294
+
const gltf = await loader.loadAsync(vrmObjectUrl);
295
+
const vrm = gltf.userData.vrm as VRM | undefined;
296
+
if (!vrm) throw new Error('Loaded file is not a VRM avatar');
297
+
298
+
VRMUtils.rotateVRM0(vrm);
299
+
vrm.scene.rotation.y = Math.PI;
300
+
scene.add(vrm.scene);
301
+
currentVrm = vrm;
302
+
cachePoseBones(vrm);
303
+
avatarStatus.textContent = file.name;
304
+
}
305
+
306
+
async function loadDefaultVrm() {
307
+
avatarStatus.textContent = 'Loading sample…';
308
+
disposeCurrentVrm();
309
+
if (vrmObjectUrl) {
310
+
URL.revokeObjectURL(vrmObjectUrl);
311
+
vrmObjectUrl = null;
312
+
}
313
+
314
+
const gltf = await loader.loadAsync('/default-avatar.vrm');
315
+
const vrm = gltf.userData.vrm as VRM | undefined;
316
+
if (!vrm) throw new Error('Default avatar is not a valid VRM');
317
+
318
+
VRMUtils.rotateVRM0(vrm);
319
+
vrm.scene.rotation.y = Math.PI;
320
+
scene.add(vrm.scene);
321
+
currentVrm = vrm;
322
+
cachePoseBones(vrm);
323
+
avatarStatus.textContent = 'default-avatar.vrm';
324
+
}
325
+
326
+
async function loadAudio(file: File) {
327
+
if (audioObjectUrl) URL.revokeObjectURL(audioObjectUrl);
328
+
329
+
audioObjectUrl = URL.createObjectURL(file);
330
+
audioEl.src = audioObjectUrl;
331
+
audioEl.load();
332
+
audioStatus.textContent = file.name;
333
+
playButton.disabled = false;
334
+
stopButton.disabled = false;
335
+
setMouth(0);
336
+
}
337
+
338
+
async function loadSampleAudio() {
339
+
if (audioObjectUrl) {
340
+
URL.revokeObjectURL(audioObjectUrl);
341
+
audioObjectUrl = null;
342
+
}
343
+
344
+
audioEl.src = '/sample-speech.ogg';
345
+
audioEl.load();
346
+
audioStatus.textContent = 'sample-speech.ogg';
347
+
playButton.disabled = false;
348
+
stopButton.disabled = false;
349
+
setMouth(0);
350
+
}
351
+
352
+
vrmInput.addEventListener('change', async () => {
353
+
const file = vrmInput.files?.[0];
354
+
if (!file) return;
355
+
356
+
avatarStatus.textContent = 'Loading…';
357
+
try {
358
+
await loadVrm(file);
359
+
} catch (error) {
360
+
console.error(error);
361
+
avatarStatus.textContent = 'Load failed';
362
+
}
363
+
});
364
+
365
+
audioInput.addEventListener('change', async () => {
366
+
const file = audioInput.files?.[0];
367
+
if (!file) return;
368
+
369
+
try {
370
+
await loadAudio(file);
371
+
} catch (error) {
372
+
console.error(error);
373
+
audioStatus.textContent = 'Load failed';
374
+
}
375
+
});
376
+
377
+
useSampleAudioButton.addEventListener('click', async () => {
378
+
try {
379
+
await loadSampleAudio();
380
+
} catch (error) {
381
+
console.error(error);
382
+
audioStatus.textContent = 'Sample load failed';
383
+
}
384
+
});
385
+
386
+
playButton.addEventListener('click', async () => {
387
+
try {
388
+
ensureAudioGraph();
389
+
if (audioContext?.state === 'suspended') await audioContext.resume();
390
+
await audioEl.play();
391
+
} catch (error) {
392
+
console.error(error);
393
+
}
394
+
});
395
+
396
+
stopButton.addEventListener('click', () => {
397
+
audioEl.pause();
398
+
audioEl.currentTime = 0;
399
+
setMouth(0);
400
+
});
401
+
402
+
audioEl.addEventListener('ended', () => setMouth(0));
403
+
404
+
window.addEventListener('resize', () => {
405
+
const width = viewport.clientWidth;
406
+
const height = viewport.clientHeight;
407
+
camera.aspect = width / height;
408
+
camera.updateProjectionMatrix();
409
+
renderer.setSize(width, height);
410
+
});
411
+
412
+
function tick() {
413
+
const delta = clock.getDelta();
414
+
const elapsed = clock.elapsedTime;
415
+
currentVrm?.update(delta);
416
+
computeMouthOpen();
417
+
animatePresence(elapsed);
418
+
controls.update();
419
+
renderer.render(scene, camera);
420
+
requestAnimationFrame(tick);
421
+
}
422
+
423
+
loadDefaultVrm().catch((error) => {
424
+
console.error(error);
425
+
avatarStatus.textContent = 'Sample load failed';
426
+
});
427
+
428
+
loadSampleAudio().catch((error) => {
429
+
console.error(error);
430
+
audioStatus.textContent = 'Sample load failed';
431
+
});
432
+
433
+
tick();
+155
src/style.css
+155
src/style.css
···
1
+
:root {
2
+
color-scheme: dark;
3
+
font-family: Inter, ui-sans-serif, system-ui, sans-serif;
4
+
background: #0b1020;
5
+
color: #e8ecf3;
6
+
}
7
+
8
+
* {
9
+
box-sizing: border-box;
10
+
}
11
+
12
+
body {
13
+
margin: 0;
14
+
min-height: 100vh;
15
+
background:
16
+
radial-gradient(circle at top, rgba(76, 123, 255, 0.18), transparent 30%),
17
+
linear-gradient(180deg, #0b1020 0%, #111827 100%);
18
+
}
19
+
20
+
#app {
21
+
display: grid;
22
+
grid-template-columns: 360px 1fr;
23
+
min-height: 100vh;
24
+
}
25
+
26
+
.panel {
27
+
padding: 20px;
28
+
border-right: 1px solid rgba(255, 255, 255, 0.08);
29
+
background: rgba(10, 15, 30, 0.78);
30
+
backdrop-filter: blur(16px);
31
+
}
32
+
33
+
h1 {
34
+
margin: 0 0 8px;
35
+
font-size: 1.6rem;
36
+
}
37
+
38
+
p,
39
+
label,
40
+
button,
41
+
input {
42
+
font: inherit;
43
+
}
44
+
45
+
.copy {
46
+
color: #9ba7bd;
47
+
line-height: 1.45;
48
+
}
49
+
50
+
.stack {
51
+
display: grid;
52
+
gap: 16px;
53
+
margin-top: 20px;
54
+
}
55
+
56
+
.card {
57
+
padding: 16px;
58
+
border: 1px solid rgba(255, 255, 255, 0.08);
59
+
border-radius: 14px;
60
+
background: rgba(255, 255, 255, 0.03);
61
+
}
62
+
63
+
.card h2 {
64
+
margin: 0 0 10px;
65
+
font-size: 1rem;
66
+
}
67
+
68
+
input[type="file"] {
69
+
width: 100%;
70
+
}
71
+
72
+
button {
73
+
border: 0;
74
+
border-radius: 10px;
75
+
padding: 10px 14px;
76
+
background: #5b8cff;
77
+
color: white;
78
+
cursor: pointer;
79
+
}
80
+
81
+
button:disabled {
82
+
opacity: 0.5;
83
+
cursor: not-allowed;
84
+
}
85
+
86
+
.stats {
87
+
display: grid;
88
+
gap: 10px;
89
+
}
90
+
91
+
.stat-row {
92
+
display: flex;
93
+
justify-content: space-between;
94
+
gap: 12px;
95
+
color: #cdd6e5;
96
+
}
97
+
98
+
.meter {
99
+
height: 10px;
100
+
width: 100%;
101
+
border-radius: 999px;
102
+
overflow: hidden;
103
+
background: rgba(255, 255, 255, 0.08);
104
+
}
105
+
106
+
.meter > div {
107
+
height: 100%;
108
+
width: 0%;
109
+
background: linear-gradient(90deg, #5b8cff, #7cf8c4);
110
+
}
111
+
112
+
.viewport {
113
+
position: relative;
114
+
}
115
+
116
+
canvas {
117
+
display: block;
118
+
width: 100%;
119
+
height: 100vh;
120
+
}
121
+
122
+
.badge {
123
+
display: inline-flex;
124
+
padding: 4px 8px;
125
+
border-radius: 999px;
126
+
background: rgba(124, 248, 196, 0.12);
127
+
color: #7cf8c4;
128
+
font-size: 0.8rem;
129
+
}
130
+
131
+
.footer {
132
+
margin-top: 16px;
133
+
color: #7f8aa3;
134
+
font-size: 0.9rem;
135
+
}
136
+
137
+
code {
138
+
font-family: ui-monospace, SFMono-Regular, monospace;
139
+
font-size: 0.9em;
140
+
}
141
+
142
+
@media (max-width: 900px) {
143
+
#app {
144
+
grid-template-columns: 1fr;
145
+
}
146
+
147
+
.panel {
148
+
border-right: 0;
149
+
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
150
+
}
151
+
152
+
canvas {
153
+
height: 60vh;
154
+
}
155
+
}
+18
tsconfig.json
+18
tsconfig.json
···
1
+
{
2
+
"compilerOptions": {
3
+
"target": "ES2020",
4
+
"useDefineForClassFields": true,
5
+
"module": "ESNext",
6
+
"lib": ["ES2020", "DOM", "DOM.Iterable"],
7
+
"skipLibCheck": true,
8
+
"moduleResolution": "Bundler",
9
+
"allowImportingTsExtensions": false,
10
+
"resolveJsonModule": true,
11
+
"isolatedModules": true,
12
+
"noEmit": true,
13
+
"strict": true,
14
+
"noUnusedLocals": true,
15
+
"noUnusedParameters": true
16
+
},
17
+
"include": ["src"]
18
+
}