nandi.uk
This repository has no description
1import './style.css';
2
3import * as THREE from 'three';
4import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
5import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js';
6import { VRM, VRMExpressionPresetName, VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
7
8function requireElement<T>(value: T | null, message: string): T {
9 if (!value) throw new Error(message);
10 return value;
11}
12
13const app = requireElement(document.querySelector<HTMLDivElement>('#app'), 'App root not found');
14
15app.innerHTML = `
16 <aside class="panel">
17 <span class="badge">TypeScript + VRM</span>
18 <h1>Avatar lip sync MVP</h1>
19 <p class="copy">Load a VRM avatar, upload an audio file, and drive mouth movement from the audio amplitude. This is amplitude-based lip sync, so it animates speaking energy rather than phoneme-accurate visemes.</p>
20
21 <div class="stack">
22 <section class="card">
23 <h2>1. Avatar</h2>
24 <p class="copy" style="margin: 0 0 12px; font-size: 0.95rem;">A sample avatar is loaded by default. Upload a different VRM to replace it.</p>
25 <input id="vrmFile" type="file" accept=".vrm,model/gltf-binary" />
26 </section>
27
28 <section class="card">
29 <h2>2. Audio</h2>
30 <p class="copy" style="margin: 0 0 12px; font-size: 0.95rem;">A public-domain speech sample is bundled by default. Use the button below or upload your own file.</p>
31 <div style="margin-bottom: 12px;">
32 <button id="useSampleAudioButton">Use sample speech</button>
33 </div>
34 <input id="audioFile" type="file" accept="audio/*" />
35 <div style="margin-top: 12px; display: flex; gap: 10px; flex-wrap: wrap;">
36 <button id="playButton" disabled>Play</button>
37 <button id="stopButton" disabled>Stop</button>
38 </div>
39 <audio id="audio" preload="auto"></audio>
40 </section>
41
42 <section class="card stats">
43 <h2>3. Live values</h2>
44 <div class="stat-row"><span>Avatar</span><strong id="avatarStatus">Not loaded</strong></div>
45 <div class="stat-row"><span>Audio</span><strong id="audioStatus">Not loaded</strong></div>
46 <div class="stat-row"><span>Mouth open</span><strong id="mouthValue">0.00</strong></div>
47 <div class="meter"><div id="mouthMeter"></div></div>
48 </section>
49 </div>
50
51 <p class="footer">If your VRM supports expression presets, this app drives <code>aa</code>. Otherwise it falls back to blendshape names like <code>A</code>, <code>aa</code>, or <code>mouthOpen</code>.</p>
52 </aside>
53 <main class="viewport"></main>
54`;
55
56const viewport = requireElement(app.querySelector<HTMLDivElement>('.viewport'), 'Missing viewport');
57const vrmInput = requireElement(app.querySelector<HTMLInputElement>('#vrmFile'), 'Missing VRM input');
58const audioInput = requireElement(app.querySelector<HTMLInputElement>('#audioFile'), 'Missing audio input');
59const useSampleAudioButton = requireElement(app.querySelector<HTMLButtonElement>('#useSampleAudioButton'), 'Missing sample audio button');
60const playButton = requireElement(app.querySelector<HTMLButtonElement>('#playButton'), 'Missing play button');
61const stopButton = requireElement(app.querySelector<HTMLButtonElement>('#stopButton'), 'Missing stop button');
62const audioEl = requireElement(app.querySelector<HTMLAudioElement>('#audio'), 'Missing audio element');
63const avatarStatus = requireElement(app.querySelector<HTMLElement>('#avatarStatus'), 'Missing avatar status');
64const audioStatus = requireElement(app.querySelector<HTMLElement>('#audioStatus'), 'Missing audio status');
65const mouthValue = requireElement(app.querySelector<HTMLElement>('#mouthValue'), 'Missing mouth value');
66const mouthMeter = requireElement(app.querySelector<HTMLElement>('#mouthMeter'), 'Missing mouth meter');
67
68const scene = new THREE.Scene();
69scene.background = new THREE.Color(0x0f172a);
70
71const camera = new THREE.PerspectiveCamera(35, viewport.clientWidth / viewport.clientHeight, 0.1, 100);
72camera.position.set(0, 1.4, 2.2);
73
74const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
75renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
76renderer.setSize(viewport.clientWidth, viewport.clientHeight);
77renderer.outputColorSpace = THREE.SRGBColorSpace;
78viewport.appendChild(renderer.domElement);
79
80const controls = new OrbitControls(camera, renderer.domElement);
81controls.target.set(0, 1.2, 0);
82controls.enableDamping = true;
83controls.minDistance = 1;
84controls.maxDistance = 5;
85controls.update();
86
87scene.add(new THREE.HemisphereLight(0xffffff, 0x1e293b, 1.4));
88const dirLight = new THREE.DirectionalLight(0xffffff, 1.4);
89dirLight.position.set(1, 1, 1);
90scene.add(dirLight);
91
92const floor = new THREE.Mesh(
93 new THREE.CircleGeometry(3, 64),
94 new THREE.MeshStandardMaterial({ color: 0x111827, roughness: 0.9, metalness: 0.05 })
95);
96floor.rotation.x = -Math.PI / 2;
97floor.position.y = -0.01;
98scene.add(floor);
99
100const clock = new THREE.Clock();
101const loader = new GLTFLoader();
102loader.register((parser) => new VRMLoaderPlugin(parser));
103
104let currentVrm: VRM | null = null;
105let audioContext: AudioContext | null = null;
106let analyser: AnalyserNode | null = null;
107let sourceNode: MediaElementAudioSourceNode | null = null;
108let dataArray: Uint8Array<ArrayBuffer> | null = null;
109let vrmObjectUrl: string | null = null;
110let audioObjectUrl: string | null = null;
111let mouthOpenAmount = 0;
112let audioEnergy = 0;
113
114type PoseBones = {
115 neck?: THREE.Object3D | null;
116 spine?: THREE.Object3D | null;
117 chest?: THREE.Object3D | null;
118 upperChest?: THREE.Object3D | null;
119 leftUpperArm?: THREE.Object3D | null;
120 rightUpperArm?: THREE.Object3D | null;
121 leftLowerArm?: THREE.Object3D | null;
122 rightLowerArm?: THREE.Object3D | null;
123 leftHand?: THREE.Object3D | null;
124 rightHand?: THREE.Object3D | null;
125};
126
127let poseBones: PoseBones = {};
128
129function disposeCurrentVrm() {
130 if (!currentVrm) return;
131 scene.remove(currentVrm.scene);
132 VRMUtils.deepDispose(currentVrm.scene);
133 currentVrm = null;
134 poseBones = {};
135}
136
137function cachePoseBones(vrm: VRM) {
138 const humanoid = vrm.humanoid;
139 poseBones = {
140 neck: humanoid?.getNormalizedBoneNode('neck'),
141 spine: humanoid?.getNormalizedBoneNode('spine'),
142 chest: humanoid?.getNormalizedBoneNode('chest'),
143 upperChest: humanoid?.getNormalizedBoneNode('upperChest'),
144 leftUpperArm: humanoid?.getNormalizedBoneNode('leftUpperArm'),
145 rightUpperArm: humanoid?.getNormalizedBoneNode('rightUpperArm'),
146 leftLowerArm: humanoid?.getNormalizedBoneNode('leftLowerArm'),
147 rightLowerArm: humanoid?.getNormalizedBoneNode('rightLowerArm'),
148 leftHand: humanoid?.getNormalizedBoneNode('leftHand'),
149 rightHand: humanoid?.getNormalizedBoneNode('rightHand'),
150 };
151}
152
153function setMouth(open: number) {
154 const clamped = THREE.MathUtils.clamp(open, 0, 1);
155 mouthOpenAmount = clamped;
156 mouthValue.textContent = clamped.toFixed(2);
157 mouthMeter.style.width = `${(clamped * 100).toFixed(1)}%`;
158
159 if (!currentVrm) return;
160
161 const manager = currentVrm.expressionManager;
162 if (!manager) return;
163
164 manager.setValue(VRMExpressionPresetName.Aa, clamped);
165 for (const name of ['A', 'a', 'aa', 'Ah', 'mouthOpen']) {
166 try {
167 manager.setValue(name as VRMExpressionPresetName, clamped);
168 } catch {
169 // ignore unsupported expression names
170 }
171 }
172}
173
174function ensureAudioGraph() {
175 if (audioContext && analyser && sourceNode && dataArray) return;
176
177 audioContext = new AudioContext();
178 analyser = audioContext.createAnalyser();
179 analyser.fftSize = 2048;
180 analyser.smoothingTimeConstant = 0.82;
181 dataArray = new Uint8Array(new ArrayBuffer(analyser.frequencyBinCount));
182 sourceNode = audioContext.createMediaElementSource(audioEl);
183 sourceNode.connect(analyser);
184 analyser.connect(audioContext.destination);
185}
186
187function computeMouthOpen() {
188 if (!analyser || !dataArray || audioEl.paused) {
189 audioEnergy = THREE.MathUtils.lerp(audioEnergy, 0, 0.18);
190 setMouth(0);
191 return;
192 }
193
194 analyser.getByteTimeDomainData(dataArray);
195 let sum = 0;
196 for (let i = 0; i < dataArray.length; i += 1) {
197 const centered = (dataArray[i] - 128) / 128;
198 sum += centered * centered;
199 }
200
201 const rms = Math.sqrt(sum / dataArray.length);
202 const boosted = THREE.MathUtils.clamp((rms - 0.012) * 10.0, 0, 1);
203 const exaggerated = THREE.MathUtils.clamp(Math.pow(boosted, 0.7) * 1.25, 0, 1);
204 audioEnergy = THREE.MathUtils.lerp(audioEnergy, exaggerated, 0.35);
205 const expressiveMouth = THREE.MathUtils.clamp(exaggerated * 1.35 + 0.02, 0, 1);
206 setMouth(expressiveMouth);
207}
208
209function animatePresence(elapsed: number) {
210 if (!currentVrm) return;
211
212 const idleSway = Math.sin(elapsed * 1.4) * 0.025;
213 const idleBob = Math.sin(elapsed * 2.1) * 0.012;
214 const talkSway = Math.sin(elapsed * 9.0) * audioEnergy * 0.035;
215 const talkNod = Math.sin(elapsed * 7.2) * audioEnergy * 0.03;
216 const talkLift = audioEnergy * 0.045;
217
218 currentVrm.scene.rotation.y = Math.PI + idleSway + talkSway;
219 currentVrm.scene.rotation.x = talkNod * 0.35;
220 currentVrm.scene.position.y = idleBob + talkLift;
221
222 const {
223 neck,
224 spine,
225 chest,
226 upperChest,
227 leftUpperArm,
228 rightUpperArm,
229 leftLowerArm,
230 rightLowerArm,
231 leftHand,
232 rightHand,
233 } = poseBones;
234
235 if (neck) {
236 neck.rotation.y = talkSway * 0.9;
237 neck.rotation.x = talkNod * 0.8;
238 neck.rotation.z = Math.sin(elapsed * 5.4) * audioEnergy * 0.04;
239 }
240
241 if (spine) {
242 spine.rotation.y = idleSway * 0.45 + talkSway * 0.4;
243 spine.rotation.x = talkNod * 0.2;
244 }
245
246 if (chest) {
247 chest.rotation.x = mouthOpenAmount * 0.05 + audioEnergy * 0.04;
248 chest.rotation.z = Math.sin(elapsed * 3.8) * 0.015 + talkSway * 0.3;
249 }
250
251 if (upperChest) {
252 upperChest.rotation.y = talkSway * 0.35;
253 upperChest.rotation.x = audioEnergy * 0.03;
254 }
255
256 if (leftUpperArm) {
257 leftUpperArm.rotation.z = -0.55 + Math.sin(elapsed * 1.7) * 0.04 + audioEnergy * 0.08;
258 leftUpperArm.rotation.x = 0.18 + Math.sin(elapsed * 2.3) * 0.03;
259 leftUpperArm.rotation.y = -0.12 - talkSway * 0.4;
260 }
261
262 if (rightUpperArm) {
263 rightUpperArm.rotation.z = 0.55 - Math.sin(elapsed * 1.9) * 0.04 - audioEnergy * 0.08;
264 rightUpperArm.rotation.x = 0.18 + Math.sin(elapsed * 2.5) * 0.03;
265 rightUpperArm.rotation.y = 0.12 - talkSway * 0.4;
266 }
267
268 if (leftLowerArm) {
269 leftLowerArm.rotation.z = -0.35 - audioEnergy * 0.08;
270 leftLowerArm.rotation.x = -0.25 + Math.sin(elapsed * 2.8) * 0.025;
271 }
272
273 if (rightLowerArm) {
274 rightLowerArm.rotation.z = 0.35 + audioEnergy * 0.08;
275 rightLowerArm.rotation.x = -0.25 + Math.sin(elapsed * 3.0) * 0.025;
276 }
277
278 if (leftHand) {
279 leftHand.rotation.y = -0.12 + Math.sin(elapsed * 3.5) * 0.03;
280 leftHand.rotation.z = -0.08;
281 }
282
283 if (rightHand) {
284 rightHand.rotation.y = 0.12 - Math.sin(elapsed * 3.3) * 0.03;
285 rightHand.rotation.z = 0.08;
286 }
287}
288
289async function loadVrm(file: File) {
290 disposeCurrentVrm();
291 if (vrmObjectUrl) URL.revokeObjectURL(vrmObjectUrl);
292
293 vrmObjectUrl = URL.createObjectURL(file);
294 const gltf = await loader.loadAsync(vrmObjectUrl);
295 const vrm = gltf.userData.vrm as VRM | undefined;
296 if (!vrm) throw new Error('Loaded file is not a VRM avatar');
297
298 VRMUtils.rotateVRM0(vrm);
299 vrm.scene.rotation.y = Math.PI;
300 scene.add(vrm.scene);
301 currentVrm = vrm;
302 cachePoseBones(vrm);
303 avatarStatus.textContent = file.name;
304}
305
306async function loadDefaultVrm() {
307 avatarStatus.textContent = 'Loading sample…';
308 disposeCurrentVrm();
309 if (vrmObjectUrl) {
310 URL.revokeObjectURL(vrmObjectUrl);
311 vrmObjectUrl = null;
312 }
313
314 const gltf = await loader.loadAsync('/default-avatar.vrm');
315 const vrm = gltf.userData.vrm as VRM | undefined;
316 if (!vrm) throw new Error('Default avatar is not a valid VRM');
317
318 VRMUtils.rotateVRM0(vrm);
319 vrm.scene.rotation.y = Math.PI;
320 scene.add(vrm.scene);
321 currentVrm = vrm;
322 cachePoseBones(vrm);
323 avatarStatus.textContent = 'default-avatar.vrm';
324}
325
326async function loadAudio(file: File) {
327 if (audioObjectUrl) URL.revokeObjectURL(audioObjectUrl);
328
329 audioObjectUrl = URL.createObjectURL(file);
330 audioEl.src = audioObjectUrl;
331 audioEl.load();
332 audioStatus.textContent = file.name;
333 playButton.disabled = false;
334 stopButton.disabled = false;
335 setMouth(0);
336}
337
338async function loadSampleAudio() {
339 if (audioObjectUrl) {
340 URL.revokeObjectURL(audioObjectUrl);
341 audioObjectUrl = null;
342 }
343
344 audioEl.src = '/sample-speech.ogg';
345 audioEl.load();
346 audioStatus.textContent = 'sample-speech.ogg';
347 playButton.disabled = false;
348 stopButton.disabled = false;
349 setMouth(0);
350}
351
352vrmInput.addEventListener('change', async () => {
353 const file = vrmInput.files?.[0];
354 if (!file) return;
355
356 avatarStatus.textContent = 'Loading…';
357 try {
358 await loadVrm(file);
359 } catch (error) {
360 console.error(error);
361 avatarStatus.textContent = 'Load failed';
362 }
363});
364
365audioInput.addEventListener('change', async () => {
366 const file = audioInput.files?.[0];
367 if (!file) return;
368
369 try {
370 await loadAudio(file);
371 } catch (error) {
372 console.error(error);
373 audioStatus.textContent = 'Load failed';
374 }
375});
376
377useSampleAudioButton.addEventListener('click', async () => {
378 try {
379 await loadSampleAudio();
380 } catch (error) {
381 console.error(error);
382 audioStatus.textContent = 'Sample load failed';
383 }
384});
385
386playButton.addEventListener('click', async () => {
387 try {
388 ensureAudioGraph();
389 if (audioContext?.state === 'suspended') await audioContext.resume();
390 await audioEl.play();
391 } catch (error) {
392 console.error(error);
393 }
394});
395
396stopButton.addEventListener('click', () => {
397 audioEl.pause();
398 audioEl.currentTime = 0;
399 setMouth(0);
400});
401
402audioEl.addEventListener('ended', () => setMouth(0));
403
404window.addEventListener('resize', () => {
405 const width = viewport.clientWidth;
406 const height = viewport.clientHeight;
407 camera.aspect = width / height;
408 camera.updateProjectionMatrix();
409 renderer.setSize(width, height);
410});
411
412function tick() {
413 const delta = clock.getDelta();
414 const elapsed = clock.elapsedTime;
415 currentVrm?.update(delta);
416 computeMouthOpen();
417 animatePresence(elapsed);
418 controls.update();
419 renderer.render(scene, camera);
420 requestAnimationFrame(tick);
421}
422
423loadDefaultVrm().catch((error) => {
424 console.error(error);
425 avatarStatus.textContent = 'Sample load failed';
426});
427
428loadSampleAudio().catch((error) => {
429 console.error(error);
430 audioStatus.textContent = 'Sample load failed';
431});
432
433tick();