nandi.uk
This repository has no description
1import './style.css';
2
3import * as THREE from 'three';
4import { GoogleGenAI, Modality, type LiveServerMessage } from '@google/genai';
5import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
6import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js';
7import { VRM, VRMExpressionPresetName, VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
8
9function requireElement<T>(value: T | null, message: string): T {
10 if (!value) throw new Error(message);
11 return value;
12}
13
14const app = requireElement(document.querySelector<HTMLDivElement>('#app'), 'App root not found');
15
16app.innerHTML = `
17 <aside class="panel">
18 <span class="badge">TypeScript + VRM</span>
19 <h1>Avatar lip sync MVP</h1>
20 <p class="copy">Load a VRM avatar, upload an audio file, and drive mouth movement from the audio amplitude. This is amplitude-based lip sync, so it animates speaking energy rather than phoneme-accurate visemes.</p>
21 <p class="copy" style="margin-top: 8px;">Gemini voice is set to <code>Kore</code>, which should read as a feminine voice.</p>
22
23 <div class="stack">
24 <section class="card">
25 <h2>1. Avatar</h2>
26 <p class="copy" style="margin: 0 0 12px; font-size: 0.95rem;">A sample avatar is loaded by default. Upload a different VRM to replace it.</p>
27 <input id="vrmFile" type="file" accept=".vrm,model/gltf-binary" />
28 </section>
29
30 <section class="card">
31 <h2>2. Audio</h2>
32 <p class="copy" style="margin: 0 0 12px; font-size: 0.95rem;">A public-domain speech sample is bundled by default. Use the button below or upload your own file.</p>
33 <div style="margin-bottom: 12px;">
34 <button id="useSampleAudioButton">Use sample speech</button>
35 </div>
36 <input id="audioFile" type="file" accept="audio/*" />
37 <div style="margin-top: 12px; display: flex; gap: 10px; flex-wrap: wrap;">
38 <button id="playButton" disabled>Play</button>
39 <button id="stopButton" disabled>Stop</button>
40 </div>
41 <audio id="audio" preload="auto"></audio>
42 </section>
43
44 <section class="card">
45 <h2>3. Gemini reply</h2>
46 <p class="copy" style="margin: 0 0 12px; font-size: 0.95rem;">Send a text prompt to Gemini Live, receive native audio, and pipe it into the avatar audio player.</p>
47 <textarea id="promptInput" rows="4" style="width: 100%; resize: vertical; border-radius: 10px; border: 1px solid rgba(255,255,255,0.12); background: rgba(255,255,255,0.04); color: inherit; padding: 10px;" placeholder="Say something as the avatar..."></textarea>
48 <div style="margin-top: 12px; display: flex; gap: 10px; flex-wrap: wrap;">
49 <button id="sendPromptButton">Send to Gemini</button>
50 </div>
51 <p id="geminiStatus" class="footer" style="margin-bottom: 0;">Gemini idle</p>
52 </section>
53
54 <section class="card stats">
55 <h2>4. Live values</h2>
56 <div class="stat-row"><span>Avatar</span><strong id="avatarStatus">Not loaded</strong></div>
57 <div class="stat-row"><span>Audio</span><strong id="audioStatus">Not loaded</strong></div>
58 <div class="stat-row"><span>Mouth open</span><strong id="mouthValue">0.00</strong></div>
59 <div class="meter"><div id="mouthMeter"></div></div>
60 </section>
61 </div>
62
63 <p class="footer">If your VRM supports expression presets, this app drives <code>aa</code>. Otherwise it falls back to blendshape names like <code>A</code>, <code>aa</code>, or <code>mouthOpen</code>.</p>
64 </aside>
65 <main class="viewport"></main>
66`;
67
68const viewport = requireElement(app.querySelector<HTMLDivElement>('.viewport'), 'Missing viewport');
69const vrmInput = requireElement(app.querySelector<HTMLInputElement>('#vrmFile'), 'Missing VRM input');
70const audioInput = requireElement(app.querySelector<HTMLInputElement>('#audioFile'), 'Missing audio input');
71const useSampleAudioButton = requireElement(app.querySelector<HTMLButtonElement>('#useSampleAudioButton'), 'Missing sample audio button');
72const promptInput = requireElement(app.querySelector<HTMLTextAreaElement>('#promptInput'), 'Missing prompt input');
73const sendPromptButton = requireElement(app.querySelector<HTMLButtonElement>('#sendPromptButton'), 'Missing Gemini send button');
74const geminiStatus = requireElement(app.querySelector<HTMLElement>('#geminiStatus'), 'Missing Gemini status');
75const playButton = requireElement(app.querySelector<HTMLButtonElement>('#playButton'), 'Missing play button');
76const stopButton = requireElement(app.querySelector<HTMLButtonElement>('#stopButton'), 'Missing stop button');
77const audioEl = requireElement(app.querySelector<HTMLAudioElement>('#audio'), 'Missing audio element');
78const avatarStatus = requireElement(app.querySelector<HTMLElement>('#avatarStatus'), 'Missing avatar status');
79const audioStatus = requireElement(app.querySelector<HTMLElement>('#audioStatus'), 'Missing audio status');
80const mouthValue = requireElement(app.querySelector<HTMLElement>('#mouthValue'), 'Missing mouth value');
81const mouthMeter = requireElement(app.querySelector<HTMLElement>('#mouthMeter'), 'Missing mouth meter');
82
83const scene = new THREE.Scene();
84scene.background = new THREE.Color(0x0f172a);
85
86const camera = new THREE.PerspectiveCamera(35, viewport.clientWidth / viewport.clientHeight, 0.1, 100);
87camera.position.set(0, 1.4, 2.2);
88
89const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
90renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
91renderer.setSize(viewport.clientWidth, viewport.clientHeight);
92renderer.outputColorSpace = THREE.SRGBColorSpace;
93viewport.appendChild(renderer.domElement);
94
95const controls = new OrbitControls(camera, renderer.domElement);
96controls.target.set(0, 1.2, 0);
97controls.enableDamping = true;
98controls.minDistance = 1;
99controls.maxDistance = 5;
100controls.update();
101
102scene.add(new THREE.HemisphereLight(0xffffff, 0x1e293b, 1.4));
103const dirLight = new THREE.DirectionalLight(0xffffff, 1.4);
104dirLight.position.set(1, 1, 1);
105scene.add(dirLight);
106
107const floor = new THREE.Mesh(
108 new THREE.CircleGeometry(3, 64),
109 new THREE.MeshStandardMaterial({ color: 0x111827, roughness: 0.9, metalness: 0.05 })
110);
111floor.rotation.x = -Math.PI / 2;
112floor.position.y = -0.01;
113scene.add(floor);
114
115const clock = new THREE.Clock();
116const loader = new GLTFLoader();
117loader.register((parser) => new VRMLoaderPlugin(parser));
118
119let currentVrm: VRM | null = null;
120let audioContext: AudioContext | null = null;
121let analyser: AnalyserNode | null = null;
122let sourceNode: MediaElementAudioSourceNode | null = null;
123let dataArray: Uint8Array<ArrayBuffer> | null = null;
124let vrmObjectUrl: string | null = null;
125let audioObjectUrl: string | null = null;
126let mouthOpenAmount = 0;
127let audioEnergy = 0;
128let geminiResponseParts: Uint8Array[] = [];
129
130type PoseBones = {
131 neck?: THREE.Object3D | null;
132 spine?: THREE.Object3D | null;
133 chest?: THREE.Object3D | null;
134 upperChest?: THREE.Object3D | null;
135 leftUpperArm?: THREE.Object3D | null;
136 rightUpperArm?: THREE.Object3D | null;
137 leftLowerArm?: THREE.Object3D | null;
138 rightLowerArm?: THREE.Object3D | null;
139 leftHand?: THREE.Object3D | null;
140 rightHand?: THREE.Object3D | null;
141};
142
143let poseBones: PoseBones = {};
144
145const geminiApiKey = import.meta.env.VITE_GEMINI_API_KEY as string | undefined;
146const geminiModel = 'gemini-3.1-flash-live-preview';
147
148function disposeCurrentVrm() {
149 if (!currentVrm) return;
150 scene.remove(currentVrm.scene);
151 VRMUtils.deepDispose(currentVrm.scene);
152 currentVrm = null;
153 poseBones = {};
154}
155
156function cachePoseBones(vrm: VRM) {
157 const humanoid = vrm.humanoid;
158 poseBones = {
159 neck: humanoid?.getNormalizedBoneNode('neck'),
160 spine: humanoid?.getNormalizedBoneNode('spine'),
161 chest: humanoid?.getNormalizedBoneNode('chest'),
162 upperChest: humanoid?.getNormalizedBoneNode('upperChest'),
163 leftUpperArm: humanoid?.getNormalizedBoneNode('leftUpperArm'),
164 rightUpperArm: humanoid?.getNormalizedBoneNode('rightUpperArm'),
165 leftLowerArm: humanoid?.getNormalizedBoneNode('leftLowerArm'),
166 rightLowerArm: humanoid?.getNormalizedBoneNode('rightLowerArm'),
167 leftHand: humanoid?.getNormalizedBoneNode('leftHand'),
168 rightHand: humanoid?.getNormalizedBoneNode('rightHand'),
169 };
170}
171
172function pcm16ToWav(pcmData: Uint8Array, sampleRate = 24000, channels = 1): Blob {
173 const bytesPerSample = 2;
174 const blockAlign = channels * bytesPerSample;
175 const buffer = new ArrayBuffer(44 + pcmData.byteLength);
176 const view = new DataView(buffer);
177
178 const writeString = (offset: number, value: string) => {
179 for (let i = 0; i < value.length; i += 1) view.setUint8(offset + i, value.charCodeAt(i));
180 };
181
182 writeString(0, 'RIFF');
183 view.setUint32(4, 36 + pcmData.byteLength, true);
184 writeString(8, 'WAVE');
185 writeString(12, 'fmt ');
186 view.setUint32(16, 16, true);
187 view.setUint16(20, 1, true);
188 view.setUint16(22, channels, true);
189 view.setUint32(24, sampleRate, true);
190 view.setUint32(28, sampleRate * blockAlign, true);
191 view.setUint16(32, blockAlign, true);
192 view.setUint16(34, 16, true);
193 writeString(36, 'data');
194 view.setUint32(40, pcmData.byteLength, true);
195 new Uint8Array(buffer, 44).set(pcmData);
196 return new Blob([buffer], { type: 'audio/wav' });
197}
198
199function concatUint8Arrays(chunks: Uint8Array[]): Uint8Array {
200 const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
201 const result = new Uint8Array(total);
202 let offset = 0;
203 for (const chunk of chunks) {
204 result.set(chunk, offset);
205 offset += chunk.byteLength;
206 }
207 return result;
208}
209
210function base64ToUint8Array(base64: string): Uint8Array {
211 const binary = atob(base64);
212 const bytes = new Uint8Array(binary.length);
213 for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i);
214 return bytes;
215}
216
217async function playGeneratedSpeech(pcmChunks: Uint8Array[]) {
218 if (audioObjectUrl) URL.revokeObjectURL(audioObjectUrl);
219 const wavBlob = pcm16ToWav(concatUint8Arrays(pcmChunks));
220 audioObjectUrl = URL.createObjectURL(wavBlob);
221 audioEl.src = audioObjectUrl;
222 audioEl.load();
223 audioStatus.textContent = 'Gemini response';
224 playButton.disabled = false;
225 stopButton.disabled = false;
226 ensureAudioGraph();
227 if (audioContext?.state === 'suspended') await audioContext.resume();
228 await audioEl.play();
229}
230
231async function sendPromptToGemini(prompt: string) {
232 if (!geminiApiKey) throw new Error('Missing VITE_GEMINI_API_KEY');
233
234 geminiStatus.textContent = 'Connecting to Gemini…';
235 sendPromptButton.disabled = true;
236 geminiResponseParts = [];
237
238 const ai = new GoogleGenAI({ apiKey: geminiApiKey });
239 const session = await ai.live.connect({
240 model: geminiModel,
241 config: {
242 responseModalities: [Modality.AUDIO],
243 speechConfig: {
244 voiceConfig: {
245 prebuiltVoiceConfig: {
246 voiceName: 'Kore',
247 },
248 },
249 },
250 systemInstruction: 'You are a concise speaking avatar. Reply conversationally in under 20 seconds.',
251 },
252 callbacks: {
253 onopen: () => {
254 geminiStatus.textContent = 'Gemini connected';
255 },
256 onmessage: async (message: LiveServerMessage) => {
257 const serverContent = message.serverContent;
258 const parts = serverContent?.modelTurn?.parts ?? [];
259 for (const part of parts) {
260 if (part.inlineData?.data) {
261 geminiResponseParts.push(base64ToUint8Array(part.inlineData.data));
262 }
263 }
264
265 if (serverContent?.turnComplete) {
266 geminiStatus.textContent = 'Playing Gemini audio';
267 if (geminiResponseParts.length > 0) await playGeneratedSpeech(geminiResponseParts);
268 await session.close();
269 sendPromptButton.disabled = false;
270 geminiStatus.textContent = 'Gemini idle';
271 }
272 },
273 onerror: (error: ErrorEvent) => {
274 console.error(error);
275 geminiStatus.textContent = 'Gemini error';
276 sendPromptButton.disabled = false;
277 },
278 onclose: () => {
279 sendPromptButton.disabled = false;
280 },
281 },
282 });
283
284 session.sendClientContent({
285 turns: [{ role: 'user', parts: [{ text: prompt }] }],
286 turnComplete: true,
287 });
288}
289
290function setMouth(open: number) {
291 const clamped = THREE.MathUtils.clamp(open, 0, 1);
292 mouthOpenAmount = clamped;
293 mouthValue.textContent = clamped.toFixed(2);
294 mouthMeter.style.width = `${(clamped * 100).toFixed(1)}%`;
295
296 if (!currentVrm) return;
297
298 const manager = currentVrm.expressionManager;
299 if (!manager) return;
300
301 manager.setValue(VRMExpressionPresetName.Aa, clamped);
302 for (const name of ['A', 'a', 'aa', 'Ah', 'mouthOpen']) {
303 try {
304 manager.setValue(name as VRMExpressionPresetName, clamped);
305 } catch {
306 // ignore unsupported expression names
307 }
308 }
309}
310
311function ensureAudioGraph() {
312 if (audioContext && analyser && sourceNode && dataArray) return;
313
314 audioContext = new AudioContext();
315 analyser = audioContext.createAnalyser();
316 analyser.fftSize = 2048;
317 analyser.smoothingTimeConstant = 0.82;
318 dataArray = new Uint8Array(new ArrayBuffer(analyser.frequencyBinCount));
319 sourceNode = audioContext.createMediaElementSource(audioEl);
320 sourceNode.connect(analyser);
321 analyser.connect(audioContext.destination);
322}
323
324function computeMouthOpen() {
325 if (!analyser || !dataArray || audioEl.paused) {
326 audioEnergy = THREE.MathUtils.lerp(audioEnergy, 0, 0.18);
327 setMouth(0);
328 return;
329 }
330
331 analyser.getByteTimeDomainData(dataArray);
332 let sum = 0;
333 for (let i = 0; i < dataArray.length; i += 1) {
334 const centered = (dataArray[i] - 128) / 128;
335 sum += centered * centered;
336 }
337
338 const rms = Math.sqrt(sum / dataArray.length);
339 const boosted = THREE.MathUtils.clamp((rms - 0.012) * 10.0, 0, 1);
340 const exaggerated = THREE.MathUtils.clamp(Math.pow(boosted, 0.7) * 1.25, 0, 1);
341 audioEnergy = THREE.MathUtils.lerp(audioEnergy, exaggerated, 0.35);
342 const expressiveMouth = THREE.MathUtils.clamp(exaggerated * 1.35 + 0.02, 0, 1);
343 setMouth(expressiveMouth);
344}
345
346function animatePresence(elapsed: number) {
347 if (!currentVrm) return;
348
349 const idleSway = Math.sin(elapsed * 1.4) * 0.025;
350 const idleBob = Math.sin(elapsed * 2.1) * 0.012;
351 const talkSway = Math.sin(elapsed * 9.0) * audioEnergy * 0.035;
352 const talkNod = Math.sin(elapsed * 7.2) * audioEnergy * 0.03;
353 const talkLift = audioEnergy * 0.045;
354
355 currentVrm.scene.rotation.y = Math.PI + idleSway + talkSway;
356 currentVrm.scene.rotation.x = talkNod * 0.35;
357 currentVrm.scene.position.y = idleBob + talkLift;
358
359 const {
360 neck,
361 spine,
362 chest,
363 upperChest,
364 leftUpperArm,
365 rightUpperArm,
366 leftLowerArm,
367 rightLowerArm,
368 leftHand,
369 rightHand,
370 } = poseBones;
371
372 if (neck) {
373 neck.rotation.y = talkSway * 0.9;
374 neck.rotation.x = talkNod * 0.8;
375 neck.rotation.z = Math.sin(elapsed * 5.4) * audioEnergy * 0.04;
376 }
377
378 if (spine) {
379 spine.rotation.y = idleSway * 0.45 + talkSway * 0.4;
380 spine.rotation.x = talkNod * 0.2;
381 }
382
383 if (chest) {
384 chest.rotation.x = mouthOpenAmount * 0.05 + audioEnergy * 0.04;
385 chest.rotation.z = Math.sin(elapsed * 3.8) * 0.015 + talkSway * 0.3;
386 }
387
388 if (upperChest) {
389 upperChest.rotation.y = talkSway * 0.35;
390 upperChest.rotation.x = audioEnergy * 0.03;
391 }
392
393 if (leftUpperArm) {
394 leftUpperArm.rotation.z = 1.12 + Math.sin(elapsed * 1.7) * 0.012 + audioEnergy * 0.02;
395 leftUpperArm.rotation.x = 0.03 + Math.sin(elapsed * 2.3) * 0.01;
396 leftUpperArm.rotation.y = -0.04 - talkSway * 0.1;
397 }
398
399 if (rightUpperArm) {
400 rightUpperArm.rotation.z = -1.12 - Math.sin(elapsed * 1.9) * 0.012 - audioEnergy * 0.02;
401 rightUpperArm.rotation.x = 0.03 + Math.sin(elapsed * 2.5) * 0.01;
402 rightUpperArm.rotation.y = 0.04 - talkSway * 0.1;
403 }
404
405 if (leftLowerArm) {
406 leftLowerArm.rotation.z = 0;
407 leftLowerArm.rotation.x = -0.14 + Math.sin(elapsed * 2.8) * 0.008;
408 }
409
410 if (rightLowerArm) {
411 rightLowerArm.rotation.z = 0;
412 rightLowerArm.rotation.x = -0.14 + Math.sin(elapsed * 3.0) * 0.008;
413 }
414
415 if (leftHand) {
416 leftHand.rotation.y = -0.08 + Math.sin(elapsed * 3.5) * 0.02;
417 leftHand.rotation.z = -0.03;
418 }
419
420 if (rightHand) {
421 rightHand.rotation.y = 0.08 - Math.sin(elapsed * 3.3) * 0.02;
422 rightHand.rotation.z = 0.03;
423 }
424}
425
426async function loadVrm(file: File) {
427 disposeCurrentVrm();
428 if (vrmObjectUrl) URL.revokeObjectURL(vrmObjectUrl);
429
430 vrmObjectUrl = URL.createObjectURL(file);
431 const gltf = await loader.loadAsync(vrmObjectUrl);
432 const vrm = gltf.userData.vrm as VRM | undefined;
433 if (!vrm) throw new Error('Loaded file is not a VRM avatar');
434
435 VRMUtils.rotateVRM0(vrm);
436 vrm.scene.rotation.y = Math.PI;
437 scene.add(vrm.scene);
438 currentVrm = vrm;
439 cachePoseBones(vrm);
440 avatarStatus.textContent = file.name;
441}
442
443async function loadDefaultVrm() {
444 avatarStatus.textContent = 'Loading sample…';
445 disposeCurrentVrm();
446 if (vrmObjectUrl) {
447 URL.revokeObjectURL(vrmObjectUrl);
448 vrmObjectUrl = null;
449 }
450
451 const gltf = await loader.loadAsync('/default-avatar.vrm');
452 const vrm = gltf.userData.vrm as VRM | undefined;
453 if (!vrm) throw new Error('Default avatar is not a valid VRM');
454
455 VRMUtils.rotateVRM0(vrm);
456 vrm.scene.rotation.y = Math.PI;
457 scene.add(vrm.scene);
458 currentVrm = vrm;
459 cachePoseBones(vrm);
460 avatarStatus.textContent = 'default-avatar.vrm';
461}
462
463async function loadAudio(file: File) {
464 if (audioObjectUrl) URL.revokeObjectURL(audioObjectUrl);
465
466 audioObjectUrl = URL.createObjectURL(file);
467 audioEl.src = audioObjectUrl;
468 audioEl.load();
469 audioStatus.textContent = file.name;
470 playButton.disabled = false;
471 stopButton.disabled = false;
472 setMouth(0);
473}
474
475async function loadSampleAudio() {
476 if (audioObjectUrl) {
477 URL.revokeObjectURL(audioObjectUrl);
478 audioObjectUrl = null;
479 }
480
481 audioEl.src = '/sample-speech.ogg';
482 audioEl.load();
483 audioStatus.textContent = 'sample-speech.ogg';
484 playButton.disabled = false;
485 stopButton.disabled = false;
486 setMouth(0);
487}
488
489vrmInput.addEventListener('change', async () => {
490 const file = vrmInput.files?.[0];
491 if (!file) return;
492
493 avatarStatus.textContent = 'Loading…';
494 try {
495 await loadVrm(file);
496 } catch (error) {
497 console.error(error);
498 avatarStatus.textContent = 'Load failed';
499 }
500});
501
502audioInput.addEventListener('change', async () => {
503 const file = audioInput.files?.[0];
504 if (!file) return;
505
506 try {
507 await loadAudio(file);
508 } catch (error) {
509 console.error(error);
510 audioStatus.textContent = 'Load failed';
511 }
512});
513
514useSampleAudioButton.addEventListener('click', async () => {
515 try {
516 await loadSampleAudio();
517 } catch (error) {
518 console.error(error);
519 audioStatus.textContent = 'Sample load failed';
520 }
521});
522
523sendPromptButton.addEventListener('click', async () => {
524 const prompt = promptInput.value.trim();
525 if (!prompt) return;
526
527 try {
528 await sendPromptToGemini(prompt);
529 } catch (error) {
530 console.error(error);
531 geminiStatus.textContent = error instanceof Error ? error.message : 'Gemini request failed';
532 sendPromptButton.disabled = false;
533 }
534});
535
536playButton.addEventListener('click', async () => {
537 try {
538 ensureAudioGraph();
539 if (audioContext?.state === 'suspended') await audioContext.resume();
540 await audioEl.play();
541 } catch (error) {
542 console.error(error);
543 }
544});
545
546stopButton.addEventListener('click', () => {
547 audioEl.pause();
548 audioEl.currentTime = 0;
549 setMouth(0);
550});
551
552audioEl.addEventListener('ended', () => setMouth(0));
553
554window.addEventListener('resize', () => {
555 const width = viewport.clientWidth;
556 const height = viewport.clientHeight;
557 camera.aspect = width / height;
558 camera.updateProjectionMatrix();
559 renderer.setSize(width, height);
560});
561
562function tick() {
563 const delta = clock.getDelta();
564 const elapsed = clock.elapsedTime;
565 currentVrm?.update(delta);
566 computeMouthOpen();
567 animatePresence(elapsed);
568 controls.update();
569 renderer.render(scene, camera);
570 requestAnimationFrame(tick);
571}
572
573loadDefaultVrm().catch((error) => {
574 console.error(error);
575 avatarStatus.textContent = 'Sample load failed';
576});
577
578loadSampleAudio().catch((error) => {
579 console.error(error);
580 audioStatus.textContent = 'Sample load failed';
581});
582
583tick();