Visemes (not available)

const client = new VoiceAgentClient({
  auth: getTicket,
  prompt: 'You are helpful.',
  visemes: true,
});

client.on('visemes', (visemes) => {
  console.log('Received', visemes.length, 'visemes');
});

await client.synthesize({
  text: 'Hello world',
  visemes: true,
  onVisemes: (visemes) => {
    console.log('Visemes:', visemes);
  },
});

interface Viseme {
  v: number;  // Character index in source text
  c: string;  // Normalized character/phoneme
  t: number;  // Timestamp in seconds from audio start
}

let timeline: Viseme[] = [];

client.on('visemes', (visemes) => {
  timeline.push(...visemes);
  timeline.sort((a, b) => a.t - b.t);
});

function updateMouth() {
  const now = audioContext.currentTime;
  let active = timeline[0];
  for (const v of timeline) {
    if (v.t <= now) active = v;
    else break;
  }
  setMouthShape(mouthShapes[active?.c || 'sil'] || 'closed');
  requestAnimationFrame(updateMouth);
}

const speed = 0.15;
currentWeight[shape] += (targetWeight[shape] - currentWeight[shape]) * speed;

client.on('status', (status) => {
  if (status === 'interrupted') timeline = []; // timeline is declared with let above
});

client.on('audio', (chunk: Uint8Array) => {
  const pcm = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.byteLength / 2);
  let sum = 0;
  for (const s of pcm) sum += (s / 32768) ** 2;
  const level = Math.sqrt(sum / pcm.length); // 0–1; schedule it for this chunk's play time
  queueMouthOpening(level);
});