Agent concept

const agent = await meeting.startAgent({
  agentId:  'support-bot-v3',
  metadata: { customerId: 'c_123', tier: 'premium' },   // → dispatch metadata.variables
});

const me = room.localParticipant;

// Executor — a normal RPC handler (runs in the browser)
me.registerRpc('book_appointment', async (req) => myApp.book(req.payload.date));

const agent = await meeting.startAgent({
  instructions: 'Help the user book appointments.',
  enterMessage: 'Hi! How can I help you today?',
  exitMessage:  'Thanks, goodbye!',

  stt: { provider: 'deepgram',   model: 'nova-3', language: 'en' },
  llm: { provider: 'openai',     model: 'gpt-4o', temperature: 0.7 },
  tts: { provider: 'elevenlabs', voice: '21m00...' },

  tools: [{
    name: 'book_appointment',
    description: 'Book an appointment for the user',
    parameters: { type: 'object', properties: { date: { type: 'string' } }, required: ['date'] },
  }],
});

agent.mode;   // 'cascade'

// Pure realtime (speech-to-speech)
await meeting.startAgent({
  instructions: '...',
  llm: { provider: 'videosdk', model: 'gemini-live', voice: 'Leda' },   // → mode 'realtime'
});

// Hybrid — realtime brain + custom voice
await meeting.startAgent({
  instructions: '...',
  llm: { provider: 'videosdk',   model: 'gemini-live' },
  tts: { provider: 'elevenlabs', voice: '21m00...' },                   // → mode 'hybrid_tts'
});

const agent = meeting.participants.get(agentId);   // kind === 'agent'

// Wait for the disclaimer to finish before enabling the mic
await agent.say('Your call is being recorded.');   // promise resolves on playback completion
mic.enable();

// Let the LLM phrase a contextual prompt
await agent.reply('Ask the user to confirm their email address');

// Cut it off if the user starts talking
agent.interrupt();

await agent.reply('Summarize');            // wait_for_playback: true  — sequence after speech
agent.reply('Summarize').catch(onErr);     // wait_for_playback: false — fire-and-forget while it speaks

const p = agent.reply('Summarize');        // start now…
doWork();                                  // …continue while it speaks…
await p;                                   // …then block until done (Python needs a handle for this)

room.on('participant-joined', (p) => {
  if (p.kind === 'agent') {
    ui.showAgentTile(p);
    // Register tools you want the agent to call (see §RPC below)
    // Subscribe to its state and transcript
  } else {
    ui.showHumanTile(p);
  }
});

room.on('participant-left', (p) => {
  // SDK auto-cleans listeners on the leaving participant — kind doesn't matter here
  ui.removeTile(p.id);
});

room.on('participant-joined', (agent) => {
  if (agent.kind !== 'agent') return;

  // Initial sync render — agentState is readable immediately
  ui.setAgentState(agent.id, agent.agentState);   // probably 'idle' or 'listening'

  // Animate transitions
  agent.on('agent-state-changed', ({ state, prev, error }) => {
    ui.setAgentState(agent.id, state);

    if (state === 'failed') {
      console.error('Agent failed:', error.code, error.message);
      if (error.retriable) ui.showRetryButton();
      else                 ui.showFatalError(error.message);
    }
  });
});

room.on('agent-text', ({ participantId, text, isFinal, timestamp }) => {
  const p = room.remoteParticipants.get(participantId) ?? room.localParticipant;
  const role = p?.kind === 'agent' ? 'assistant' : 'user';

  if (!isFinal) {
    ui.updateStreamingMessage(role, text);    // overwrite the in-progress line
  } else {
    ui.commitMessage(role, text, timestamp);  // append final to history
  }
});

// Three function tools the agent's LLM can invoke
me.registerRpc('current_location', async () => ({ lat, lng }));
me.registerRpc('upcoming_events',  async (req) => ({ events: await calendar.list(req.payload.range) }));
me.registerRpc('book_appointment', async (req) => {
  if (!(await ui.confirmBooking(req.payload))) return { booked: false, reason: 'user_declined' };
  const r = await calendar.book(req.payload);
  return { booked: true, id: r.id };
});

// Tear down at unmount / leave
function cleanup() {
  me.unregisterRpc('current_location');
  me.unregisterRpc('upcoming_events');
  me.unregisterRpc('book_appointment');
}

// Build a small per-turn dashboard
const turnMetrics = new Map();   // turnId → { llm_ttft, llm_latency, ... }

agent.on('agent-metric', (m) => {
  if (!m.turn) return;                                    // skip session-level metrics for this UI
  const acc = turnMetrics.get(m.turn) ?? {};
  acc[m.name] = m.value;
  turnMetrics.set(m.turn, acc);

  // When the turn finishes (e2e_latency arrives on the 'turn' component), render
  if (m.component === 'turn' && m.name === 'e2e_latency') {
    ui.showTurnBadge(m.turn, {
      ttft:       acc.llm_ttft    != null ? `${acc.llm_ttft} ms`    : '—',
      llmLatency: acc.llm_latency != null ? `${acc.llm_latency} ms` : '—',
      total:      `${m.value} ms`,
    });
    turnMetrics.delete(m.turn);
  }
});

let sessionTokensIn = 0, sessionTokensOut = 0;

agent.on('agent-metric', (m) => {
  if (m.name === 'prompt_tokens')     sessionTokensIn  += m.value;
  if (m.name === 'completion_tokens') sessionTokensOut += m.value;

  if (m.component === 'turn' && m.name === 'e2e_latency') {
    ui.updateCostBadge({
      tokensIn:  sessionTokensIn,
      tokensOut: sessionTokensOut,
      // metadata may carry provider-specific cost data
      model:     m.metadata?.model ?? 'unknown',
    });
  }
});

const SLOW_TTFT_MS = 1500;

agent.on('agent-metric', (m) => {
  if (m.name === 'llm_ttft' && m.value > SLOW_TTFT_MS) {
    console.warn(`[agent] slow TTFT: ${m.value}ms turn=${m.turn} model=${m.metadata?.model}`);
    if (process.env.NODE_ENV === 'development') ui.showDevToast(`Slow TTFT (${m.value}ms)`);
  }
});

const me = room.localParticipant;

// Tool: get the user's current location (browser-only data)
me.registerRpc('current_location', async () => {
  const pos = await new Promise((resolve, reject) =>
    navigator.geolocation.getCurrentPosition(resolve, reject)
  );
  return { lat: pos.coords.latitude, lng: pos.coords.longitude };
});

// Tool: query the user's calendar (client-only auth context)
me.registerRpc('upcoming_events', async (req) => {
  const { range } = req.payload;             // { range: 'today' | 'week' }
  const events = await myCalendarAPI.list(range);
  return { events };
});

// Tool: book an appointment (needs explicit user confirmation)
me.registerRpc('book_appointment', async (req) => {
  const { date, time, title } = req.payload;
  const ok = await ui.confirmBooking({ date, time, title });
  if (!ok) return { booked: false, reason: 'user_declined' };
  const result = await myCalendarAPI.book({ date, time, title });
  return { booked: true, id: result.id };
});

// Participant exposes a confirmation handler
me.registerRpc('confirm_unmute', async (req) => {
  const choice = await ui.showDialog({
    title: 'Host is asking to unmute you',
    body:  `${req.from} would like you to speak.`,
    actions: ['Allow', 'Deny'],
  });
  return { allowed: choice === 'Allow' };
});

// Host side
async function inviteToSpeak(participantId) {
  const target = room.remoteParticipants.get(participantId);
  try {
    const { allowed } = await target.callRpc('confirm_unmute', undefined, { timeout: 30_000 });
    if (allowed) await target.requestPublishAudio();
    else         ui.toast(`${target.displayName} declined`);
  } catch (err) {
    if (err.code === 'RPC_TIMEOUT') ui.toast(`${target.displayName} didn't respond`);
  }
}

me.registerRpc('attention_check', async () => ({
  tabFocused:      !document.hidden,
  lastInteraction: Date.now() - lastUserActionAt,
}));

const results = await Promise.allSettled(
  [...room.remoteParticipants.values()]
    .filter(p => p.kind === 'human')
    .map(async p => {
      const data = await p.callRpc('attention_check', undefined, { timeout: 2_000 });
      return { id: p.id, name: p.displayName, ...data };
    })
);

const focused    = results.filter(r => r.status === 'fulfilled' && r.value.tabFocused).length;
const distracted = results.filter(r => r.status === 'fulfilled' && !r.value.tabFocused).length;
const noReply    = results.filter(r => r.status === 'rejected').length;
ui.showAttentionBar({ focused, distracted, noReply });

let inProgressMessageId = null;

room.on('agent-text', ({ participantId, text, isFinal }) => {
  const p = room.remoteParticipants.get(participantId);
  if (p?.kind !== 'agent') return;

  if (!isFinal) {
    // Streaming chunk — append to the in-progress message
    inProgressMessageId ??= ui.createStreamingMessage(p.displayName);
    ui.appendTo(inProgressMessageId, text);
  } else {
    // Final — commit and reset
    ui.finalizeMessage(inProgressMessageId, text);
    inProgressMessageId = null;
  }
});

const agents = new Map();   // id → agent participant

room.on('participant-joined', (p) => {
  if (p.kind !== 'agent') return;
  agents.set(p.id, p);

  p.on('agent-state-changed', ({ state }) => {
    ui.setAgentState(p.id, state);   // independent per agent
  });
});

room.on('participant-left', (p) => {
  agents.delete(p.id);
});

room.on('agent-text', ({ participantId, text, isFinal }) => {
  // participantId tells you WHO the text represents.
  // To know which agent PRODUCED it (when text represents a human), the agents
  // themselves should publish a per-agent attribute or use a routing convention.
  ui.appendTranscript(participantId, text, isFinal);
});