Live Interview

// InterviewWebSocketService.connect()
const wsUrl = process.env.NEXT_PUBLIC_WS_URL;
const url = `${wsUrl}/ws?sessionId=${encodeURIComponent(sessionId)}&tenantId=${encodeURIComponent(tenantId)}`;

const ws = new WebSocket(url);

ws.onopen = () => {
  startAudioRecording(ws);   // AudioStreamService.startRecording()
  fireGreeting();            // speakText() via backend /tts/synthesize
};

ws.onmessage = (event) => {
  const msg = JSON.parse(event.data);
  // AGENT_RESPONSE with audio metadata → handleAudioResponse()
  // AGENT_RESPONSE text-only → speakText() via TTS, then handleTextResponse()
  // TRANSCRIPTION → addCandidateMessage() + light STT indicator for 4 s
};

MediaStream (getUserMedia: echoCancellation + noiseSuppression + autoGainControl)
    │
    ├── AudioContext → AnalyserNode (fftSize=256) → mic waveform visualizer (never stopped)
    │
    └── MediaRecorder (audio/webm;codecs=opus OR audio/webm OR audio/mp4)
          │
          ├── Start new recorder immediately (gap-free)
          ├── After 5 s: stop old recorder → dataavailable fires with ONE complete WebM blob
          ├── Skip if blob < 100 bytes (silence)
          ├── Skip if muted (mic track disabled)
          └── arrayBuffer() → sendAudioChunk(buffer, chunkIndex)
                                  │
                                  ▼
                   WebSocket: { type: "AUDIO", data: base64, sampleRate: 16000,
                                encoding: "LINEAR16", chunkIndex, sessionId, tenantId }

Candidate Browser
    │
    ├── MediaRecorder (WebM/Opus, 5-second blobs)
    │     └── base64 → WebSocket { type: "AUDIO" }
    │
    ▼
Vert.x WebSocket Edge (port 8080)
    │     └── Kafka → audio.candidate.spoken (key=sessionId, snappy)
    │
    ▼
Agno Interviewer Agent "Maya" (port 7778)
    ├── Kafka consumer (ThreadPoolExecutor background thread)
    ├── Google Cloud Speech-to-Text (STT)
    │     └── Config: LINEAR16, en-US, enhanced model, auto-punctuation
    │           Google awaits SILENCE_TIMEOUT (~2-3 s) for complete utterance
    ├── Vertex AI Gemini 2.0 Flash (fine-tuned) — response generation with interview plan context
    │     └── Only responds if utterance >= 10 words (short answers buffered)
    │           Confusion / off-topic detection built in
    └── Gemini 2.5 Flash TTS (Vertex AI, voice: Kore), PCM L16 24 kHz mono
          └── Supports emotion tags: [encouraging], [thoughtful], [pause], [laugh], etc.
          └── Kafka → audio.agent.spoken (key=sessionId)
                  │
                  ▼
          Vert.x Kafka Consumer → WebSocket { type: "AGENT_RESPONSE", data: base64_pcm }
                  │
                  ▼
          AgentResponseHandler
              ├── pcmToAudioBuffer(): Int16 → Float32 → AudioBuffer (24kHz, zero decode overhead)
              ├── Sequential audio queue (never overlapping)
              ├── AnalyserNode → Maya waveform visualizer
              └── MediaStreamDestinationNode → combined interview recording track

// Fires once per session mount (greetingFiredRef guards against double-fire)
if (!greetingFiredRef.current) {
  greetingFiredRef.current = true;
  setTimeout(() => {
    responseHandlerRef.current
      ?.speakText(
        "Hello! Welcome to your AI interview. I'm your interviewer today. " +
          "Please take a moment to get comfortable, then tell me about yourself " +
          "and what excites you about this role.",
      )
      .catch(() => {});
  }, 800);
}

// Starts when camera + WebSocket + mic recording are all active (isRecording)
const tracks = [
  ...cameraStream.getVideoTracks(),      // camera from getUserMedia
  ...micStream.getAudioTracks(),          // mic from AudioStreamService.getMicStream()
  ...(agentStream?.getAudioTracks() ?? []), // TTS from AgentResponseHandler.getAgentAudioStream()
];

const combined = new MediaStream(tracks);
const recorder = new MediaRecorder(combined, {
  mimeType: 'video/webm;codecs=vp8,opus',
  videoBitsPerSecond: 500_000,
  audioBitsPerSecond: 64_000,
});

recorder.ondataavailable = (e) => {
  // Sends as { type: "VIDEO", data: base64 }
  wsService.sendVideoChunk(await e.data.arrayBuffer());
};

recorder.start(30_000); // One chunk every 30 seconds

// FaceDetector runs on a canvas snapshot of the video element
const detector = new FaceDetector({ maxDetectedFaces: 5, fastMode: true });

setInterval(async () => {
  ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
  const faces = await detector.detect(canvas);

  if (faces.length === 0) recordViolation('No face detected...');
  else if (faces.length >= 2) recordViolation(`Multiple faces detected (${faces.length})...`);
}, 5000);

// InterviewWebSocketService — linear backoff: 1s, 2s, 3s, 4s, 5s
private reconnect(): void {
  this.reconnectAttempts++;
  const delay = this.reconnectDelay * this.reconnectAttempts; // 1000 ms * attempt
  setTimeout(() => this.connect().catch(() => {}), delay);
}