Technical Architecture: Building a Low-Latency Real-Time AI Video Dubbing & Translation Pipeline

[Client Capture] --- WebRTC Audio ---> [Ingestion Gateway] 
                                              |
                                     (Socket stream / ts chunks)
                                              v
[Voice Dubbed Output] <-- WebRTC/TTS --- [AI Orchestration Pipeline]
                                         - Whisper ASR (1-2s window)
                                         - LLM Context Translation
                                         - Neural Voice Cloning (TTS)
                                         - FFmpeg Sync / Lip-Sync
``/

---

### 2. Audio Ingestion Setup using WebRTC
To capture raw audio bytes directly from the user's microphone with absolute minimal overhead, we configure a **WebRTC PeerConnection** rather than typical HTTP POST requests. Here is the client-side TypeScript initialization:

```typescript
// client-webrtc-streamer.ts
export async function initializeWebRTCAudioStream(serverUrl: string) {
  const localStream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
  const peerConnection = new RTCPeerConnection({
    iceServers: [{ urls: "stun:stun.l.google.com:19302" }]
  });

  // Add track
  localStream.getTracks().forEach(track => peerConnection.addTrack(track, localStream));

  // Handle connection
  peerConnection.onicecandidate = (event) => {
    if (event.candidate) {
      // Send candidate to server signaling gateway
      sendCandidateToServer(event.candidate);
    }
  };

  return peerConnection;
}

# fastapi_socket_asr.py
from fastapi import FastAPI, WebSocket
import asyncio

app = FastAPI()

@app.websocket("/ws/stream-audio")
async def websocket_audio_endpoint(websocket: WebSocket):
    await websocket.accept()
    audio_buffer = bytearray()
    
    try:
        while True:
            data = await websocket.receive_bytes()
            audio_buffer.extend(data)
            
            # If buffer accumulates 2 seconds of audio (64KB at 16kHz PCM)
            if len(audio_buffer) >= 64000:
                chunk = audio_buffer[:64000]
                del audio_buffer[:64000]
                
                # Asynchronously invoke Whisper ASR inference
                text = await run_whisper_inference(chunk)
                if text.strip():
                    await websocket.send_json({"type": "ASR_RESULT", "text": text})
    except Exception as e:
        print(f"Connection closed: {e}")

# FFmpeg command to stretch cloned audio speed by 1.25x without changing the pitch
ffmpeg -i cloned_voice.wav -filter:a "atempo=1.25" -vn synced_voice.wav