+Audio Stream Combining:
+
+```javascript
+microphoneStream = await navigator.mediaDevices.getUserMedia({
+ audio: true,
+});
+systemStream = await navigator.mediaDevices.getDisplayMedia({
+ video: true,
+ audio: true,
+});
+
+if (systemStream.getAudioTracks().length === 0) {
+ console.error("No screen share audio track. Microphone only.");
+ systemStream.getTracks().forEach((track) => track.stop());
+ systemStream = null;
+}
+
+audioContext = new AudioContext();
+mixedDestination = audioContext.createMediaStreamDestination();
+
+const micSource = audioContext.createMediaStreamSource(microphoneStream);
+micSource.connect(mixedDestination);
+if (systemStream) {
+ const systemSource = audioContext.createMediaStreamSource(systemStream);
+ systemSource.connect(mixedDestination);
+}
+
+mixedAudioStream = mixedDestination.stream;
+```
+
+
+=== Push audio on new audio available:
+```javascript
+audioChunks = [];
+
+mediaRecorder.addEventListener("dataavailable", (event) => {
+ if (event.data.size > 0) {
+ audioChunks.push(event.data);
+ }
+});
+```
+
+#pagebreak()
+=== Handle WebSocket:
+```javascript
+const socket = new WebSocket(BACKEND_WS_URL);
+socket.binaryType = "arraybuffer";
+
+socket.addEventListener("open", () => {
+ transcriptionSocket = socket;
+ resolve(socket);
+});
+
+socket.addEventListener("message", (event) => {
+ if (typeof event.data !== "string") {
+ return;
+ }
+
+ let parsed;
+ try {
+ parsed = JSON.parse(event.data);
+ } catch {
+ return;
+ }
+
+ if (typeof parsed.error === "string") {
+ console.error("transcription error:", parsed.error);
+ return;
+ }
+
+ if (parsed.type === "Turn") {
+ handleTurnEvent(parsed);
+ } else if (parsed.type === "SpeakerRevision") {
+ handleSpeakerRevisionEvent(parsed);
+ }
+});
+```
+
+
+=== Sending Audio across WebSocket:
+```javascript
+streamProcessorNode.onaudioprocess = (audioProcessEvent) => {
+ if (
+ !transcriptionSocket ||
+ transcriptionSocket.readyState !== WebSocket.OPEN
+ ) {
+ return;
+ }
+
+ const channelSamples = audioProcessEvent.inputBuffer.getChannelData(0);
+ const downsampled = downsampleToTargetSampleRate(
+ channelSamples,
+ audioContext.sampleRate,
+ TARGET_SAMPLE_RATE,
+ );
+ const buffer = float32ToPcm16Buffer(downsampled);
+ transcriptionSocket.send(buffer);
+};
+```
+
+== Backend WebSocket Pipe
+
+=== Router:
+```rust
+let state = AppState {
+ whisper_api_key: Arc::new(whisper_api_key),
+ whisper_api_base: Arc::new(whisper_api_base),
+};
+
+let cors = CorsLayer::new()
+ .allow_origin(Any)
+ .allow_headers(Any)
+ .allow_methods(Any);
+
+let app = Router::new()
+ .route("/ws/transcribe", get(ws_transcribe))
+ .with_state(state)
+ .layer(cors);
+```
+
+=== Websocket Handling (upstream to client is omitted because it is basically the same code, just flipped):
+#codly(skips: ((7,56),(45,86)))
+```rust
+async fn ws_transcribe(ws: WebSocketUpgrade, State(state): State<AppState>) -> impl IntoResponse {
+ ws.on_upgrade(move |socket| proxy_ws_to_whisper(socket, state))
+}
+
+
+async fn proxy_ws_to_whisper(client_socket: WebSocket, state: AppState) {
+ let client_to_upstream = async {
+ while let Some(message_result) = client_rx.next().await {
+ let message = match message_result {
+ Ok(message) => message,
+ Err(error) => {
+ return Err(format!("Client socket read error: {error}"));
+ }
+ };
+
+ let upstream_message = match message {
+ Message::Binary(bytes) => UpstreamMessage::Binary(bytes),
+ Message::Text(text) => UpstreamMessage::Text(text.to_string().into()),
+ Message::Ping(bytes) => UpstreamMessage::Ping(bytes),
+ Message::Pong(bytes) => UpstreamMessage::Pong(bytes),
+ Message::Close(frame) => {
+ let close_frame =
+ frame.map(|f| tokio_tungstenite::tungstenite::protocol::CloseFrame {
+ code: f.code.into(),
+ reason: f.reason.to_string().into(),
+ });
+ let _ = upstream_tx.send(UpstreamMessage::Close(close_frame)).await;
+ return Ok(());
+ }
+ };
+
+ if let Err(error) = upstream_tx.send(upstream_message).await {
+ return Err(format!("Upstream socket write error: {error}"));
+ }
+ }
+
+ let _ = upstream_tx
+ .send(UpstreamMessage::Text(
+ "{\"type\":\"Terminate\"}".to_string().into(),
+ ))
+ .await;
+ let _ = upstream_tx.send(UpstreamMessage::Close(None)).await;
+ Ok(())
+ };
+}
+```