From 7bacd98a86bda5178f98e4b1b1a889da94c9689e Mon Sep 17 00:00:00 2001 From: Taran Nathan Date: Tue, 21 Jul 2026 12:58:25 -0700 Subject: [PATCH] speech recognition --- index.html | 2 + script.js | 272 ++++++- transcription-backend/.gitignore | 3 + transcription-backend/Cargo.lock | 1106 +++++++++++++++++++++++++++++ transcription-backend/Cargo.toml | 14 + transcription-backend/src/main.rs | 238 +++++++ 6 files changed, 1627 insertions(+), 8 deletions(-) create mode 100644 transcription-backend/.gitignore create mode 100644 transcription-backend/Cargo.lock create mode 100644 transcription-backend/Cargo.toml create mode 100644 transcription-backend/src/main.rs diff --git a/index.html b/index.html index e2cad11..1ee9d54 100644 --- a/index.html +++ b/index.html @@ -18,6 +18,8 @@

+

+

\ No newline at end of file diff --git a/script.js b/script.js index ec65453..c1a1991 100644 --- a/script.js +++ b/script.js @@ -6,13 +6,46 @@ let systemStream; let mixedAudioStream; let audioContext; let mixedDestination; +let transcriptionSocket; +let streamSourceNode; +let streamProcessorNode; +let streamMuteNode; +let committedTranscription = ""; +let partialTranscription = ""; const startButton = document.getElementById("btnStart"); const stopButton = document.getElementById("btnStop"); const playButton = document.getElementById("btnPlay"); const audioElement = document.getElementById("audioPlay"); +const transcriptionElement = document.getElementById("transcription"); + +const BACKEND_WS_URL = "ws://localhost:3000/ws/transcribe"; +const CHUNK_MS = 3000; +const TARGET_SAMPLE_RATE = 16000; + +function renderTranscription() { + const committed = committedTranscription.trim(); + const partial = partialTranscription.trim(); + + if (!committed && !partial) { + transcriptionElement.textContent = ""; + return; + } + + transcriptionElement.textContent = partial ? `${committed} ${partial}`.trim() : committed; +} + +function cleanupCaptureResources(closeTranscriptionSocket = true) { + if (streamProcessorNode) { + streamProcessorNode.disconnect(); + } + if (streamSourceNode) { + streamSourceNode.disconnect(); + } + if (streamMuteNode) { + streamMuteNode.disconnect(); + } -function cleanupCaptureResources() { [microphoneStream, systemStream, mixedAudioStream].forEach((stream) => { if (stream) { stream.getTracks().forEach((track) => track.stop()); @@ -23,15 +56,187 @@ function cleanupCaptureResources() { audioContext.close(); } + if (closeTranscriptionSocket && transcriptionSocket && transcriptionSocket.readyState === WebSocket.OPEN) { + transcriptionSocket.close(); + } + microphoneStream = null; systemStream = null; mixedAudioStream = null; audioContext = null; mixedDestination = null; + streamSourceNode = null; + streamProcessorNode = null; + streamMuteNode = null; + if (closeTranscriptionSocket) { + transcriptionSocket = null; + } +} + +function appendCommittedTranscription(text) { + if (!text) { + return; + } + + const trimmedText = text.trim(); + if (!trimmedText) { + return; + } + + committedTranscription = committedTranscription + ? `${committedTranscription} ${trimmedText}` + : trimmedText; + partialTranscription = ""; + renderTranscription(); +} + +function updatePartialTranscription(text) { + const trimmedText = text?.trim() || ""; + partialTranscription = trimmedText; + renderTranscription(); +} + +function extractTranscriptFromRealtimeEvent(payload) { + if (!payload || typeof payload !== "object") { + return null; + } + + if (typeof payload.error === "string") { + return { + kind: "error", + text: payload.error + }; + } + + const typeValue = typeof payload.type === "string" ? payload.type.toLowerCase() : ""; + const finalByType = + typeValue.includes("termination") || + typeValue.includes("final") || + typeValue.includes("completed"); + + const candidates = [ + payload.text, + payload.transcript, + payload.delta, + payload.partial, + payload.final, + payload.turn?.text, + payload.turn?.transcript, + payload.turn?.final, + payload.result?.text, + payload.result?.transcript + ]; + + const transcriptText = candidates.find((value) => typeof value === "string" && value.trim()); + if (!transcriptText) { + return null; + } + + return { + kind: finalByType || payload.is_final === true || payload.final === true ? "final" : "partial", + text: transcriptText + }; +} + +function downsampleToTargetSampleRate(float32Samples, inputSampleRate, targetSampleRate) { + if (targetSampleRate >= inputSampleRate) { + return float32Samples; + } + + const ratio = inputSampleRate / targetSampleRate; + const outputLength = Math.round(float32Samples.length / ratio); + const result = new Float32Array(outputLength); + + let outputIndex = 0; + let inputIndex = 0; + while (outputIndex < outputLength) { + const nextInputIndex = Math.round((outputIndex + 1) * ratio); + let total = 0; + let count = 0; + + for (let i = inputIndex; i < nextInputIndex && i < float32Samples.length; i += 1) { + total += float32Samples[i]; + count += 1; + } + + result[outputIndex] = count > 0 ? total / count : 0; + outputIndex += 1; + inputIndex = nextInputIndex; + } + + return result; +} + +function float32ToPcm16Buffer(float32Samples) { + const arrayBuffer = new ArrayBuffer(float32Samples.length * 2); + const dataView = new DataView(arrayBuffer); + + for (let i = 0; i < float32Samples.length; i += 1) { + const clamped = Math.max(-1, Math.min(1, float32Samples[i])); + const int16Value = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + dataView.setInt16(i * 2, int16Value, true); + } + + return arrayBuffer; +} + +function connectTranscriptionSocket() { + return new Promise((resolve, reject) => { + const socket = new WebSocket(BACKEND_WS_URL); + socket.binaryType = "arraybuffer"; + + socket.addEventListener("open", () => { + transcriptionSocket = socket; + resolve(socket); + }); + + socket.addEventListener("message", (event) => { + if (typeof event.data !== "string") { + return; + } + + let parsed; + try { + parsed = JSON.parse(event.data); + } catch { + return; + } + + const transcriptEvent = extractTranscriptFromRealtimeEvent(parsed); + if (!transcriptEvent) { + return; + } + + if (transcriptEvent.kind === "error") { + console.error("Realtime transcription error:", transcriptEvent.text); + return; + } + + if (transcriptEvent.kind === "final") { + appendCommittedTranscription(transcriptEvent.text); + } else { + updatePartialTranscription(transcriptEvent.text); + } + }); + + socket.addEventListener("error", (event) => { + reject(new Error(`Transcription socket error: ${event.type}`)); + }); + + socket.addEventListener("close", () => { + if (transcriptionSocket === socket) { + transcriptionSocket = null; + } + }); + }); } startButton.addEventListener("click", async () => { try { + committedTranscription = ""; + partialTranscription = ""; + renderTranscription(); + microphoneStream = await navigator.mediaDevices.getUserMedia({ audio: true }); systemStream = await navigator.mediaDevices.getDisplayMedia({ video: true, @@ -41,7 +246,7 @@ startButton.addEventListener("click", async () => { console.log(systemStream.getTracks()); if (systemStream.getAudioTracks().length === 0) { - console.error( "No screen share audio track. Microphone only." ); + console.error("No screen share audio track. Microphone only."); systemStream.getTracks().forEach((track) => track.stop()); systemStream = null; } @@ -57,7 +262,44 @@ startButton.addEventListener("click", async () => { } mixedAudioStream = mixedDestination.stream; - mediaRecorder = new MediaRecorder(mixedAudioStream); + await connectTranscriptionSocket(); + + streamSourceNode = audioContext.createMediaStreamSource(mixedAudioStream); + streamProcessorNode = audioContext.createScriptProcessor(4096, 1, 1); + streamMuteNode = audioContext.createGain(); + streamMuteNode.gain.value = 0; + + streamSourceNode.connect(streamProcessorNode); + streamProcessorNode.connect(streamMuteNode); + streamMuteNode.connect(audioContext.destination); + + streamProcessorNode.onaudioprocess = (audioProcessEvent) => { + if (!transcriptionSocket || transcriptionSocket.readyState !== WebSocket.OPEN) { + return; + } + + const channelSamples = audioProcessEvent.inputBuffer.getChannelData(0); + const downsampled = downsampleToTargetSampleRate( + channelSamples, + audioContext.sampleRate, + TARGET_SAMPLE_RATE + ); + const pcm16Buffer = float32ToPcm16Buffer(downsampled); + transcriptionSocket.send(pcm16Buffer); + }; + + const preferredMimeTypes = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/ogg;codecs=opus", + "audio/ogg" + ]; + const selectedMimeType = preferredMimeTypes.find((type) => MediaRecorder.isTypeSupported(type)); + console.log("selected mime type:", selectedMimeType); + mediaRecorder = selectedMimeType + ? new MediaRecorder(mixedAudioStream, { mimeType: selectedMimeType }) + : new MediaRecorder(mixedAudioStream); + //Array audioChunks = []; mediaRecorder.addEventListener("dataavailable", (event) => { @@ -66,9 +308,9 @@ startButton.addEventListener("click", async () => { } }); - mediaRecorder.addEventListener("stop", async () => { - const audioBlob = new Blob(audioChunks, { type: "audio/ogg" }); - console.log(await audioBlob.text()); + mediaRecorder.addEventListener("stop", () => { + const audioBlob = new Blob(audioChunks, { type: mediaRecorder.mimeType || "audio/webm" }); + console.log(audioBlob) if (recordedAudioUrl) { URL.revokeObjectURL(recordedAudioUrl); @@ -79,10 +321,20 @@ startButton.addEventListener("click", async () => { audioElement.src = recordedAudioUrl; playButton.disabled = false; - cleanupCaptureResources(); + const socketToClose = transcriptionSocket; + if (socketToClose && socketToClose.readyState === WebSocket.OPEN) { + socketToClose.send(JSON.stringify({ type: "Terminate" })); + setTimeout(() => { + if (socketToClose.readyState === WebSocket.OPEN) { + socketToClose.close(); + } + }, 1500); + } + + cleanupCaptureResources(false); }); - mediaRecorder.start(); + mediaRecorder.start(CHUNK_MS); startButton.disabled = true; stopButton.disabled = false; playButton.disabled = true; @@ -100,6 +352,10 @@ stopButton.addEventListener("click", () => { return; } + if (transcriptionSocket && transcriptionSocket.readyState === WebSocket.OPEN) { + transcriptionSocket.send(JSON.stringify({ type: "Terminate" })); + } + mediaRecorder.stop(); stopButton.disabled = true; startButton.disabled = false; diff --git a/transcription-backend/.gitignore b/transcription-backend/.gitignore new file mode 100644 index 0000000..a145785 --- /dev/null +++ b/transcription-backend/.gitignore @@ -0,0 +1,3 @@ +build +.env + diff --git a/transcription-backend/Cargo.lock b/transcription-backend/Cargo.lock new file mode 100644 index 0000000..d35c31e --- /dev/null +++ b/transcription-backend/Cargo.lock @@ -0,0 +1,1106 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "axum-macros", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "multer", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "sync_wrapper", + "tokio", + "tokio-tungstenite 0.29.0", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22053b6a34f84abc97f9129e61334f40174659a1b9bd18c970b83db6a9a6348b" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "native-tls", + "tokio", + "tokio-native-tls", + "tungstenite 0.27.0", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.29.0", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "http", + "pin-project-lite", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "transcription-backend" +version = "0.1.0" +dependencies = [ + "axum", + "dotenvy", + "futures-util", + "serde", + "serde_json", + "tokio", + "tokio-tungstenite 0.27.0", + "tower-http", +] + +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "native-tls", + "rand", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1", + "thiserror", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/transcription-backend/Cargo.toml b/transcription-backend/Cargo.toml new file mode 100644 index 0000000..c235065 --- /dev/null +++ b/transcription-backend/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "transcription-backend" +version = "0.1.0" +edition = "2024" + +[dependencies] +axum = { version = "0.8.9", features = ["macros", "multipart", "ws"] } +futures-util = "0.3.31" +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread"] } +tokio-tungstenite = { version = "0.27.0", features = ["native-tls"] } +tower-http = { version = "0.6.6", features = ["cors"] } +dotenvy = "0.15.7" diff --git a/transcription-backend/src/main.rs b/transcription-backend/src/main.rs new file mode 100644 index 0000000..92e9a00 --- /dev/null +++ b/transcription-backend/src/main.rs @@ -0,0 +1,238 @@ +use std::{env, net::SocketAddr, sync::Arc}; + +use axum::{ + Router, + extract::{ + State, + ws::{Message, WebSocket, WebSocketUpgrade}, + }, + response::IntoResponse, + routing::get, +}; +use dotenvy::dotenv; +use futures_util::{SinkExt, StreamExt}; +use serde::Serialize; +use tokio_tungstenite::{ + connect_async, + tungstenite::{Message as UpstreamMessage, client::IntoClientRequest}, +}; +use tower_http::cors::{Any, CorsLayer}; + +#[derive(Clone)] +struct AppState { + whisper_api_key: Arc, + whisper_api_base: Arc, +} + +#[derive(Serialize)] +struct HealthResponse<'a> { + status: &'a str, +} + +#[tokio::main] +async fn main() { + dotenv().ok(); + + let whisper_api_key = env::var("WHISPER_API_KEY") + .expect("WHISPER_API_KEY is required (server-side API key, starts with wai_)"); + let whisper_api_base = + env::var("WHISPER_API_URL").unwrap_or_else(|_| "https://api.whisperai.com".to_string()); + + let state = AppState { + whisper_api_key: Arc::new(whisper_api_key), + whisper_api_base: Arc::new(whisper_api_base), + }; + + let cors = CorsLayer::new() + .allow_origin(Any) + .allow_headers(Any) + .allow_methods(Any); + + let app = Router::new() + .route("/health", get(health)) + .route("/ws/transcribe", get(ws_transcribe)) + .with_state(state) + .layer(cors); + + let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); + let listener = tokio::net::TcpListener::bind(addr) + .await + .expect("failed to bind TCP listener"); + + println!("Transcription backend listening on http://{addr}"); + axum::serve(listener, app) + .await + .expect("axum server failed"); +} + +async fn health() -> impl IntoResponse { + axum::Json(HealthResponse { status: "ok" }) +} + +async fn ws_transcribe(ws: WebSocketUpgrade, State(state): State) -> impl IntoResponse { + ws.on_upgrade(move |socket| proxy_ws_to_whisper(socket, state)) +} + +fn build_upstream_ws_url(base_url: &str) -> Result { + let trimmed = base_url.trim().trim_end_matches('/'); + let without_v1 = trimmed.strip_suffix("/v1").unwrap_or(trimmed); + + let ws_base = if let Some(rest) = without_v1.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = without_v1.strip_prefix("http://") { + format!("ws://{rest}") + } else if without_v1.starts_with("wss://") || without_v1.starts_with("ws://") { + without_v1.to_string() + } else { + return Err("WHISPER_API_URL must start with http(s):// or ws(s)://".to_string()); + }; + + Ok(format!( + "{ws_base}/v1/realtime/ws?speech_model=whisperai-realtime-pro&encoding=pcm_s16le&sample_rate=16000&format_turns=true&speaker_labels=true" + )) +} + +async fn proxy_ws_to_whisper(client_socket: WebSocket, state: AppState) { + let upstream_ws_url = match build_upstream_ws_url(&state.whisper_api_base) { + Ok(url) => url, + Err(error_message) => { + let mut socket = client_socket; + let _ = socket.send(Message::Text(error_message.into())).await; + let _ = socket.close().await; + return; + } + }; + + let mut request = match upstream_ws_url.into_client_request() { + Ok(request) => request, + Err(error) => { + let mut socket = client_socket; + let _ = socket + .send(Message::Text( + format!("Failed to build upstream WS request: {error}").into(), + )) + .await; + let _ = socket.close().await; + return; + } + }; + + let auth_header = match state.whisper_api_key.parse() { + Ok(header) => header, + Err(error) => { + let mut socket = client_socket; + let _ = socket + .send(Message::Text( + format!("Invalid WHISPER_API_KEY header value: {error}").into(), + )) + .await; + let _ = socket.close().await; + return; + } + }; + request.headers_mut().insert("Authorization", auth_header); + + let (upstream_socket, _) = match connect_async(request).await { + Ok(result) => result, + Err(error) => { + let mut socket = client_socket; + let _ = socket + .send(Message::Text( + format!("Failed to connect to Whisper realtime API: {error}").into(), + )) + .await; + let _ = socket.close().await; + return; + } + }; + + let (mut client_tx, mut client_rx) = client_socket.split(); + let (mut upstream_tx, mut upstream_rx) = upstream_socket.split(); + + let client_to_upstream = async { + while let Some(message_result) = client_rx.next().await { + let message = match message_result { + Ok(message) => message, + Err(error) => { + return Err(format!("Client socket read error: {error}")); + } + }; + + let upstream_message = match message { + Message::Binary(bytes) => UpstreamMessage::Binary(bytes), + Message::Text(text) => UpstreamMessage::Text(text.to_string().into()), + Message::Ping(bytes) => UpstreamMessage::Ping(bytes), + Message::Pong(bytes) => UpstreamMessage::Pong(bytes), + Message::Close(frame) => { + let close_frame = + frame.map(|f| tokio_tungstenite::tungstenite::protocol::CloseFrame { + code: f.code.into(), + reason: f.reason.to_string().into(), + }); + let _ = upstream_tx.send(UpstreamMessage::Close(close_frame)).await; + return Ok(()); + } + }; + + if let Err(error) = upstream_tx.send(upstream_message).await { + return Err(format!("Upstream socket write error: {error}")); + } + } + + let _ = upstream_tx + .send(UpstreamMessage::Text( + "{\"type\":\"Terminate\"}".to_string().into(), + )) + .await; + let _ = upstream_tx.send(UpstreamMessage::Close(None)).await; + Ok(()) + }; + + let upstream_to_client = async { + while let Some(message_result) = upstream_rx.next().await { + let message = match message_result { + Ok(message) => message, + Err(error) => { + return Err(format!("Upstream socket read error: {error}")); + } + }; + + let client_message = match message { + UpstreamMessage::Binary(bytes) => Message::Binary(bytes), + UpstreamMessage::Text(text) => Message::Text(text.to_string().into()), + UpstreamMessage::Ping(bytes) => Message::Ping(bytes), + UpstreamMessage::Pong(bytes) => Message::Pong(bytes), + UpstreamMessage::Close(frame) => { + let close_frame = frame.map(|f| axum::extract::ws::CloseFrame { + code: f.code.into(), + reason: f.reason.to_string().into(), + }); + let _ = client_tx.send(Message::Close(close_frame)).await; + return Ok(()); + } + UpstreamMessage::Frame(_) => continue, + }; + + if let Err(error) = client_tx.send(client_message).await { + return Err(format!("Client socket write error: {error}")); + } + } + + Ok(()) + }; + + tokio::select! { + result = client_to_upstream => { + if let Err(error_message) = result { + let _ = client_tx.send(Message::Text(error_message.into())).await; + } + let _ = client_tx.close().await; + } + result = upstream_to_client => { + if let Err(error_message) = result { + let _ = client_tx.send(Message::Text(error_message.into())).await; + } + let _ = client_tx.close().await; + } + } +} -- 2.47.3