8058
Ir
irvc / files
irvc / web / src / lib / voice.ts
17.7 KB
1import { useCallback, useEffect, useRef, useState } from "react"; 2import { 3 AudioPresets, 4 Room, 5 RoomEvent, 6 ScreenSharePresets, 7 Track, 8 VideoPresets, 9} from "livekit-client"; 10import type { 11 AudioCaptureOptions, 12 AudioProcessorOptions, 13 Participant, 14 ScreenShareCaptureOptions, 15 TrackProcessor, 16 TrackPublishOptions, 17} from "livekit-client"; 18import { api } from "./api"; 19import type { Settings } from "./settings"; 20import type { Channel } from "./types"; 21 22interface VoiceToken { 23 url: string; 24 token: string; 25 room: string; 26} 27 28// beep plays a short tone for call roster changes (someone joined or left the 29// call you are in) and for an incoming DM ring. Web Audio needs no asset; it 30// unlocks on the click that joined the call. 31export function beep(freq: number) { 32 const ctx = new AudioContext(); 33 ctx.resume().catch(() => {}); 34 const osc = ctx.createOscillator(); 35 const gain = ctx.createGain(); 36 gain.gain.value = 0.1; 37 osc.frequency.value = freq; 38 osc.connect(gain).connect(ctx.destination); 39 osc.start(); 40 osc.stop(ctx.currentTime + 0.12); 41 osc.onended = () => ctx.close(); 42} 43 44// dbfs computes the RMS level, in dBFS, of an analyser's latest time-domain 45// samples. Near-silence clamps toward -inf via the epsilon. 46function dbfs(analyser: AnalyserNode, buf: Float32Array<ArrayBuffer>): number { 47 analyser.getFloatTimeDomainData(buf); 48 let sum = 0; 49 for (const v of buf) sum += v * v; 50 return 20 * Math.log10(Math.sqrt(sum / buf.length) + 1e-8); 51} 52 53// A hard gate that keeps the microphone silent while its level is below the 54// user's threshold, so room noise and keyboard clatter are not transmitted. 55// The threshold lives in a ref, so the slider takes effect without republishing 56// the track. `level` reports dBFS for the settings meter. 57interface MicGate { 58 processor: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>; 59 threshold: { current: number }; 60 level: () => number; 61} 62 63function micGate(): MicGate { 64 const threshold = { current: -60 }; 65 const state = { level: -100 }; 66 let timer = 0; 67 68 const processor: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions> = { 69 name: "irvc-mic-gate", 70 async init({ track, audioContext }) { 71 clearInterval(timer); 72 const source = audioContext.createMediaStreamSource(new MediaStream([track])); 73 const analyser = audioContext.createAnalyser(); 74 analyser.fftSize = 2048; 75 const gain = audioContext.createGain(); 76 const dest = audioContext.createMediaStreamDestination(); 77 source.connect(analyser); 78 source.connect(gain); 79 gain.connect(dest); 80 processor.processedTrack = dest.stream.getAudioTracks()[0]; 81 82 const buf = new Float32Array(analyser.fftSize); 83 let hold = 0; 84 timer = window.setInterval(() => { 85 const db = dbfs(analyser, buf); 86 state.level = db; 87 const now = performance.now(); 88 if (db >= threshold.current) hold = now + 250; 89 gain.gain.value = now < hold ? 1 : 0; 90 }, 20); 91 }, 92 async restart(opts) { 93 await processor.destroy(); 94 await processor.init(opts); 95 }, 96 async destroy() { 97 clearInterval(timer); 98 timer = 0; 99 state.level = -100; 100 }, 101 }; 102 103 return { processor, threshold, level: () => state.level }; 104} 105 106// useMicPreview opens the microphone on its own so the settings meter works 107// before joining a call. It returns a dBFS level getter. Nothing is published 108// or played; the stream exists only for metering. 109export function useMicPreview(settings: Settings, enabled: boolean): () => number { 110 const level = useRef(-100); 111 const settingsRef = useRef(settings); 112 settingsRef.current = settings; 113 114 useEffect(() => { 115 if (!enabled) return; 116 let ctx: AudioContext | null = null; 117 let stream: MediaStream | null = null; 118 let raf = 0; 119 let cancelled = false; 120 (async () => { 121 try { 122 ctx = new AudioContext(); 123 const mic = await navigator.mediaDevices.getUserMedia({ 124 audio: capture(settingsRef.current), 125 }); 126 if (cancelled) { 127 mic.getTracks().forEach((t) => t.stop()); 128 ctx.close(); 129 return; 130 } 131 stream = mic; 132 ctx.resume().catch(() => {}); 133 const analyser = ctx.createAnalyser(); 134 analyser.fftSize = 2048; 135 ctx.createMediaStreamSource(mic).connect(analyser); 136 const buf = new Float32Array(analyser.fftSize); 137 const tick = () => { 138 level.current = dbfs(analyser, buf); 139 raf = requestAnimationFrame(tick); 140 }; 141 tick(); 142 } catch { 143 level.current = -100; 144 } 145 })(); 146 return () => { 147 cancelled = true; 148 cancelAnimationFrame(raf); 149 stream?.getTracks().forEach((t) => t.stop()); 150 ctx?.close(); 151 level.current = -100; 152 }; 153 // Restart on the device the user actually selected; other processing 154 // toggles shift the level only slightly and should not re-prompt. 155 // eslint-disable-next-line react-hooks/exhaustive-deps 156 }, [enabled, settings.micDeviceId]); 157 158 return useCallback(() => level.current, []); 159} 160 161// capture maps our settings to LiveKit/browser microphone constraints. 162function capture(s: Settings): AudioCaptureOptions { 163 return { 164 echoCancellation: s.echoCancellation, 165 noiseSuppression: s.noiseSuppression, 166 autoGainControl: s.autoGainControl, 167 voiceIsolation: s.voiceIsolation, 168 ...(s.micDeviceId ? { deviceId: s.micDeviceId } : {}), 169 }; 170} 171 172function publish(s: Settings) { 173 return { audioPreset: AudioPresets[s.audioPreset] }; 174} 175 176// screenShare maps the quality setting to a capture resolution and encoding. 177// The simulcast layers are ordered low to high, below the preset's own layer. 178const screenSharePresets = { 179 "720p": ScreenSharePresets.h720fps30, 180 "1080p": ScreenSharePresets.h1080fps30, 181}; 182const screenShareLayers = { 183 "720p": [VideoPresets.h180, VideoPresets.h360], 184 "1080p": [VideoPresets.h360, VideoPresets.h720], 185}; 186 187function screenShare(s: Settings): [ScreenShareCaptureOptions, TrackPublishOptions] { 188 const preset = screenSharePresets[s.screenShareQuality]; 189 return [ 190 { audio: true, resolution: preset.resolution }, 191 { 192 screenShareEncoding: preset.encoding, 193 screenShareSimulcastLayers: screenShareLayers[s.screenShareQuality], 194 }, 195 ]; 196} 197 198// A Share is a screen-share publication, ours or a peer's. LiveKit models it as 199// a track on a participant rather than a participant, so the sidebar synthesizes 200// one row per share. 201export interface Share { 202 id: string; 203 name: string; 204 self: boolean; 205 track: Track; 206} 207 208function sharesOf(room: Room): Share[] { 209 const shares: Share[] = []; 210 const collect = (p: Participant, self: boolean) => { 211 for (const pub of p.trackPublications.values()) { 212 if (pub.source === Track.Source.ScreenShare && pub.track?.kind === Track.Kind.Video) { 213 shares.push({ 214 id: `${p.identity}:${pub.trackSid}`, 215 name: p.name || p.identity, 216 self, 217 track: pub.track, 218 }); 219 } 220 } 221 }; 222 collect(room.localParticipant, true); 223 for (const p of room.remoteParticipants.values()) collect(p, false); 224 return shares; 225} 226 227// A camera publication, ours or a peer's, in a DM video call. 228export interface Camera { 229 id: string; 230 name: string; 231 self: boolean; 232 track: Track; 233} 234 235function camerasOf(room: Room): Camera[] { 236 const cameras: Camera[] = []; 237 const collect = (p: Participant, self: boolean) => { 238 for (const pub of p.trackPublications.values()) { 239 if ( 240 pub.source === Track.Source.Camera && 241 pub.track?.kind === Track.Kind.Video && 242 !pub.isMuted 243 ) { 244 cameras.push({ 245 id: `${p.identity}:${pub.trackSid}`, 246 name: p.name || p.identity, 247 self, 248 track: pub.track, 249 }); 250 } 251 } 252 }; 253 collect(room.localParticipant, true); 254 for (const p of room.remoteParticipants.values()) collect(p, false); 255 return cameras; 256} 257 258// micSignature identifies the applied capture + encoding config, so live 259// changes can be detected without republishing on every render. 260function micSignature(s: Settings): string { 261 return JSON.stringify([ 262 s.echoCancellation, 263 s.noiseSuppression, 264 s.autoGainControl, 265 s.voiceIsolation, 266 s.audioPreset, 267 s.micDeviceId, 268 ]); 269} 270 271export type VoiceStatus = "idle" | "connecting" | "live" | "error"; 272 273// VoiceState is what we tell the server about our own call: which channel we 274// are in (0 for none) and whether we can hear or be heard. It drives the mute 275// and deafen markers everyone else sees in their roster. 276export interface VoiceState { 277 channelId: number; 278 muted: boolean; 279 deafened: boolean; 280} 281 282export interface Voice { 283 channel: Channel | null; 284 status: VoiceStatus; 285 error: string; 286 peers: string[]; 287 speaking: string[]; 288 shares: Share[]; 289 cameras: Camera[]; 290 sharing: boolean; 291 cameraOn: boolean; 292 muted: boolean; 293 join: (c: Channel) => void; 294 leave: () => void; 295 toggleMute: () => void; 296 toggleCamera: () => void; 297 toggleShare: () => void; 298 setPeerVolume: (name: string, volume: number) => void; 299 micLevel: () => number; 300} 301 302// useVoice owns the single LiveKit connection for the session so it survives 303// switching panes. Chat holds it and passes it to the sidebar controls. 304// Voice settings are read from a ref so `join` always uses the latest values, 305// while live changes (device, processing, quality) are applied via effects. 306export function useVoice(settings: Settings, onVoice?: (state: VoiceState) => void): Voice { 307 const [channel, setChannel] = useState<Channel | null>(null); 308 const [status, setStatus] = useState<VoiceStatus>("idle"); 309 const [error, setError] = useState(""); 310 const [peers, setPeers] = useState<string[]>([]); 311 const [speaking, setSpeaking] = useState<string[]>([]); 312 const [shares, setShares] = useState<Share[]>([]); 313 const [cameras, setCameras] = useState<Camera[]>([]); 314 const [cameraOn, setCameraOn] = useState(false); 315 const [muted, setMuted] = useState(false); 316 const roomRef = useRef<Room | null>(null); 317 const audioRef = useRef<HTMLAudioElement[]>([]); 318 const genRef = useRef(0); 319 const readyRef = useRef(false); 320 const settingsRef = useRef(settings); 321 const mutedRef = useRef(false); 322 const appliedRef = useRef(""); 323 const gateRef = useRef<MicGate | null>(null); 324 const onVoiceRef = useRef(onVoice); 325 settingsRef.current = settings; 326 mutedRef.current = muted; 327 onVoiceRef.current = onVoice; 328 329 if (!gateRef.current) gateRef.current = micGate(); 330 const gate = gateRef.current; 331 gate.threshold.current = settings.gateThreshold; 332 const micLevel = useCallback(() => gate.level(), [gate]); 333 334 const leave = useCallback(() => { 335 genRef.current++; 336 const room = roomRef.current; 337 roomRef.current = null; 338 appliedRef.current = ""; 339 readyRef.current = false; 340 audioRef.current.forEach((el) => el.remove()); 341 audioRef.current = []; 342 room?.disconnect(); 343 setChannel(null); 344 setStatus("idle"); 345 setError(""); 346 setPeers([]); 347 setSpeaking([]); 348 setShares([]); 349 setCameras([]); 350 setCameraOn(false); 351 setMuted(false); 352 }, []); 353 354 const join = useCallback( 355 async (c: Channel) => { 356 leave(); 357 const gen = genRef.current; 358 const s = settingsRef.current; 359 setChannel(c); 360 setStatus("connecting"); 361 try { 362 const t = await api<VoiceToken>(`/channels/${c.id}/voice`, { method: "POST" }); 363 if (gen !== genRef.current) return; 364 const room = new Room({ 365 audioCaptureDefaults: capture(s), 366 publishDefaults: publish(s), 367 }); 368 roomRef.current = room; 369 const sync = () => { 370 setPeers([...room.remoteParticipants.values()].map((p) => p.name || p.identity)); 371 setShares(sharesOf(room)); 372 setCameras(camerasOf(room)); 373 }; 374 room.on(RoomEvent.ParticipantConnected, () => { 375 if (readyRef.current && settingsRef.current.joinSound) beep(440); 376 sync(); 377 }); 378 room.on(RoomEvent.ParticipantDisconnected, () => { 379 if (readyRef.current && settingsRef.current.joinSound) beep(660); 380 sync(); 381 }); 382 room.on(RoomEvent.ActiveSpeakersChanged, (speakers: Participant[]) => 383 setSpeaking(speakers.map((p) => p.name || p.identity)), 384 ); 385 room.on(RoomEvent.TrackSubscribed, (track) => { 386 if (track.kind !== Track.Kind.Audio) return; 387 const el = track.attach() as HTMLAudioElement; 388 el.muted = settingsRef.current.deafened; 389 audioRef.current.push(el); 390 }); 391 // Screen shares surface as a track on a participant, so every event that 392 // can add or drop one — publish, subscribe, unpublish — resyncs the rows. 393 for (const ev of [ 394 RoomEvent.TrackPublished, 395 RoomEvent.TrackUnpublished, 396 RoomEvent.TrackSubscribed, 397 RoomEvent.TrackUnsubscribed, 398 RoomEvent.TrackMuted, 399 RoomEvent.TrackUnmuted, 400 RoomEvent.LocalTrackPublished, 401 RoomEvent.LocalTrackUnpublished, 402 ]) { 403 room.on(ev, sync); 404 } 405 room.on(RoomEvent.Disconnected, () => { 406 if (roomRef.current === room) { 407 roomRef.current = null; 408 setStatus("error"); 409 setError("disconnected"); 410 } 411 }); 412 await room.connect(t.url, t.token); 413 if (gen !== genRef.current) { 414 room.disconnect(); 415 return; 416 } 417 // Existing participants fired ParticipantConnected during connect; only 418 // later connect/disconnect events should beep. 419 readyRef.current = true; 420 const mic = await room.localParticipant.setMicrophoneEnabled(true, capture(s), publish(s)); 421 if (mic?.audioTrack) await mic.audioTrack.setProcessor(gate.processor); 422 // A DM call is a video call: publish the camera too, falling back to 423 // audio only if the camera is unavailable or denied. 424 if (c.type === "dm") { 425 await room.localParticipant 426 .setCameraEnabled(true) 427 .then(() => setCameraOn(true)) 428 .catch(() => {}); 429 } 430 if (s.speakerDeviceId) { 431 await room.switchActiveDevice("audiooutput", s.speakerDeviceId).catch(() => {}); 432 } 433 appliedRef.current = micSignature(s); 434 setStatus("live"); 435 sync(); 436 } catch (err) { 437 if (gen !== genRef.current) return; 438 setStatus("error"); 439 setError((err as Error).message); 440 } 441 }, 442 [leave, gate], 443 ); 444 445 // Apply mic device / processing / quality changes to a live call. LiveKit 446 // only honours new options on publish, so the mic track is republished. 447 useEffect(() => { 448 const room = roomRef.current; 449 if (!room || status !== "live") return; 450 const sig = micSignature(settings); 451 if (sig === appliedRef.current) return; 452 appliedRef.current = sig; 453 (async () => { 454 const pub = room.localParticipant.getTrackPublication(Track.Source.Microphone); 455 if (pub?.track) await room.localParticipant.unpublishTrack(pub.track); 456 const p = await room.localParticipant.setMicrophoneEnabled( 457 true, 458 capture(settings), 459 publish(settings), 460 ); 461 if (p?.audioTrack) await p.audioTrack.setProcessor(gate.processor); 462 if (mutedRef.current) await p?.mute(); 463 })().catch((err) => setError((err as Error).message)); 464 }, [settings, status, gate]); 465 466 // Report our own call state whenever any of it changes: joining, leaving, 467 // muting, deafening. The server rebroadcasts it as presence. 468 useEffect(() => { 469 onVoiceRef.current?.({ 470 channelId: channel?.id ?? 0, 471 muted, 472 deafened: settings.deafened, 473 }); 474 }, [channel, muted, settings.deafened]); 475 476 // Speaker changes route through setSinkId on browsers that support it. 477 useEffect(() => { 478 const room = roomRef.current; 479 if (!room || status !== "live" || !settings.speakerDeviceId) return; 480 room.switchActiveDevice("audiooutput", settings.speakerDeviceId).catch(() => {}); 481 }, [settings.speakerDeviceId, status]); 482 483 // Deafen mutes all remote audio locally, without affecting the server. 484 useEffect(() => { 485 audioRef.current.forEach((el) => { 486 el.muted = settings.deafened; 487 }); 488 }, [settings.deafened]); 489 490 const toggleMute = useCallback(async () => { 491 const room = roomRef.current; 492 if (!room) return; 493 const next = !muted; 494 await room.localParticipant.setMicrophoneEnabled(!next); 495 setMuted(next); 496 }, [muted]); 497 498 const toggleCamera = useCallback(async () => { 499 const room = roomRef.current; 500 if (!room) return; 501 const next = !cameraOn; 502 // A denied camera stays off; do not claim it is on. 503 await room.localParticipant.setCameraEnabled(next).then( 504 () => setCameraOn(next), 505 () => setCameraOn(false), 506 ); 507 }, [cameraOn]); 508 509 const toggleShare = useCallback(async () => { 510 const room = roomRef.current; 511 if (!room) return; 512 const on = room.localParticipant.getTrackPublication(Track.Source.ScreenShare); 513 const [capture, publishOptions] = screenShare(settingsRef.current); 514 try { 515 await room.localParticipant.setScreenShareEnabled(!on, capture, publishOptions); 516 } catch (err) { 517 // Dismissing the picker rejects with NotAllowedError; that is not a fault. 518 if ((err as Error).name !== "NotAllowedError") setError((err as Error).message); 519 } 520 setShares(sharesOf(room)); 521 }, []); 522 523 const setPeerVolume = useCallback((name: string, volume: number) => { 524 const room = roomRef.current; 525 if (!room) return; 526 for (const p of room.remoteParticipants.values()) { 527 if ((p.name || p.identity) === name) p.setVolume(volume); 528 } 529 }, []); 530 531 useEffect( 532 () => () => { 533 roomRef.current?.disconnect(); 534 }, 535 [], 536 ); 537 538 return { 539 channel, 540 status, 541 error, 542 peers, 543 speaking, 544 shares, 545 cameras, 546 sharing: shares.some((s) => s.self), 547 cameraOn, 548 muted, 549 join, 550 leave, 551 toggleMute, 552 toggleCamera, 553 toggleShare, 554 setPeerVolume, 555 micLevel, 556 }; 557}