xonecas.io
8058
Ir

irvc / files

irvc / web / src / lib / voice.ts

17.7 KB
1import { useCallback, useEffect, useRef, useState } from "react";
2import {
3  AudioPresets,
4  Room,
5  RoomEvent,
6  ScreenSharePresets,
7  Track,
8  VideoPresets,
9} from "livekit-client";
10import type {
11  AudioCaptureOptions,
12  AudioProcessorOptions,
13  Participant,
14  ScreenShareCaptureOptions,
15  TrackProcessor,
16  TrackPublishOptions,
17} from "livekit-client";
18import { api } from "./api";
19import type { Settings } from "./settings";
20import type { Channel } from "./types";
21
22interface VoiceToken {
23  url: string;
24  token: string;
25  room: string;
26}
27
28// beep plays a short tone for call roster changes (someone joined or left the
29// call you are in) and for an incoming DM ring. Web Audio needs no asset; it
30// unlocks on the click that joined the call.
31export function beep(freq: number) {
32  const ctx = new AudioContext();
33  ctx.resume().catch(() => {});
34  const osc = ctx.createOscillator();
35  const gain = ctx.createGain();
36  gain.gain.value = 0.1;
37  osc.frequency.value = freq;
38  osc.connect(gain).connect(ctx.destination);
39  osc.start();
40  osc.stop(ctx.currentTime + 0.12);
41  osc.onended = () => ctx.close();
42}
43
44// dbfs computes the RMS level, in dBFS, of an analyser's latest time-domain
45// samples. Near-silence clamps toward -inf via the epsilon.
46function dbfs(analyser: AnalyserNode, buf: Float32Array<ArrayBuffer>): number {
47  analyser.getFloatTimeDomainData(buf);
48  let sum = 0;
49  for (const v of buf) sum += v * v;
50  return 20 * Math.log10(Math.sqrt(sum / buf.length) + 1e-8);
51}
52
53// A hard gate that keeps the microphone silent while its level is below the
54// user's threshold, so room noise and keyboard clatter are not transmitted.
55// The threshold lives in a ref, so the slider takes effect without republishing
56// the track. `level` reports dBFS for the settings meter.
57interface MicGate {
58  processor: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>;
59  threshold: { current: number };
60  level: () => number;
61}
62
63function micGate(): MicGate {
64  const threshold = { current: -60 };
65  const state = { level: -100 };
66  let timer = 0;
67
68  const processor: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions> = {
69    name: "irvc-mic-gate",
70    async init({ track, audioContext }) {
71      clearInterval(timer);
72      const source = audioContext.createMediaStreamSource(new MediaStream([track]));
73      const analyser = audioContext.createAnalyser();
74      analyser.fftSize = 2048;
75      const gain = audioContext.createGain();
76      const dest = audioContext.createMediaStreamDestination();
77      source.connect(analyser);
78      source.connect(gain);
79      gain.connect(dest);
80      processor.processedTrack = dest.stream.getAudioTracks()[0];
81
82      const buf = new Float32Array(analyser.fftSize);
83      let hold = 0;
84      timer = window.setInterval(() => {
85        const db = dbfs(analyser, buf);
86        state.level = db;
87        const now = performance.now();
88        if (db >= threshold.current) hold = now + 250;
89        gain.gain.value = now < hold ? 1 : 0;
90      }, 20);
91    },
92    async restart(opts) {
93      await processor.destroy();
94      await processor.init(opts);
95    },
96    async destroy() {
97      clearInterval(timer);
98      timer = 0;
99      state.level = -100;
100    },
101  };
102
103  return { processor, threshold, level: () => state.level };
104}
105
106// useMicPreview opens the microphone on its own so the settings meter works
107// before joining a call. It returns a dBFS level getter. Nothing is published
108// or played; the stream exists only for metering.
109export function useMicPreview(settings: Settings, enabled: boolean): () => number {
110  const level = useRef(-100);
111  const settingsRef = useRef(settings);
112  settingsRef.current = settings;
113
114  useEffect(() => {
115    if (!enabled) return;
116    let ctx: AudioContext | null = null;
117    let stream: MediaStream | null = null;
118    let raf = 0;
119    let cancelled = false;
120    (async () => {
121      try {
122        ctx = new AudioContext();
123        const mic = await navigator.mediaDevices.getUserMedia({
124          audio: capture(settingsRef.current),
125        });
126        if (cancelled) {
127          mic.getTracks().forEach((t) => t.stop());
128          ctx.close();
129          return;
130        }
131        stream = mic;
132        ctx.resume().catch(() => {});
133        const analyser = ctx.createAnalyser();
134        analyser.fftSize = 2048;
135        ctx.createMediaStreamSource(mic).connect(analyser);
136        const buf = new Float32Array(analyser.fftSize);
137        const tick = () => {
138          level.current = dbfs(analyser, buf);
139          raf = requestAnimationFrame(tick);
140        };
141        tick();
142      } catch {
143        level.current = -100;
144      }
145    })();
146    return () => {
147      cancelled = true;
148      cancelAnimationFrame(raf);
149      stream?.getTracks().forEach((t) => t.stop());
150      ctx?.close();
151      level.current = -100;
152    };
153    // Restart on the device the user actually selected; other processing
154    // toggles shift the level only slightly and should not re-prompt.
155    // eslint-disable-next-line react-hooks/exhaustive-deps
156  }, [enabled, settings.micDeviceId]);
157
158  return useCallback(() => level.current, []);
159}
160
161// capture maps our settings to LiveKit/browser microphone constraints.
162function capture(s: Settings): AudioCaptureOptions {
163  return {
164    echoCancellation: s.echoCancellation,
165    noiseSuppression: s.noiseSuppression,
166    autoGainControl: s.autoGainControl,
167    voiceIsolation: s.voiceIsolation,
168    ...(s.micDeviceId ? { deviceId: s.micDeviceId } : {}),
169  };
170}
171
172function publish(s: Settings) {
173  return { audioPreset: AudioPresets[s.audioPreset] };
174}
175
176// screenShare maps the quality setting to a capture resolution and encoding.
177// The simulcast layers are ordered low to high, below the preset's own layer.
178const screenSharePresets = {
179  "720p": ScreenSharePresets.h720fps30,
180  "1080p": ScreenSharePresets.h1080fps30,
181};
182const screenShareLayers = {
183  "720p": [VideoPresets.h180, VideoPresets.h360],
184  "1080p": [VideoPresets.h360, VideoPresets.h720],
185};
186
187function screenShare(s: Settings): [ScreenShareCaptureOptions, TrackPublishOptions] {
188  const preset = screenSharePresets[s.screenShareQuality];
189  return [
190    { audio: true, resolution: preset.resolution },
191    {
192      screenShareEncoding: preset.encoding,
193      screenShareSimulcastLayers: screenShareLayers[s.screenShareQuality],
194    },
195  ];
196}
197
198// A Share is a screen-share publication, ours or a peer's. LiveKit models it as
199// a track on a participant rather than a participant, so the sidebar synthesizes
200// one row per share.
201export interface Share {
202  id: string;
203  name: string;
204  self: boolean;
205  track: Track;
206}
207
208function sharesOf(room: Room): Share[] {
209  const shares: Share[] = [];
210  const collect = (p: Participant, self: boolean) => {
211    for (const pub of p.trackPublications.values()) {
212      if (pub.source === Track.Source.ScreenShare && pub.track?.kind === Track.Kind.Video) {
213        shares.push({
214          id: `${p.identity}:${pub.trackSid}`,
215          name: p.name || p.identity,
216          self,
217          track: pub.track,
218        });
219      }
220    }
221  };
222  collect(room.localParticipant, true);
223  for (const p of room.remoteParticipants.values()) collect(p, false);
224  return shares;
225}
226
227// A camera publication, ours or a peer's, in a DM video call.
228export interface Camera {
229  id: string;
230  name: string;
231  self: boolean;
232  track: Track;
233}
234
235function camerasOf(room: Room): Camera[] {
236  const cameras: Camera[] = [];
237  const collect = (p: Participant, self: boolean) => {
238    for (const pub of p.trackPublications.values()) {
239      if (
240        pub.source === Track.Source.Camera &&
241        pub.track?.kind === Track.Kind.Video &&
242        !pub.isMuted
243      ) {
244        cameras.push({
245          id: `${p.identity}:${pub.trackSid}`,
246          name: p.name || p.identity,
247          self,
248          track: pub.track,
249        });
250      }
251    }
252  };
253  collect(room.localParticipant, true);
254  for (const p of room.remoteParticipants.values()) collect(p, false);
255  return cameras;
256}
257
258// micSignature identifies the applied capture + encoding config, so live
259// changes can be detected without republishing on every render.
260function micSignature(s: Settings): string {
261  return JSON.stringify([
262    s.echoCancellation,
263    s.noiseSuppression,
264    s.autoGainControl,
265    s.voiceIsolation,
266    s.audioPreset,
267    s.micDeviceId,
268  ]);
269}
270
271export type VoiceStatus = "idle" | "connecting" | "live" | "error";
272
273// VoiceState is what we tell the server about our own call: which channel we
274// are in (0 for none) and whether we can hear or be heard. It drives the mute
275// and deafen markers everyone else sees in their roster.
276export interface VoiceState {
277  channelId: number;
278  muted: boolean;
279  deafened: boolean;
280}
281
282export interface Voice {
283  channel: Channel | null;
284  status: VoiceStatus;
285  error: string;
286  peers: string[];
287  speaking: string[];
288  shares: Share[];
289  cameras: Camera[];
290  sharing: boolean;
291  cameraOn: boolean;
292  muted: boolean;
293  join: (c: Channel) => void;
294  leave: () => void;
295  toggleMute: () => void;
296  toggleCamera: () => void;
297  toggleShare: () => void;
298  setPeerVolume: (name: string, volume: number) => void;
299  micLevel: () => number;
300}
301
302// useVoice owns the single LiveKit connection for the session so it survives
303// switching panes. Chat holds it and passes it to the sidebar controls.
304// Voice settings are read from a ref so `join` always uses the latest values,
305// while live changes (device, processing, quality) are applied via effects.
306export function useVoice(settings: Settings, onVoice?: (state: VoiceState) => void): Voice {
307  const [channel, setChannel] = useState<Channel | null>(null);
308  const [status, setStatus] = useState<VoiceStatus>("idle");
309  const [error, setError] = useState("");
310  const [peers, setPeers] = useState<string[]>([]);
311  const [speaking, setSpeaking] = useState<string[]>([]);
312  const [shares, setShares] = useState<Share[]>([]);
313  const [cameras, setCameras] = useState<Camera[]>([]);
314  const [cameraOn, setCameraOn] = useState(false);
315  const [muted, setMuted] = useState(false);
316  const roomRef = useRef<Room | null>(null);
317  const audioRef = useRef<HTMLAudioElement[]>([]);
318  const genRef = useRef(0);
319  const readyRef = useRef(false);
320  const settingsRef = useRef(settings);
321  const mutedRef = useRef(false);
322  const appliedRef = useRef("");
323  const gateRef = useRef<MicGate | null>(null);
324  const onVoiceRef = useRef(onVoice);
325  settingsRef.current = settings;
326  mutedRef.current = muted;
327  onVoiceRef.current = onVoice;
328
329  if (!gateRef.current) gateRef.current = micGate();
330  const gate = gateRef.current;
331  gate.threshold.current = settings.gateThreshold;
332  const micLevel = useCallback(() => gate.level(), [gate]);
333
334  const leave = useCallback(() => {
335    genRef.current++;
336    const room = roomRef.current;
337    roomRef.current = null;
338    appliedRef.current = "";
339    readyRef.current = false;
340    audioRef.current.forEach((el) => el.remove());
341    audioRef.current = [];
342    room?.disconnect();
343    setChannel(null);
344    setStatus("idle");
345    setError("");
346    setPeers([]);
347    setSpeaking([]);
348    setShares([]);
349    setCameras([]);
350    setCameraOn(false);
351    setMuted(false);
352  }, []);
353
354  const join = useCallback(
355    async (c: Channel) => {
356      leave();
357      const gen = genRef.current;
358      const s = settingsRef.current;
359      setChannel(c);
360      setStatus("connecting");
361      try {
362        const t = await api<VoiceToken>(`/channels/${c.id}/voice`, { method: "POST" });
363        if (gen !== genRef.current) return;
364        const room = new Room({
365          audioCaptureDefaults: capture(s),
366          publishDefaults: publish(s),
367        });
368        roomRef.current = room;
369        const sync = () => {
370          setPeers([...room.remoteParticipants.values()].map((p) => p.name || p.identity));
371          setShares(sharesOf(room));
372          setCameras(camerasOf(room));
373        };
374        room.on(RoomEvent.ParticipantConnected, () => {
375          if (readyRef.current && settingsRef.current.joinSound) beep(440);
376          sync();
377        });
378        room.on(RoomEvent.ParticipantDisconnected, () => {
379          if (readyRef.current && settingsRef.current.joinSound) beep(660);
380          sync();
381        });
382        room.on(RoomEvent.ActiveSpeakersChanged, (speakers: Participant[]) =>
383          setSpeaking(speakers.map((p) => p.name || p.identity)),
384        );
385        room.on(RoomEvent.TrackSubscribed, (track) => {
386          if (track.kind !== Track.Kind.Audio) return;
387          const el = track.attach() as HTMLAudioElement;
388          el.muted = settingsRef.current.deafened;
389          audioRef.current.push(el);
390        });
391        // Screen shares surface as a track on a participant, so every event that
392        // can add or drop one — publish, subscribe, unpublish — resyncs the rows.
393        for (const ev of [
394          RoomEvent.TrackPublished,
395          RoomEvent.TrackUnpublished,
396          RoomEvent.TrackSubscribed,
397          RoomEvent.TrackUnsubscribed,
398          RoomEvent.TrackMuted,
399          RoomEvent.TrackUnmuted,
400          RoomEvent.LocalTrackPublished,
401          RoomEvent.LocalTrackUnpublished,
402        ]) {
403          room.on(ev, sync);
404        }
405        room.on(RoomEvent.Disconnected, () => {
406          if (roomRef.current === room) {
407            roomRef.current = null;
408            setStatus("error");
409            setError("disconnected");
410          }
411        });
412        await room.connect(t.url, t.token);
413        if (gen !== genRef.current) {
414          room.disconnect();
415          return;
416        }
417        // Existing participants fired ParticipantConnected during connect; only
418        // later connect/disconnect events should beep.
419        readyRef.current = true;
420        const mic = await room.localParticipant.setMicrophoneEnabled(true, capture(s), publish(s));
421        if (mic?.audioTrack) await mic.audioTrack.setProcessor(gate.processor);
422        // A DM call is a video call: publish the camera too, falling back to
423        // audio only if the camera is unavailable or denied.
424        if (c.type === "dm") {
425          await room.localParticipant
426            .setCameraEnabled(true)
427            .then(() => setCameraOn(true))
428            .catch(() => {});
429        }
430        if (s.speakerDeviceId) {
431          await room.switchActiveDevice("audiooutput", s.speakerDeviceId).catch(() => {});
432        }
433        appliedRef.current = micSignature(s);
434        setStatus("live");
435        sync();
436      } catch (err) {
437        if (gen !== genRef.current) return;
438        setStatus("error");
439        setError((err as Error).message);
440      }
441    },
442    [leave, gate],
443  );
444
445  // Apply mic device / processing / quality changes to a live call. LiveKit
446  // only honours new options on publish, so the mic track is republished.
447  useEffect(() => {
448    const room = roomRef.current;
449    if (!room || status !== "live") return;
450    const sig = micSignature(settings);
451    if (sig === appliedRef.current) return;
452    appliedRef.current = sig;
453    (async () => {
454      const pub = room.localParticipant.getTrackPublication(Track.Source.Microphone);
455      if (pub?.track) await room.localParticipant.unpublishTrack(pub.track);
456      const p = await room.localParticipant.setMicrophoneEnabled(
457        true,
458        capture(settings),
459        publish(settings),
460      );
461      if (p?.audioTrack) await p.audioTrack.setProcessor(gate.processor);
462      if (mutedRef.current) await p?.mute();
463    })().catch((err) => setError((err as Error).message));
464  }, [settings, status, gate]);
465
466  // Report our own call state whenever any of it changes: joining, leaving,
467  // muting, deafening. The server rebroadcasts it as presence.
468  useEffect(() => {
469    onVoiceRef.current?.({
470      channelId: channel?.id ?? 0,
471      muted,
472      deafened: settings.deafened,
473    });
474  }, [channel, muted, settings.deafened]);
475
476  // Speaker changes route through setSinkId on browsers that support it.
477  useEffect(() => {
478    const room = roomRef.current;
479    if (!room || status !== "live" || !settings.speakerDeviceId) return;
480    room.switchActiveDevice("audiooutput", settings.speakerDeviceId).catch(() => {});
481  }, [settings.speakerDeviceId, status]);
482
483  // Deafen mutes all remote audio locally, without affecting the server.
484  useEffect(() => {
485    audioRef.current.forEach((el) => {
486      el.muted = settings.deafened;
487    });
488  }, [settings.deafened]);
489
490  const toggleMute = useCallback(async () => {
491    const room = roomRef.current;
492    if (!room) return;
493    const next = !muted;
494    await room.localParticipant.setMicrophoneEnabled(!next);
495    setMuted(next);
496  }, [muted]);
497
498  const toggleCamera = useCallback(async () => {
499    const room = roomRef.current;
500    if (!room) return;
501    const next = !cameraOn;
502    // A denied camera stays off; do not claim it is on.
503    await room.localParticipant.setCameraEnabled(next).then(
504      () => setCameraOn(next),
505      () => setCameraOn(false),
506    );
507  }, [cameraOn]);
508
509  const toggleShare = useCallback(async () => {
510    const room = roomRef.current;
511    if (!room) return;
512    const on = room.localParticipant.getTrackPublication(Track.Source.ScreenShare);
513    const [capture, publishOptions] = screenShare(settingsRef.current);
514    try {
515      await room.localParticipant.setScreenShareEnabled(!on, capture, publishOptions);
516    } catch (err) {
517      // Dismissing the picker rejects with NotAllowedError; that is not a fault.
518      if ((err as Error).name !== "NotAllowedError") setError((err as Error).message);
519    }
520    setShares(sharesOf(room));
521  }, []);
522
523  const setPeerVolume = useCallback((name: string, volume: number) => {
524    const room = roomRef.current;
525    if (!room) return;
526    for (const p of room.remoteParticipants.values()) {
527      if ((p.name || p.identity) === name) p.setVolume(volume);
528    }
529  }, []);
530
531  useEffect(
532    () => () => {
533      roomRef.current?.disconnect();
534    },
535    [],
536  );
537
538  return {
539    channel,
540    status,
541    error,
542    peers,
543    speaking,
544    shares,
545    cameras,
546    sharing: shares.some((s) => s.self),
547    cameraOn,
548    muted,
549    join,
550    leave,
551    toggleMute,
552    toggleCamera,
553    toggleShare,
554    setPeerVolume,
555    micLevel,
556  };
557}