xonecas.io
4828
Sr

stream-together / commit b87bab2

Adds AI Noise supression

To be able to expose `node_modules/agora-extension-ai-denoiser/external public/external`
to the client, the folder is copied over to public at build time.

sean2024-06-28 b87bab281865f9655adf3f2e10f000813154b9f2 parent e9e1dcb

7 files changed

app/[slug]/host/page.tsx+3−10
@@ -1,10 +1,10 @@
11 "use client";
22
33 import dynamic from "next/dynamic";
4-import { useCallback, useEffect, useState } from "react";
4+import { useEffect, useState } from "react";
55 import { FaRegCopy as CopyIcon } from "react-icons/fa";
66 import { IoMdCheckmark as CheckmarkIcon } from "react-icons/io";
7-import { useCopyToClipboard, useLocalStorage } from "usehooks-ts";
7+import { useCopyToClipboard } from "usehooks-ts";
88
99 const AgoraProvider = dynamic(() => import("@/components/agora-provider"), {
1010 ssr: false,
@@ -14,14 +14,7 @@ const AgoraHost = dynamic(() => import("@/components/agora-host-video"), {
1414 ssr: false,
1515 });
1616
17-export default function StreamSetupPage({
18- params,
19-}: {
20- params: { slug: string };
21-}) {
22- const [appId] = useLocalStorage("app-id", "");
23- const [token] = useLocalStorage("token", "");
24- const [channel] = useLocalStorage("channel", "");
17+export default function StreamSetupPage() {
2518 const [currentUrl, setCurrentUrl] = useState("");
2619 const [copied, copy] = useCopyToClipboard();
2720
components/agora-host-video.tsx+54−47
@@ -19,6 +19,8 @@ import { GoGear as GearIcon } from "react-icons/go";
1919 import { IoVideocamOutline as VideoOnIcon } from "react-icons/io5";
2020 import { IoVideocamOffOutline as VideoOffIcon } from "react-icons/io5";
2121
22+import useNoiseReduction from "@/hooks/agora-ai-noise-reduction";
23+
2224 import { Modal } from "./modal";
2325 import PrimaryButton from "./primary-button";
2426 import SecondaryButton from "./secondary-button";
@@ -51,11 +53,13 @@ export default function AgoraHost() {
5153 getCameras();
5254 }, []);
5355
54- const { localMicrophoneTrack } = useLocalMicrophoneTrack(audioStarted);
5556 const { localCameraTrack } = useLocalCameraTrack(videoStarted, {
5657 cameraId: selectedCameraId,
5758 });
5859
60+ const { localMicrophoneTrack } = useLocalMicrophoneTrack(audioStarted);
61+ useNoiseReduction({ localMicrophoneTrack });
62+
5963 const isConnected = useIsConnected();
6064
6165 const handleStarCall = useCallback(() => {
@@ -133,7 +137,7 @@ export default function AgoraHost() {
133137 ) : callStarted ? (
134138 <div>Loading...</div>
135139 ) : (
136- <div className="text-center">
140+ <div className="flex flex-row justify-center gap-4">
137141 <button onClick={handleStarCall} title="Go live!">
138142 <PrimaryButton>
139143 <div className="flex flex-row items-center px-2 gap-2">
@@ -142,6 +146,54 @@ export default function AgoraHost() {
142146 </div>
143147 </PrimaryButton>
144148 </button>
149+ <Modal
150+ openButton={
151+ <button
152+ onClick={() => setShowSettingsModal(!showSettingsModal)}
153+ >
154+ <SecondaryButton>
155+ <div className="py-2 px-4 flex flex-row gap-2 items-center">
156+ <GearIcon size="2em" />
157+ <p>Settings</p>
158+ </div>
159+ </SecondaryButton>
160+ </button>
161+ }
162+ showModal={showSettingsModal}
163+ >
164+ <div className="p-4 bg-stone-800 rounded-2xl flex flex-col gap-4">
165+ <div>
166+ <h4 className="font-bold pb-4">
167+ Select your video device:
168+ </h4>
169+ {availableCameras.map((c) => {
170+ return (
171+ <label key={c.id} className="flex flex-row gap-2">
172+ <input
173+ type="radio"
174+ name="selected-camera"
175+ value={c.id}
176+ checked={c.id === selectedCameraId}
177+ onChange={(e) => {
178+ const id = e.target.value;
179+ setSelectedCameraId(id);
180+ }}
181+ />
182+ <div>{c.label}</div>
183+ </label>
184+ );
185+ })}
186+ </div>
187+
188+ <div className="text-center">
189+ <button onClick={() => setShowSettingsModal(false)}>
190+ <PrimaryButton>
191+ <p className="px-4 py-2">Close</p>
192+ </PrimaryButton>
193+ </button>
194+ </div>
195+ </div>
196+ </Modal>
145197 </div>
146198 )}
147199 </div>
@@ -161,51 +213,6 @@ export default function AgoraHost() {
161213 </div>
162214 );
163215 })}
164-
165- <Modal
166- openButton={
167- <button onClick={() => setShowSettingsModal(!showSettingsModal)}>
168- <SecondaryButton>
169- <div className="py-2 px-4 flex flex-row gap-2 items-center">
170- <GearIcon size="2em" />
171- <p>Settings</p>
172- </div>
173- </SecondaryButton>
174- </button>
175- }
176- showModal={showSettingsModal}
177- >
178- <div className="p-4 bg-stone-800 rounded-2xl flex flex-col gap-4">
179- <div>
180- <h4 className="font-bold pb-4">Select your video device:</h4>
181- {availableCameras.map((c) => {
182- return (
183- <label key={c.id} className="flex flex-row gap-2">
184- <input
185- type="radio"
186- name="selected-camera"
187- value={c.id}
188- checked={c.id === selectedCameraId}
189- onChange={(e) => {
190- const id = e.target.value;
191- setSelectedCameraId(id);
192- }}
193- />
194- <div>{c.label}</div>
195- </label>
196- );
197- })}
198- </div>
199-
200- <div className="text-center">
201- <button onClick={() => setShowSettingsModal(false)}>
202- <PrimaryButton>
203- <p className="px-4 py-2">Close</p>
204- </PrimaryButton>
205- </button>
206- </div>
207- </div>
208- </Modal>
209216 </div>
210217 );
211218 }
hooks/agora-ai-noise-reduction.ts+53−0
@@ -0,0 +1,53 @@
1+"use client";
2+
3+import {
4+ AIDenoiserExtension,
5+ IAIDenoiserProcessor,
6+} from "agora-extension-ai-denoiser";
7+import AgoraRTC, { IMicrophoneAudioTrack } from "agora-rtc-react";
8+import { useEffect, useRef } from "react";
9+
10+export default function useNoiseReduction({
11+ localMicrophoneTrack,
12+}: {
13+ localMicrophoneTrack: IMicrophoneAudioTrack | null;
14+}) {
15+ const extension = useRef(
16+ new AIDenoiserExtension({
17+ assetsPath: "/external",
18+ }),
19+ );
20+ const processor = useRef<IAIDenoiserProcessor>();
21+
22+ useEffect(() => {
23+ const initializeExtension = async () => {
24+ AgoraRTC.registerExtensions([extension.current]);
25+ if (!extension.current.checkCompatibility()) {
26+ console.error("Your browser does not support AI Denoiser!");
27+ return;
28+ }
29+
30+ if (localMicrophoneTrack) {
31+ try {
32+ processor.current = extension.current.createProcessor();
33+ localMicrophoneTrack
34+ .pipe(processor.current)
35+ .pipe(localMicrophoneTrack.processorDestination);
36+ await processor.current.enable();
37+ } catch (error) {
38+ console.error("Failed to apply ai denoiser.");
39+ }
40+ }
41+ };
42+ void initializeExtension();
43+
44+ return () => {
45+ const disableExtension = async () => {
46+ processor.current?.unpipe();
47+ localMicrophoneTrack?.unpipe();
48+ await processor.current?.disable();
49+ };
50+ void disableExtension();
51+ };
52+ }, [localMicrophoneTrack]);
53+}
package-lock.json+10−0
@@ -8,6 +8,7 @@
88 "name": "stream-together",
99 "version": "0.1.0",
1010 "dependencies": {
11+ "agora-extension-ai-denoiser": "^1.1.0",
1112 "agora-rtc-react": "^2.2.0",
1213 "agora-rtc-sdk-ng": "^4.21.0",
1314 "next": "14.2.4",
@@ -665,6 +666,15 @@
665666 "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
666667 }
667668 },
669+ "node_modules/agora-extension-ai-denoiser": {
670+ "version": "1.1.0",
671+ "resolved": "https://registry.npmjs.org/agora-extension-ai-denoiser/-/agora-extension-ai-denoiser-1.1.0.tgz",
672+ "integrity": "sha512-g4klzzz7IQblNt0c+gJ/DM7sa5oMZ/7GWE4c9aqs3snHRAo+CjDieT84iAc1KUhAiY84RrfhyFdtpvbbEoYBcA==",
673+ "license": "MIT",
674+ "peerDependencies": {
675+ "agora-rtc-sdk-ng": ">=4.15.0"
676+ }
677+ },
668678 "node_modules/agora-rtc-react": {
669679 "version": "2.2.0",
670680 "resolved": "https://registry.npmjs.org/agora-rtc-react/-/agora-rtc-react-2.2.0.tgz",
package.json+3−2
@@ -3,12 +3,13 @@
33 "version": "0.1.0",
44 "private": true,
55 "scripts": {
6- "dev": "next dev",
7- "build": "next build",
6+ "dev": "cp -r node_modules/agora-extension-ai-denoiser/external public/external && next dev",
7+ "build": "cp -r node_modules/agora-extension-ai-denoiser/external public/external && next build",
88 "start": "next start",
99 "lint": "next lint"
1010 },
1111 "dependencies": {
12+ "agora-extension-ai-denoiser": "^1.1.0",
1213 "agora-rtc-react": "^2.2.0",
1314 "agora-rtc-sdk-ng": "^4.21.0",
1415 "next": "14.2.4",
public/external/denoiser-wasm-simd.wasm+0−0
Binary file, not shown.
public/external/denoiser-wasm.wasm+0−0
Binary file, not shown.