@mistralai/mistralai3.5.016.3 kB
import { VoiceAgentConnectionError, VoiceAgentDependencyError, } from "../errors.js";
import { BYTES_PER_SAMPLE, CHANNELS, FRAME_DURATION_MS, SAMPLE_RATE, } from "./audio.js";
const SOX_INSTALL_HINTS = {
darwin: "Install SoX: brew install sox",
linux: "Install SoX: apt install sox",
win32: "Install SoX: scoop install sox. The official SoX build for Windows ships " +
"only sox.exe, so a manual install also needs copy sox.exe rec.exe and " +
"copy sox.exe play.exe in the SoX directory",
};
const DEFAULT_SOX_INSTALL_HINT = "Install SoX: brew install sox (macOS), apt install sox (Linux) or " +
"scoop install sox (Windows)";
export function soxInstallHint(platform) {
return SOX_INSTALL_HINTS[platform] ?? DEFAULT_SOX_INSTALL_HINT;
}
const MICROPHONE_PERMISSION_PATHS = {
darwin: "System Settings > Privacy & Security > Microphone",
win32: "Settings > Privacy & security > Microphone",
};
const DEVICE_NOUNS = { input: "microphone", output: "speaker" };
const NO_DEVICE_PATTERN = /no default audio device|can ?not find audio device|no such device|no audio device/i;
const DEVICE_BUSY_PATTERN = /resource busy|device is busy|already allocated|already in use|used by another/i;
export function audioDeviceHint(options) {
const noun = DEVICE_NOUNS[options.device];
if (NO_DEVICE_PATTERN.test(options.stderr)) {
return `No ${noun} is available, connect one and retry`;
}
if (DEVICE_BUSY_PATTERN.test(options.stderr)) {
return (`The ${noun} is held by another application, close whatever else is ` +
"using it and retry");
}
const permissionPath = MICROPHONE_PERMISSION_PATHS[options.platform];
if (options.device === "input" && permissionPath != null) {
return ("The microphone could not be opened, the operating system may not have " +
`granted access. Allow it under ${permissionPath}, then rerun`);
}
return `The default audio ${options.device} device is unavailable or could not be opened`;
}
const RTC_NODE_PACKAGE = "@livekit/rtc-node";
const BYTES_PER_MS = (SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE) / 1000;
const SOX_BUFFER_BYTES = BYTES_PER_MS * FRAME_DURATION_MS;
/**
* Reported to the echo canceller on top of our own queue, to cover `play`'s
* own buffering and the speaker-to-microphone path we cannot measure.
*
* Measured against the WebRTC canceller: reporting at or above the real delay
* attenuates by 16-24 dB and overshooting costs nothing, while reporting under
* it collapses to 4-13 dB because the echo falls outside the search window. So
* this errs high on purpose.
*/
const PLAYBACK_PATH_LATENCY_MS = 250;
const APM_FRAME_BYTES = (SAMPLE_RATE / 100) * CHANNELS * BYTES_PER_SAMPLE;
const OUTPUT_QUEUE_MAX_MS = 300;
const OUTPUT_QUEUE_MAX_BYTES = (SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * OUTPUT_QUEUE_MAX_MS) / 1000;
/** Input args for `rec`: record raw PCM from mic to stdout. */
const REC_ARGS = [
"--buffer",
String(SOX_BUFFER_BYTES),
"-q",
"-L",
"-t",
"raw",
"-b",
"16",
"-e",
"signed-integer",
"-r",
String(SAMPLE_RATE),
"-c",
String(CHANNELS),
"-",
];
/** Args for `play`: read raw PCM from stdin, play to default audio device. */
const PLAY_ARGS = [
"--buffer",
String(SOX_BUFFER_BYTES),
"-q",
"-L",
"--ignore-length",
"-t",
"raw",
"-b",
"16",
"-e",
"signed-integer",
"-r",
String(SAMPLE_RATE),
"-c",
String(CHANNELS),
"-",
];
/** Flush interval in ms — matches the frame duration for minimal latency. */
const FLUSH_INTERVAL_MS = 20;
const STARTUP_GRACE_PERIOD_MS = 250;
const STDERR_MAX_CHARACTERS = 4096;
function isMissingBinaryError(err) {
return (typeof err === "object" &&
err !== null &&
"code" in err &&
err.code === "ENOENT");
}
function errorMessage(error) {
return error instanceof Error ? error.message : String(error);
}
function monitorAudioProcess(child, options) {
let stderr = "";
let failure = null;
let resolveTermination;
const terminated = new Promise((resolve) => {
resolveTermination = resolve;
});
const recordFailure = (error) => {
if (failure != null)
return;
failure = error;
resolveTermination(error);
};
child.stderr?.setEncoding("utf8");
child.stderr?.on("data", (chunk) => {
stderr = (stderr + chunk).slice(-STDERR_MAX_CHARACTERS);
});
const spawned = new Promise((resolve, reject) => {
child.once("spawn", resolve);
child.once("error", (error) => {
const exception = isMissingBinaryError(error)
? new VoiceAgentDependencyError(`'${options.command}' not found. ${soxInstallHint(options.platform)}`)
: new VoiceAgentConnectionError(`Failed to start '${options.command}' for default audio ${options.device}: ${errorMessage(error)}`, { cause: error });
recordFailure(exception);
reject(exception);
});
});
child.stdin?.on("error", (error) => {
recordFailure(new VoiceAgentConnectionError(`The default audio ${options.device} device became unavailable: ${errorMessage(error)}`, { cause: error }));
});
child.once("exit", (code, signal) => {
const details = stderr.trim();
const outcome = signal == null ? `exit code ${String(code)}` : `signal ${signal}`;
const hint = audioDeviceHint({
device: options.device,
platform: options.platform,
stderr,
});
recordFailure(new VoiceAgentConnectionError(`${hint}. '${options.command}' exited with ${outcome}` +
(details ? `. ${details}` : "")));
});
return {
child,
spawned,
terminated,
failure: () => failure,
};
}
async function waitForStartup(processes) {
const rejectOnTermination = processes.map((process) => process.terminated.then((error) => {
throw error;
}));
await Promise.race([
new Promise((resolve) => setTimeout(resolve, STARTUP_GRACE_PERIOD_MS)),
...rejectOnTermination,
]);
}
async function assertSoxBinary(command) {
const { spawn } = await import("node:child_process");
const child = spawn(command, ["--version"], { stdio: "ignore" });
await new Promise((resolve, reject) => {
child.once("spawn", resolve);
child.once("error", (error) => {
reject(isMissingBinaryError(error)
? new VoiceAgentDependencyError(`'${command}' not found. ${soxInstallHint(process.platform)}`)
: new VoiceAgentConnectionError(`Failed to start '${command}': ${errorMessage(error)}`, { cause: error }));
});
});
child.unref();
}
async function loadFfiBindings() {
const { createRequire } = await import("node:module");
try {
const rtcNodePath = createRequire(import.meta.url).resolve(RTC_NODE_PACKAGE);
return createRequire(rtcNodePath)("@livekit/rtc-ffi-bindings");
}
catch (error) {
throw new VoiceAgentDependencyError(`Echo cancellation needs an optional peer dependency: npm install ${RTC_NODE_PACKAGE}`, { cause: error });
}
}
class EchoCanceller {
_ffi;
_handle;
_capture = Buffer.alloc(0);
_render = Buffer.alloc(0);
_failure = null;
constructor(ffi, handle) {
this._ffi = ffi;
this._handle = handle;
}
static async create(options) {
const ffi = await loadFfiBindings();
const response = EchoCanceller._request(ffi, {
case: "newApm",
value: {
echoCancellerEnabled: true,
gainControllerEnabled: options.autoGainControl,
highPassFilterEnabled: options.highPassFilter,
noiseSuppressionEnabled: options.noiseSuppression,
},
});
const handle = response?.apm?.handle?.id;
if (handle == null) {
throw new VoiceAgentConnectionError("Failed to create the echo canceller");
}
return new EchoCanceller(ffi, handle);
}
processCapture(audio, delayMs) {
if (this._failure != null)
throw this._failure;
this._capture = Buffer.concat([this._capture, audio]);
const frames = this._takeFrames("_capture");
if (frames.byteLength === 0)
return frames;
// Without the delay the canceller aligns the speaker stream against the
// wrong point in time and cancels almost nothing.
EchoCanceller._request(this._ffi, {
case: "apmSetStreamDelay",
value: { apmHandle: this._handle, delayMs },
});
const failure = this._processFrames("apmProcessStream", frames);
if (failure != null)
throw failure;
return frames;
}
processRender(audio) {
this._render = Buffer.concat([this._render, audio]);
this._failure ?? (this._failure = this._processFrames("apmProcessReverseStream", this._takeFrames("_render")));
}
close() {
new this._ffi.FfiHandle(this._handle).dispose();
}
_takeFrames(buffer) {
const pending = this[buffer];
const length = pending.byteLength - (pending.byteLength % APM_FRAME_BYTES);
this[buffer] = Buffer.from(pending.subarray(length));
return Buffer.from(pending.subarray(0, length));
}
_processFrames(kind, frames) {
for (let offset = 0; offset < frames.byteLength; offset += APM_FRAME_BYTES) {
const frame = frames.subarray(offset, offset + APM_FRAME_BYTES);
const response = EchoCanceller._request(this._ffi, {
case: kind,
value: {
apmHandle: this._handle,
dataPtr: this._ffi.livekitRetrievePtr(frame),
size: frame.byteLength,
sampleRate: SAMPLE_RATE,
numChannels: CHANNELS,
},
});
if (response?.error) {
return new VoiceAgentConnectionError(`Echo cancellation failed: ${response.error}`);
}
}
return null;
}
static _request(ffi, message) {
const request = new ffi.FfiRequest({ message });
return ffi.FfiResponse.fromBinary(ffi.livekitFfiRequest(request.toBinary()))
.message.value;
}
}
export class DefaultAudioInterface {
_echoCancellation;
_audioProcessing;
_echoCanceller = null;
_recorder = null;
_player = null;
_running = false;
_outputQueue = [];
_flushTimer = null;
_playbackEndsAt = 0;
_echoCancellationDelayMs;
constructor(options = {}) {
this._echoCancellation = options.echoCancellation ?? true;
this._echoCancellationDelayMs = options.echoCancellationDelayMs;
this._audioProcessing = {
noiseSuppression: options.noiseSuppression ?? true,
highPassFilter: options.highPassFilter ?? true,
autoGainControl: options.autoGainControl ?? true,
};
}
async preflight() {
await assertSoxBinary("rec");
if (this._echoCancellation)
await loadFfiBindings();
}
async start() {
if (this._running) {
throw new VoiceAgentConnectionError("Default audio interface is already running");
}
this._running = true;
try {
if (this._echoCancellation) {
this._echoCanceller = await EchoCanceller.create(this._audioProcessing);
}
const { spawn } = await import("node:child_process");
this._recorder = monitorAudioProcess(spawn("rec", REC_ARGS, {
stdio: ["ignore", "pipe", "pipe"],
}), {
command: "rec",
device: "input",
platform: process.platform,
});
await this._recorder.spawned;
const recorderFailure = this._recorder.failure();
if (recorderFailure != null)
throw recorderFailure;
this._player = this._spawnPlayer(spawn);
await this._player.spawned;
const playerFailure = this._player.failure();
if (playerFailure != null)
throw playerFailure;
await waitForStartup([this._recorder, this._player]);
this._flushTimer = setInterval(() => this._flushOutput(), FLUSH_INTERVAL_MS);
}
catch (error) {
await this.stop();
throw error;
}
}
async *inputAudio() {
const recorder = this._recorder;
const stdout = recorder?.child.stdout;
if (recorder == null || stdout == null) {
throw new VoiceAgentConnectionError("Audio recorder is not running");
}
for await (const chunk of stdout) {
if (!this._running)
return;
const audio = this._echoCanceller?.processCapture(chunk, this._streamDelayMs()) ?? chunk;
if (audio.byteLength === 0)
continue;
yield new Uint8Array(audio);
}
if (this._running) {
throw (recorder.failure() ??
new VoiceAgentConnectionError("The default audio input stream stopped unexpectedly"));
}
}
outputAudio(audio) {
if (!this._running)
return Promise.resolve();
const player = this._player;
if (player == null) {
// The transport can receive audio while start() is still spawning the
// player. Keep those first frames so they can be flushed once ready.
this._enqueueOutput(audio);
return Promise.resolve();
}
const failure = player.failure();
if (failure != null)
return Promise.reject(failure);
if (player.child.stdin == null || player.child.stdin.destroyed) {
return Promise.reject(new VoiceAgentConnectionError("The default audio output device is unavailable"));
}
this._enqueueOutput(audio);
return Promise.resolve();
}
stop() {
this._running = false;
if (this._flushTimer != null) {
clearInterval(this._flushTimer);
this._flushTimer = null;
}
this._outputQueue.length = 0;
this._playbackEndsAt = 0;
this._killProcess(this._player);
this._player = null;
this._killProcess(this._recorder);
this._recorder = null;
this._echoCanceller?.close();
this._echoCanceller = null;
return Promise.resolve();
}
_spawnPlayer(spawn) {
return monitorAudioProcess(spawn("play", PLAY_ARGS, {
stdio: ["pipe", "ignore", "pipe"],
}), {
command: "play",
device: "output",
platform: process.platform,
});
}
_enqueueOutput(audio) {
this._outputQueue.push(Buffer.from(audio));
let queuedBytes = this._outputQueue.reduce((total, frame) => total + frame.byteLength, 0);
while (queuedBytes > OUTPUT_QUEUE_MAX_BYTES) {
const oldest = this._outputQueue.shift();
if (oldest == null)
return;
queuedBytes -= oldest.byteLength;
}
}
_killProcess(process) {
if (process == null || process.child.killed)
return;
process.child.stdin?.end();
process.child.kill("SIGTERM");
}
_streamDelayMs() {
if (this._echoCancellationDelayMs !== undefined) {
return this._echoCancellationDelayMs;
}
const queuedMs = Math.max(this._playbackEndsAt - performance.now(), 0);
return Math.round(PLAYBACK_PATH_LATENCY_MS + queuedMs);
}
/** Drain the output queue to the play process. */
_flushOutput() {
const player = this._player;
const stdin = player?.child.stdin;
if (player?.failure() != null || stdin == null || stdin.destroyed) {
this._outputQueue.length = 0;
return;
}
while (this._outputQueue.length > 0) {
const chunk = this._outputQueue.shift();
if (chunk == null)
break;
stdin.write(chunk);
// The canceller is fed here, not in outputAudio: it has to see the
// speaker stream on the timeline it is actually played on.
this._playbackEndsAt =
Math.max(this._playbackEndsAt, performance.now()) +
chunk.byteLength / BYTES_PER_MS;
this._echoCanceller?.processRender(chunk);
}
}
}
//# sourceMappingURL=node.js.map
Shown as plain text from the published tarball, after apm checked it against the version's checksum. Nothing in it runs on this page.