# Media stream

> The media-stream helpers of @morevoice/sdk.

Helpers exported by `@morevoice/sdk` from `src/lib/media-stream.ts`.

## API

Exported from `@morevoice/sdk` (source: `packages/sdk-ts/src/lib/media-stream.ts`).

### `ConnectedMessage`

*interface*

```ts
interface ConnectedMessage {
	event: "connected";
	protocol?: string;
	version?: string;
}
```

### `StreamStart`

*interface*

```ts
interface StreamStart {
	streamSid: string;
	/** The MoreVoice call (`call_…`). */
	callSid: string;
	accountSid?: string;
	/** `inbound` (the caller), `outbound` (what the call plays), or both. */
	tracks: string[];
	/** The `custom_parameters` the stream was started with (flow node, inbound route or POST /v1/calls). */
	customParameters: Record<string, string>;
	mediaFormat: MediaFormat;
	/** `connect` (you speak back) or `fork` (listen only), when the server says; MoreVoice ignores what a fork stream sends. */
	mode?: "connect" | "fork" | (string & {});
	[field: string]: unknown;
}
```

### `StartMessage`

*interface*

```ts
interface StartMessage {
	event: "start";
	sequenceNumber?: string;
	streamSid?: string;
	start: StreamStart;
}
```

### `MediaMessage`

*interface*

```ts
interface MediaMessage {
	event: "media";
	sequenceNumber?: string;
	streamSid?: string;
	media: { track?: string; chunk?: string; timestamp?: string; payload: string };
}
```

### `DtmfMessage`

*interface*

```ts
interface DtmfMessage {
	event: "dtmf";
	sequenceNumber?: string;
	streamSid?: string;
	dtmf: { track?: string; digit: string };
}
```

### `MarkMessage`

*interface*

```ts
interface MarkMessage {
	event: "mark";
	sequenceNumber?: string;
	streamSid?: string;
	mark: { name: string };
}
```

### `StopMessage`

*interface*

```ts
interface StopMessage {
	event: "stop";
	sequenceNumber?: string;
	streamSid?: string;
	stop?: { accountSid?: string; callSid?: string; reason?: string; [field: string]: unknown };
}
```

### `InboundMediaMessage`

*type*

Everything MoreVoice sends. Unknown events (a newer server) reach `message` listeners untouched.

```ts
type InboundMediaMessage = ConnectedMessage | StartMessage | MediaMessage | DtmfMessage | MarkMessage | StopMessage;
```

### `TransferTarget`

*type*

Where `transfer` sends the call (doc 07 §2.4d's transfer shapes).

```ts
type TransferTarget = ({ number: string } | { queue_id: string } | { assistant_id: string }) & {
	/** Warm: the target hears a summary first. Default: cold. */
	mode?: "warm" | "cold";
};
```

### `OutboundMediaMessage`

*type*

What this helper sends.

```ts
type OutboundMediaMessage =
	| { event: "media"; streamSid: string; media: { payload: string } }
	| { event: "mark"; streamSid: string; mark: { name: string } }
	| { event: "clear"; streamSid: string }
	| { event: "transfer"; streamSid: string; transfer: TransferTarget }
	| { event: "hangup"; streamSid: string; hangup: { reason?: string } }
	| { event: "metadata"; streamSid: string; metadata: Record<string, string> };
```

### `MediaSocket`

*interface*

The parts of a WebSocket the helper uses. Both shapes work: the `ws` package (on/off, send with a callback) and the
WHATWG WebSocket (addEventListener, `data` on message events).

```ts
interface MediaSocket {
	readyState: number;
	bufferedAmount?: number;
	send(data: string, cb?: (err?: Error) => void): void;
	close(code?: number, reason?: string): void;
	on?(event: string, listener: (...args: any[]) => void): unknown;
	off?(event: string, listener: (...args: any[]) => void): unknown;
	removeListener?(event: string, listener: (...args: any[]) => void): unknown;
	addEventListener?(event: string, listener: (ev: any) => void): void;
	removeEventListener?(event: string, listener: (ev: any) => void): void;
}
```

### `AudioMeta`

*interface*

Facts about one inbound audio frame.

```ts
interface AudioMeta {
	/** `inbound` (the caller) or `outbound` (what the call plays: fork mode with both tracks). */
	track: string;
	/** Sample rate of `frame` (the stream's, or the `sampleRate` you asked for). */
	sampleRate: number;
	/** Milliseconds since the stream started, as MoreVoice stamped it. */
	timestamp: number | null;
	chunk: number | null;
	sequenceNumber: number | null;
	/** The undecoded payload bytes, in the stream's own format. */
	payload: Uint8Array;
}
```

### `MediaStreamEvents`

*interface*

```ts
interface MediaStreamEvents {
	connected: [ConnectedMessage];
	start: [StreamStart];
	/** Decoded PCM16, one call per media message. */
	audio: [Int16Array, AudioMeta];
	dtmf: [string, { track: string | null }];
	/** A mark you sent was played (or cleared). */
	mark: [string];
	stop: [StopMessage];
	/** Every parsed message, known or not. */
	message: [Record<string, unknown>];
	close: [{ code: number; reason: string }];
	error: [Error];
}
```

### `MediaStreamEvent`

*type*

```ts
type MediaStreamEvent = keyof MediaStreamEvents;
```

### `MediaStreamHandlers`

*interface*

```ts
interface MediaStreamHandlers {
	onConnected?: Listener<"connected">;
	onStart?: Listener<"start">;
	onAudio?: Listener<"audio">;
	onDtmf?: Listener<"dtmf">;
	onMark?: Listener<"mark">;
	onStop?: Listener<"stop">;
	onMessage?: Listener<"message">;
	onClose?: Listener<"close">;
	onError?: Listener<"error">;
}
```

### `AttachOptions`

*interface*

```ts
interface AttachOptions extends MediaStreamHandlers {
	/** Deliver `audio` at this rate (resampled, e.g. 16000 for a speech model), instead of the stream's own. */
	sampleRate?: number;
	/** `sendAudio` resolves once the socket's send buffer is at or below this many bytes (default 0: drained). */
	highWaterMark?: number;
}
```

### `SendAudioOptions`

*interface*

```ts
interface SendAudioOptions {
	/** The rate of the PCM16 you pass (default: the stream's). Other rates are resampled, continuously across calls. */
	sampleRate?: number;
	/** Release the resampler's few held-back milliseconds now (the end of an utterance). mark() does it too. */
	flush?: boolean;
}
```

### `MediaStreamError`

*class*

```ts
class MediaStreamError extends Error {
	readonly name = "MediaStreamError";
	constructor(
		readonly code: "not_started" | "closed" | "fork_mode" | "unsupported_format" | "bad_message" | "send_failed",
		message: string,
	);
}
```

### `MediaStreamSession`

*class*

One media stream on an open WebSocket. Created by `mediaStream.attach()` (or by `createMediaStreamServer` for each
connection). Listen with `on(...)` or the `on…` handlers; speak with `sendAudio`, `mark`, `clear`; control the call
with `transfer`, `hangup` and `metadata`.

```ts
class MediaStreamSession {
	/** Resolves with the `start` message (rejects if the socket closes first). */
	readonly started: Promise<StreamStart>;
	/** Resolves when the socket closes. */
	readonly closed: Promise<{ code: number; reason: string }>;
	constructor(socket: MediaSocket, options: AttachOptions = {});
	/** The `start` message, once it arrived. */
	get start(): StreamStart | null;
	get streamSid(): string | null;
	/** The MoreVoice call id (`call_…`). */
	get callId(): string | null;
	get mediaFormat(): MediaFormat;
	get customParameters(): Record<string, string>;
	get isClosed(): boolean;
	on<K extends MediaStreamEvent>(event: K, listener: Listener<K>): this;
	off<K extends MediaStreamEvent>(event: K, listener: Listener<K>): this;
	/** Play audio to the caller (connect mode). PCM16 (`Int16Array`, at `sampleRate`, default the stream's) is resampled and encoded to the stream's format; a `Uint8Array` is already-encoded payload in the stream's own format (μ-law bytes for a μ-law stream: what a Twilio bot has). Sent as 20 ms `media` messages; resolves when the socket has drained (backpressure: await it before sending more). */
	async sendAudio(audio: Int16Array | Uint8Array, options: SendAudioOptions = {}): Promise<void>;
	/** Release audio the resampler is still holding (only when you send PCM at a rate other than the stream's). */
	async flushAudio(): Promise<void>;
	/** Put a named mark after the audio sent so far. Resolves `true` when MoreVoice reports that playback reached it, or that it was cleared; `false` when the stream ended first. Flushes the resampler first. Like every send, the mark goes out in call order (`mark(); clear()` sends the mark first). */
	async mark(name: string): Promise<boolean>;
	/** Stop playing everything sent so far (barge-in). Pending marks come back as played (MoreVoice echoes them). */
	clear(): void;
	/** Hand the call to a person, a queue or a MoreVoice assistant (MoreVoice extension). */
	transfer(target: TransferTarget): void;
	/** End the call (MoreVoice extension). */
	hangup(reason?: string): void;
	/** Merge key/values into the call's metadata (shown on the call, sent with its events; MoreVoice extension). String values only, at most 50 keys of 1–40 characters, values up to 500 (the API's metadata limits). */
	metadata(data: Record<string, string>): void;
	/** Close the WebSocket (MoreVoice then applies the stream's fallback: an assistant, or hanging up). */
	close(code = 1000, reason = "bye"): void;
}
```

### `attachMediaStream()`

*function*

Wire a media stream onto an open WebSocket.

```ts
function attachMediaStream(socket: MediaSocket, options: AttachOptions = {}): MediaStreamSession;
```

### `UpgradeRequestLike`

*interface*

An HTTP upgrade request, as Node's `http.IncomingMessage` (or anything with a url and headers).

```ts
interface UpgradeRequestLike {
	url?: string;
	headers: HeadersLike;
}
```

### `UpgradeVerifier`

*type*

```ts
type UpgradeVerifier = WebhookSecret | readonly WebhookSecret[] | ((req: UpgradeRequestLike) => boolean | Promise<boolean>);
```

### `verifyMediaStreamUpgrade()`

*function*

Check a media-stream upgrade request. With a secret: the Standard Webhooks headers over the request target (path and
query). Throws a WebhookVerificationError (missing_header, timestamp_out_of_range, bad_signature, …) when it fails.

```ts
function verifyMediaStreamUpgrade(req: UpgradeRequestLike, verify: UpgradeVerifier, options: VerifyOptions = {}): Promise<void>;
```

### `MediaStreamServerOptions`

*interface*

```ts
interface MediaStreamServerOptions extends AttachOptions {
	/** Only upgrades on this path (default: any). */
	path?: string;
	/** Required: your signing secret (whsec_…), or a function. Pass `false` only for local experiments. */
	verify: UpgradeVerifier | false;
	/** Signature checks (tolerance, clock). */
	verifyOptions?: VerifyOptions;
	/** Each new stream (after the upgrade); the session is already attached. */
	onStream(stream: MediaStreamSession, req: UpgradeRequestLike): void | Promise<void>;
	/** Use your own `http.Server` (its `upgrade` event is taken over for `path`); default: a new one. */
	server?: NodeHttpServerLike;
	/** Refuse messages larger than this (default 1 MiB). */
	maxPayload?: number;
}
```

### `NodeHttpServerLike`

*interface*

The parts of node:http's Server used here (structural, so the SDK's types don't need @types/node).

```ts
interface NodeHttpServerLike {
	on(event: "upgrade", listener: (req: any, socket: any, head: any) => void): unknown;
	off?(event: "upgrade", listener: (req: any, socket: any, head: any) => void): unknown;
	listen(port: number, host: string | undefined, cb: () => void): unknown;
	close(cb?: (err?: Error) => void): unknown;
	address(): unknown;
}
```

### `MediaStreamServer`

*interface*

```ts
interface MediaStreamServer {
	/** Start listening (only for the server created here); resolves with the bound port. */
	listen(port?: number, host?: string): Promise<{ port: number; host: string }>;
	/** Handle one upgrade yourself (e.g. from your framework's server). */
	handleUpgrade(req: any, socket: any, head: any): void;
	/** Every open stream. */
	readonly streams: ReadonlySet<MediaStreamSession>;
	/** Close every stream and stop listening. */
	close(): Promise<void>;
}
```

### `createMediaStreamServer()`

*function*

A media-stream endpoint for Node: verifies each upgrade, attaches a session and hands it to `onStream`. Needs the
`ws` package (`npm i ws`), loaded on first use.

```ts
function createMediaStreamServer(options: MediaStreamServerOptions): MediaStreamServer;
```

### `mediaStream`

*constant*

`mv.mediaStream` / `import { mediaStream }`: attach to a socket, or verify an upgrade.

```ts
const mediaStream: {
	attach: typeof attachMediaStream;
	verifyUpgrade: typeof verifyMediaStreamUpgrade;
	createServer: typeof createMediaStreamServer;
};
```