Media stream

Helpers exported by @morevoice/sdk from src/lib/media-stream.ts.

API

Exported from @morevoice/sdk (source: packages/sdk-ts/src/lib/media-stream.ts).

ConnectedMessage

interface

interface ConnectedMessage {
  event: "connected";
  protocol?: string;
  version?: string;
}

StreamStart

interface

interface StreamStart {
  streamSid: string;
  /** The MoreVoice call (`call_…`). */
  callSid: string;
  accountSid?: string;
  /** `inbound` (the caller), `outbound` (what the call plays), or both. */
  tracks: string[];
  /** The `custom_parameters` the stream was started with (flow node, inbound route or POST /v1/calls). */
  customParameters: Record<string, string>;
  mediaFormat: MediaFormat;
  /** `connect` (you speak back) or `fork` (listen only), when the server says; MoreVoice ignores what a fork stream sends. */
  mode?: "connect" | "fork" | (string & {});
  [field: string]: unknown;
}

StartMessage

interface

interface StartMessage {
  event: "start";
  sequenceNumber?: string;
  streamSid?: string;
  start: StreamStart;
}

MediaMessage

interface

interface MediaMessage {
  event: "media";
  sequenceNumber?: string;
  streamSid?: string;
  media: { track?: string; chunk?: string; timestamp?: string; payload: string };
}

DtmfMessage

interface

interface DtmfMessage {
  event: "dtmf";
  sequenceNumber?: string;
  streamSid?: string;
  dtmf: { track?: string; digit: string };
}

MarkMessage

interface

interface MarkMessage {
  event: "mark";
  sequenceNumber?: string;
  streamSid?: string;
  mark: { name: string };
}

StopMessage

interface

interface StopMessage {
  event: "stop";
  sequenceNumber?: string;
  streamSid?: string;
  stop?: { accountSid?: string; callSid?: string; reason?: string; [field: string]: unknown };
}

InboundMediaMessage

type

Everything MoreVoice sends. Unknown events (a newer server) reach message listeners untouched.

type InboundMediaMessage = ConnectedMessage | StartMessage | MediaMessage | DtmfMessage | MarkMessage | StopMessage;

TransferTarget

type

Where transfer sends the call (doc 07 §2.4d’s transfer shapes).

type TransferTarget = ({ number: string } | { queue_id: string } | { assistant_id: string }) & {
  /** Warm: the target hears a summary first. Default: cold. */
  mode?: "warm" | "cold";
};

OutboundMediaMessage

type

What this helper sends.

type OutboundMediaMessage =
  | { event: "media"; streamSid: string; media: { payload: string } }
  | { event: "mark"; streamSid: string; mark: { name: string } }
  | { event: "clear"; streamSid: string }
  | { event: "transfer"; streamSid: string; transfer: TransferTarget }
  | { event: "hangup"; streamSid: string; hangup: { reason?: string } }
  | { event: "metadata"; streamSid: string; metadata: Record<string, string> };

MediaSocket

interface

The parts of a WebSocket the helper uses. Both shapes work: the ws package (on/off, send with a callback) and the WHATWG WebSocket (addEventListener, data on message events).

interface MediaSocket {
  readyState: number;
  bufferedAmount?: number;
  send(data: string, cb?: (err?: Error) => void): void;
  close(code?: number, reason?: string): void;
  on?(event: string, listener: (...args: any[]) => void): unknown;
  off?(event: string, listener: (...args: any[]) => void): unknown;
  removeListener?(event: string, listener: (...args: any[]) => void): unknown;
  addEventListener?(event: string, listener: (ev: any) => void): void;
  removeEventListener?(event: string, listener: (ev: any) => void): void;
}

AudioMeta

interface

Facts about one inbound audio frame.

interface AudioMeta {
  /** `inbound` (the caller) or `outbound` (what the call plays: fork mode with both tracks). */
  track: string;
  /** Sample rate of `frame` (the stream's, or the `sampleRate` you asked for). */
  sampleRate: number;
  /** Milliseconds since the stream started, as MoreVoice stamped it. */
  timestamp: number | null;
  chunk: number | null;
  sequenceNumber: number | null;
  /** The undecoded payload bytes, in the stream's own format. */
  payload: Uint8Array;
}

MediaStreamEvents

interface

interface MediaStreamEvents {
  connected: [ConnectedMessage];
  start: [StreamStart];
  /** Decoded PCM16, one call per media message. */
  audio: [Int16Array, AudioMeta];
  dtmf: [string, { track: string | null }];
  /** A mark you sent was played (or cleared). */
  mark: [string];
  stop: [StopMessage];
  /** Every parsed message, known or not. */
  message: [Record<string, unknown>];
  close: [{ code: number; reason: string }];
  error: [Error];
}

MediaStreamEvent

type

type MediaStreamEvent = keyof MediaStreamEvents;

MediaStreamHandlers

interface

interface MediaStreamHandlers {
  onConnected?: Listener<"connected">;
  onStart?: Listener<"start">;
  onAudio?: Listener<"audio">;
  onDtmf?: Listener<"dtmf">;
  onMark?: Listener<"mark">;
  onStop?: Listener<"stop">;
  onMessage?: Listener<"message">;
  onClose?: Listener<"close">;
  onError?: Listener<"error">;
}

AttachOptions

interface

interface AttachOptions extends MediaStreamHandlers {
  /** Deliver `audio` at this rate (resampled, e.g. 16000 for a speech model), instead of the stream's own. */
  sampleRate?: number;
  /** `sendAudio` resolves once the socket's send buffer is at or below this many bytes (default 0: drained). */
  highWaterMark?: number;
}

SendAudioOptions

interface

interface SendAudioOptions {
  /** The rate of the PCM16 you pass (default: the stream's). Other rates are resampled, continuously across calls. */
  sampleRate?: number;
  /** Release the resampler's few held-back milliseconds now (the end of an utterance). mark() does it too. */
  flush?: boolean;
}

MediaStreamError

class

class MediaStreamError extends Error {
  readonly name = "MediaStreamError";
  constructor(
    readonly code: "not_started" | "closed" | "fork_mode" | "unsupported_format" | "bad_message" | "send_failed",
    message: string,
  );
}

MediaStreamSession

class

One media stream on an open WebSocket. Created by mediaStream.attach() (or by createMediaStreamServer for each connection). Listen with on(...) or the on… handlers; speak with sendAudio, mark, clear; control the call with transfer, hangup and metadata.

class MediaStreamSession {
  /** Resolves with the `start` message (rejects if the socket closes first). */
  readonly started: Promise<StreamStart>;
  /** Resolves when the socket closes. */
  readonly closed: Promise<{ code: number; reason: string }>;
  constructor(socket: MediaSocket, options: AttachOptions = {});
  /** The `start` message, once it arrived. */
  get start(): StreamStart | null;
  get streamSid(): string | null;
  /** The MoreVoice call id (`call_…`). */
  get callId(): string | null;
  get mediaFormat(): MediaFormat;
  get customParameters(): Record<string, string>;
  get isClosed(): boolean;
  on<K extends MediaStreamEvent>(event: K, listener: Listener<K>): this;
  off<K extends MediaStreamEvent>(event: K, listener: Listener<K>): this;
  /** Play audio to the caller (connect mode). PCM16 (`Int16Array`, at `sampleRate`, default the stream's) is resampled and encoded to the stream's format; a `Uint8Array` is already-encoded payload in the stream's own format (μ-law bytes for a μ-law stream: what a Twilio bot has). Sent as 20 ms `media` messages; resolves when the socket has drained (backpressure: await it before sending more). */
  async sendAudio(audio: Int16Array | Uint8Array, options: SendAudioOptions = {}): Promise<void>;
  /** Release audio the resampler is still holding (only when you send PCM at a rate other than the stream's). */
  async flushAudio(): Promise<void>;
  /** Put a named mark after the audio sent so far. Resolves `true` when MoreVoice reports that playback reached it, or that it was cleared; `false` when the stream ended first. Flushes the resampler first. Like every send, the mark goes out in call order (`mark(); clear()` sends the mark first). */
  async mark(name: string): Promise<boolean>;
  /** Stop playing everything sent so far (barge-in). Pending marks come back as played (MoreVoice echoes them). */
  clear(): void;
  /** Hand the call to a person, a queue or a MoreVoice assistant (MoreVoice extension). */
  transfer(target: TransferTarget): void;
  /** End the call (MoreVoice extension). */
  hangup(reason?: string): void;
  /** Merge key/values into the call's metadata (shown on the call, sent with its events; MoreVoice extension). String values only, at most 50 keys of 1–40 characters, values up to 500 (the API's metadata limits). */
  metadata(data: Record<string, string>): void;
  /** Close the WebSocket (MoreVoice then applies the stream's fallback: an assistant, or hanging up). */
  close(code = 1000, reason = "bye"): void;
}

attachMediaStream()

function

Wire a media stream onto an open WebSocket.

function attachMediaStream(socket: MediaSocket, options: AttachOptions = {}): MediaStreamSession;

UpgradeRequestLike

interface

An HTTP upgrade request, as Node’s http.IncomingMessage (or anything with a url and headers).

interface UpgradeRequestLike {
  url?: string;
  headers: HeadersLike;
}

UpgradeVerifier

type

type UpgradeVerifier = WebhookSecret | readonly WebhookSecret[] | ((req: UpgradeRequestLike) => boolean | Promise<boolean>);

verifyMediaStreamUpgrade()

function

Check a media-stream upgrade request. With a secret: the Standard Webhooks headers over the request target (path and query). Throws a WebhookVerificationError (missing_header, timestamp_out_of_range, bad_signature, …) when it fails.

function verifyMediaStreamUpgrade(req: UpgradeRequestLike, verify: UpgradeVerifier, options: VerifyOptions = {}): Promise<void>;

MediaStreamServerOptions

interface

interface MediaStreamServerOptions extends AttachOptions {
  /** Only upgrades on this path (default: any). */
  path?: string;
  /** Required: your signing secret (whsec_…), or a function. Pass `false` only for local experiments. */
  verify: UpgradeVerifier | false;
  /** Signature checks (tolerance, clock). */
  verifyOptions?: VerifyOptions;
  /** Each new stream (after the upgrade); the session is already attached. */
  onStream(stream: MediaStreamSession, req: UpgradeRequestLike): void | Promise<void>;
  /** Use your own `http.Server` (its `upgrade` event is taken over for `path`); default: a new one. */
  server?: NodeHttpServerLike;
  /** Refuse messages larger than this (default 1 MiB). */
  maxPayload?: number;
}

NodeHttpServerLike

interface

The parts of node:http’s Server used here (structural, so the SDK’s types don’t need @types/node).

interface NodeHttpServerLike {
  on(event: "upgrade", listener: (req: any, socket: any, head: any) => void): unknown;
  off?(event: "upgrade", listener: (req: any, socket: any, head: any) => void): unknown;
  listen(port: number, host: string | undefined, cb: () => void): unknown;
  close(cb?: (err?: Error) => void): unknown;
  address(): unknown;
}

MediaStreamServer

interface

interface MediaStreamServer {
  /** Start listening (only for the server created here); resolves with the bound port. */
  listen(port?: number, host?: string): Promise<{ port: number; host: string }>;
  /** Handle one upgrade yourself (e.g. from your framework's server). */
  handleUpgrade(req: any, socket: any, head: any): void;
  /** Every open stream. */
  readonly streams: ReadonlySet<MediaStreamSession>;
  /** Close every stream and stop listening. */
  close(): Promise<void>;
}

createMediaStreamServer()

function

A media-stream endpoint for Node: verifies each upgrade, attaches a session and hands it to onStream. Needs the ws package (npm i ws), loaded on first use.

function createMediaStreamServer(options: MediaStreamServerOptions): MediaStreamServer;

mediaStream

constant

mv.mediaStream / import { mediaStream }: attach to a socket, or verify an upgrade.

const mediaStream: {
  attach: typeof attachMediaStream;
  verifyUpgrade: typeof verifyMediaStreamUpgrade;
  createServer: typeof createMediaStreamServer;
};