Streaming text to speech (WebSocket)

{
  "object": "tts_stream_token",
  "token": "Hq3x9Zt0bL7wKc2Vn5RdQe8yUf1aJm4sPo6iTg0hBkE",
  "expires_at": "2026-10-07T05:31:00Z",
  "url": "/v1/audio/speech/stream?token=Hq3x9Zt0bL7wKc2Vn5RdQe8yUf1aJm4sPo6iTg0hBkE"
}

{
  "type": "session.ready",
  "session_id": "ttsws_01K6Z8M3Q4R7T9V2X5Y8A1B4C7",
  "model": "mira-tts",
  "voice": "neha",
  "response_format": "pcm",
  "sample_rate": 48000,
  "encoding": "pcm_s16le",
  "audio_transport": "binary",
  "limits": {
    "max_segment_chars": 250,
    "max_queued_chars": 10000,
    "max_contexts": 64,
    "idle_timeout_secs": 120,
    "max_session_secs": 3600
  }
}

{
  "type": "audio.done",
  "context_id": "turn-1",
  "request_id": "tts_01K6Z8N0B2D4F6H8K0M2P4R6T8",
  "chars_billed": 41,
  "cost_paise": 8,
  "bytes": 19840,
  "first_byte_ms": 212
}

# pip install "websockets>=14"
"""Stream a reply into Mirai and save 8 kHz μ-law audio, sentence by sentence.

    export MIRA_API_KEY=sk_live_...
    python stream_tts.py
    ffplay -f mulaw -ar 8000 reply.ulaw     # listen to the result
"""
import asyncio
import json
import os

import websockets

URL = "wss://sandbox.voice.miraiminds.co/v1/audio/speech/stream"


async def llm_tokens():
    """Stand-in for your LLM's streamed reply."""
    reply = "नमस्ते! आपका order कल शाम तक पहुँच जाएगा। Delivery से पहले हमारा agent आपको call करेगा।"
    for word in reply.split(" "):
        yield word + " "
        await asyncio.sleep(0.03)


async def main():
    headers = {"Authorization": f"Bearer {os.environ['MIRA_API_KEY']}"}
    async with websockets.connect(URL, additional_headers=headers) as ws:
        await ws.send(json.dumps(
            {"type": "session.update", "voice": "neha", "response_format": "ulaw_8000"}))

        async def send_reply(context_id):
            async for token in llm_tokens():
                await ws.send(json.dumps({"type": "text", "context_id": context_id, "text": token}))
            await ws.send(json.dumps({"type": "flush", "context_id": context_id}))

        sender = asyncio.create_task(send_reply("turn-1"))
        with open("reply.ulaw", "wb") as out:
            async for message in ws:
                if isinstance(message, bytes):
                    out.write(message)  # raw 8 kHz μ-law, ready for a phone line
                    continue
                event = json.loads(message)
                if event["type"] == "audio.start":
                    print("speaking:", event["text"])
                elif event["type"] == "audio.done":
                    print(f"  delivered: {event['chars_billed']} characters, {event['cost_paise']} paise")
                elif event["type"] == "error":
                    print("error:", event["code"], event["message"])
                elif event["type"] == "context.done" and event["context_id"] == "turn-1":
                    break
        await sender


asyncio.run(main())

import asyncio
import base64
import json
import time

LEAD_SECS = 0.4   # how far ahead of the caller the provider may be
CHUNK = 160       # 20 ms of 8 kHz μ-law


class PhoneLeg:
    """Plays Mirai's ulaw_8000 audio on one Twilio Media Stream."""

    def __init__(self, twilio_ws, stream_sid):
        self.twilio_ws = twilio_ws      # the Media Stream WebSocket in your FastAPI app
        self.stream_sid = stream_sid
        self.until = 0.0                # when the audio sent so far finishes playing
        self.speaking = None            # context_id of the latest audio.start
        self.cancelled = set()

    async def on_mirai_message(self, message):
        """Call this for every message you read from the Mirai socket."""
        if isinstance(message, str):
            event = json.loads(message)
            if event["type"] == "audio.start":
                self.speaking = event["context_id"]
            return
        for i in range(0, len(message), CHUNK):
            chunk = message[i:i + CHUNK]
            await self._pace(len(chunk) / 8000)     # μ-law: 8,000 bytes per second
            if self.speaking in self.cancelled:
                return                              # the caller interrupted: drop the rest
            await self.twilio_ws.send_text(json.dumps({
                "event": "media",
                "streamSid": self.stream_sid,
                "media": {"payload": base64.b64encode(chunk).decode()},
            }))

    async def barge_in(self, mirai_ws, context_id):
        """Call this when your speech detection hears the caller talk over the bot."""
        self.cancelled.add(context_id)
        await mirai_ws.send(json.dumps({"type": "cancel", "context_id": context_id}))
        await self.twilio_ws.send_text(json.dumps({"event": "clear", "streamSid": self.stream_sid}))
        self.until = 0.0

    async def _pace(self, secs):
        now = time.monotonic()
        self.until = max(self.until, now)
        await asyncio.sleep(max(0.0, self.until - LEAD_SECS - now))
        self.until += secs

// Your backend (Express). The API key never leaves it.
app.post("/api/tts-token", requireSignedIn, async (req, res) => {
  const r = await fetch("https://sandbox.voice.miraiminds.co/v2/tts/stream/tokens", {
    method: "POST",
    headers: { Authorization: `Bearer ${process.env.MIRA_API_KEY}` },
  });
  if (!r.ok) return res.status(502).json({ error: "could not mint a stream token" });
  const { url } = await r.json();
  res.json({ url: "wss://sandbox.voice.miraiminds.co" + url });
});

<textarea id="text">नमस्ते! आपका order कल शाम तक पहुँच जाएगा।</textarea>
<button id="speak">Speak</button>
<button id="stop">Stop</button>

<script type="module">
  const RATE = 24000;                  // pcm_s16le mono at 24 kHz
  const PREBUFFER = RATE * 0.15;       // samples held before a sentence starts

  const PLAYER = `
    class Player extends AudioWorkletProcessor {
      constructor() {
        super();
        this.queue = []; this.queued = 0; this.playing = false;
        this.port.onmessage = ({ data }) => {
          if (data === "clear") { this.queue = []; this.queued = 0; this.playing = false; return; }
          const pcm = new Int16Array(data);
          this.queue.push({ pcm, pos: 0 });
          this.queued += pcm.length;
        };
      }
      process(_inputs, [output]) {
        const out = output[0];
        out.fill(0);
        if (!this.playing && this.queued < ${PREBUFFER}) return true;
        this.playing = true;
        for (let i = 0; i < out.length; i++) {
          const head = this.queue[0];
          if (!head) { this.playing = false; break; }   // ran dry: buffer again first
          out[i] = head.pcm[head.pos++] / 0x8000;
          this.queued--;
          if (head.pos === head.pcm.length) this.queue.shift();
        }
        return true;
      }
    }
    registerProcessor("player", Player);
  `;

  const $ = (id) => document.getElementById(id);
  const ctx = new AudioContext({ sampleRate: RATE });
  await ctx.audioWorklet.addModule(
    URL.createObjectURL(new Blob([PLAYER], { type: "text/javascript" })));
  const player = new AudioWorkletNode(ctx, "player", { numberOfInputs: 0, outputChannelCount: [1] });
  player.connect(ctx.destination);

  let ws, turn = 0, speaking = null;
  const cancelled = new Set();

  async function connect() {
    const { url } = await fetch("/api/tts-token", { method: "POST" }).then((r) => r.json());
    ws = new WebSocket(url);
    ws.binaryType = "arraybuffer";
    ws.onmessage = ({ data }) => {
      if (typeof data !== "string") {
        if (!cancelled.has(speaking)) player.port.postMessage(data, [data]);
        return;
      }
      const event = JSON.parse(data);
      if (event.type === "audio.start") speaking = event.context_id;
      if (event.type === "error") console.warn(event.code, event.message);
    };
    await new Promise((resolve, reject) => {
      ws.addEventListener("open", resolve, { once: true });
      ws.addEventListener("error", reject, { once: true });
    });
    ws.send(JSON.stringify({ type: "session.update", voice: "neha", response_format: "pcm", sample_rate: RATE }));
  }

  $("speak").onclick = async () => {
    await ctx.resume();   // browsers start audio only after a click
    if (!ws || ws.readyState !== WebSocket.OPEN) await connect();
    turn += 1;
    ws.send(JSON.stringify({ type: "text", context_id: `turn-${turn}`, text: $("text").value, flush: true }));
  };

  $("stop").onclick = () => {
    const id = `turn-${turn}`;
    cancelled.add(id);
    ws?.send(JSON.stringify({ type: "cancel", context_id: id }));
    player.port.postMessage("clear");
  };
</script>