This guide adds a streaming chat to a Next.js App Router project. The browser talks to a Route Handler in your app; the handler talks to OrbContext with your secret key. The key never reaches the client.
Install and configure
Shell npm install @orbcontext/sdk.env.local ORBCONTEXT_API_URL=https://api-staging.orbcontext.com ORBCONTEXT_API_KEY=sk_live_... ORBCONTEXT_BOT_ID=b_...These variables have no
NEXT_PUBLIC_prefix, so Next keeps them on the server.Create one shared client
lib/orbcontext.ts import 'server-only'; import { OrbContext } from '@orbcontext/sdk'; // One client per process: it owns the connection limits, the cache and the 429 gate. export const orb = new OrbContext({ apiKey: process.env.ORBCONTEXT_API_KEY!, baseURL: process.env.ORBCONTEXT_API_URL!, }); export const BOT_ID = process.env.ORBCONTEXT_BOT_ID!;Stream answers from a Route Handler
The handler validates input, then forwards each event as one line of JSON (NDJSON), which is easy to read in any client. Passing
req.signalcancels generation when the visitor closes the tab, so you stop paying for an answer nobody reads.app/api/chat/route.ts
import { APIError, OrbContextError } from '@orbcontext/sdk'; import { BOT_ID, orb } from '@/lib/orbcontext'; export const runtime = 'nodejs'; export async function POST(req: Request) { const { message, conversationId } = await req.json().catch(() => ({})); if (typeof message !== 'string' || !message.trim() || message.length > 4000) { return Response.json({ error: 'A message of 1–4000 characters is required.' }, { status: 400 }); } const encoder = new TextEncoder(); const body = new ReadableStream({ async start(controller) { const send = (event: unknown) => controller.enqueue(encoder.encode(JSON.stringify(event) + '\n')); try { for await (const event of orb.chat.stream( { botId: BOT_ID, message, conversationId }, { signal: req.signal }, )) { send(event); } } catch (err) { // Never forward internal details. Tell the UI something it can show. const busy = err instanceof APIError && err.status === 429; send({ type: 'error', message: busy ? 'The service is busy. Please try again in a moment.' : 'The answer failed. Please try again.', }); if (!(err instanceof OrbContextError)) console.error(err); } finally { controller.close(); } }, }); return new Response(body, { headers: { 'Content-Type': 'application/x-ndjson; charset=utf-8', 'Cache-Control': 'no-store' }, }); }Build the chat component
A client component reads the stream and appends tokens as they arrive. It keeps the
conversationIdso follow-up questions have context.components/Chat.tsx 'use client'; import { useRef, useState } from 'react'; type Turn = { role: 'user' | 'assistant'; text: string; refused?: boolean }; export function Chat() { const [turns, setTurns] = useState<Turn[]>([]); const [busy, setBusy] = useState(false); const conversationId = useRef<string | undefined>(undefined); async function ask(message: string) { setBusy(true); setTurns((t) => [...t, { role: 'user', text: message }, { role: 'assistant', text: '' }]); const patch = (fn: (turn: Turn) => Turn) => setTurns((t) => t.map((turn, i) => (i === t.length - 1 ? fn(turn) : turn))); try { const res = await fetch('/api/chat', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ message, conversationId: conversationId.current }), }); if (!res.ok || !res.body) throw new Error('Request failed'); const reader = res.body.pipeThrough(new TextDecoderStream()).getReader(); let buffer = ''; for (;;) { const { value, done } = await reader.read(); if (done) break; buffer += value; const lines = buffer.split('\n'); buffer = lines.pop() ?? ''; for (const line of lines.filter(Boolean)) { const event = JSON.parse(line); if (event.type === 'started') conversationId.current = event.conversationId; if (event.type === 'token') patch((t) => ({ ...t, text: t.text + event.value })); if (event.type === 'done') patch((t) => ({ ...t, refused: event.refused })); if (event.type === 'error') patch((t) => ({ ...t, text: event.message })); } } } catch { patch((t) => ({ ...t, text: 'Something went wrong. Please try again.' })); } finally { setBusy(false); } } return ( <div> {turns.map((turn, i) => ( <p key={i} data-role={turn.role}>{turn.text}</p> ))} <form onSubmit={(e) => { e.preventDefault(); const input = e.currentTarget.elements.namedItem('q') as HTMLInputElement; if (input.value.trim() && !busy) ask(input.value.trim()); input.value = ''; }} > <input name="q" placeholder="Ask a question" disabled={busy} /> <button disabled={busy}>Send</button> </form> </div> ); }app/page.tsx import { Chat } from '@/components/Chat'; export default function Page() { return <Chat />; }
Production checklist
- Authenticate your visitors in the Route Handler before you call OrbContext, and rate-limit it. Anyone who can reach
/api/chatspends your credits. - Do not retry chat automatically. A retry can charge twice. Let the user press send again.
- Track the conversation yourself if users can have several: store the
conversationIdagainst your own user and check ownership before reusing it. - Use the
refusedflag to show a “talk to a person” option when the bot could not find an answer.