Streaming response

A response part-way through generation, with a caret on the line still arriving. First token lands in about a second and the whole response takes thirty, so the reader spends twenty-nine of those reading rather than waiting.

npx shadcn@latest add button
npm i lucide-react

Tokens this needs: --foreground, --card, --muted, --muted-foreground, --border, --chart-1

────────────────────────────────────────────────────────────────────────
// StreamingResponse.tsx
────────────────────────────────────────────────────────────────────────

import { useEffect, useRef, useState, type ReactNode } from "react";
import { ArrowDown } from "lucide-react";
import { Button } from "@/components/ui/button";

type Props = {
  /** Everything received so far. The caller appends each text_delta. */
  text: string;
  /** True once message_stop arrives. */
  done: boolean;
  /** Your markdown renderer. Only closed blocks reach it. */
  render: (markdown: string) => ReactNode;
  /** Time to first token and total, when the product wants to show them. */
  timing?: { firstToken: number; total: number };
};

/**
 * Markdown is only meaningful once its delimiter arrives. A fence opened on
 * the last delta would render as an empty code block that reflows on every
 * token, so split at the last unclosed one and hold the tail as plain text.
 */
function splitOpenFence(text: string) {
  const fences = [...text.matchAll(/^```/gm)];
  if (fences.length % 2 === 0) return { closed: text, open: "" };
  const at = fences[fences.length - 1].index!;
  return { closed: text.slice(0, at), open: text.slice(at) };
}

export function StreamingResponse({ text, done, render, timing }: Props) {
  const box = useRef<HTMLDivElement>(null);
  const [pinned, setPinned] = useState(true);
  const { closed, open } = splitOpenFence(text);

  // Pin the viewport to the live edge until the reader scrolls up. Any
  // upward scroll detaches; the button below is the way back.
  useEffect(() => {
    const el = box.current;
    if (el && pinned) el.scrollTop = el.scrollHeight;
  }, [text, pinned]);
  const onScroll = () => {
    const el = box.current!;
    setPinned(el.scrollTop + el.clientHeight >= el.scrollHeight - 8);
  };

  return (
    <div className="relative">
      <div ref={box} onScroll={onScroll} className="max-h-96 overflow-y-auto text-sm leading-relaxed text-foreground">
        {render(closed)}
        {open && <pre className="whitespace-pre-wrap font-mono text-xs text-muted-foreground">{open}</pre>}
        {!done && <span aria-hidden="true" className="ml-0.5 inline-block h-3.5 w-0.5 animate-pulse bg-foreground align-middle" />}
      </div>

      {!pinned && !done && (
        <Button size="sm" variant="outline" className="absolute bottom-2 left-1/2 -translate-x-1/2 shadow" onClick={() => setPinned(true)}>
          <ArrowDown className="size-4" /> back to live
        </Button>
      )}

      {/* A live region fed per token interrupts itself forever. Announce the
          start and the end as discrete cues, and the whole text once. */}
      <div className="sr-only" aria-live="polite">
        {done ? text : text ? "Responding" : ""}
      </div>

      {timing && (
        <div className="mt-4">
          <div className="flex h-3 w-full overflow-hidden rounded-[2px]" role="img" aria-label={`first token after ${timing.firstToken}s, ${timing.total}s total`}>
            <span className="bg-chart-1" style={{ width: `${(100 * timing.firstToken) / timing.total}%` }} />
            <span className="flex-1 bg-muted" />
          </div>
          <div className="mt-1.5 flex justify-between text-xs tabular-nums text-muted-foreground">
            <span>{timing.firstToken}s</span><span>{timing.total}s total</span>
          </div>
        </div>
      )}
    </div>
  );
}

────────────────────────────────────────────────────────────────────────
// demo.tsx
────────────────────────────────────────────────────────────────────────

import { useEffect, useState } from "react";
import { StreamingResponse } from "./StreamingResponse";

/**
 * The answer arrives a few words at a time. It starts part-way through, with
 * a fence just opened, which is the moment the wireframe draws: a closed
 * paragraph, a line still arriving, and markdown held as plain text until
 * its delimiter comes.
 */
const HEAD =
  "Three things are stacking up. The TypeScript check runs twice, once in the lint step and again in the build, because both scripts call tsc and neither passes --incremental. The image pipeline re-encodes every asset on every run, since the cache key includes a timestamp. And the test step sits on the critical path even though nothing downstream depends on it.\n\nThe quickest win is the double type-check. Run it once, up front, and reuse the result:\n\n```bash npm run build";
const TAIL =
  " --skip-typecheck\n```\n\nThen key the image cache on a content hash rather than mtime, and move the tests into a parallel job that gates the deploy rather than the build.";

/** Plain paragraphs. A real app hands this to its markdown renderer. */
const render = (md: string) =>
  md.split(/\n\n+/).filter(Boolean).map((p, i) => (
    p.startsWith("```")
      ? <pre key={i} className="mt-3 rounded-md bg-muted p-3 font-mono text-xs">{p.replace(/^```\w*\s?|```$/g, "").trim()}</pre>
      : <p key={i} className={i ? "mt-3" : ""}>{p}</p>
  ));

export default function Demo() {
  const [text, setText] = useState(HEAD);
  const [done, setDone] = useState(false);

  useEffect(() => {
    const words = TAIL.split(/(?<=\s)/);
    let i = 0;
    const id = setInterval(() => {
      if (i >= words.length) { setDone(true); clearInterval(id); return; }
      setText((t) => t + words[i++]);
    }, 120);
    return () => clearInterval(id);
  }, []);

  return (
    <div className="rounded-lg border bg-card p-4">
      <div className="flex justify-end">
        <p className="max-w-md rounded-xl bg-muted px-3.5 py-2 text-sm text-foreground">Why is the build slow?</p>
      </div>
      <div className="mt-5">
        <StreamingResponse text={text} done={done} render={render} timing={{ firstToken: 1, total: 30 }} />
      </div>
    </div>
  );
}