Benchmarking Inference

import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

public class TokenTimings {
    record Chunk(long atMs, int tokens) {}            // when a streamed chunk arrived, and how many tokens it held

    public static void main(String[] args) {
        long sentMs = 0;
        List<Chunk> chunks = List.of(new Chunk(412, 1), new Chunk(436, 1), new Chunk(461, 1), new Chunk(485, 1),
                new Chunk(557, 2), new Chunk(581, 1), new Chunk(830, 1), new Chunk(854, 1));
        int tokens = chunks.stream().mapToInt(Chunk::tokens).sum();
        long ttft = chunks.get(0).atMs() - sentMs;
        long e2e = chunks.get(chunks.size() - 1).atMs() - sentMs;
        List<Long> itl = new ArrayList<>();
        for (int i = 1; i < chunks.size(); i++) itl.add(chunks.get(i).atMs() - chunks.get(i - 1).atMs());
        double tpot = (double) (e2e - ttft) / (tokens - 1);
        double meanItl = itl.stream().mapToLong(Long::longValue).average().orElse(0);
        long maxItl = itl.stream().mapToLong(Long::longValue).max().orElse(0);
        System.out.println("chunks=" + chunks.size() + " tokens=" + tokens);
        System.out.println("TTFT      = " + ttft + " ms");
        System.out.println("E2E       = " + e2e + " ms");
        System.out.println("ITL list  = " + itl + " ms");
        System.out.printf(Locale.ROOT, "mean ITL  = %.2f ms, max ITL = %d ms%n", meanItl, maxItl);
        System.out.printf(Locale.ROOT, "TPOT      = (%d - %d) / (%d - 1) = %.2f ms%n", e2e, ttft, tokens, tpot);
        System.out.printf(Locale.ROOT, "decode speed seen by this user = %.1f tokens/s%n", 1000 / tpot);
        System.out.printf(Locale.ROOT, "tokens / E2E                   = %.1f tokens/s%n", tokens * 1000.0 / e2e);
    }
}

import java.util.List;
import java.util.Locale;

public class ThroughputVsUser {
    record Req(double startS, double ttftS, double endS, int outTokens) {}

    public static void main(String[] args) {
        System.out.println("batch  step ms  tok/s per user  tok/s total");
        for (int b : new int[] {1, 2, 4, 8, 16, 32}) {
            double stepMs = 12 + 0.6 * b;                     // decode step cost model: fixed weights read + per-sequence work
            System.out.printf(Locale.ROOT, "%5d  %7.1f  %14.1f  %11.1f%n", b, stepMs, 1000 / stepMs, b * 1000 / stepMs);
        }
        List<Req> trace = List.of(new Req(0.0, 0.4, 6.4, 200), new Req(0.5, 0.9, 4.9, 120),
                new Req(1.0, 1.6, 9.6, 250), new Req(7.0, 7.3, 9.8, 100));
        double first = trace.stream().mapToDouble(Req::startS).min().orElse(0);
        double last = trace.stream().mapToDouble(Req::endS).max().orElse(0);
        int tokens = trace.stream().mapToInt(Req::outTokens).sum();
        double perUser = trace.stream().mapToDouble(r -> (r.outTokens() - 1) / (r.endS() - r.ttftS())).average().orElse(0);
        System.out.printf(Locale.ROOT, "trace: %d tokens in %.1f s -> output throughput %.1f tok/s%n", tokens, last - first, tokens / (last - first));
        System.out.printf(Locale.ROOT, "trace: mean per-request decode speed %.1f tok/s%n", perUser);
        for (Req r : trace)
            System.out.printf(Locale.ROOT, "  request at %.1f s: %d tokens, decode %.1f tok/s%n", r.startS(), r.outTokens(), (r.outTokens() - 1) / (r.endS() - r.ttftS()));
    }
}

import java.util.Arrays;
import java.util.Locale;
import java.util.Random;
import org.HdrHistogram.Histogram;

public class HdrPercentiles {
    public static void main(String[] args) {
        Random random = new Random(42);
        Histogram h = new Histogram(60_000_000L, 3);        // values up to 60 s in microseconds, 3 significant digits
        long[] all = new long[1000];
        for (int i = 0; i < all.length; i++) {
            long us = random.nextDouble() < 0.05
                    ? 1_500_000 + random.nextInt(1_000_000)                          // 5%: queued behind a long prompt
                    : Math.max(50_000, 180_000 + (long) (random.nextGaussian() * 30_000)); // 95%: normal TTFT
            all[i] = us;
            h.recordValue(us);
        }
        Arrays.sort(all);
        System.out.printf(Locale.ROOT, "count %d, mean %.3f ms%n", h.getTotalCount(), h.getMean() / 1000);
        System.out.println("percentile   exact ms     hdr ms");
        for (double p : new double[] {50, 90, 95, 99, 99.9, 100}) {
            int rank = (int) Math.ceil(Math.round(p * 10) * all.length / 1000.0) - 1;   // nearest rank
            System.out.printf(Locale.ROOT, "%10s  %9.3f  %9.3f%n", "p" + (p == 100 ? "max" : (p % 1 == 0 ? String.valueOf((int) p) : String.valueOf(p))),
                    all[rank] / 1000.0, h.getValueAtPercentile(p) / 1000.0);
        }
        System.out.println("bucket of 1,234,567 us: " + h.lowestEquivalentValue(1_234_567) + " .. " + h.highestEquivalentValue(1_234_567));
    }
}

import java.util.List;
import java.util.Locale;

public class Goodput {
    record Done(int id, double ttftMs, double e2eMs, int outTokens) {
        double tpotMs() { return outTokens > 1 ? (e2eMs - ttftMs) / (outTokens - 1) : 0; }   // vLLM: 0 for one token
    }

    static long good(List<Done> done, double ttftSlo, double tpotSlo) {
        return done.stream().filter(d -> d.ttftMs() <= ttftSlo && d.tpotMs() <= tpotSlo).count();
    }

    public static void main(String[] args) {
        List<Done> done = List.of(
                new Done(1, 310, 6_310, 121), new Done(2, 420, 9_420, 151), new Done(3, 1_250, 7_250, 101),
                new Done(4, 380, 12_380, 161), new Done(5, 290, 290, 1), new Done(6, 2_900, 9_900, 141),
                new Done(7, 450, 5_450, 101), new Done(8, 330, 4_330, 41), new Done(9, 610, 6_610, 121),
                new Done(10, 395, 8_795, 141));
        double durationS = 20.0;
        System.out.println("id  TTFT ms  TPOT ms  TTFT<=500  TPOT<=60");
        for (Done d : done)
            System.out.printf(Locale.ROOT, "%2d  %7.0f  %7.2f  %9s  %8s%n", d.id(), d.ttftMs(), d.tpotMs(),
                    d.ttftMs() <= 500 ? "yes" : "no", d.tpotMs() <= 60 ? "yes" : "no");
        System.out.printf(Locale.ROOT, "request throughput: %d / %.0f s = %.2f req/s%n", done.size(), durationS, done.size() / durationS);
        for (double[] slo : new double[][] {{500, 60}, {1000, 60}, {500, 100}, {300, 50}}) {
            long g = good(done, slo[0], slo[1]);
            System.out.printf(Locale.ROOT, "goodput TTFT<=%4.0f ms, TPOT<=%3.0f ms: %2d good -> %.2f req/s%n", slo[0], slo[1], g, g / durationS);
        }
    }
}

import java.util.Locale;
import java.util.Random;

public class PoissonArrivals {
    public static void main(String[] args) {
        double rate = 4.0;                                   // requests per second
        Random random = new Random(7);
        double t = 0;
        StringBuilder first = new StringBuilder();
        int n = 10_000, shortGaps = 0;
        int[] perSecond = new int[3000];
        for (int i = 0; i < n; i++) {
            double gap = -Math.log(1 - random.nextDouble()) / rate;   // exponential gap: inverse of the CDF
            t += gap;
            if (gap < 0.05) shortGaps++;
            if (i < 8) first.append(String.format(Locale.ROOT, "%.3f ", t));
            if (t < perSecond.length) perSecond[(int) t]++;
        }
        System.out.println("first 8 arrival times (s): " + first.toString().trim());
        System.out.printf(Locale.ROOT, "%d arrivals in %.1f s: mean rate %.3f req/s, mean gap %.4f s%n", n, t, n / t, t / n);
        System.out.printf(Locale.ROOT, "gaps under 50 ms: %.1f%% (a fixed 250 ms schedule has 0%%)%n", 100.0 * shortGaps / n);
        int full = (int) t, min = Integer.MAX_VALUE, max = 0;
        int[] hist = new int[16];
        for (int s = 0; s < full; s++) { min = Math.min(min, perSecond[s]); max = Math.max(max, perSecond[s]); hist[Math.min(15, perSecond[s])]++; }
        System.out.println("requests in one 1-second window: min " + min + ", max " + max + " (fixed schedule: always 4)");
        StringBuilder line = new StringBuilder("windows with k requests (k:count):");
        for (int k = 0; k <= max; k++) line.append(' ').append(k).append(':').append(hist[k]);
        System.out.println(line);
    }
}

import java.util.Locale;
import org.HdrHistogram.Histogram;

public class OpenVsClosed {
    // one request at a time, 50 ms each; the server is frozen from 10,000 ms to 12,000 ms
    static double serve(double start) {
        if (start >= 10_000 && start < 12_000) start = 12_000;
        double end = start + 50;
        if (start < 10_000 && end > 10_000) end += 2_000;
        return end;
    }

    static void report(String name, Histogram h) {
        System.out.printf(Locale.ROOT, "%-34s n=%3d  p50=%7.1f  p90=%7.1f  p99=%7.1f  max=%7.1f ms%n", name, h.getTotalCount(),
                h.getValueAtPercentile(50) / 1000.0, h.getValueAtPercentile(90) / 1000.0,
                h.getValueAtPercentile(99) / 1000.0, h.getMaxValue() / 1000.0);
    }

    public static void main(String[] args) {
        Histogram closedNaive = new Histogram(60_000_000L, 3), closedHdr = new Histogram(60_000_000L, 3);
        Histogram closedIntended = new Histogram(60_000_000L, 3), open = new Histogram(60_000_000L, 3);
        double free = 0;                                       // closed loop: one client, waits for each response
        for (double planned = 0; planned < 30_000; planned += 100) {
            double sent = Math.max(planned, free);
            free = serve(sent);
            closedNaive.recordValue((long) ((free - sent) * 1000));                  // from the actual send
            closedHdr.recordValueWithExpectedInterval((long) ((free - sent) * 1000), 100_000);
            closedIntended.recordValue((long) ((free - planned) * 1000));            // from the intended send
        }
        double serverFree = 0;                                  // open loop: send on schedule, server queues
        for (double planned = 0; planned < 30_000; planned += 100) {
            serverFree = serve(Math.max(planned, serverFree));
            open.recordValue((long) ((serverFree - planned) * 1000));
        }
        report("closed loop, from actual send", closedNaive);
        report("closed loop, expected-interval fix", closedHdr);
        report("closed loop, from intended send", closedIntended);
        report("open loop, from intended send", open);
    }
}

import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

public class LittleClosedLoop {
    static final int PROMPT = 200, OUTPUT = 50;

    static final class Req { double arrival; int done; Req(double a) { arrival = a; } }

    // n clients; each waits think ms after a response before sending the next request
    static void run(int n, double think) {
        List<Req> active = new ArrayList<>();
        double[] nextSend = new double[n];
        Req[] ofClient = new Req[n];
        double now = 0, end = 120_000, busySum = 0, latencySum = 0;
        long finished = 0;
        while (now < end) {
            for (int c = 0; c < n; c++)
                if (ofClient[c] == null && nextSend[c] <= now) { ofClient[c] = new Req(nextSend[c]); active.add(ofClient[c]); }
            if (active.isEmpty()) { double next = Double.MAX_VALUE; for (double s : nextSend) next = Math.min(next, s); now = next; continue; }
            int prefills = 0;
            for (Req r : active) if (r.done == 0) prefills++;
            double step = 10 + 0.5 * active.size() + 0.05 * PROMPT * prefills;    // ms for this forward pass
            busySum += active.size() * step;                                     // request-ms spent in the system
            now += step;
            for (int c = 0; c < n; c++) {
                Req r = ofClient[c];
                if (r == null || !active.contains(r)) continue;
                if (++r.done == OUTPUT) {
                    active.remove(r); ofClient[c] = null; finished++;
                    latencySum += now - r.arrival; nextSend[c] = now + think;
                }
            }
        }
        double lambda = finished / (now / 1000), w = latencySum / finished / 1000, l = busySum / now;
        System.out.printf(Locale.ROOT, "N=%2d think=%4.1f s: lambda=%6.2f req/s  W=%5.3f s  L=%5.2f  lambda*W=%5.2f  lambda*(W+Z)=%5.2f%n",
                n, think / 1000, lambda, w, l, lambda * w, lambda * (w + think / 1000));
    }

    public static void main(String[] args) {
        for (int n : new int[] {1, 4, 16}) run(n, 0);
        for (int n : new int[] {4, 16}) run(n, 2_000);
    }
}

import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.Locale;
import java.util.Random;

public class ConcurrencySweep {
    static final int PROMPT = 200, SLOTS = 16;

    static final class Req {
        final int client, output; final double arrival; double firstToken; int done;
        Req(int c, double a, int out) { client = c; arrival = a; output = out; }
    }

    static double pct(List<Double> v, double p) {
        List<Double> s = new ArrayList<>(v); s.sort(null);
        return s.get((int) Math.ceil(p / 100 * s.size()) - 1);
    }

    static void run(int n) {
        Random random = new Random(1);                                    // output lengths 30 to 70 tokens
        Deque<Req> waiting = new ArrayDeque<>();
        List<Req> running = new ArrayList<>();
        List<Double> ttft = new ArrayList<>(), tpot = new ArrayList<>(), e2e = new ArrayList<>();
        for (int c = 0; c < n; c++) waiting.add(new Req(c, 0, 30 + random.nextInt(41)));          // all clients send at time 0
        double now = 0, end = 120_000;
        long tokens = 0;
        while (now < end) {
            while (running.size() < SLOTS && !waiting.isEmpty()) running.add(waiting.poll());   // admit into free slots
            int prefills = 0;
            for (Req r : running) if (r.done == 0) prefills++;
            now += 10 + 0.5 * running.size() + 0.05 * PROMPT * prefills;
            for (Req r : new ArrayList<>(running)) {
                if (r.done++ == 0) r.firstToken = now;
                tokens++;
                if (r.done == r.output) {
                    running.remove(r);
                    ttft.add(r.firstToken - r.arrival); e2e.add(now - r.arrival);
                    tpot.add((now - r.firstToken) / (r.output - 1));
                    waiting.add(new Req(r.client, now, 30 + random.nextInt(41)));                     // closed loop: send again at once
                }
            }
        }
        System.out.printf(Locale.ROOT, "%3d  %7.2f  %8.0f  %8.0f  %8.0f  %7.1f  %7.0f%n", n, e2e.size() / (now / 1000), tokens / (now / 1000),
                pct(ttft, 50), pct(ttft, 99), tpot.stream().mapToDouble(Double::doubleValue).average().orElse(0),
                e2e.stream().mapToDouble(Double::doubleValue).average().orElse(0));
    }

    public static void main(String[] args) {
        System.out.println("  N    req/s     tok/s  TTFTp50  TTFTp99  TPOT ms   E2E ms");
        for (int n : new int[] {1, 2, 4, 8, 16, 24, 32, 64}) run(n);
    }
}

import java.util.Arrays;
import java.util.Locale;
import java.util.Random;

public class PromptMix {
    static final int BLOCK = 16;                         // the prefix cache stores whole 16-token blocks

    // prompt = shared prefix + unique part; cachedPrefix says whether the shared prefix is already in the cache
    static double ttftMs(int prefix, int unique, boolean cachedPrefix) {
        int cached = cachedPrefix ? (prefix / BLOCK) * BLOCK : 0;
        if (cached == prefix + unique) cached -= 1;      // the last prompt token is always computed again
        int computed = prefix + unique - cached;
        return 25 + computed / 4.0;                      // 25 ms overhead, 4,000 prompt tokens per second
    }

    static void report(String name, double[] ttft, long promptTokens, long computedTokens) {
        Arrays.sort(ttft);
        double mean = Arrays.stream(ttft).average().orElse(0);
        System.out.printf(Locale.ROOT, "%-30s mean %6.1f  p50 %6.1f  p99 %6.1f ms  cache hit %5.1f%%%n", name, mean,
                ttft[ttft.length / 2 - 1], ttft[(int) Math.ceil(0.99 * ttft.length) - 1], 100.0 * (promptTokens - computedTokens) / promptTokens);
    }

    static int logNormal(Random r, double median, double sigma, int max) {
        return (int) Math.min(max, Math.max(8, Math.round(median * Math.exp(sigma * r.nextGaussian()))));
    }

    public static void main(String[] args) {
        int n = 1000;
        Random random = new Random(3);
        double[][] t = new double[4][n];
        long[][] tok = new long[4][2];
        for (int i = 0; i < n; i++) {
            int len = logNormal(random, 600, 1.0, 8192), user = logNormal(random, 200, 0.8, 4096);
            int[][] cases = {{1024, 0, i > 0 ? 1 : 0}, {0, 1024, 0}, {0, len, 0}, {1500, user, i > 0 ? 1 : 0}};
            for (int c = 0; c < 4; c++) {
                int prefix = cases[c][0], unique = cases[c][1];
                boolean hit = cases[c][2] == 1;
                t[c][i] = ttftMs(prefix, unique, hit);
                tok[c][0] += prefix + unique;
                tok[c][1] += Math.round((t[c][i] - 25) * 4);
            }
        }
        report("same 1024-token prompt", t[0], tok[0][0], tok[0][1]);
        report("random 1024-token prompts", t[1], tok[1][0], tok[1][1]);
        report("lognormal lengths, median 600", t[2], tok[2][0], tok[2][1]);
        report("1500 shared + lognormal 200", t[3], tok[3][0], tok[3][1]);
    }
}

import java.util.Locale;

public class BatchedBenchColumns {
    // formulas from tools/batched-bench/batched-bench.cpp
    static void row(int pp, int tg, int pl, boolean ppShared, boolean kvUnified, double tPp, double tTg) {
        int nKv = ppShared ? (kvUnified ? pp : pl * pp) + pl * tg : pl * (pp + tg);
        double speedPp = ppShared ? pp / tPp : pl * pp / tPp;
        double speedTg = pl * tg / tTg;
        double t = tPp + tTg;
        double speed = (ppShared ? pp + pl * tg : pl * (pp + tg)) / t;
        System.out.printf(Locale.ROOT, "|%6d | %6d | %4d | %6d | %8.3f | %8.2f | %8.3f | %8.2f | %8.3f | %8.2f |%n",
                pp, tg, pl, nKv, tPp, speedPp, tTg, speedTg, t, speed);
    }

    public static void main(String[] args) {
        System.out.println("|    PP |     TG |    B |   N_KV |   T_PP s | S_PP t/s |   T_TG s | S_TG t/s |      T s |    S t/s |");
        row(128, 128, 1, false, false, 0.108, 3.079);     // times from the README sample table
        row(128, 128, 8, false, false, 0.751, 7.344);
        row(128, 128, 32, false, false, 3.408, 8.801);
        row(128, 128, 32, true, false, 0.108, 8.801);   // assumed times: -pps, one prefill of 128 tokens
        row(128, 128, 32, true, true, 0.108, 8.801);    // the same with -kvu
        double perSeq1 = 128 / 3.079, perSeq32 = 128 / 8.801;
        System.out.printf(Locale.ROOT, "per-sequence decode speed: B=1 %.2f t/s, B=32 %.2f t/s%n", perSeq1, perSeq32);
    }
}

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.Locale;

public class LlamaBenchJson {
    // one line of `llama-bench -o jsonl` from the pinned build (tiny random model; some fields removed)
    static final String LINE = """
        {"build_commit": "0c1e570", "backends": "CPU", "model_type": "llama ?B all F32", "model_size": 322446336, \
        "model_n_params": 80611584, "n_batch": 2048, "n_ubatch": 512, "n_threads": 2, "type_k": "f16", "type_v": "f16", \
        "n_gpu_layers": -1, "flash_attn": -1, "n_prompt": 64, "n_gen": 16, "n_depth": 0, \
        "avg_ns": 77120449, "stddev_ns": 9304004, "avg_ts": 1044.942729, "stddev_ts": 126.064519, \
        "samples_ns": [ 83699374, 70541524 ],"samples_ts": [ 955.802, 1134.08 ]}""";

    public static void main(String[] args) throws Exception {
        JsonNode r = new ObjectMapper().readTree(LINE);
        int prompt = r.get("n_prompt").asInt(), gen = r.get("n_gen").asInt(), depth = r.get("n_depth").asInt();
        String test = prompt > 0 && gen > 0 ? "pp" + prompt + "+tg" + gen : prompt > 0 ? "pp" + prompt : "tg" + gen;
        System.out.println("test " + test + (depth > 0 ? " @ d" + depth : "") + ", threads " + r.get("n_threads").asInt()
                + ", K/V cache " + r.get("type_k").asText() + "/" + r.get("type_v").asText());
        double sum = 0;
        for (JsonNode ns : r.get("samples_ns")) {
            double ts = 1e9 * (prompt + gen) / ns.asLong();                  // llama-bench: (n_prompt + n_gen) / time
            sum += ts;
            System.out.printf(Locale.ROOT, "  sample %.3f ms -> %.3f t/s%n", ns.asLong() / 1e6, ts);
        }
        System.out.printf(Locale.ROOT, "avg_ts (mean of samples) %.3f, reported %.3f%n", sum / r.get("samples_ns").size(), r.get("avg_ts").asDouble());
        System.out.printf(Locale.ROOT, "tokens / avg_ns          %.3f%n", 1e9 * (prompt + gen) / r.get("avg_ns").asLong());
        System.out.printf(Locale.ROOT, "model %.2f MiB, %.2f M parameters, %.2f bytes per parameter%n", r.get("model_size").asLong() / 1048576.0,
                r.get("model_n_params").asLong() / 1e6, r.get("model_size").asDouble() / r.get("model_n_params").asLong());
    }
}

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;

public class VllmReport {
    record Out(boolean success, double start, int promptLen, double ttft, double[] itl, int outputTokens) {
        double latency() { return ttft + Arrays.stream(itl).sum(); }
    }

    static double pct(List<Double> v, double p) {                 // numpy's default: linear interpolation
        double[] a = v.stream().mapToDouble(Double::doubleValue).sorted().toArray();
        double idx = p / 100 * (a.length - 1);
        int lo = (int) Math.floor(idx), hi = (int) Math.ceil(idx);
        return a[lo] + (a[hi] - a[lo]) * (idx - lo);
    }

    static void line(String k, Object v) {
        System.out.println(String.format(Locale.ROOT, v instanceof Double ? "%-40s %-10.2f" : "%-40s %-10s", k, v).stripTrailing());
    }

    static void metric(String header, String name, List<Double> seconds) {
        int left = (50 - header.length()) / 2;                   // Python's "{s:-^50}" centering
        System.out.println("-".repeat(left) + header + "-".repeat(50 - header.length() - left));
        line("Mean " + name + " (ms):", seconds.stream().mapToDouble(Double::doubleValue).average().orElse(0) * 1000);
        line("Median " + name + " (ms):", pct(seconds, 50) * 1000);
        line("P99 " + name + " (ms):", pct(seconds, 99) * 1000);
    }

    public static void main(String[] args) {
        List<Out> outs = new ArrayList<>();
        for (int i = 0; i < 8; i++) {
            double[] itl = new double[19 + 5 * i];
            Arrays.fill(itl, 0.040 + 0.002 * i);
            outs.add(new Out(i != 7, 0.5 * i, 100 + 50 * i, 0.200 + 0.050 * i, itl, 20 + 5 * i));
        }
        double duration = 6.0;
        List<Double> ttft = new ArrayList<>(), tpot = new ArrayList<>(), itl = new ArrayList<>(), e2el = new ArrayList<>();
        int completed = 0, input = 0, output = 0, good = 0;
        for (Out o : outs) {
            if (!o.success()) continue;
            completed++; input += o.promptLen(); output += o.outputTokens();
            double t = o.outputTokens() > 1 ? (o.latency() - o.ttft()) / (o.outputTokens() - 1) : 0;
            if (o.outputTokens() > 1) tpot.add(t);
            for (double x : o.itl()) itl.add(x);
            ttft.add(o.ttft()); e2el.add(o.latency());
            if (o.ttft() <= 0.400 && t <= 0.050) good++;          // --goodput ttft:400 tpot:50
        }
        System.out.println("=".repeat(12) + " Serving Benchmark Result " + "=".repeat(12));
        line("Successful requests:", completed);
        line("Failed requests:", outs.size() - completed);
        line("Benchmark duration (s):", duration);
        line("Total input tokens:", input);
        line("Total generated tokens:", output);
        line("Request throughput (req/s):", completed / duration);
        line("Request goodput (req/s):", good / duration);
        line("Output token throughput (tok/s):", output / duration);
        line("Total token throughput (tok/s):", (input + output) / duration);
        metric("Time to First Token", "TTFT", ttft);
        metric("Time per Output Token (excl. 1st token)", "TPOT", tpot);
        metric("Inter-token Latency", "ITL", itl);
        metric("End-to-end Latency", "E2EL", e2el);
        System.out.println("=".repeat(50));
    }
}

import java.util.ArrayList;
import java.util.List;

public class BenchChecklist {
    record Plan(String name, int warmupRequests, int requestsPerLevel, boolean uniquePrompts, boolean fixedOutputLength,
                boolean streaming, boolean openLoopOrQueueTime, boolean freshServerPerConfig, int clientCpuPercent,
                boolean usageTokenCounts, boolean sameMachineAsProduction) {}

    static List<String> check(Plan p) {
        List<String> w = new ArrayList<>();
        if (p.warmupRequests() == 0) w.add("no warmup: the first requests pay for loading, graph capture, and cold caches");
        if (p.requestsPerLevel() < 200) w.add("only " + p.requestsPerLevel() + " requests per level: p99 is set by the slowest " + Math.max(1, p.requestsPerLevel() / 100) + " request(s)");
        if (!p.uniquePrompts()) w.add("identical prompts: the prefix cache serves almost every prompt token");
        if (!p.fixedOutputLength()) w.add("output length not fixed: an early end-of-sequence changes tokens per request between runs");
        if (!p.streaming()) w.add("not streaming: TTFT and ITL cannot be measured, only end-to-end latency");
        if (!p.openLoopOrQueueTime()) w.add("closed loop without queue time: stalls are hidden (coordinated omission)");
        if (!p.freshServerPerConfig()) w.add("server reused across configurations: cache contents carry over");
        if (p.clientCpuPercent() > 70) w.add("load generator at " + p.clientCpuPercent() + "% CPU: the client may be the bottleneck");
        if (!p.usageTokenCounts()) w.add("tokens counted as chunks: servers may put several tokens in one chunk");
        if (!p.sameMachineAsProduction()) w.add("different hardware or engine version than production: numbers do not transfer");
        return w;
    }

    public static void main(String[] args) {
        List<Plan> plans = List.of(
                new Plan("quick laptop run", 0, 20, false, false, true, false, false, 85, false, false),
                new Plan("release benchmark", 10, 500, true, true, true, true, true, 30, true, true));
        for (Plan p : plans) {
            List<String> warnings = check(p);
            System.out.println(p.name() + ": " + warnings.size() + " warning(s)");
            for (String w : warnings) System.out.println("  - " + w);
        }
    }
}

import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.HdrHistogram.Histogram;

public class BenchCore {
    interface Stopwatch { long micros(); }                  // time since the stopwatch started
    interface Clock { Stopwatch start(); }

    static Clock systemClock() {                            // production: System.nanoTime
        return () -> { long t0 = System.nanoTime(); return () -> (System.nanoTime() - t0) / 1000; };
    }

    /** Test clock: reading i of a stopwatch adds deltas[i % deltas.length] microseconds. */
    record FakeClock(long... deltas) implements Clock {
        public Stopwatch start() {
            long[] state = {0, 0};                           // {readings so far, current time}
            return () -> state[1] += deltas[(int) (state[0]++ % deltas.length)];
        }
    }

    /** One request: when it was sent, when each content chunk arrived, tokens from usage. */
    record Timing(long sentUs, List<Long> chunkUs, int tokens) {
        long ttft() { return chunkUs.get(0) - sentUs; }
        long e2e() { return chunkUs.get(chunkUs.size() - 1) - sentUs; }
        long tpot() { return tokens > 1 ? (e2e() - ttft()) / (tokens - 1) : 0; }
    }

    static final class Stats {
        final Histogram ttft = new Histogram(3_600_000_000L, 3), tpot = new Histogram(3_600_000_000L, 3);
        final Histogram itl = new Histogram(3_600_000_000L, 3), e2e = new Histogram(3_600_000_000L, 3);
        long tokens;
        void add(Timing t) {
            ttft.recordValue(t.ttft()); e2e.recordValue(t.e2e()); tokens += t.tokens();
            if (t.tokens() > 1) tpot.recordValue(t.tpot());
            for (int i = 1; i < t.chunkUs().size(); i++) itl.recordValue(t.chunkUs().get(i) - t.chunkUs().get(i - 1));
        }
        static String ms(Histogram h, double p) { return String.format(Locale.ROOT, "%.1f", h.getValueAtPercentile(p) / 1000.0); }
    }

    public static void main(String[] args) {
        Clock clock = new FakeClock(0, 300_000, 20_000, 20_000, 20_000, 60_000);  // send, TTFT, then 4 gaps
        Stopwatch sw = clock.start();
        Stats stats = new Stats();
        for (int r = 0; r < 3; r++) {                        // one worker, three requests back to back
            long sent = sw.micros();
            List<Long> chunks = new ArrayList<>();
            for (int c = 0; c < 5; c++) chunks.add(sw.micros());   // one reading per content chunk
            Timing t = new Timing(sent, chunks, 5);
            System.out.println("request " + r + ": sent " + sent + " us, chunks " + chunks + ", ttft " + t.ttft()
                    + ", tpot " + t.tpot() + ", e2e " + t.e2e());
            stats.add(t);
        }
        System.out.println("TTFT p50 " + Stats.ms(stats.ttft, 50) + " ms, TPOT p50 " + Stats.ms(stats.tpot, 50)
                + " ms, ITL p50 " + Stats.ms(stats.itl, 50) + " ms, ITL max " + Stats.ms(stats.itl, 100)
                + " ms, E2E p50 " + Stats.ms(stats.e2e, 50) + " ms, tokens " + stats.tokens);
        long t0 = systemClock().start().micros();
        System.out.println("system clock reading right after start is non-negative: " + (t0 >= 0));
    }
}

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.sun.net.httpserver.HttpServer;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Stream;

public class BenchLevel {
    // ---- clock and statistics (stage 1) ----
    interface Stopwatch { long micros(); }
    interface Clock { Stopwatch start(); }
    static Clock systemClock() { return () -> { long t0 = System.nanoTime(); return () -> (System.nanoTime() - t0) / 1000; }; }
    record FakeClock(long... deltas) implements Clock {
        public Stopwatch start() { long[] s = {0, 0}; return () -> s[1] += deltas[(int) (s[0]++ % deltas.length)]; }
    }
    record Timing(long sentUs, List<Long> chunkUs, int tokens) {
        long ttft() { return chunkUs.get(0) - sentUs; }
        long e2e() { return chunkUs.get(chunkUs.size() - 1) - sentUs; }
        long tpot() { return tokens > 1 ? (e2e() - ttft()) / (tokens - 1) : 0; }
    }

    // ---- trimmed ChatClient from Project 11: streaming text only ----
    @JsonInclude(JsonInclude.Include.NON_NULL)
    record Message(String role, String content) {}
    @JsonInclude(JsonInclude.Include.NON_NULL)
    record Request(String model, List<Message> messages, Double temperature, Integer maxTokens, List<Object> tools) {}
    record Usage(int promptTokens, int completionTokens, int totalTokens) {}
    record Result(String content, String finishReason, Usage usage) {}
    interface Listener { void onContent(String text); }
    static final class Cancel { volatile boolean cancelled; }

    static final class ChatClient implements AutoCloseable {
        final HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build();
        final ObjectMapper json = new ObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE)
                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
        final URI endpoint;
        ChatClient(String baseUrl, String apiKey) { endpoint = URI.create(baseUrl + "/chat/completions"); }

        Result stream(Request request, Listener listener, Cancel cancel) throws IOException, InterruptedException {
            ObjectNode body = json.valueToTree(request);
            body.put("stream", true);
            body.putObject("stream_options").put("include_usage", true);
            HttpRequest req = HttpRequest.newBuilder(endpoint).timeout(Duration.ofMinutes(10)).header("Content-Type", "application/json")
                    .POST(HttpRequest.BodyPublishers.ofString(json.writeValueAsString(body))).build();
            HttpResponse<Stream<String>> res = http.send(req, HttpResponse.BodyHandlers.ofLines());
            if (res.statusCode() >= 400) throw new IOException("HTTP " + res.statusCode());
            StringBuilder content = new StringBuilder();
            String finish = null;
            Usage usage = null;
            try (Stream<String> lines = res.body()) {
                Iterator<String> it = lines.iterator();
                while (it.hasNext() && !cancel.cancelled) {
                    String line = it.next();
                    if (!line.startsWith("data:")) continue;
                    String data = line.substring(5).trim();
                    if (data.equals("[DONE]")) break;
                    JsonNode chunk = json.readTree(data);
                    JsonNode u = chunk.path("usage");
                    if (u.isObject()) usage = new Usage(u.path("prompt_tokens").asInt(), u.path("completion_tokens").asInt(), u.path("total_tokens").asInt());
                    for (JsonNode choice : chunk.path("choices")) {
                        String text = choice.path("delta").path("content").asText("");
                        if (!text.isEmpty()) { content.append(text); listener.onContent(text); }
                        if (choice.hasNonNull("finish_reason")) finish = choice.get("finish_reason").asText();
                    }
                }
            }
            return new Result(content.toString(), finish, usage);
        }
        @Override public void close() { http.close(); }
    }

    static Timing once(ChatClient client, Stopwatch sw, int id, int maxTokens) throws Exception {
        List<Long> chunks = new ArrayList<>();
        long sent = sw.micros();                                    // reading 1: the send
        Result r = client.stream(new Request("tiny", List.of(new Message("user", "Request " + id)), 0.0, maxTokens, null),
                text -> chunks.add(sw.micros()), new Cancel());      // one reading per content chunk
        return new Timing(sent, chunks, r.usage().completionTokens());
    }

    // ---- fake streaming server: a role chunk, max_tokens one-token chunks, a usage chunk ----
    static HttpServer fakeServer() throws IOException {
        ObjectMapper json = new ObjectMapper();
        HttpServer server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
        server.setExecutor(Executors.newVirtualThreadPerTaskExecutor());
        server.createContext("/v1/chat/completions", ex -> {
            int n = json.readTree(ex.getRequestBody().readAllBytes()).path("max_tokens").asInt(16);
            ex.getResponseHeaders().set("Content-Type", "text/event-stream");
            ex.sendResponseHeaders(200, 0);
            try (OutputStream out = ex.getResponseBody()) {
                List<String> events = new ArrayList<>();
                events.add("{\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null},\"finish_reason\":null}]}");
                for (int i = 0; i < n; i++)
                    events.add("{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" tok" + i + "\"},\"finish_reason\":" + (i == n - 1 ? "\"length\"" : "null") + "}]}");
                events.add("{\"choices\":[],\"usage\":{\"prompt_tokens\":210,\"completion_tokens\":" + n + ",\"total_tokens\":" + (210 + n) + "}}");
                events.add("[DONE]");
                for (String e : events) { out.write(("data: " + e + "\n\n").getBytes(StandardCharsets.UTF_8)); out.flush(); }
            }
        });
        server.start();
        return server;
    }

    public static void main(String[] args) throws Exception {
        HttpServer server = fakeServer();
        Clock clock = new FakeClock(0, 200_000, 30_000, 30_000, 30_000);   // send, TTFT 200 ms, 3 gaps of 30 ms
        int n = 2, requests = 5;
        try (ChatClient client = new ChatClient("http://127.0.0.1:" + server.getAddress().getPort() + "/v1", null);
             ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor()) {
            List<Future<List<String>>> workers = new ArrayList<>();
            for (int w = 0; w < n; w++) {
                int worker = w;
                workers.add(pool.submit(() -> {
                    Stopwatch sw = clock.start();                          // each worker has its own stopwatch
                    List<String> lines = new ArrayList<>();
                    for (int id = worker; id < requests; id += n) {        // closed loop: next request after the last chunk
                        Timing t = once(client, sw, id, 4);
                        lines.add("worker " + worker + " request " + id + ": sent " + t.sentUs() / 1000 + " ms, chunks "
                                + t.chunkUs().stream().map(c -> c / 1000).toList() + " ms, tokens " + t.tokens()
                                + ", ttft " + t.ttft() / 1000 + ", tpot " + t.tpot() / 1000 + ", e2e " + t.e2e() / 1000);
                    }
                    return lines;
                }));
            }
            for (Future<List<String>> f : workers) f.get().forEach(System.out::println);
        } finally {
            server.stop(0);
        }
    }
}

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.sun.net.httpserver.HttpServer;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.IntFunction;
import java.util.stream.Stream;
import org.HdrHistogram.Histogram;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;

@Command(name = "bench", mixinStandardHelpOptions = true, description = "Load test an OpenAI-compatible server.")
public class LoopBench implements Callable<Integer> {
    @Option(names = "--url", required = true, description = "Base URL ending in /v1") String url;
    @Option(names = "--model", defaultValue = "default") String model;
    @Option(names = "--concurrency", split = ",", defaultValue = "1,2,4", description = "Closed-loop clients per level") int[] levels;
    @Option(names = "--requests", defaultValue = "16", description = "Requests per level") int requests;
    @Option(names = "--warmup", defaultValue = "2") int warmup;
    @Option(names = "--max-tokens", defaultValue = "32") int maxTokens;
    @Option(names = "--prompt-words", defaultValue = "200") int promptWords;
    @Option(names = "--slo-ttft-ms", defaultValue = "1000") double sloTtftMs;
    @Option(names = "--slo-tpot-ms", defaultValue = "100") double sloTpotMs;
    IntFunction<Clock> clocks = level -> systemClock();      // tests inject a fake clock per level

    // ---- clock and statistics (stage 1) ----
    interface Stopwatch { long micros(); }
    interface Clock { Stopwatch start(); }
    static Clock systemClock() { return () -> { long t0 = System.nanoTime(); return () -> (System.nanoTime() - t0) / 1000; }; }
    record FakeClock(long... deltas) implements Clock {
        public Stopwatch start() { long[] s = {0, 0}; return () -> s[1] += deltas[(int) (s[0]++ % deltas.length)]; }
    }
    record Timing(long sentUs, List<Long> chunkUs, int tokens) {
        long ttft() { return chunkUs.get(0) - sentUs; }
        long e2e() { return chunkUs.get(chunkUs.size() - 1) - sentUs; }
        long tpot() { return tokens > 1 ? (e2e() - ttft()) / (tokens - 1) : 0; }
    }

    // ---- trimmed ChatClient from Project 11: streaming text only ----
    @JsonInclude(JsonInclude.Include.NON_NULL)
    record Message(String role, String content) {}
    @JsonInclude(JsonInclude.Include.NON_NULL)
    record Request(String model, List<Message> messages, Double temperature, Integer maxTokens, List<Object> tools) {}
    record Usage(int promptTokens, int completionTokens, int totalTokens) {}
    record Result(String content, String finishReason, Usage usage) {}
    interface Listener { void onContent(String text); }
    static final class Cancel { volatile boolean cancelled; }

    static final class ChatClient implements AutoCloseable {
        final HttpClient http = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build();
        final ObjectMapper json = new ObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE)
                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
        final URI endpoint;
        ChatClient(String baseUrl, String apiKey) { endpoint = URI.create(baseUrl + "/chat/completions"); }

        Result stream(Request request, Listener listener, Cancel cancel) throws IOException, InterruptedException {
            ObjectNode body = json.valueToTree(request);
            body.put("stream", true);
            body.putObject("stream_options").put("include_usage", true);
            HttpRequest req = HttpRequest.newBuilder(endpoint).timeout(Duration.ofMinutes(10)).header("Content-Type", "application/json")
                    .POST(HttpRequest.BodyPublishers.ofString(json.writeValueAsString(body))).build();
            HttpResponse<Stream<String>> res = http.send(req, HttpResponse.BodyHandlers.ofLines());
            if (res.statusCode() >= 400) throw new IOException("HTTP " + res.statusCode());
            StringBuilder content = new StringBuilder();
            String finish = null;
            Usage usage = null;
            try (Stream<String> lines = res.body()) {
                Iterator<String> it = lines.iterator();
                while (it.hasNext() && !cancel.cancelled) {
                    String line = it.next();
                    if (!line.startsWith("data:")) continue;
                    String data = line.substring(5).trim();
                    if (data.equals("[DONE]")) break;
                    JsonNode chunk = json.readTree(data);
                    JsonNode u = chunk.path("usage");
                    if (u.isObject()) usage = new Usage(u.path("prompt_tokens").asInt(), u.path("completion_tokens").asInt(), u.path("total_tokens").asInt());
                    for (JsonNode choice : chunk.path("choices")) {
                        String text = choice.path("delta").path("content").asText("");
                        if (!text.isEmpty()) { content.append(text); listener.onContent(text); }
                        if (choice.hasNonNull("finish_reason")) finish = choice.get("finish_reason").asText();
                    }
                }
            }
            return new Result(content.toString(), finish, usage);
        }
        @Override public void close() { http.close(); }
    }

    // ---- one request, one level ----
    Timing once(ChatClient client, Stopwatch sw, int id) throws Exception {
        String prompt = "Request " + id + ". " + "Summarize the build log. ".repeat(Math.max(1, promptWords / 4));
        List<Long> chunks = new ArrayList<>();
        long sent = sw.micros();
        Result r = client.stream(new Request(model, List.of(new Message("user", prompt)), 0.0, maxTokens, null),
                text -> chunks.add(sw.micros()), new Cancel());
        if (chunks.isEmpty()) throw new IOException("no content chunk");
        return new Timing(sent, chunks, r.usage() != null ? r.usage().completionTokens() : chunks.size());
    }

    String level(ChatClient client, int n) throws Exception {
        Clock clock = clocks.apply(n);
        Histogram ttft = new Histogram(3_600_000_000L, 3), tpot = new Histogram(3_600_000_000L, 3), itl = new Histogram(3_600_000_000L, 3);
        long tokens = 0, durationUs = 0;
        int ok = 0, failed = 0, good = 0;
        try (ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor()) {
            List<Future<List<Timing>>> workers = new ArrayList<>();
            for (int w = 0; w < n; w++) {
                int worker = w;
                workers.add(pool.submit(() -> {                          // closed loop: requests w, w+n, w+2n, ...
                    Stopwatch sw = clock.start();
                    List<Timing> done = new ArrayList<>();
                    for (int id = worker; id < requests; id += n) {
                        try { done.add(once(client, sw, n * 1000 + id)); } catch (IOException e) { done.add(null); }
                    }
                    return done;
                }));
            }
            for (Future<List<Timing>> f : workers) {
                for (Timing t : f.get()) {
                    if (t == null) { failed++; continue; }
                    ok++; tokens += t.tokens();
                    ttft.recordValue(t.ttft());
                    if (t.tokens() > 1) tpot.recordValue(t.tpot());
                    for (int i = 1; i < t.chunkUs().size(); i++) itl.recordValue(t.chunkUs().get(i) - t.chunkUs().get(i - 1));
                    if (t.ttft() <= sloTtftMs * 1000 && t.tpot() <= sloTpotMs * 1000) good++;
                    durationUs = Math.max(durationUs, t.chunkUs().get(t.chunkUs().size() - 1));
                }
            }
        }
        double s = durationUs / 1e6;
        return String.format(Locale.ROOT, "%4d  %3d  %4d  %6.2f  %7.1f  %8.1f  %8.1f  %8.1f  %8.1f  %7.1f  %7.2f",
                n, ok, failed, ok / s, tokens / s, ttft.getValueAtPercentile(50) / 1000.0, ttft.getValueAtPercentile(99) / 1000.0,
                tpot.getValueAtPercentile(50) / 1000.0, tpot.getValueAtPercentile(99) / 1000.0, itl.getValueAtPercentile(99) / 1000.0, good / s);
    }

    public Integer call() throws Exception {
        try (ChatClient client = new ChatClient(url, null)) {
            Stopwatch warm = systemClock().start();
            for (int i = 0; i < warmup; i++) once(client, warm, -1 - i);
            System.out.println("conc   ok  fail   req/s    tok/s  TTFTp50  TTFTp99  TPOTp50  TPOTp99  ITLp99  goodput");
            for (int n : levels) System.out.println(level(client, n));
        }
        return 0;
    }

    // ---- fake streaming server: a role chunk, max_tokens one-token chunks, a usage chunk ----
    static HttpServer fakeServer() throws IOException {
        ObjectMapper json = new ObjectMapper();
        HttpServer server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
        server.setExecutor(Executors.newVirtualThreadPerTaskExecutor());
        server.createContext("/v1/chat/completions", ex -> {
            int n = json.readTree(ex.getRequestBody().readAllBytes()).path("max_tokens").asInt(16);
            ex.getResponseHeaders().set("Content-Type", "text/event-stream");
            ex.sendResponseHeaders(200, 0);
            try (OutputStream out = ex.getResponseBody()) {
                List<String> events = new ArrayList<>();
                events.add("{\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null},\"finish_reason\":null}]}");
                for (int i = 0; i < n; i++)
                    events.add("{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" tok" + i + "\"},\"finish_reason\":" + (i == n - 1 ? "\"length\"" : "null") + "}]}");
                events.add("{\"choices\":[],\"usage\":{\"prompt_tokens\":210,\"completion_tokens\":" + n + ",\"total_tokens\":" + (210 + n) + "}}");
                events.add("[DONE]");
                for (String e : events) { out.write(("data: " + e + "\n\n").getBytes(StandardCharsets.UTF_8)); out.flush(); }
            }
        });
        server.start();
        return server;
    }

    public static void main(String[] args) throws Exception {
        if (args.length > 0) System.exit(new CommandLine(new LoopBench()).execute(args));
        HttpServer server = fakeServer();
        try {
            LoopBench bench = new LoopBench();
            bench.clocks = n -> {                                  // the "server" slows down as concurrency grows
                long[] d = new long[1 + 8];                          // send, first chunk, 7 gaps (max_tokens 8)
                d[1] = (150 + 25L * n) * 1000;
                for (int i = 2; i < d.length; i++) d[i] = (20 + 2L * n) * 1000;
                return new FakeClock(d);
            };
            int code = new CommandLine(bench).execute("--url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1",
                    "--concurrency", "1,2,4,8", "--requests", "16", "--max-tokens", "8", "--slo-ttft-ms", "300", "--slo-tpot-ms", "30");
            System.out.println("exit code " + code);
        } finally {
            server.stop(0);
        }
    }
}