Workshop: loop serve, run, and chat

import com.sun.net.httpserver.HttpServer;
import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;

public class BackendStage {
    interface Backend extends AutoCloseable {
        String name();
        String baseUrl();                       // ends with /v1
        void ensureReady() throws Exception;    // start if needed, then wait until healthy
        @Override void close();                 // stop only what this backend started
    }

    record RemoteBackend(String baseUrl, String apiKey) implements Backend {
        public String name() { return "remote"; }
        public void ensureReady() throws Exception {
            HttpRequest.Builder b = HttpRequest.newBuilder(URI.create(baseUrl + "/models"));
            if (apiKey != null) b.header("Authorization", "Bearer " + apiKey);
            HttpResponse<String> r;
            try {
                r = HttpClient.newHttpClient().send(b.build(), HttpResponse.BodyHandlers.ofString());
            } catch (ConnectException e) {
                throw new IllegalStateException("no server is listening at the configured URL");
            }
            if (r.statusCode() == 401) throw new IllegalStateException("the server rejected the API key");
            if (r.statusCode() != 200) throw new IllegalStateException("unexpected status " + r.statusCode());
        }
        public void close() { }                 // the server is not ours to stop
    }

    public static void main(String[] args) throws Exception {
        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
        server.createContext("/v1/models", ex -> {
            boolean ok = "Bearer sk-1".equals(ex.getRequestHeaders().getFirst("Authorization"));
            byte[] body = (ok ? "{\"data\":[{\"id\":\"coder\"}]}" : "{\"error\":\"Unauthorized\"}").getBytes(StandardCharsets.UTF_8);
            ex.sendResponseHeaders(ok ? 200 : 401, body.length);
            ex.getResponseBody().write(body);
            ex.close();
        });
        server.start();
        String url = "http://127.0.0.1:" + server.getAddress().getPort() + "/v1";
        for (String key : new String[] {"sk-1", "wrong"}) {
            try (Backend b = new RemoteBackend(url, key)) {
                b.ensureReady();
                System.out.println(b.name() + " backend ready with key " + key);
            } catch (IllegalStateException e) {
                System.out.println(e.getMessage());
            }
        }
        server.stop(0);
        try (Backend b = new RemoteBackend(url, "sk-1")) { b.ensureReady(); } catch (IllegalStateException e) { System.out.println(e.getMessage()); }
    }
}

import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

public class LlamaServerCommand {
    record ModelInfo(String name, String gguf, int blockCount, long fileBytes, int trainCtx, long kvBytesPerToken) {}
    record Fit(int gpuLayers, int ctx) {}

    static final long MIB = 1L << 20;

    static Fit fit(ModelInfo m, long gpuBytes, int wantedCtx) {
        int ctx = Math.min(wantedCtx, m.trainCtx());
        if (gpuBytes == 0) return new Fit(0, ctx);
        long perLayer = m.fileBytes() / (m.blockCount() + 1);      // blocks plus the output layer
        long free = gpuBytes - m.kvBytesPerToken() * ctx - 512 * MIB;  // KV cache and a compute reserve
        int layers = (int) Math.max(0, Math.min(m.blockCount() + 1, free / perLayer));
        return new Fit(layers, ctx);
    }

    static List<String> command(String binary, ModelInfo m, Fit f, int port, int parallel) {
        List<String> c = new ArrayList<>(List.of(binary, "-m", m.gguf(), "--host", "127.0.0.1", "--port", Integer.toString(port)));
        c.addAll(List.of("-c", Integer.toString(f.ctx() * parallel), "-np", Integer.toString(parallel)));
        c.addAll(List.of("-ngl", Integer.toString(f.gpuLayers()), "--alias", m.name(), "--no-webui"));
        return c;
    }

    public static void main(String[] args) {
        ModelInfo coder = new ModelInfo("qwen2.5-coder-7b:q4_k_m", "/home/u/.loop/blobs/sha256-6a07", 28,
                4_683_087_332L, 32768, 2L * 28 * 4 * 128 * 2);
        long[] gpus = {0, 4096 * MIB, 24_576 * MIB};
        for (long gpu : gpus) {
            Fit f = fit(coder, gpu, 16384);
            System.out.printf(Locale.ROOT, "GPU %5d MiB -> -ngl %2d of %d%n", gpu / MIB, f.gpuLayers(), coder.blockCount() + 1);
        }
        System.out.println(String.join(" ", command("llama-server", coder, fit(coder, 4096 * MIB, 16384), 41237, 1)));
        System.out.println("context capped at training length: " + fit(coder, 0, 131072).ctx());
    }
}

import java.io.IOException;
import java.net.BindException;
import java.net.InetAddress;
import java.net.ServerSocket;

public class FreePortStage {
    static final InetAddress LOOPBACK = InetAddress.getLoopbackAddress();

    static int freePort() throws IOException {
        try (ServerSocket s = new ServerSocket(0, 0, LOOPBACK)) { return s.getLocalPort(); }
    }

    /** Stands in for an engine that binds the port it is given. */
    static ServerSocket startEngine(int port) throws IOException { return new ServerSocket(port, 0, LOOPBACK); }

    static ServerSocket startWithRetry(int attempts, int firstPort) throws IOException {
        int port = firstPort;
        for (int i = 1; ; i++) {
            try {
                ServerSocket engine = startEngine(port);
                System.out.println("attempt " + i + ": started");
                return engine;
            } catch (BindException e) {
                System.out.println("attempt " + i + ": port taken, choosing another");
                if (i == attempts) throw e;
                port = freePort();
            }
        }
    }

    public static void main(String[] args) throws IOException {
        int port = freePort();
        System.out.println("port is in the ephemeral range: " + (port >= 1024 && port <= 65535));
        try (ServerSocket held = new ServerSocket(0, 0, LOOPBACK)) {
            try (ServerSocket second = new ServerSocket(held.getLocalPort(), 0, LOOPBACK)) {
                System.out.println("second bind succeeded");
            } catch (BindException e) {
                System.out.println("second bind on a used port: " + e.getClass().getSimpleName());
            }
            try (ServerSocket engine = startWithRetry(3, held.getLocalPort())) {
                System.out.println("engine port differs from the taken one: " + (engine.getLocalPort() != held.getLocalPort()));
            }
        }
    }
}

import com.sun.net.httpserver.HttpServer;
import java.io.IOException;
import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpTimeoutException;
import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

public class SupervisorStage {
    static class Supervisor {
        final List<String> command; final int port; Process process; Thread reader;
        final ArrayDeque<String> log = new ArrayDeque<>();
        final HttpClient http = HttpClient.newHttpClient();
        Supervisor(List<String> command, int port) { this.command = command; this.port = port; }

        void start() throws IOException {
            ProcessBuilder pb = new ProcessBuilder(command).redirectErrorStream(true);
            pb.environment().remove("JAVA_TOOL_OPTIONS");
            process = pb.start();
            reader = Thread.ofVirtual().start(() -> process.inputReader().lines().forEach(this::remember));
        }
        synchronized void remember(String line) { if (log.size() == 20) log.removeFirst(); log.addLast(line); }
        synchronized String lastLog() { return log.isEmpty() ? "(no output)" : log.peekLast(); }

        int probe() throws Exception {
            try {
                return http.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/health"))
                        .timeout(Duration.ofSeconds(5)).build(), HttpResponse.BodyHandlers.discarding()).statusCode();
            } catch (ConnectException e) { return -1; }                      // not listening yet
            catch (HttpTimeoutException e) { return 0; }                     // listening but not answering
        }

        int awaitHealthy(Duration timeout) throws Exception {
            long deadline = System.nanoTime() + timeout.toNanos(), sleep = 50;
            int loadingAnswers = 0;
            while (true) {
                if (!process.isAlive()) {
                    reader.join();
                    throw new IllegalStateException("engine exited with code " + process.exitValue() + " during startup: " + lastLog());
                }
                int status = probe();
                if (status == 200) return loadingAnswers;
                if (status == 503) loadingAnswers++;
                if (System.nanoTime() > deadline) throw new IllegalStateException("engine not ready after " + timeout);
                Thread.sleep(sleep);
                sleep = Math.min(sleep * 2, 1000);
            }
        }
    }

    static List<String> fakeEngine(int port, String mode) {
        return List.of(ProcessHandle.current().info().command().orElse("java"), "-cp", System.getProperty("java.class.path"),
                "FakeEngine", Integer.toString(port), mode);
    }

    static int freePort() throws IOException { try (ServerSocket s = new ServerSocket(0)) { return s.getLocalPort(); } }

    public static void main(String[] args) throws Exception {
        int port = freePort();
        Supervisor ok = new Supervisor(fakeEngine(port, "ok"), port);
        ok.start();
        System.out.println("ready after " + ok.awaitHealthy(Duration.ofSeconds(30)) + " 'loading' answers; last log: " + ok.lastLog());
        ok.process.toHandle().destroy();
        ok.process.waitFor();

        int port2 = freePort();
        Supervisor bad = new Supervisor(fakeEngine(port2, "crash"), port2);
        bad.start();
        try { bad.awaitHealthy(Duration.ofSeconds(30)); } catch (IllegalStateException e) { System.out.println(e.getMessage()); }
    }
}

class FakeEngine {
    public static void main(String[] args) throws Exception {
        if (args[1].equals("crash")) {
            System.out.println("llama_model_load: error loading model: failed to open missing.gguf");
            System.exit(1);
        }
        AtomicInteger checks = new AtomicInteger();
        HttpServer s = HttpServer.create(new InetSocketAddress("127.0.0.1", Integer.parseInt(args[0])), 0);
        s.createContext("/health", ex -> { ex.sendResponseHeaders(checks.incrementAndGet() <= 2 ? 503 : 200, -1); ex.close(); });
        s.start();
        System.out.println("llama_server: model loaded");
    }
}

import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.util.List;

public class IdleUnloadStage {
    static class ManagedEngine {
        final long idleTimeoutSeconds;
        Process process;
        long lastUsed;
        int active, starts;
        ManagedEngine(long idleTimeoutSeconds) { this.idleTimeoutSeconds = idleTimeoutSeconds; }

        void beginRequest(long now) throws Exception {
            if (process == null || !process.isAlive()) {
                int port;
                try (ServerSocket s = new ServerSocket(0)) { port = s.getLocalPort(); }
                ProcessBuilder pb = new ProcessBuilder(ProcessHandle.current().info().command().orElse("java"), "-cp",
                        System.getProperty("java.class.path"), "FakeEngine", Integer.toString(port));
                pb.environment().remove("JAVA_TOOL_OPTIONS");
                process = pb.redirectOutput(ProcessBuilder.Redirect.DISCARD).redirectErrorStream(true).start();
                starts++;
                System.out.printf("t=%4ds start engine (start #%d)%n", now, starts);
            }
            active++;
            lastUsed = now;
        }

        void endRequest(long now) { active--; lastUsed = now; }

        void tick(long now) throws InterruptedException {
            boolean idle = process != null && process.isAlive() && active == 0 && now - lastUsed >= idleTimeoutSeconds;
            if (idle) {
                process.toHandle().destroy();
                process.waitFor();
                System.out.printf("t=%4ds idle for %ds, engine stopped%n", now, now - lastUsed);
            } else {
                System.out.printf("t=%4ds tick: %s%n", now, process != null && process.isAlive()
                        ? (active > 0 ? "busy, keep" : "idle " + (now - lastUsed) + "s, keep") : "not running");
            }
        }
    }

    public static void main(String[] args) throws Exception {
        ManagedEngine engine = new ManagedEngine(300);
        engine.beginRequest(0);
        engine.tick(120);                 // a long generation is still running
        engine.endRequest(200);
        engine.tick(360);
        engine.tick(500);
        engine.tick(600);
        engine.beginRequest(650);
        engine.endRequest(660);
        System.out.println("engine starts: " + engine.starts);
        engine.process.toHandle().destroy();
    }
}

class FakeEngine {
    public static void main(String[] args) throws Exception {
        HttpServer.create(new InetSocketAddress("127.0.0.1", Integer.parseInt(args[0])), 0).start();
    }
}

import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class ShutdownStage {
    static Process startEngine() throws Exception {
        int port;
        try (ServerSocket s = new ServerSocket(0)) { port = s.getLocalPort(); }
        ProcessBuilder pb = new ProcessBuilder(javaCommand("FakeEngine", Integer.toString(port)));
        pb.environment().remove("JAVA_TOOL_OPTIONS");
        Process p = pb.redirectOutput(ProcessBuilder.Redirect.DISCARD).redirectErrorStream(true).start();
        for (int i = 0; i < 100 && p.toHandle().descendants().findAny().isEmpty(); i++) Thread.sleep(50);  // wait for the worker
        return p;
    }

    static List<String> javaCommand(String... args) {
        List<String> c = new java.util.ArrayList<>(List.of(ProcessHandle.current().info().command().orElse("java"),
                "-cp", System.getProperty("java.class.path")));
        c.addAll(List.of(args));
        return c;
    }

    static void killTree(ProcessHandle root) throws Exception {
        List<ProcessHandle> children = root.descendants().toList();
        root.destroy();                                         // SIGTERM to the engine first
        root.onExit().get(10, TimeUnit.SECONDS);
        for (ProcessHandle c : children) { c.destroy(); c.onExit().get(10, TimeUnit.SECONDS); }
    }

    public static void main(String[] args) throws Exception {
        Process first = startEngine();
        ProcessHandle worker = first.toHandle().descendants().findFirst().orElseThrow();
        first.toHandle().destroy();
        first.waitFor();
        Thread.sleep(200);
        System.out.println("parent killed alone -> worker still alive: " + worker.isAlive());
        worker.destroy();
        worker.onExit().get(10, TimeUnit.SECONDS);

        Process second = startEngine();
        ProcessHandle worker2 = second.toHandle().descendants().findFirst().orElseThrow();
        killTree(second.toHandle());
        System.out.println("tree kill -> engine alive: " + second.isAlive() + ", worker alive: " + worker2.isAlive());

        Process third = startEngine();
        ProcessHandle worker3 = third.toHandle().descendants().findFirst().orElseThrow();
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            try {
                killTree(third.toHandle());
                System.out.println("shutdown hook -> engine alive: " + third.isAlive() + ", worker alive: " + worker3.isAlive());
            } catch (Exception e) { System.out.println("shutdown hook failed: " + e); }
        }));
        System.out.println("main returns with the third engine running");
    }
}

class FakeEngine {
    public static void main(String[] args) throws Exception {
        if (args[0].equals("worker")) { Thread.sleep(120_000); return; }
        ProcessBuilder pb = new ProcessBuilder(ShutdownStage.javaCommand("FakeEngine", "worker"));
        pb.environment().remove("JAVA_TOOL_OPTIONS");
        pb.redirectOutput(ProcessBuilder.Redirect.DISCARD).redirectErrorStream(true).start();
        HttpServer.create(new InetSocketAddress("127.0.0.1", Integer.parseInt(args[0])), 0).start();
    }
}

import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

public class VllmCommand {
    record VllmOptions(String model, String servedName, int port, int maxModelLen, double gpuMemoryUtilization,
                       String toolParser, String reasoningParser) {}

    static List<String> command(VllmOptions o) {
        List<String> c = new ArrayList<>(List.of("vllm", "serve", o.model(), "--host", "127.0.0.1",
                "--port", Integer.toString(o.port()), "--served-model-name", o.servedName(),
                "--max-model-len", Integer.toString(o.maxModelLen()),
                "--gpu-memory-utilization", String.format(Locale.ROOT, "%.2f", o.gpuMemoryUtilization())));
        if (o.toolParser() != null) c.addAll(List.of("--enable-auto-tool-choice", "--tool-call-parser", o.toolParser()));
        if (o.reasoningParser() != null) c.addAll(List.of("--reasoning-parser", o.reasoningParser()));
        c.add("--enable-prompt-tokens-details");
        return c;
    }

    record Timing(Duration startupDeadline, boolean refusedWhileLoading, boolean status503WhileLoading) {}

    public static void main(String[] args) {
        System.out.println(String.join(" ", command(new VllmOptions("Qwen/Qwen2.5-Coder-7B-Instruct", "coder", 41300,
                32768, 0.90, "hermes", null))));
        System.out.println(String.join(" ", command(new VllmOptions("Qwen/Qwen3-8B", "qwen3", 41301,
                16384, 0.85, "hermes", "qwen3"))));
        System.out.println("llama-server: " + new Timing(Duration.ofMinutes(2), true, true));
        System.out.println("vllm serve:   " + new Timing(Duration.ofMinutes(15), true, false));
    }
}

import com.sun.net.httpserver.HttpServer;
import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;

@Command(name = "serve", mixinStandardHelpOptions = true, description = "Start and supervise an inference engine.")
public class LoopServe implements Callable<Integer> {
    @Parameters(paramLabel = "MODEL", description = "Path of a GGUF file") String model;
    @Option(names = {"-c", "--ctx-size"}, defaultValue = "4096", description = "Context per slot") int ctx;
    @Option(names = "--parallel", defaultValue = "1", description = "Slots") int parallel;
    @Option(names = "--gpu-layers", defaultValue = "0", description = "Layers on the GPU (from loop fit)") int gpuLayers;
    @Option(names = "--engine-command", split = " ", defaultValue = "llama-server", description = "Engine launcher") List<String> engine;
    @Option(names = "--once", hidden = true) boolean once;

    public Integer call() throws Exception {
        int port;
        try (ServerSocket s = new ServerSocket(0)) { port = s.getLocalPort(); }
        List<String> cmd = new ArrayList<>(engine);
        cmd.addAll(List.of("-m", model, "--host", "127.0.0.1", "--port", Integer.toString(port), "-c",
                Integer.toString(ctx * parallel), "-np", Integer.toString(parallel), "-ngl", Integer.toString(gpuLayers), "--no-webui"));
        ProcessBuilder pb = new ProcessBuilder(cmd).redirectErrorStream(true);
        pb.environment().remove("JAVA_TOOL_OPTIONS");
        Process p = pb.start();
        List<String> log = new ArrayList<>();
        Thread reader = Thread.ofVirtual().start(() -> p.inputReader().lines().forEach(l -> { synchronized (log) { log.add(l); } }));
        Thread hook = new Thread(() -> { p.toHandle().descendants().forEach(ProcessHandle::destroy); p.destroy(); });
        Runtime.getRuntime().addShutdownHook(hook);

        HttpClient http = HttpClient.newHttpClient();
        URI health = URI.create("http://127.0.0.1:" + port + "/health");
        long deadline = System.nanoTime() + Duration.ofMinutes(2).toNanos(), sleep = 50;
        while (true) {
            if (!p.isAlive()) { reader.join(); System.err.println("engine exited: " + (log.isEmpty() ? "" : log.get(log.size() - 1))); return 1; }
            int status;
            try { status = http.send(HttpRequest.newBuilder(health).build(), HttpResponse.BodyHandlers.discarding()).statusCode(); }
            catch (ConnectException e) { status = -1; }
            if (status == 200) break;
            if (System.nanoTime() > deadline) { System.err.println("engine did not become ready"); return 1; }
            Thread.sleep(sleep);
            sleep = Math.min(sleep * 2, 1000);
        }
        System.out.println("serving " + model + " (ctx " + ctx + " x " + parallel + " slots, " + gpuLayers + " GPU layers)");
        if (once) { System.out.println("engine ready"); return 0; }
        System.out.println("base URL http://127.0.0.1:" + port + "/v1");
        return p.waitFor();                                      // runs until Ctrl+C or the engine exits
    }

    public static void main(String[] args) {
        String fake = ProcessHandle.current().info().command().orElse("java") + " -cp " + System.getProperty("java.class.path") + " FakeEngine";
        String[] demo = args.length > 0 ? args : new String[] {"/models/tiny.gguf", "--ctx-size", "2048", "--parallel", "2",
                "--engine-command", fake, "--once"};
        int code = new CommandLine(new LoopServe()).execute(demo);
        System.out.println("exit code " + code);
    }
}

class FakeEngine {
    public static void main(String[] args) throws Exception {
        int port = Integer.parseInt(args[List.of(args).indexOf("--port") + 1]);
        HttpServer s = HttpServer.create(new InetSocketAddress("127.0.0.1", port), 0);
        s.createContext("/health", ex -> { ex.sendResponseHeaders(200, -1); ex.close(); });
        s.start();
    }
}

import java.util.ArrayList;
import java.util.List;

public class ConversationStage {
    record Message(String role, String content) {}

    static class Conversation {
        String system;
        final List<Message> turns = new ArrayList<>();
        Conversation(String system) { this.system = system; }

        void add(String role, String content) { turns.add(new Message(role, content)); }

        static int estimateTokens(Message m) { return 4 + (m.content().length() + 3) / 4; }   // about 4 characters per token

        /** The messages for the next request: system prompt, then as many recent turns as fit the budget. */
        List<Message> forRequest(int budget) {
            List<Message> out = new ArrayList<>();
            Message sys = new Message("system", system);
            int used = estimateTokens(sys);
            int first = turns.size();
            while (first > 0 && used + estimateTokens(turns.get(first - 1)) <= budget) used += estimateTokens(turns.get(--first));
            if (first < turns.size() && turns.get(first).role().equals("assistant")) first++;   // start at a user turn
            out.add(sys);
            out.addAll(turns.subList(first, turns.size()));
            return out;
        }
    }

    public static void main(String[] args) {
        Conversation c = new Conversation("You are a concise Java assistant.");
        c.add("user", "What does Optional.orElseThrow() do with an empty Optional?");
        c.add("assistant", "It throws NoSuchElementException.");
        c.add("user", "And orElseGet?");
        c.add("assistant", "It calls the supplier and returns its result, so the default is computed only when empty.");
        c.add("user", "Show a one-line example.");
        for (int budget : new int[] {200, 60, 30}) {
            List<Message> msgs = c.forRequest(budget);
            System.out.println("budget " + budget + ": " + msgs.size() + " messages, first turn: "
                    + (msgs.size() > 1 ? msgs.get(1).content() : "(none)"));
        }
    }
}

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.sun.net.httpserver.HttpServer;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.List;

public class StreamingStage {
    static final ObjectMapper JSON = new ObjectMapper();

    record Result(String content, String finishReason, JsonNode usage) {}

    interface Listener {                       // as in Project 11
        void onContent(String text);
        default void onReasoning(String text) { }
    }

    static class ChatClient {
        final HttpClient http = HttpClient.newHttpClient();
        final String baseUrl;
        ChatClient(String baseUrl) { this.baseUrl = baseUrl; }

        Result stream(ObjectNode request, Listener listener) throws Exception {
            request.put("stream", true).putObject("stream_options").put("include_usage", true);
            HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + "/chat/completions"))
                    .header("Content-Type", "application/json")
                    .POST(HttpRequest.BodyPublishers.ofString(JSON.writeValueAsString(request))).build();
            HttpResponse<InputStream> resp = http.send(req, HttpResponse.BodyHandlers.ofInputStream());
            StringBuilder text = new StringBuilder();
            String finish = null;
            JsonNode usage = null;
            try (BufferedReader in = new BufferedReader(new InputStreamReader(resp.body(), StandardCharsets.UTF_8))) {
                for (String line; (line = in.readLine()) != null; ) {
                    if (!line.startsWith("data: ")) continue;             // blank separators and comments
                    if (line.equals("data: [DONE]")) break;
                    JsonNode chunk = JSON.readTree(line.substring(6));
                    if (chunk.hasNonNull("usage")) usage = chunk.get("usage");
                    for (JsonNode choice : chunk.path("choices")) {
                        String delta = choice.path("delta").path("content").asText("");
                        if (!delta.isEmpty()) { text.append(delta); listener.onContent(delta); }
                        if (choice.hasNonNull("finish_reason")) finish = choice.get("finish_reason").asText();
                    }
                }
            }
            return new Result(text.toString(), finish, usage);
        }
    }

    static HttpServer fakeServer(List<String> tokens) throws Exception {
        HttpServer s = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
        s.createContext("/v1/chat/completions", ex -> {
            ex.getResponseHeaders().add("Content-Type", "text/event-stream");
            ex.sendResponseHeaders(200, 0);
            try (OutputStream out = ex.getResponseBody()) {
                send(out, "{\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null},\"finish_reason\":null}]}");
                for (String t : tokens) send(out, "{\"choices\":[{\"index\":0,\"delta\":{\"content\":" + JSON.writeValueAsString(t) + "},\"finish_reason\":null}]}");
                send(out, "{\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}");
                send(out, "{\"choices\":[],\"usage\":{\"prompt_tokens\":21,\"completion_tokens\":" + tokens.size() + ",\"total_tokens\":" + (21 + tokens.size()) + "}}");
                out.write("data: [DONE]\n\n".getBytes(StandardCharsets.UTF_8));
            }
        });
        s.start();
        return s;
    }

    static void send(OutputStream out, String json) throws java.io.IOException {
        out.write(("data: " + json + "\n\n").getBytes(StandardCharsets.UTF_8));
        out.flush();
    }

    public static void main(String[] args) throws Exception {
        HttpServer server = fakeServer(List.of("Use", " `", "List", ".of", "(1, 2, 3)", "`", "."));
        ChatClient client = new ChatClient("http://127.0.0.1:" + server.getAddress().getPort() + "/v1");
        ObjectNode request = JSON.createObjectNode().put("model", "coder").put("max_tokens", 64);
        request.putArray("messages").addObject().put("role", "user").put("content", "An immutable list of 1, 2, 3?");
        Result r = client.stream(request, delta -> System.out.print("[" + delta + "]"));
        System.out.println();
        System.out.println("text: " + r.content());
        System.out.println("finish: " + r.finishReason() + ", usage: " + r.usage());
        server.stop(0);
    }
}

import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

public class SlashCommands {
    record Message(String role, String content) {}
    record Saved(String system, Map<String, Object> settings, List<Message> turns) {}

    String system = "You are a helpful assistant.";
    final Map<String, Object> settings = new TreeMap<>(Map.of("temperature", 0.7));
    List<Message> turns = new ArrayList<>(List.of(new Message("user", "hi"), new Message("assistant", "Hello.")));
    final Path dir;
    SlashCommands(Path dir) { this.dir = dir; }

    /** Returns false when the chat should end. */
    boolean handle(String line) throws Exception {
        String[] p = line.strip().split("\\s+", 3);
        switch (p[0]) {
            case "/set" -> {
                if (p.length < 3) { System.out.println("usage: /set <name> <value>"); return true; }
                switch (p[1]) {
                    case "temperature", "top_p", "min_p" -> settings.put(p[1], Double.parseDouble(p[2]));
                    case "max_tokens", "top_k", "seed" -> settings.put(p[1], Integer.parseInt(p[2]));
                    default -> { System.out.println("unknown setting " + p[1]); return true; }
                }
                System.out.println("set " + p[1] + " = " + settings.get(p[1]));
            }
            case "/system" -> { system = line.strip().substring("/system".length()).strip(); System.out.println("system prompt replaced"); }
            case "/clear" -> { turns = new ArrayList<>(); System.out.println("history cleared"); }
            case "/save" -> {
                new ObjectMapper().writerWithDefaultPrettyPrinter().writeValue(dir.resolve(p[1] + ".json").toFile(), new Saved(system, settings, turns));
                System.out.println("saved " + turns.size() + " messages as " + p[1]);
            }
            case "/load" -> {
                Saved s = new ObjectMapper().readValue(dir.resolve(p[1] + ".json").toFile(), Saved.class);
                system = s.system(); settings.clear(); settings.putAll(s.settings()); turns = new ArrayList<>(s.turns());
                System.out.println("loaded " + turns.size() + " messages from " + p[1]);
            }
            case "/bye" -> { return false; }
            default -> System.out.println("unknown command " + p[0] + "; commands: /set /system /clear /save /load /bye");
        }
        return true;
    }

    public static void main(String[] args) throws Exception {
        SlashCommands chat = new SlashCommands(Files.createTempDirectory("loop-sessions"));
        for (String line : List.of("/set temperature 0.2", "/set max_tokens 256", "/set colour blue", "/system You review Java code.",
                "/save review", "/clear", "/load review", "/help", "/bye", "/set temperature 1.0")) {
            System.out.println("> " + line);
            if (!chat.handle(line)) { System.out.println("bye"); break; }
        }
        System.out.println("settings " + chat.settings + ", system: " + chat.system + ", turns: " + chat.turns.size());
    }
}

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import org.jline.reader.EndOfFileException;
import org.jline.reader.LineReader;
import org.jline.reader.LineReaderBuilder;
import org.jline.terminal.Terminal;
import org.jline.terminal.impl.DumbTerminal;

public class LineInputStage {
    /** Reads one message; a line that starts with """ begins a block that ends at a line ending with """. */
    static String readMessage(LineReader reader) {
        String first = reader.readLine(">>> ");
        if (!first.startsWith("\"\"\"")) return first;
        StringBuilder block = new StringBuilder(first.substring(3));
        while (!block.toString().endsWith("\"\"\"")) {
            String next = reader.readLine("... ");
            block.append(block.isEmpty() ? "" : "\n").append(next);
        }
        return block.substring(0, block.length() - 3).strip();
    }

    public static void main(String[] args) throws Exception {
        String typed = "explain this\n\"\"\"\nfor (int i = 0; i <= n; i++)\n    sum += a[i];\n\"\"\"\n/set temperature 0.2\nexplain this\n";
        ByteArrayOutputStream screen = new ByteArrayOutputStream();
        Terminal terminal = new DumbTerminal(new ByteArrayInputStream(typed.getBytes(StandardCharsets.UTF_8)), screen);
        LineReader reader = LineReaderBuilder.builder().terminal(terminal)
                .option(LineReader.Option.DISABLE_EVENT_EXPANSION, true).build();
        try {
            while (true) {
                String msg = readMessage(reader);
                System.out.println("message (" + msg.lines().count() + " lines): " + msg.replace("\n", "\\n"));
            }
        } catch (EndOfFileException e) {
            System.out.println("end of input");
        }
        reader.getHistory().forEach(h -> System.out.println("history " + h.index() + ": " + h.line()));
        System.out.println("prompts written: " + (screen.toString(StandardCharsets.UTF_8).split(">>> |\\.\\.\\. ", -1).length - 1));
    }
}

import java.nio.file.Path;
import org.jline.reader.EndOfFileException;
import org.jline.reader.LineReader;
import org.jline.reader.LineReaderBuilder;
import org.jline.reader.UserInterruptException;
import org.jline.terminal.Terminal;
import org.jline.terminal.TerminalBuilder;

public class TerminalInput {
    public static void main(String[] args) throws Exception {
        Terminal terminal = TerminalBuilder.builder().system(true).build();
        LineReader reader = LineReaderBuilder.builder().terminal(terminal)
                .variable(LineReader.HISTORY_FILE, Path.of(System.getProperty("user.home"), ".loop", "history"))
                .option(LineReader.Option.DISABLE_EVENT_EXPANSION, true)
                .build();
        while (true) {
            try {
                String line = reader.readLine(">>> ");
                if (line.equals("/bye")) break;
                terminal.writer().println("you typed: " + line);
            } catch (UserInterruptException e) {          // Ctrl+C at the prompt clears the line
                terminal.writer().println("(use /bye or Ctrl+D to exit)");
            } catch (EndOfFileException e) {             // Ctrl+D at an empty prompt
                break;
            }
        }
        reader.getHistory().save();
    }
}

import com.fasterxml.jackson.databind.ObjectMapper;
import com.sun.net.httpserver.HttpServer;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.jline.terminal.Terminal;
import org.jline.terminal.impl.DumbTerminal;

public class CancelStage {
    public static void main(String[] args) throws Exception {
        ObjectMapper json = new ObjectMapper();
        AtomicInteger served = new AtomicInteger();
        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
        server.createContext("/v1/chat/completions", ex -> {
            ex.sendResponseHeaders(200, 0);
            try (OutputStream out = ex.getResponseBody()) {
                for (int i = 0; i < 200; i++) {                       // a long answer, one word every 20 ms
                    out.write(("data: {\"choices\":[{\"delta\":{\"content\":\"word" + i + " \"}}]}\n\n").getBytes(StandardCharsets.UTF_8));
                    out.flush();
                    served.incrementAndGet();
                    Thread.sleep(20);
                }
            } catch (Exception clientWentAway) { }
        });
        server.start();

        Terminal terminal = new DumbTerminal(new ByteArrayInputStream(new byte[0]), new ByteArrayOutputStream());
        AtomicBoolean cancel = new AtomicBoolean();
        terminal.handle(Terminal.Signal.INT, signal -> cancel.set(true));     // Ctrl+C sets a flag only

        HttpResponse<InputStream> resp = HttpClient.newHttpClient().send(HttpRequest.newBuilder(
                        URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/v1/chat/completions"))
                .POST(HttpRequest.BodyPublishers.ofString("{\"stream\":true,\"messages\":[]}")).build(), HttpResponse.BodyHandlers.ofInputStream());
        StringBuilder shown = new StringBuilder();
        int deltas = 0;
        try (BufferedReader in = new BufferedReader(new InputStreamReader(resp.body(), StandardCharsets.UTF_8))) {
            for (String line; (line = in.readLine()) != null; ) {
                if (cancel.get()) break;                                    // leaving the block closes the stream
                if (!line.startsWith("data: ")) continue;
                shown.append(json.readTree(line.substring(6)).at("/choices/0/delta/content").asText());
                if (++deltas == 3) terminal.raise(Terminal.Signal.INT);    // the user presses Ctrl+C here
            }
        }
        System.out.println("shown: " + shown.toString().strip() + " ^C");
        System.out.println("generation cancelled after " + deltas + " deltas; program still running: true");
        Thread.sleep(300);
        System.out.println("server stopped sending early: " + (served.get() < 200));
        server.stop(0);
    }
}

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.List;

public class ReasoningStage {
    static final ObjectMapper JSON = new ObjectMapper();

    static class Display {
        final StringBuilder thinking = new StringBuilder(), answer = new StringBuilder();
        boolean insideTag;

        void onDelta(JsonNode delta) {
            String r = delta.hasNonNull("reasoning_content") ? delta.get("reasoning_content").asText()
                    : delta.path("reasoning").asText("");
            if (!r.isEmpty()) thinking.append(r);
            String c = delta.path("content").asText("");
            while (!c.isEmpty()) {                       // split <think>...</think> inside content
                String marker = insideTag ? "</think>" : "<think>";
                int at = c.indexOf(marker);
                String part = at < 0 ? c : c.substring(0, at);
                (insideTag ? thinking : answer).append(part);
                if (at < 0) break;
                insideTag = !insideTag;
                c = c.substring(at + marker.length());
            }
        }

        void print(String label) {
            System.out.println(label);
            if (!thinking.isEmpty()) System.out.println("  thinking: " + thinking.toString().strip());
            System.out.println("  answer:   " + answer.toString().strip());
        }
    }

    static void run(String label, List<String> deltas) throws Exception {
        Display d = new Display();
        for (String delta : deltas) d.onDelta(JSON.readTree(delta));
        d.print(label);
    }

    public static void main(String[] args) throws Exception {
        run("llama-server (reasoning_content):", List.of("{\"reasoning_content\":\"n starts at 0, \"}",
                "{\"reasoning_content\":\"so <= reads one past the end.\"}", "{\"content\":\"Use i < n.\"}"));
        run("vLLM (reasoning):", List.of("{\"reasoning\":\"Strings are immutable.\"}", "{\"content\":\"Use StringBuilder.\"}"));
        run("no parser (<think> in content):", List.of("{\"content\":\"<think>\"}", "{\"content\":\"Check null first.\"}",
                "{\"content\":\"</think>\"}", "{\"content\":\"Add Objects.requireNonNull(x).\"}"));
        run("plain model:", List.of("{\"content\":\"Use a record.\"}"));
    }
}

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.nio.file.Files;
import java.nio.file.Path;

public class ProfilesStage {
    @JsonIgnoreProperties(ignoreUnknown = true)
    @JsonInclude(JsonInclude.Include.NON_NULL)
    record Profile(String system, Double temperature, @JsonProperty("top_p") Double topP,
                   @JsonProperty("max_tokens") Integer maxTokens, Integer seed) {}

    static final ObjectMapper JSON = new ObjectMapper();

    static ObjectNode request(String model, Profile p, Double cliTemperature) {
        ObjectNode r = JSON.createObjectNode().put("model", model);
        if (p.temperature() != null) r.put("temperature", p.temperature());
        if (p.topP() != null) r.put("top_p", p.topP());
        if (p.maxTokens() != null) r.put("max_tokens", p.maxTokens());
        if (p.seed() != null) r.put("seed", p.seed());
        if (cliTemperature != null) r.put("temperature", cliTemperature);     // the command line wins
        r.putArray("messages").addObject().put("role", "system").put("content", p.system());
        return r;
    }

    public static void main(String[] args) throws Exception {
        Path dir = Files.createTempDirectory("profiles");
        Files.writeString(dir.resolve("reviewer.json"), """
                {"system": "You review Java code. List bugs first, then style.", "temperature": 0.2,
                 "top_p": 0.9, "max_tokens": 1024, "seed": 42, "notes": "used for PR reviews"}""");
        JSON.writeValue(dir.resolve("ideas.json").toFile(), new Profile("Suggest three alternatives.", 1.0, null, 400, null));

        try (var files = Files.list(dir)) {
            files.map(f -> f.getFileName().toString().replace(".json", "")).sorted().forEach(n -> System.out.println("profile: " + n));
        }
        Profile reviewer = JSON.readValue(dir.resolve("reviewer.json").toFile(), Profile.class);
        System.out.println(reviewer);
        System.out.println(JSON.writeValueAsString(request("coder", reviewer, null)));
        System.out.println(JSON.writeValueAsString(request("coder", reviewer, 0.0)));
        System.out.println(Files.readString(dir.resolve("ideas.json")));
    }
}

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.sun.net.httpserver.HttpServer;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;

@Command(name = "chat", mixinStandardHelpOptions = true, description = "Chat with an OpenAI-compatible server.")
public class LoopChat implements Callable<Integer> {
    @Option(names = "--url", required = true, description = "Base URL ending in /v1") String url;
    @Option(names = "--model", defaultValue = "default", description = "Model name sent in requests") String model;
    @Option(names = "--system", defaultValue = "You are a helpful coding assistant.") String system;
    @Option(names = "--temperature") Double temperature;
    @Option(names = "--max-tokens", defaultValue = "256") int maxTokens;
    @Option(names = "--context", defaultValue = "4096", description = "Context size for trimming") int context;

    static final ObjectMapper JSON = new ObjectMapper();
    record Message(String role, String content) {}
    final List<Message> turns = new ArrayList<>();
    int lastPromptTokens;

    public Integer call() throws Exception {
        BufferedReader in = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8));
        for (String line; (line = in.readLine()) != null; ) {
            if (line.isBlank()) continue;
            System.out.println(">>> " + line);
            if (line.startsWith("/")) {
                String[] p = line.split("\\s+", 3);
                switch (p[0]) {
                    case "/bye" -> { return 0; }
                    case "/clear" -> { turns.clear(); System.out.println("history cleared"); }
                    case "/set" -> { if (p[1].equals("temperature")) temperature = Double.valueOf(p[2]); System.out.println("set " + p[1] + " = " + p[2]); }
                    case "/system" -> { system = line.substring(8).strip(); System.out.println("system prompt replaced"); }
                    default -> System.out.println("unknown command " + p[0]);
                }
                continue;
            }
            turns.add(new Message("user", line));
            String answer = streamAnswer();
            turns.add(new Message("assistant", answer));
        }
        return 0;
    }

    String streamAnswer() throws Exception {
        ObjectNode req = JSON.createObjectNode().put("model", model).put("max_tokens", maxTokens).put("stream", true);
        if (temperature != null) req.put("temperature", temperature);
        req.putObject("stream_options").put("include_usage", true);
        ArrayNode msgs = req.putArray("messages");
        msgs.addObject().put("role", "system").put("content", system);
        int budget = context - maxTokens, used = 0, first = turns.size();
        while (first > 0 && used + 4 + turns.get(first - 1).content().length() / 4 <= budget) used += 4 + turns.get(--first).content().length() / 4;
        for (Message m : turns.subList(first, turns.size())) msgs.addObject().put("role", m.role()).put("content", m.content());

        HttpResponse<InputStream> resp = HttpClient.newHttpClient().send(HttpRequest.newBuilder(URI.create(url + "/chat/completions"))
                .header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofString(JSON.writeValueAsString(req))).build(),
                HttpResponse.BodyHandlers.ofInputStream());
        if (resp.statusCode() != 200) { String err = new String(resp.body().readAllBytes(), StandardCharsets.UTF_8); System.out.println("error " + resp.statusCode() + ": " + err); return ""; }
        StringBuilder answer = new StringBuilder();
        String finish = "?";
        JsonNode usage = null;
        try (BufferedReader sse = new BufferedReader(new InputStreamReader(resp.body(), StandardCharsets.UTF_8))) {
            for (String line; (line = sse.readLine()) != null && !line.equals("data: [DONE]"); ) {
                if (!line.startsWith("data: ")) continue;
                JsonNode chunk = JSON.readTree(line.substring(6));
                if (chunk.hasNonNull("usage")) usage = chunk.get("usage");
                for (JsonNode c : chunk.path("choices")) {
                    JsonNode d = c.path("delta");
                    String thinking = d.hasNonNull("reasoning_content") ? d.get("reasoning_content").asText() : d.path("reasoning").asText("");
                    if (!thinking.isEmpty()) System.out.print("(thinking: " + thinking + ") ");
                    String text = d.path("content").asText("");
                    System.out.print(text);
                    System.out.flush();
                    answer.append(text);
                    if (c.hasNonNull("finish_reason")) finish = c.get("finish_reason").asText();
                }
            }
        }
        System.out.println();
        if (usage != null) {
            lastPromptTokens = usage.path("prompt_tokens").asInt();
            System.out.println("[" + finish + ", " + msgs.size() + " messages, prompt " + lastPromptTokens + " tokens, cached "
                    + usage.path("prompt_tokens_details").path("cached_tokens").asInt(0) + ", output " + usage.path("completion_tokens").asInt() + "]");
        }
        return answer.toString();
    }

    public static void main(String[] args) throws Exception {
        if (args.length > 0) { System.exit(new CommandLine(new LoopChat()).execute(args)); }
        HttpServer fake = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
        fake.createContext("/v1/chat/completions", ex -> {
            JsonNode req = JSON.readTree(ex.getRequestBody());
            JsonNode msgs = req.get("messages");
            String reply = "You said '" + msgs.get(msgs.size() - 1).get("content").asText() + "' (temperature " + req.path("temperature").asText("default") + ").";
            ex.sendResponseHeaders(200, 0);
            try (OutputStream out = ex.getResponseBody()) {
                for (String w : reply.split("(?<= )")) out.write(("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":" + JSON.writeValueAsString(w) + "}}]}\n\n").getBytes(StandardCharsets.UTF_8));
                out.write(("data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n").getBytes(StandardCharsets.UTF_8));
                out.write(("data: {\"choices\":[],\"usage\":{\"prompt_tokens\":" + (12 * msgs.size()) + ",\"completion_tokens\":" + reply.split(" ").length
                        + ",\"prompt_tokens_details\":{\"cached_tokens\":" + (12 * (msgs.size() - 1)) + "}}}\n\ndata: [DONE]\n\n").getBytes(StandardCharsets.UTF_8));
            }
        });
        fake.start();
        System.setIn(new ByteArrayInputStream("hello\n/set temperature 0.2\nhow are you?\n/clear\nagain\n/bye\nnever read\n".getBytes(StandardCharsets.UTF_8)));
        int code = new CommandLine(new LoopChat()).execute("--url", "http://127.0.0.1:" + fake.getAddress().getPort() + "/v1", "--model", "coder");
        System.out.println("exit code " + code);
        fake.stop(0);
    }
}