fix(server): make pre-stream inference failures return 503

docs/failure-injection.md

@@ -29,9 +29,9 @@ needed.
 | `/inference` upstream is not configured | HTTP 500 with a JSON error | `server/internalApiEndpointServerHook.test.ts` › environment configuration |
 | `/inference` cannot list upstream models | HTTP 500 with a JSON error | `server/internalApiEndpointServerHook.test.ts` › streaming path |
 | Upstream model fails but another one is available | Retried on the next model, answer still streams | `server/internalApiEndpointServerHook.test.ts` › streaming path |
-| Upstream model fails on every attempt | SSE error frame, so the client stops waiting | `server/internalApiEndpointServerHook.test.ts` › streaming path |
+| Every model fails before a single token is sent | HTTP 503 with a JSON error | `server/internalApiEndpointServerHook.test.ts` › streaming path |
 | Upstream stream ends without a finish part | `Stream ended unexpectedly` error frame | `server/internalApiEndpointServerHook.test.ts` › streaming path |
-| Upstream fails after content was already sent | No retry, the partial answer stands | `server/internalApiEndpointServerHook.test.ts` › streaming path |
+| Upstream fails after content was already sent | No retry, SSE error frame, the partial answer stands | `server/internalApiEndpointServerHook.test.ts` › streaming path |
 | Response is already unwritable when streaming would start | No headers set, the upstream is never called | `server/internalApiEndpointServerHook.test.ts` › streaming path |
 | `/inference` answers 503 | Generation state becomes `failed`, nothing persisted | `client/modules/textGeneration.degradation.test.ts` |
 | Generation interrupted mid-stream | Partial answer preserved, state stays `interrupted` | `client/modules/textGeneration.degradation.test.ts` |

server/internalApiEndpointServerHook.test.ts

@@ -571,7 +571,39 @@ describe("internalApiEndpointServerHook", () => {
       }
     });
 
-    it("emits SSE error when the only model fails", async () => {
+    it("responds 503 JSON when the only model's stream fails before any content", async () => {
+      vi.mocked(streamText).mockImplementation(
+        () =>
+          streamOf([
+            { type: "error", error: new Error("upstream down") },
+          ]) as never,
+      );
+      const handler = getRegisteredHandler();
+      const response = createResponse();
+      await handler(
+        createRequest({ messages: [{ role: "user", content: "hi" }] }),
+        response,
+        vi.fn(),
+      );
+      expect(response.statusCode).toBe(503);
+      expect(response.setHeader).toHaveBeenCalledWith(
+        "Content-Type",
+        "application/json",
+      );
+      const payload = JSON.parse(response.end.mock.calls[0][0]);
+      expect(payload).toEqual({
+        error: "Service unavailable - all models failed",
+        lastError: "upstream down",
+      });
+      expect(response.setHeader).not.toHaveBeenCalledWith(
+        "Content-Type",
+        "text/event-stream",
+      );
+      expect(response.write).not.toHaveBeenCalled();
+      expect(vi.mocked(streamText)).toHaveBeenCalledTimes(1);
+    });
+
+    it("responds 503 JSON when the only model cannot start", async () => {
       vi.mocked(streamText).mockImplementation(() => {
         throw new Error("upstream down");
       });
@@ -582,8 +614,17 @@ describe("internalApiEndpointServerHook", () => {
         response,
         vi.fn(),
       );
-      const writes = response.write.mock.calls.map((c) => c[0]).join("");
-      expect(writes).toContain("all models failed: upstream down");
+      expect(response.statusCode).toBe(503);
+      expect(response.setHeader).toHaveBeenCalledWith(
+        "Content-Type",
+        "application/json",
+      );
+      const payload = JSON.parse(response.end.mock.calls[0][0]);
+      expect(payload).toEqual({
+        error: "Service unavailable - all models failed",
+        lastError: "upstream down",
+      });
+      expect(response.write).not.toHaveBeenCalled();
       expect(vi.mocked(streamText)).toHaveBeenCalledTimes(1);
     });
 
@@ -671,6 +712,7 @@ describe("internalApiEndpointServerHook", () => {
       const writes = response.write.mock.calls.map((c) => c[0]).join("");
       expect(writes).not.toContain("A different answer");
       expect(writes).toContain("all models failed");
+      expect(writes).toContain("[DONE]");
     });
 
     it("responds 500 when model list fetch fails and no env model set", async () => {

server/internalApiEndpointServerHook.ts

@@ -95,7 +95,16 @@ function sendJsonError(
   safeEndResponse(response, JSON.stringify(payload));
 }
 
+function ensureSseHeaders(response: ServerResponse): void {
+  if (response.headersSent) return;
+
+  response.setHeader("Content-Type", "text/event-stream");
+  response.setHeader("Cache-Control", "no-cache");
+  response.setHeader("Connection", "keep-alive");
+}
+
 function sendSseData(response: ServerResponse, data: unknown): void {
+  ensureSseHeaders(response);
   safeWriteResponse(response, `data: ${JSON.stringify(data)}\n\n`);
 }
 
@@ -109,12 +118,6 @@ function sendSseError(
   message: string,
   model?: string,
 ): void {
-  if (!response.headersSent) {
-    response.setHeader("Content-Type", "text/event-stream");
-    response.setHeader("Cache-Control", "no-cache");
-    response.setHeader("Connection", "keep-alive");
-  }
-
   sendSseData(response, {
     error: message,
     ...(model ? { model } : {}),
@@ -258,7 +261,6 @@ export function internalApiEndpointServerHook<
         const config = getModelConfig();
         const maxAttempts = 5;
         let lastError: unknown = null;
-        let hasStartedStreaming = false;
 
         const clampedMaxTokens =
           requestBody.max_tokens !== undefined
@@ -282,13 +284,6 @@ export function internalApiEndpointServerHook<
             return;
           }
 
-          if (!hasStartedStreaming) {
-            response.setHeader("Content-Type", "text/event-stream");
-            response.setHeader("Cache-Control", "no-cache");
-            response.setHeader("Connection", "keep-alive");
-            hasStartedStreaming = true;
-          }
-
           try {
             const stream = streamText({
               model: openaiProvider.chatModel(model),
@@ -355,7 +350,7 @@ export function internalApiEndpointServerHook<
         const lastErrorMessage =
           lastError instanceof Error ? lastError.message : "Unknown error";
 
-        if (!hasStartedStreaming) {
+        if (!response.headersSent) {
           sendJsonError(response, 503, {
             error: "Service unavailable - all models failed",
             lastError: lastErrorMessage,