Diagnose setup by what is certain, and deliver errors the way the client asked

Review of the branch found the persona diagnosis was mostly truncation
wearing a name tag. The <.> marker sits at the start of the system
message and the persona marker after the card, so a card Janitor cut
short arrives with the prompt present and the persona gone — and was
told its persona was wrong, with truncation as an afterthought. The
original message was only ever about context size, which is the best
evidence there is about what actually happens.

diagnoseSetup() now says only what the markers make certain. A missing
<.> is a missing prompt, since truncation cannot remove the first three
characters. A persona marker that is present but not "." is a wrong
description, quoted back so there is nothing to guess. A persona marker
that is gone, or cut off mid-tag, leads with truncation and mentions the
persona second.

The error path decided "streaming?" from the Accept header while the
success path used body.stream, so a client that sent stream:true without
that header got JSON back for every error and never displayed it. The
flag is hoisted above the try; the header is only a fallback for when
the body could not be read.

Token search is restricted to after the greeting, in order. A card built
from a sucker-proxied chat could carry an expired token in its text, and
a lookup on it would fail forever. Conversation order also means the
token a chat was started with always wins over one pasted later. The
system fallback now takes a role-less first message only; a request with
roles but no system turn is reported as such rather than misdiagnosed as
a missing prompt. The echo feature reads the parser's lastUser instead of
its own case-sensitive lookup. "Nothing was saved" is dropped from the
unknown-error message, since it is not guaranteed.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NX2SsmUqQftWTZAY4BDuFa
This commit is contained in:
Severian
2026-09-11 06:46:52 +08:00
parent a8b3669c17
commit 7569fb8e82
5 changed files with 201 additions and 84 deletions
+61 -21
View File
@@ -1,5 +1,6 @@
/**
* Which failures a user can act on, and which just need reporting.
* Which failures a user can act on, which just need reporting, and whether
* the setup diagnosis names the right step.
*/
import { test } from "node:test";
import assert from "node:assert/strict";
@@ -10,31 +11,71 @@ import {
NO_GREETING,
SuckerError,
TOO_FEW_MESSAGES,
missingSetup,
diagnoseSetup,
isImpossibleRequest,
isSuckerError,
userFacingMessage,
} from "../src/lib/errors.ts";
test("a missing setting is named, not left to the reader to find", () => {
const noPrompt = missingSetup(false, true).message;
assert.match(noPrompt, /custom prompt is not coming through as <\.>/);
assert.ok(!/persona needs the name/.test(noPrompt), "must not blame the setting that is fine");
const CARD = "<Gaius's Persona>a tired innkeeper</Gaius's Persona>";
const ok = (prompt: string, persona: string) => `${prompt}\n${CARD}\n${persona}`;
const noPersona = missingSetup(true, false).message;
assert.match(noPersona, /persona is not coming through/);
assert.ok(!/custom prompt is not coming through/.test(noPersona));
const neither = missingSetup(false, false).message;
assert.match(neither, /neither of the two required settings/);
test("a correctly set up card has nothing to diagnose", () => {
assert.equal(diagnoseSetup(ok("<.>", "<UserPersona>.</UserPersona>")), null);
});
test("every setup message still offers truncation as the second cause", () => {
for (const m of [missingSetup(false, true), missingSetup(true, false), missingSetup(false, false)]) {
assert.match(m.message, /Context Size/);
assert.ok(m.message.includes(CONTACT));
// Shown verbatim: this is the one class of failure a user can act on.
assert.equal(userFacingMessage(m), m.message);
test("a missing custom prompt is named with certainty, not blamed on truncation", () => {
const msg = diagnoseSetup(ok("", "<UserPersona>.</UserPersona>"))!.message;
assert.match(msg, /custom prompt is not coming through as <\.>/);
assert.ok(!/cut it short|Context Size/.test(msg), "truncation cannot remove the first three characters");
assert.ok(!/persona/i.test(msg), "the persona is fine and must not be mentioned");
});
test("a wrong persona description is quoted back, so there is nothing to guess", () => {
const msg = diagnoseSetup(ok("<.>", "<UserPersona>I am a knight</UserPersona>"))!.message;
assert.match(msg, /persona description is "I am a knight"/);
assert.match(msg, /just a dot/);
assert.ok(!/Context Size/.test(msg), "this case is certain; truncation is not the story");
});
test("an empty persona description says so", () => {
assert.match(diagnoseSetup(ok("<.>", "<UserPersona></UserPersona>"))!.message, /description is empty/);
});
test("a long persona description is trimmed to one readable line", () => {
const long = "x".repeat(80) + "\n\n y";
const msg = diagnoseSetup(ok("<.>", `<UserPersona>${long}</UserPersona>`))!.message;
assert.match(msg, /"x{40}…"/);
assert.ok(!msg.includes("\n"), "no raw newlines from user content");
});
test("a persona marker that is gone leads with truncation, the common cause", () => {
const msg = diagnoseSetup(ok("<.>", ""))!.message;
assert.match(msg, /end of it is missing/);
assert.ok(msg.indexOf("Context Size") < msg.indexOf("persona"), "truncation first, persona second");
});
test("a persona marker cut off mid-tag is truncation, not a wrong description", () => {
const msg = diagnoseSetup(ok("<.>", "<UserPersona>.</UserPe"))!.message;
assert.match(msg, /Context Size/);
assert.ok(!/description is/.test(msg));
});
test("both settings wrong at once are both named", () => {
const msg = diagnoseSetup(ok("", "<UserPersona>knight</UserPersona>"))!.message;
assert.match(msg, /custom prompt is not coming through/);
assert.match(msg, /description is "knight" as well/);
const neither = diagnoseSetup(ok("", ""))!.message;
assert.match(neither, /custom prompt is not coming through/);
assert.match(neither, /persona did not come through either/);
});
test("every setup message is shown verbatim and says where to go when stuck", () => {
for (const card of [ok("", "<UserPersona>.</UserPersona>"), ok("<.>", "<UserPersona>x</UserPersona>"), ok("<.>", ""), ok("", "")]) {
const e = diagnoseSetup(card)!;
assert.ok(isSuckerError(e));
assert.equal(userFacingMessage(e), e.message);
assert.ok(e.message.includes(CONTACT));
}
});
@@ -43,8 +84,7 @@ test("shapes Janitor cannot produce never tell the user to fix them", () => {
const msg = userFacingMessage(e);
assert.match(msg, /not your fault and there is nothing for you to fix/i);
assert.ok(msg.includes(CONTACT), "must say where to report it");
// The old failure mode: instructions for something they cannot do.
assert.ok(!/open the chat|try again|point sucker/i.test(msg), `actionable advice leaked into: ${msg}`);
assert.ok(!/open the chat|try again|point sucker|nothing was saved/i.test(msg), `overclaim or advice leaked into: ${msg}`);
}
});
@@ -63,7 +103,7 @@ test("a genuine bug reports itself without leaking internals", () => {
});
test("nobody is called a dingus", () => {
for (const e of [missingSetup(false, false), NO_CARD, NO_GREETING, TOO_FEW_MESSAGES, new Error("x")]) {
for (const e of [diagnoseSetup("nothing")!, NO_CARD, NO_GREETING, TOO_FEW_MESSAGES, new Error("x")]) {
assert.ok(!/dingus/i.test(userFacingMessage(e)));
}
});
+26 -2
View File
@@ -62,13 +62,26 @@ test("the token is still found once later replies stop carrying it", () => {
assert.match(found!, /\[sucker:conv=abc123\]/);
});
test("a token the user just pasted outranks an older one", () => {
test("the token a chat was started with outranks one pasted later, so a chat cannot flip between cards", () => {
const candidates = tokenCandidates([
...withProbe,
{ role: "assistant", content: `created (v1). ${TOKEN}` },
{ role: "user", content: "{{user}}: [sucker:conv=zzz999]" },
]);
assert.match(candidates[0], /zzz999/);
const first = candidates.find((c) => c.includes("sucker:conv="));
assert.match(first!, /abc123/);
});
test("a token in the card or the greeting is never a candidate", () => {
const stale = "[sucker:conv=stale00]";
const candidates = tokenCandidates([
{ role: "system", content: `${CARD} made with sucker: ${stale}` },
{ role: "user", content: "." },
{ role: "assistant", content: `${GREETING} ${stale}` },
{ role: "user", content: NAME },
]);
assert.ok(!candidates.some((c) => c.includes(stale)), "an expired token baked into a card would make it unextractable forever");
assert.deepEqual(candidates, [NAME]);
});
test("roles are matched whatever their casing", () => {
@@ -106,6 +119,17 @@ test("messages with no usable content are skipped, not treated as slots", () =>
assert.equal(slots.nameSlot, NAME);
});
test("a request with roles but no system turn has no card, rather than a misdiagnosed one", () => {
const slots = selectMessageSlots([
{ role: "assistant", content: GREETING },
{ role: "user", content: NAME },
]);
// Taking the greeting as the card would produce a "custom prompt not set"
// message for a request the user could never have fixed.
assert.equal(slots.system, undefined);
assert.equal(slots.greeting, GREETING);
});
test("a card with no role on the system turn still resolves", () => {
const slots = selectMessageSlots([
{ content: CARD },