mirror of
https://github.com/severian-dev/sucker.severian.dev.git
synced 2026-10-18 01:20:29 +00:00
Diagnose setup by what is certain, and deliver errors the way the client asked
Review of the branch found the persona diagnosis was mostly truncation wearing a name tag. The <.> marker sits at the start of the system message and the persona marker after the card, so a card Janitor cut short arrives with the prompt present and the persona gone — and was told its persona was wrong, with truncation as an afterthought. The original message was only ever about context size, which is the best evidence there is about what actually happens. diagnoseSetup() now says only what the markers make certain. A missing <.> is a missing prompt, since truncation cannot remove the first three characters. A persona marker that is present but not "." is a wrong description, quoted back so there is nothing to guess. A persona marker that is gone, or cut off mid-tag, leads with truncation and mentions the persona second. The error path decided "streaming?" from the Accept header while the success path used body.stream, so a client that sent stream:true without that header got JSON back for every error and never displayed it. The flag is hoisted above the try; the header is only a fallback for when the body could not be read. Token search is restricted to after the greeting, in order. A card built from a sucker-proxied chat could carry an expired token in its text, and a lookup on it would fail forever. Conversation order also means the token a chat was started with always wins over one pasted later. The system fallback now takes a role-less first message only; a request with roles but no system turn is reported as such rather than misdiagnosed as a missing prompt. The echo feature reads the parser's lastUser instead of its own case-sensitive lookup. "Nothing was saved" is dropped from the unknown-error message, since it is not guaranteed. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NX2SsmUqQftWTZAY4BDuFa
This commit is contained in:
+61
-21
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* Which failures a user can act on, and which just need reporting.
|
||||
* Which failures a user can act on, which just need reporting, and whether
|
||||
* the setup diagnosis names the right step.
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
@@ -10,31 +11,71 @@ import {
|
||||
NO_GREETING,
|
||||
SuckerError,
|
||||
TOO_FEW_MESSAGES,
|
||||
missingSetup,
|
||||
diagnoseSetup,
|
||||
isImpossibleRequest,
|
||||
isSuckerError,
|
||||
userFacingMessage,
|
||||
} from "../src/lib/errors.ts";
|
||||
|
||||
test("a missing setting is named, not left to the reader to find", () => {
|
||||
const noPrompt = missingSetup(false, true).message;
|
||||
assert.match(noPrompt, /custom prompt is not coming through as <\.>/);
|
||||
assert.ok(!/persona needs the name/.test(noPrompt), "must not blame the setting that is fine");
|
||||
const CARD = "<Gaius's Persona>a tired innkeeper</Gaius's Persona>";
|
||||
const ok = (prompt: string, persona: string) => `${prompt}\n${CARD}\n${persona}`;
|
||||
|
||||
const noPersona = missingSetup(true, false).message;
|
||||
assert.match(noPersona, /persona is not coming through/);
|
||||
assert.ok(!/custom prompt is not coming through/.test(noPersona));
|
||||
|
||||
const neither = missingSetup(false, false).message;
|
||||
assert.match(neither, /neither of the two required settings/);
|
||||
test("a correctly set up card has nothing to diagnose", () => {
|
||||
assert.equal(diagnoseSetup(ok("<.>", "<UserPersona>.</UserPersona>")), null);
|
||||
});
|
||||
|
||||
test("every setup message still offers truncation as the second cause", () => {
|
||||
for (const m of [missingSetup(false, true), missingSetup(true, false), missingSetup(false, false)]) {
|
||||
assert.match(m.message, /Context Size/);
|
||||
assert.ok(m.message.includes(CONTACT));
|
||||
// Shown verbatim: this is the one class of failure a user can act on.
|
||||
assert.equal(userFacingMessage(m), m.message);
|
||||
test("a missing custom prompt is named with certainty, not blamed on truncation", () => {
|
||||
const msg = diagnoseSetup(ok("", "<UserPersona>.</UserPersona>"))!.message;
|
||||
assert.match(msg, /custom prompt is not coming through as <\.>/);
|
||||
assert.ok(!/cut it short|Context Size/.test(msg), "truncation cannot remove the first three characters");
|
||||
assert.ok(!/persona/i.test(msg), "the persona is fine and must not be mentioned");
|
||||
});
|
||||
|
||||
test("a wrong persona description is quoted back, so there is nothing to guess", () => {
|
||||
const msg = diagnoseSetup(ok("<.>", "<UserPersona>I am a knight</UserPersona>"))!.message;
|
||||
assert.match(msg, /persona description is "I am a knight"/);
|
||||
assert.match(msg, /just a dot/);
|
||||
assert.ok(!/Context Size/.test(msg), "this case is certain; truncation is not the story");
|
||||
});
|
||||
|
||||
test("an empty persona description says so", () => {
|
||||
assert.match(diagnoseSetup(ok("<.>", "<UserPersona></UserPersona>"))!.message, /description is empty/);
|
||||
});
|
||||
|
||||
test("a long persona description is trimmed to one readable line", () => {
|
||||
const long = "x".repeat(80) + "\n\n y";
|
||||
const msg = diagnoseSetup(ok("<.>", `<UserPersona>${long}</UserPersona>`))!.message;
|
||||
assert.match(msg, /"x{40}…"/);
|
||||
assert.ok(!msg.includes("\n"), "no raw newlines from user content");
|
||||
});
|
||||
|
||||
test("a persona marker that is gone leads with truncation, the common cause", () => {
|
||||
const msg = diagnoseSetup(ok("<.>", ""))!.message;
|
||||
assert.match(msg, /end of it is missing/);
|
||||
assert.ok(msg.indexOf("Context Size") < msg.indexOf("persona"), "truncation first, persona second");
|
||||
});
|
||||
|
||||
test("a persona marker cut off mid-tag is truncation, not a wrong description", () => {
|
||||
const msg = diagnoseSetup(ok("<.>", "<UserPersona>.</UserPe"))!.message;
|
||||
assert.match(msg, /Context Size/);
|
||||
assert.ok(!/description is/.test(msg));
|
||||
});
|
||||
|
||||
test("both settings wrong at once are both named", () => {
|
||||
const msg = diagnoseSetup(ok("", "<UserPersona>knight</UserPersona>"))!.message;
|
||||
assert.match(msg, /custom prompt is not coming through/);
|
||||
assert.match(msg, /description is "knight" as well/);
|
||||
const neither = diagnoseSetup(ok("", ""))!.message;
|
||||
assert.match(neither, /custom prompt is not coming through/);
|
||||
assert.match(neither, /persona did not come through either/);
|
||||
});
|
||||
|
||||
test("every setup message is shown verbatim and says where to go when stuck", () => {
|
||||
for (const card of [ok("", "<UserPersona>.</UserPersona>"), ok("<.>", "<UserPersona>x</UserPersona>"), ok("<.>", ""), ok("", "")]) {
|
||||
const e = diagnoseSetup(card)!;
|
||||
assert.ok(isSuckerError(e));
|
||||
assert.equal(userFacingMessage(e), e.message);
|
||||
assert.ok(e.message.includes(CONTACT));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -43,8 +84,7 @@ test("shapes Janitor cannot produce never tell the user to fix them", () => {
|
||||
const msg = userFacingMessage(e);
|
||||
assert.match(msg, /not your fault and there is nothing for you to fix/i);
|
||||
assert.ok(msg.includes(CONTACT), "must say where to report it");
|
||||
// The old failure mode: instructions for something they cannot do.
|
||||
assert.ok(!/open the chat|try again|point sucker/i.test(msg), `actionable advice leaked into: ${msg}`);
|
||||
assert.ok(!/open the chat|try again|point sucker|nothing was saved/i.test(msg), `overclaim or advice leaked into: ${msg}`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -63,7 +103,7 @@ test("a genuine bug reports itself without leaking internals", () => {
|
||||
});
|
||||
|
||||
test("nobody is called a dingus", () => {
|
||||
for (const e of [missingSetup(false, false), NO_CARD, NO_GREETING, TOO_FEW_MESSAGES, new Error("x")]) {
|
||||
for (const e of [diagnoseSetup("nothing")!, NO_CARD, NO_GREETING, TOO_FEW_MESSAGES, new Error("x")]) {
|
||||
assert.ok(!/dingus/i.test(userFacingMessage(e)));
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user