Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .agents/skills/visual-qa/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ npm run qa:visual -- --only 09,15,17 --variants light
npm run qa:visual -- --only 07 --variants narrow --calibrate
```

For another interface language, start the app with `KINDLING_LOCALE=<tag> npm run tauri:qa`
(restarting Vite first) and follow the README's "Other interface languages": every image
differs from the English baselines, so judge the audits and review the captures instead.

Translate requested IDs such as `09 15 17` into `--only 09,15,17`. Unsupported IDs
remain manual/unexecuted unless actually performed. Use `--calibrate` only for
requested repeatability work or diagnosing capture instability, not every pass.
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"format:rust:check": "cd src-tauri && cargo fmt --all -- --check",
"format:all": "npm run format && npm run format:rust",
"test": "vitest run",
"test:qa-harness": "node --test qa/visual/harness.test.mjs qa/visual/socket.test.mjs qa/visual/capture.test.mjs qa/visual/baselines.test.mjs qa/visual/context.test.mjs",
"test:qa-harness": "node --test qa/visual/harness.test.mjs qa/visual/run.test.mjs qa/visual/socket.test.mjs qa/visual/capture.test.mjs qa/visual/baselines.test.mjs qa/visual/context.test.mjs",
"qa:visual": "node qa/visual/run.mjs",
"qa:baselines:check": "node qa/visual/baselines.mjs",
"test:demo-seed": "node --test qa/demo/seed.test.mjs",
Expand Down
28 changes: 28 additions & 0 deletions qa/visual/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,34 @@ The latest 147-checkpoint comparison (after the macOS 27.2 baseline refresh) too
**248 seconds**; all images matched and every checkpoint passed. The earlier
240-capture calibration took **379 seconds**. See VALIDATION.md for measured evidence.

### Other interface languages

The suites are written against English copy and also run in every shipped locale.
Start the QA app with the locale override, which reaches both the webview (through
Vite) and the native menus:

```bash
KINDLING_LOCALE=ru npm run tauri:qa # or zh-CN, es, pt-BR, de
npm run qa:visual
```

Vite reads the override when it starts, so stop the QA app and any Vite server on
port 1420 before switching language. At install the harness loads the app's own
i18n module and maps each English string the suites use (button text, messages,
`aria-label`, `title` and `placeholder` values, including one-placeholder
templates such as "Close {name}") to the running locale's copy, via
`q.L`/`q.Ls`/`q.has`/`q.is` in the page and `localize()`/`tr()` in the runner.
Fixture content stays English. When adding a step, name controls by their English
copy or a test id as usual; compare visible text with `window.__qa.has/is` rather
than `textContent.includes('…')`.

Accepted baselines are English, so a non-English pass reports every image as
changed (exit 2). Its value is the audits (axe, Press, overflow, console) and a
visual review of the captures for clipped text, untranslated copy and font
fallback. The overflow audit only sees scroll overflow, not text cut off inside
a fixed-size control. Native menus are not part of the WebKit captures; check
them by hand in each locale.

Baselines are tied to the OS/WebKit build that captured them (the manifest's
`environment.os`): a macOS update can shift text metrics by a pixel or two on
every screen and flag every image as changed. Before bisecting source commits,
Expand Down
87 changes: 86 additions & 1 deletion qa/visual/harness.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,91 @@
return els.find((e) => e.textContent.trim().includes(text)) || null;
};
qa.nth = (testid, n) => all(testid)[n] || null;

// ---------------------------------------------------------------- locale
// Suites are written against English copy. Resolve that copy to the running
// interface locale through the app's own i18n module, so the same suites run
// under KINDLING_LOCALE=<tag>. English text with no catalog entry (fixture
// data, product names) resolves to itself.
const GROUPS = [
"common",
"counts",
"editor",
"editorial",
"export",
"find",
"logic",
"onboarding",
"references",
"settings",
"shell",
"sidebar",
];
qa.loadI18n = async () => {
// Same URL as the app's own import, so this is the app's module instance and locale.
// Vite warns about these dynamic imports; leave them unannotated, because a
// @vite-ignore makes it inject a module import that breaks this classic script.
const i18n = await import("/src/lib/i18n/index.ts");
const byText = new Map();
const templates = [];
for (const group of GROUPS) {
const { default: catalog } = await import(`/src/lib/i18n/locales/en/${group}.ts`);
for (const [key, value] of Object.entries(catalog)) {
const keys = byText.get(value) || [];
keys.push(`${group}.${key}`);
byText.set(value, keys);
// One-placeholder copy such as "Close {name}" or "Export {format}", for
// labels a component fills in at runtime.
const parts = value.split(/\{(\w+)\}/);
if (parts.length === 3 && (parts[0] + parts[2]).trim().length >= 3)
templates.push({
key: `${group}.${key}`,
before: parts[0],
name: parts[1],
after: parts[2],
});
}
}
qa.i18n = { t: i18n.t, tn: i18n.tn, locale: i18n.getLocale(), byText, templates };
return qa.i18n.locale;
};
/** Every rendering of this English string in the running locale (several keys can share it). */
qa.Ls = (en) => {
if (!qa.i18n) return [en];
const keys = qa.i18n.byText.get(en);
if (keys) return [...new Set(keys.map((key) => qa.i18n.t(key)))];
// A filled-in template: translate the template and, recursively, the value.
const forms = qa.i18n.templates.flatMap(({ key, before, name, after }) => {
if (
!en.startsWith(before) ||
!en.endsWith(after) ||
en.length <= before.length + after.length
)
return [];
const value = en.slice(before.length, en.length - after.length);
return qa.Ls(value).map((v) => qa.i18n.t(key, { [name]: v }));
});
return forms.length ? [...new Set(forms)] : [en];
};
qa.L = (en) => qa.Ls(en)[0];
/** True when `text` contains any rendering of the English string. */
qa.has = (text, en) => qa.Ls(en).some((s) => (text ?? "").includes(s));
/** True when `text` (whitespace-normalised) equals a rendering of the English string. */
qa.is = (text, en) => qa.Ls(en).includes((text ?? "").replace(/\s+/g, " ").trim());
/**
* Attribute selector for a templated catalog string, e.g. "Shortcut for {label}"
* with any label: matches the text before and after the placeholder.
*/
qa.template = (attr, key, name) => {
const [before, after = ""] = qa.i18n.t(key).split(`{${name}}`);
return `${before ? `[${attr}^=${JSON.stringify(before)}]` : ""}${after ? `[${attr}$=${JSON.stringify(after)}]` : ""}`;
};
/** Localise English values in [aria-label="…"], [title="…"] and [placeholder="…"] selectors. */
qa.sel = (css) =>
css.replace(/\[(aria-label|title|placeholder)="([^"]*)"\]/g, (_, attr, en) => {
const forms = qa.Ls(en).map((s) => `[${attr}=${JSON.stringify(s)}]`);
return forms.length === 1 ? forms[0] : `:is(${forms.join(",")})`;
});
qa.titles = (testid) => all(testid).map((e) => e.textContent.trim());

// ---------------------------------------------------------------- actions
Expand Down Expand Up @@ -473,7 +558,7 @@
theme: document.documentElement.dataset.theme,
body: probe("body", document.body),
sidebar: probe("sidebar", $('[data-testid="sidebar"]')),
references: probe("references", $('aside[aria-label="References"]')),
references: probe("references", $(qa.sel('aside[aria-label="References"]'))),
select: probe("select", $('[data-testid="scene-panel"] select')),
prose: probe("prose", $(".novel-editor-content")),
chapterTitle: probe("chapterTitle", $('[data-testid="chapter-title"]')),
Expand Down
48 changes: 48 additions & 0 deletions qa/visual/harness.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -385,3 +385,51 @@ test("PNG masters use explicit baseline paths and deterministic pixel thresholds
close();
}
});

// Fake the app's i18n module: German copy, with two keys sharing one English string.
function withGerman(w) {
const de = {
"export.saveProfile": "Profil speichern",
"export.lineSpacing": "Zeilenabstand",
"logic.lineSpacing": "Zeilenabst.",
"settings.keyboard.shortcutFor": "Kürzel für {label}",
"zh.shortcutFor": "{label} 的快捷键",
};
const byText = new Map([
["Save profile", ["export.saveProfile"]],
["Line spacing", ["export.lineSpacing", "logic.lineSpacing"]],
]);
de["common.closeNamed"] = "{name} schließen";
de["find.title"] = "Suchen und ersetzen";
byText.set("Find and Replace", ["find.title"]);
const t = (key, params = {}) =>
Object.entries(params).reduce((s, [k, v]) => s.replaceAll(`{${k}}`, v), de[key] ?? key);
const templates = [{ key: "common.closeNamed", before: "Close ", name: "name", after: "" }];
w.__qa.i18n = { t, locale: "de", byText, templates };
}

test("locale helpers resolve English suite copy to the running locale", () => {
const { w, close } = setup();
const q = w.__qa;
assert.deepEqual([...q.Ls("Save profile")], ["Save profile"], "English before i18n loads");
withGerman(w);
assert.equal(q.L("Save profile"), "Profil speichern");
assert.deepEqual([...q.Ls("Line spacing")], ["Zeilenabstand", "Zeilenabst."]);
assert.deepEqual([...q.Ls("QA Mara")], ["QA Mara"], "fixture text has no catalog entry");
assert.ok(q.has("Bitte Zeilenabst. wählen", "Line spacing"));
assert.ok(!q.has("Line spacing", "Line spacing"), "English copy no longer matches");
assert.ok(q.is(" Profil \n speichern ", "Save profile"));
assert.equal(q.sel('button[aria-label="Save profile"]'), 'button[aria-label="Profil speichern"]');
assert.equal(
q.sel('[title="Line spacing"]'),
':is([title="Zeilenabstand"],[title="Zeilenabst."])'
);
assert.equal(
q.template("aria-label", "settings.keyboard.shortcutFor", "label"),
'[aria-label^="Kürzel für "]'
);
assert.equal(q.template("aria-label", "zh.shortcutFor", "label"), '[aria-label$=" 的快捷键"]');
assert.deepEqual([...q.Ls("Close Find and Replace")], ["Suchen und ersetzen schließen"]);
assert.deepEqual([...q.Ls("Close ")], ["Close "], "a template needs a non-empty value");
close();
});
50 changes: 43 additions & 7 deletions qa/visual/run.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import { collectContext, contextMarkdown } from "./context.mjs";

const repo = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
const quote = JSON.stringify;
// [aria-label="…"], [title="…"] or [placeholder="…"], with plain or escaped quotes.
const ATTR = /\[(aria-label|title|placeholder)(\^?)=(\\?["'])(.*?)\3\]/g;
export const variants = {
light: { theme: "light", width: 1600, height: 968 },
dark: { theme: "dark", width: 1600, height: 968 },
Expand Down Expand Up @@ -51,11 +53,40 @@ export class Runner {
this.owned = [];
this.native = new SocketCapture(app, output);
}
js(code) {
return this.app.evaluate(`const q = window.__qa; ${code}`);
async js(code) {
return this.app.evaluate(`const q = window.__qa; ${await this.localize(code)}`);
}
wait(expression) {
return this.app.wait(expression);
async wait(expression) {
return this.app.wait(await this.localize(expression));
}
/**
* Suites name controls by their English aria-label, title or placeholder. Rewrite
* those attribute selectors to the running locale's copy (see harness `qa.Ls`).
* Quotes may be plain or escaped, because the selectors sit inside JS source.
*/
async localize(code) {
if (!this.strings) return code;
// Resolve labels that are not plain catalog strings ("Close Find and Replace")
// through the page's template lookup once, then cache them.
const unknown = [...code.matchAll(ATTR)].map((m) => m[4]).filter((en) => !(en in this.strings));
if (unknown.length) {
const resolved = await this.app.evaluate(
`return Object.fromEntries(${quote([...new Set(unknown)])}.map((en) => [en, window.__qa.Ls(en)]));`
);
Object.assign(this.strings, resolved);
}
return code.replace(ATTR, (match, attr, op, q, en) => {
const forms = this.strings[en];
if (!forms || (forms.length === 1 && forms[0] === en)) return match;
// Escape apostrophes too: the selector may sit inside a single-quoted JS string.
const esc = (s) => s.replaceAll("'", "\\'").replaceAll(q.at(-1), `\\${q.at(-1)}`);
const sel = (s) => `[${attr}${op}=${q}${esc(s)}${q}]`;
return forms.length === 1 ? sel(forms[0]) : `:is(${forms.map(sel).join(",")})`;
});
}
/** The running locale's rendering of an English string (itself when it has no catalog entry). */
tr(en) {
return this.strings?.[en]?.[0] ?? en;
}
async check(expression, message = expression) {
if (this.interrupted) throw new Error("Run interrupted; cleaning up owned fixtures");
Expand All @@ -69,7 +100,7 @@ export class Runner {
async text(text, root = "document") {
await this.js(`const root = ${root};
const el = [...root.querySelectorAll('button,[role=button]')].find(e =>
e.textContent.replace(/\\s+/g,' ').trim() === ${quote(text)} && !e.disabled && e.getClientRects().length);
q.is(e.textContent, ${quote(text)}) && !e.disabled && e.getClientRects().length);
if (!el) throw new Error('Missing button: ' + ${quote(text)});
el.scrollIntoView({block:'nearest'}); el.click();`);
}
Expand Down Expand Up @@ -123,7 +154,12 @@ export class Runner {
await this.app.evaluate(`if(!window.__qa) { const s=document.createElement('script');
s.src=${quote(`/@fs${repo}/qa/visual/harness.js`)} + '?t=' + Date.now();document.head.append(s); }`);
await this.wait("window.__qa?.importFixture && window.__qa?.diff");
await this.js(`q.repoRoot = ${quote(repo)};`);
await this.js(`q.repoRoot = ${quote(repo)}; q.loadI18n();`);
await this.wait("window.__qa.i18n");
this.locale = await this.js("return q.i18n.locale;");
// English strings whose rendering differs in this locale, for localize().
this.strings = await this.js(`return Object.fromEntries([...q.i18n.byText.keys()]
.map((en) => [en, q.Ls(en)]).filter(([en, forms]) => forms.length > 1 || forms[0] !== en));`);
}
async fixture() {
await this.js(`q.importFixture(${quote(join(repo, "test-data/simple-story.pltr"))});`);
Expand Down Expand Up @@ -173,7 +209,7 @@ export class Runner {
const stats = await this.app.invoke("get_writing_stats", { projectId: this.project.id });
await this.click('[aria-controls="writing-statistics-panel"]');
await this.wait(
`document.querySelector('[aria-controls="writing-statistics-panel"]')?.closest('footer').textContent.includes(${quote(`Project: ${stats.project_words.toLocaleString("en-US")} words`)})`
`document.querySelector('[aria-controls="writing-statistics-panel"]')?.closest('footer').textContent.includes(window.__qa.i18n.t("editor.writingStatusBar.projectWords", {count: (${stats.project_words}).toLocaleString()}))`
);
await this.click('[aria-controls="writing-statistics-panel"]');
}
Expand Down
62 changes: 62 additions & 0 deletions qa/visual/run.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { Runner } from "./run.mjs";

test("localize rewrites English attribute selectors in any quoting, and leaves the rest", async () => {
const asked = [];
const app = {
evaluate: async (code) => {
asked.push(code);
return {
"Close Find and Replace": ["Suchen und ersetzen schließen"],
"QA Mara": ["QA Mara"],
"Writer\\'s notes": ["Writer\\'s notes"],
};
},
};
const r = new Runner(app, "/tmp/qa-out");
const code = `document.querySelector('[aria-label="Save profile"]')`;
assert.equal(await r.localize(code), code, "no rewrite before the locale strings load");
assert.equal(r.tr("Save profile"), "Save profile");

r.strings = {
"Save profile": ["Profil speichern"],
"Line spacing": ["Zeilenabstand", "Zeilenabst."],
"Writer's notes": ["Notizen „des Autors“", "l'auteur"],
};
assert.equal(r.tr("Save profile"), "Profil speichern");
assert.equal(r.tr("QA Mara"), "QA Mara");
assert.equal(
await r.localize(`document.querySelector('[aria-label="Save profile"]')`),
`document.querySelector('[aria-label="Profil speichern"]')`
);
assert.equal(
await r.localize(JSON.stringify('nav [title="Line spacing"] button')),
JSON.stringify('nav :is([title="Zeilenabstand"],[title="Zeilenabst."]) button')
);
assert.equal(
await r.localize(`'[placeholder="Writer\\'s notes"]'`),
`'[placeholder="Writer\\'s notes"]'`,
"an escaped English apostrophe is a different string and is left alone"
);
assert.equal(
(await r.localize(`x('[aria-label="Writer's notes"]')`)).includes(`l\\'auteur`),
true,
"apostrophes in translations are escaped for single-quoted JS strings"
);
assert.equal(
await r.localize(`[aria-labelledby="Save profile"]`),
`[aria-labelledby="Save profile"]`,
"aria-labelledby is an id reference, not copy"
);
assert.equal(await r.localize(`[data-testid="Save profile"]`), `[data-testid="Save profile"]`);

assert.equal(
await r.localize(`[aria-label="Close Find and Replace"] [title="QA Mara"]`),
`[aria-label="Suchen und ersetzen schließen"] [title="QA Mara"]`,
"labels outside the catalog are resolved through the page once and cached"
);
const calls = asked.length;
await r.localize(`[aria-label="Close Find and Replace"]`);
assert.equal(asked.length, calls);
});
Loading
Loading