Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions docusaurus.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,21 @@ module.exports = {
],
'docusaurus-plugin-sass',
require.resolve('docusaurus-plugin-image-zoom'),
[
'docusaurus-plugin-llms',
{
title: 'KubeVela',
description:
'KubeVela is a modern software delivery platform that makes deploying and operating applications across hybrid, multi-cloud environments easier, faster and more reliable.',
// Write a markdown copy of every doc next to its HTML page so agents
// can fetch <page-url>.md; llms.txt links point at these copies.
generateMarkdownFiles: true,
// Generate per-version output matching the site's routing: the latest
// released version (served at /docs/) gets the root llms.txt, while
// next and older versions get their own under /docs/<version>/.
versions: 'auto',
},
],
],
scripts: [
{
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"docusaurus": "docusaurus",
"start": "docusaurus start",
"build": "NODE_OPTIONS=\"--max-old-space-size=4096\" docusaurus build",
"postbuild": "node scripts/llms-postprocess.js",
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
"swizzle": "docusaurus swizzle",
"deploy": "NODE_OPTIONS=\"--max-old-space-size=4096\" docusaurus deploy",
"clear": "docusaurus clear",
Expand All @@ -26,6 +27,7 @@
"asciinema-player": "^3.8.0",
"clsx": "^1.2.1",
"docusaurus-plugin-image-zoom": "^1.0.1",
"docusaurus-plugin-llms": "^0.6.1",
"docusaurus-plugin-sass": "^0.2.5",
"mdx-mermaid": "^2.0.3",
"mermaid": "^10.9.1",
Expand Down
219 changes: 219 additions & 0 deletions scripts/llms-postprocess.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
/**
* Post-processes the agent-facing files that docusaurus-plugin-llms writes
* into the build output. Runs as the `postbuild` lifecycle script, after
* `docusaurus build` (the plugin writes its files in postBuild, which
* Docusaurus runs concurrently across plugins, so this cannot safely be
* another plugin).
*
* 1. Prepends a pointer to the llms.txt documentation index to every
* generated markdown page, so agents that fetch a single page can
* discover the full index.
* 2. Restructures the indexes into the llmstxt.org nested layout so each
* file stays within the ~100K character budget agents can ingest in one
* read: the CLI subcommand man-pages move into a nested index at
* <version>/cli/llms.txt, per-version indexes move under their real
* route (/docs/<version>/llms.txt), and the root index links to all of
* them. Overly long entry descriptions are shortened.
*/
const fs = require('fs');
const path = require('path');

const BUILD_DIR = path.join(__dirname, '..', 'build');
const SITE_URL = 'https://kubevela.io';
const DIRECTIVE =
'> For the complete KubeVela documentation index, see [llms.txt](https://kubevela.io/llms.txt).\n\n';
// CLI subcommand man-pages (vela_*.md); the main `cli/vela.md` page stays.
const CLI_SUBCOMMAND = /\]\([^)]*\/cli\/vela_[^)]*\)/;
const MAX_DESCRIPTION = 160;
// llms-txt-size fails at >= 100K characters (afdocs AGENT_TRUNCATION_LIMIT).
const INDEX_SIZE_LIMIT = 100000;

function walkMarkdown(dir, files = []) {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) {
walkMarkdown(full, files);
} else if (entry.name.endsWith('.md')) {
files.push(full);
}
}
return files;
}

function trimDescription(line) {
const match = line.match(/^(- \[[^\]]*\]\([^)]*\)): (.*)$/);
if (!match || match[2].length <= MAX_DESCRIPTION) {
return line;
}
const cut = match[2].slice(0, MAX_DESCRIPTION).replace(/\s+\S*$/, '');
return `${match[1]}: ${cut}...`;
}

/**
* Split the CLI man-page entries out of an index into a nested index file,
* trim long descriptions, and append the extra sections provided. Writes the
* resulting index to destFile and returns its size.
*/
function restructureIndex({ srcFile, destFile, label, routePrefix, extraSections }) {
const lines = fs.readFileSync(srcFile, 'utf8').split('\n');
const cliLines = [];
const kept = [];
for (const line of lines) {
if (CLI_SUBCOMMAND.test(line)) {
cliLines.push(trimDescription(line));
} else {
kept.push(trimDescription(line));
}
}

const sections = [...extraSections];
if (cliLines.length > 0) {
// The CLI pages live under the version's route (/docs[/<version>]/cli/),
// which for the root index is not the directory llms.txt sits in.
const cliIndexFile = path.join(BUILD_DIR, ...routePrefix.split('/').filter(Boolean), 'cli', 'llms.txt');
const cliIndex = [
`# KubeVela CLI reference (${label})`,
'',
`> Manual pages for every subcommand of the vela command-line tool (${label} documentation).`,
'',
'## Commands',
'',
...cliLines,
'',
].join('\n');
fs.mkdirSync(path.dirname(cliIndexFile), { recursive: true });
fs.writeFileSync(cliIndexFile, cliIndex);
sections.unshift({
heading: '## CLI command reference',
lines: [
`- [vela CLI command index](${SITE_URL}${routePrefix}/cli/llms.txt): Nested index of the ${cliLines.length} vela subcommand manual pages.`,
],
});
}

let content = kept.join('\n').replace(/\n+$/, '\n');
for (const section of sections) {
content += `\n${section.heading}\n\n${section.lines.join('\n')}\n`;
}

fs.mkdirSync(path.dirname(destFile), { recursive: true });
fs.writeFileSync(destFile, content);
if (srcFile !== destFile) {
fs.unlinkSync(srcFile);
}
return { size: content.length, cliCount: cliLines.length };
}

if (!fs.existsSync(BUILD_DIR)) {
console.error(`llms-postprocess: build directory not found at ${BUILD_DIR}`);
process.exit(1);
}

// Discover the non-root versions the plugin generated (top-level <version>/
// output directories holding an llms.txt, e.g. next/, v1.10/). Their real
// routes live under /docs/<version>/.
const versions = fs
.readdirSync(BUILD_DIR, { withFileTypes: true })
.filter(
(e) =>
e.isDirectory() &&
/^(next|v\d+\.\d+)$/.test(e.name) &&
fs.existsSync(path.join(BUILD_DIR, e.name, 'llms.txt'))
)
.map((e) => e.name)
.sort((a, b) => {
if (a === 'next') return -1;
if (b === 'next') return 1;
const [aMaj, aMin] = a.slice(1).split('.').map(Number);
const [bMaj, bMin] = b.slice(1).split('.').map(Number);
return bMaj - aMaj || bMin - aMin;
});

let failed = false;
const report = (name, result) => {
console.log(
`llms-postprocess: ${name} — ${result.size} chars, ${result.cliCount} CLI entries nested`
);
if (result.size >= INDEX_SIZE_LIMIT) {
console.error(
`llms-postprocess: ${name} is over the ${INDEX_SIZE_LIMIT} character agent ingestion budget`
);
failed = true;
}
};

// Per-version indexes: move under /docs/<version>/ and nest their CLI pages.
for (const version of versions) {
const label = version === 'next' ? 'next, unreleased' : version;
report(
`docs/${version}/llms.txt`,
restructureIndex({
srcFile: path.join(BUILD_DIR, version, 'llms.txt'),
destFile: path.join(BUILD_DIR, 'docs', version, 'llms.txt'),
label,
routePrefix: `/docs/${version}`,
extraSections: [],
})
);
const full = path.join(BUILD_DIR, version, 'llms-full.txt');
if (fs.existsSync(full)) {
fs.renameSync(full, path.join(BUILD_DIR, 'docs', version, 'llms-full.txt'));
}
}

// Intermediate index listing the per-version indexes, one level below the
// root so the hierarchy mirrors how the docs are organized: the root index
// covers the latest release, and other versions hang off their own indexes.
const versionsIndexFile = path.join(BUILD_DIR, 'docs', 'versions', 'llms.txt');
fs.mkdirSync(path.dirname(versionsIndexFile), { recursive: true });
fs.writeFileSync(
versionsIndexFile,
[
'# KubeVela documentation versions',
'',
`> Indexes of the KubeVela documentation for the in-development (next) and previously released versions. The latest release is indexed at ${SITE_URL}/llms.txt.`,
'',
'## Versions',
'',
...versions.map((version) => {
const label = version === 'next' ? 'Next (unreleased)' : version;
return `- [${label} documentation index](${SITE_URL}/docs/${version}/llms.txt): Nested index of the ${label} docs.`;
}),
'',
].join('\n')
);

// Root index (latest released docs): nest its CLI pages and link the
// versions index so agents can find every other version from the root.
report(
'llms.txt',
restructureIndex({
srcFile: path.join(BUILD_DIR, 'llms.txt'),
destFile: path.join(BUILD_DIR, 'llms.txt'),
label: 'latest release',
routePrefix: '/docs',
extraSections: [
{
heading: '## Other documentation versions',
lines: [
`- [Documentation versions index](${SITE_URL}/docs/versions/llms.txt): Nested index of all other documentation versions (next and released).`,
],
},
],
})
);

// Pointer to the root index on every generated markdown page.
let markdownUpdated = 0;
for (const file of walkMarkdown(BUILD_DIR)) {
const content = fs.readFileSync(file, 'utf8');
if (!content.startsWith(DIRECTIVE)) {
fs.writeFileSync(file, DIRECTIVE + content);
markdownUpdated++;
}
}
console.log(`llms-postprocess: added llms.txt pointer to ${markdownUpdated} markdown file(s)`);

if (failed) {
process.exit(1);
}
30 changes: 30 additions & 0 deletions src/theme/Root.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import React from 'react';

// Visually hidden (but machine-readable) note telling AI agents where the
// markdown documentation index lives. Rendered on every page, server-side,
// near the top of the document body.
const hiddenStyle = {
position: 'absolute',
width: '1px',
height: '1px',
padding: 0,
margin: '-1px',
overflow: 'hidden',
clip: 'rect(0, 0, 0, 0)',
whiteSpace: 'nowrap',
border: 0,
};

export default function Root({children}) {
return (
<>
<div style={hiddenStyle}>
A machine-readable index of this documentation for AI agents is
available at <a href="/llms.txt">https://kubevela.io/llms.txt</a>.
Markdown versions of documentation pages are available by appending
.md to their URLs.
</div>
{children}
</>
);
}
30 changes: 29 additions & 1 deletion yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3895,6 +3895,13 @@ brace-expansion@^1.1.7:
balanced-match "^1.0.0"
concat-map "0.0.1"

brace-expansion@^2.0.1:
version "2.1.8"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.1.8.tgz#b4cf1e18069418e97732d5f30b39bb321c41bd9b"
integrity sha512-BnlDzDYQoplTsZSvPi86CNzmbcb07X8ufR8fk1zivaiEJlXJkiq1MQyDCVmlWcsWbj6aWXqHnrwwevbrm/Saig==
dependencies:
balanced-match "^1.0.0"

braces@^3.0.2, braces@~3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
Expand Down Expand Up @@ -5263,6 +5270,15 @@ docusaurus-plugin-image-zoom@^1.0.1:
medium-zoom "^1.0.6"
validate-peer-dependencies "^2.2.0"

docusaurus-plugin-llms@^0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/docusaurus-plugin-llms/-/docusaurus-plugin-llms-0.6.1.tgz#08e67711885addc9138c433ef49eaf025a1a877d"
integrity sha512-elqTn7httrRG20g1mfzTtWZjLVpXMNKO2IS9R84dxiIp7QjAcDuubLp6/bvpe3umHdPnEXHJKhNOpTY+wnQa1Q==
dependencies:
gray-matter "4.0.3"
minimatch "9.0.3"
yaml "2.8.1"

docusaurus-plugin-sass@^0.2.5:
version "0.2.5"
resolved "https://registry.yarnpkg.com/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz#6bfb8a227ac6265be685dcbc24ba1989e27b8005"
Expand Down Expand Up @@ -6186,7 +6202,7 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11,
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"
integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==

gray-matter@^4.0.3:
gray-matter@4.0.3, gray-matter@^4.0.3:
version "4.0.3"
resolved "https://registry.yarnpkg.com/gray-matter/-/gray-matter-4.0.3.tgz#e893c064825de73ea1f5f7d88c7a9f7274288798"
integrity sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==
Expand Down Expand Up @@ -8551,6 +8567,13 @@ minimatch@3.1.2, minimatch@^3.1.1:
dependencies:
brace-expansion "^1.1.7"

minimatch@9.0.3:
version "9.0.3"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825"
integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==
dependencies:
brace-expansion "^2.0.1"

minimist@^1.2.0:
version "1.2.8"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
Expand Down Expand Up @@ -11938,6 +11961,11 @@ yallist@^4.0.0:
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==

yaml@2.8.1:
version "2.8.1"
resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.1.tgz#1870aa02b631f7e8328b93f8bc574fac5d6c4d79"
integrity sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==

yargs-parser@^21.1.1:
version "21.1.1"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35"
Expand Down
Loading