diff --git a/Dockerfile b/Dockerfile index deb1eca40d0..86cf37bac24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ COPY package.json package.json COPY yarn.lock yarn.lock RUN yarn install COPY . /workspace -RUN NODE_OPTIONS="--openssl-legacy-provider --max-old-space-size=4096" yarn docusaurus build +RUN NODE_OPTIONS="--openssl-legacy-provider --max-old-space-size=4096" yarn docusaurus build && node scripts/llms-postprocess.js FROM nginx:1.21 WORKDIR / diff --git a/docusaurus.config.js b/docusaurus.config.js index c6d69e1c1fa..5e48dae5a81 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -308,6 +308,21 @@ module.exports = { ], 'docusaurus-plugin-sass', require.resolve('docusaurus-plugin-image-zoom'), + [ + 'docusaurus-plugin-llms', + { + title: 'KubeVela', + description: + 'KubeVela is a modern software delivery platform that makes deploying and operating applications across hybrid, multi-cloud environments easier, faster and more reliable.', + // Write a markdown copy of every doc next to its HTML page so agents + // can fetch .md; llms.txt links point at these copies. + generateMarkdownFiles: true, + // Generate per-version output matching the site's routing: the latest + // released version (served at /docs/) gets the root llms.txt, while + // next and older versions get their own under /docs//. + versions: 'auto', + }, + ], ], scripts: [ { diff --git a/package.json b/package.json index 4336001106c..0f3c9403ada 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "NODE_OPTIONS=\"--max-old-space-size=4096\" docusaurus build", + "postbuild": "node scripts/llms-postprocess.js", "swizzle": "docusaurus swizzle", "deploy": "NODE_OPTIONS=\"--max-old-space-size=4096\" docusaurus deploy", "clear": "docusaurus clear", @@ -26,6 +27,7 @@ "asciinema-player": "^3.8.0", "clsx": "^1.2.1", "docusaurus-plugin-image-zoom": "^1.0.1", + "docusaurus-plugin-llms": "^0.6.1", "docusaurus-plugin-sass": "^0.2.5", "mdx-mermaid": "^2.0.3", "mermaid": "^10.9.1", diff --git a/scripts/llms-postprocess.js b/scripts/llms-postprocess.js new file mode 100644 index 00000000000..b7b71b04509 --- /dev/null +++ b/scripts/llms-postprocess.js @@ -0,0 +1,219 @@ +/** + * Post-processes the agent-facing files that docusaurus-plugin-llms writes + * into the build output. Runs as the `postbuild` lifecycle script, after + * `docusaurus build` (the plugin writes its files in postBuild, which + * Docusaurus runs concurrently across plugins, so this cannot safely be + * another plugin). + * + * 1. Prepends a pointer to the llms.txt documentation index to every + * generated markdown page, so agents that fetch a single page can + * discover the full index. + * 2. Restructures the indexes into the llmstxt.org nested layout so each + * file stays within the ~100K character budget agents can ingest in one + * read: the CLI subcommand man-pages move into a nested index at + * /cli/llms.txt, per-version indexes move under their real + * route (/docs//llms.txt), and the root index links to all of + * them. Overly long entry descriptions are shortened. + */ +const fs = require('fs'); +const path = require('path'); + +const BUILD_DIR = path.join(__dirname, '..', 'build'); +const SITE_URL = 'https://kubevela.io'; +const DIRECTIVE = + '> For the complete KubeVela documentation index, see [llms.txt](https://kubevela.io/llms.txt).\n\n'; +// CLI subcommand man-pages (vela_*.md); the main `cli/vela.md` page stays. +const CLI_SUBCOMMAND = /\]\([^)]*\/cli\/vela_[^)]*\)/; +const MAX_DESCRIPTION = 160; +// llms-txt-size fails at >= 100K characters (afdocs AGENT_TRUNCATION_LIMIT). +const INDEX_SIZE_LIMIT = 100000; + +function walkMarkdown(dir, files = []) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + walkMarkdown(full, files); + } else if (entry.name.endsWith('.md')) { + files.push(full); + } + } + return files; +} + +function trimDescription(line) { + const match = line.match(/^(- \[[^\]]*\]\([^)]*\)): (.*)$/); + if (!match || match[2].length <= MAX_DESCRIPTION) { + return line; + } + const cut = match[2].slice(0, MAX_DESCRIPTION).replace(/\s+\S*$/, ''); + return `${match[1]}: ${cut}...`; +} + +/** + * Split the CLI man-page entries out of an index into a nested index file, + * trim long descriptions, and append the extra sections provided. Writes the + * resulting index to destFile and returns its size. + */ +function restructureIndex({ srcFile, destFile, label, routePrefix, extraSections }) { + const lines = fs.readFileSync(srcFile, 'utf8').split('\n'); + const cliLines = []; + const kept = []; + for (const line of lines) { + if (CLI_SUBCOMMAND.test(line)) { + cliLines.push(trimDescription(line)); + } else { + kept.push(trimDescription(line)); + } + } + + const sections = [...extraSections]; + if (cliLines.length > 0) { + // The CLI pages live under the version's route (/docs[/]/cli/), + // which for the root index is not the directory llms.txt sits in. + const cliIndexFile = path.join(BUILD_DIR, ...routePrefix.split('/').filter(Boolean), 'cli', 'llms.txt'); + const cliIndex = [ + `# KubeVela CLI reference (${label})`, + '', + `> Manual pages for every subcommand of the vela command-line tool (${label} documentation).`, + '', + '## Commands', + '', + ...cliLines, + '', + ].join('\n'); + fs.mkdirSync(path.dirname(cliIndexFile), { recursive: true }); + fs.writeFileSync(cliIndexFile, cliIndex); + sections.unshift({ + heading: '## CLI command reference', + lines: [ + `- [vela CLI command index](${SITE_URL}${routePrefix}/cli/llms.txt): Nested index of the ${cliLines.length} vela subcommand manual pages.`, + ], + }); + } + + let content = kept.join('\n').replace(/\n+$/, '\n'); + for (const section of sections) { + content += `\n${section.heading}\n\n${section.lines.join('\n')}\n`; + } + + fs.mkdirSync(path.dirname(destFile), { recursive: true }); + fs.writeFileSync(destFile, content); + if (srcFile !== destFile) { + fs.unlinkSync(srcFile); + } + return { size: content.length, cliCount: cliLines.length }; +} + +if (!fs.existsSync(BUILD_DIR)) { + console.error(`llms-postprocess: build directory not found at ${BUILD_DIR}`); + process.exit(1); +} + +// Discover the non-root versions the plugin generated (top-level / +// output directories holding an llms.txt, e.g. next/, v1.10/). Their real +// routes live under /docs//. +const versions = fs + .readdirSync(BUILD_DIR, { withFileTypes: true }) + .filter( + (e) => + e.isDirectory() && + /^(next|v\d+\.\d+)$/.test(e.name) && + fs.existsSync(path.join(BUILD_DIR, e.name, 'llms.txt')) + ) + .map((e) => e.name) + .sort((a, b) => { + if (a === 'next') return -1; + if (b === 'next') return 1; + const [aMaj, aMin] = a.slice(1).split('.').map(Number); + const [bMaj, bMin] = b.slice(1).split('.').map(Number); + return bMaj - aMaj || bMin - aMin; + }); + +let failed = false; +const report = (name, result) => { + console.log( + `llms-postprocess: ${name} — ${result.size} chars, ${result.cliCount} CLI entries nested` + ); + if (result.size >= INDEX_SIZE_LIMIT) { + console.error( + `llms-postprocess: ${name} is over the ${INDEX_SIZE_LIMIT} character agent ingestion budget` + ); + failed = true; + } +}; + +// Per-version indexes: move under /docs// and nest their CLI pages. +for (const version of versions) { + const label = version === 'next' ? 'next, unreleased' : version; + report( + `docs/${version}/llms.txt`, + restructureIndex({ + srcFile: path.join(BUILD_DIR, version, 'llms.txt'), + destFile: path.join(BUILD_DIR, 'docs', version, 'llms.txt'), + label, + routePrefix: `/docs/${version}`, + extraSections: [], + }) + ); + const full = path.join(BUILD_DIR, version, 'llms-full.txt'); + if (fs.existsSync(full)) { + fs.renameSync(full, path.join(BUILD_DIR, 'docs', version, 'llms-full.txt')); + } +} + +// Intermediate index listing the per-version indexes, one level below the +// root so the hierarchy mirrors how the docs are organized: the root index +// covers the latest release, and other versions hang off their own indexes. +const versionsIndexFile = path.join(BUILD_DIR, 'docs', 'versions', 'llms.txt'); +fs.mkdirSync(path.dirname(versionsIndexFile), { recursive: true }); +fs.writeFileSync( + versionsIndexFile, + [ + '# KubeVela documentation versions', + '', + `> Indexes of the KubeVela documentation for the in-development (next) and previously released versions. The latest release is indexed at ${SITE_URL}/llms.txt.`, + '', + '## Versions', + '', + ...versions.map((version) => { + const label = version === 'next' ? 'Next (unreleased)' : version; + return `- [${label} documentation index](${SITE_URL}/docs/${version}/llms.txt): Nested index of the ${label} docs.`; + }), + '', + ].join('\n') +); + +// Root index (latest released docs): nest its CLI pages and link the +// versions index so agents can find every other version from the root. +report( + 'llms.txt', + restructureIndex({ + srcFile: path.join(BUILD_DIR, 'llms.txt'), + destFile: path.join(BUILD_DIR, 'llms.txt'), + label: 'latest release', + routePrefix: '/docs', + extraSections: [ + { + heading: '## Other documentation versions', + lines: [ + `- [Documentation versions index](${SITE_URL}/docs/versions/llms.txt): Nested index of all other documentation versions (next and released).`, + ], + }, + ], + }) +); + +// Pointer to the root index on every generated markdown page. +let markdownUpdated = 0; +for (const file of walkMarkdown(BUILD_DIR)) { + const content = fs.readFileSync(file, 'utf8'); + if (!content.startsWith(DIRECTIVE)) { + fs.writeFileSync(file, DIRECTIVE + content); + markdownUpdated++; + } +} +console.log(`llms-postprocess: added llms.txt pointer to ${markdownUpdated} markdown file(s)`); + +if (failed) { + process.exit(1); +} diff --git a/src/theme/Root.js b/src/theme/Root.js new file mode 100644 index 00000000000..5bc0caf235a --- /dev/null +++ b/src/theme/Root.js @@ -0,0 +1,30 @@ +import React from 'react'; + +// Visually hidden (but machine-readable) note telling AI agents where the +// markdown documentation index lives. Rendered on every page, server-side, +// near the top of the document body. +const hiddenStyle = { + position: 'absolute', + width: '1px', + height: '1px', + padding: 0, + margin: '-1px', + overflow: 'hidden', + clip: 'rect(0, 0, 0, 0)', + whiteSpace: 'nowrap', + border: 0, +}; + +export default function Root({children}) { + return ( + <> +
+ A machine-readable index of this documentation for AI agents is + available at https://kubevela.io/llms.txt. + Markdown versions of documentation pages are available by appending + .md to their URLs. +
+ {children} + + ); +} diff --git a/yarn.lock b/yarn.lock index c84a66cac99..c44b5dbad65 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3895,6 +3895,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.1.8" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.1.8.tgz#b4cf1e18069418e97732d5f30b39bb321c41bd9b" + integrity sha512-BnlDzDYQoplTsZSvPi86CNzmbcb07X8ufR8fk1zivaiEJlXJkiq1MQyDCVmlWcsWbj6aWXqHnrwwevbrm/Saig== + dependencies: + balanced-match "^1.0.0" + braces@^3.0.2, braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" @@ -5263,6 +5270,15 @@ docusaurus-plugin-image-zoom@^1.0.1: medium-zoom "^1.0.6" validate-peer-dependencies "^2.2.0" +docusaurus-plugin-llms@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/docusaurus-plugin-llms/-/docusaurus-plugin-llms-0.6.1.tgz#08e67711885addc9138c433ef49eaf025a1a877d" + integrity sha512-elqTn7httrRG20g1mfzTtWZjLVpXMNKO2IS9R84dxiIp7QjAcDuubLp6/bvpe3umHdPnEXHJKhNOpTY+wnQa1Q== + dependencies: + gray-matter "4.0.3" + minimatch "9.0.3" + yaml "2.8.1" + docusaurus-plugin-sass@^0.2.5: version "0.2.5" resolved "https://registry.yarnpkg.com/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz#6bfb8a227ac6265be685dcbc24ba1989e27b8005" @@ -6186,7 +6202,7 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== -gray-matter@^4.0.3: +gray-matter@4.0.3, gray-matter@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/gray-matter/-/gray-matter-4.0.3.tgz#e893c064825de73ea1f5f7d88c7a9f7274288798" integrity sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q== @@ -8551,6 +8567,13 @@ minimatch@3.1.2, minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" +minimatch@9.0.3: + version "9.0.3" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825" + integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.0: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" @@ -11938,6 +11961,11 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== +yaml@2.8.1: + version "2.8.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.1.tgz#1870aa02b631f7e8328b93f8bc574fac5d6c4d79" + integrity sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw== + yargs-parser@^21.1.1: version "21.1.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35"