Transfer skill trees by manifest, fetching only missing files

apps/host-daemon/src/app.ts

@@ -62,6 +62,7 @@ import {
   type HostWatcher,
 } from "@bb/host-watcher";
 import { PluginHostManager } from "./plugin-host-manager.js";
+import type { SkillTreeTransport } from "./skill-trees.js";
 import { writeMachineSuspensionMarker } from "./suspension-marker.js";
 import {
   defaultServerMoveServiceOptions,
@@ -537,16 +538,24 @@ export async function createHostDaemonApp(
       sendServerMessage({ type: "connect-next.status", status });
     },
   });
+  const skillTreeTransport: SkillTreeTransport = {
+    fetchManifest: (treeHash) =>
+      runSessionRequest({
+        source: "fetchSkillTree",
+        request: () => serverClient.fetchSkillTreeManifest(treeHash),
+      }),
+    fetchContent: (treeHash, paths) =>
+      runSessionRequest({
+        source: "fetchSkillTree",
+        request: () => serverClient.fetchSkillTreeContent(treeHash, paths),
+      }),
+  };
   runtimeManager = new RuntimeManager({
     bridgeBundleDir: options.bridgeBundleDir,
     createRuntime: options.createRuntime,
     dataDir: options.dataDir,
     dataDirSkillsRootPath,
-    fetchSkillTree: (treeHash) =>
-      runSessionRequest({
-        source: "fetchSkillTree",
-        request: () => serverClient.fetchSkillTree(treeHash),
-      }),
+    skillTreeTransport,
     hostWatcher: options.hostWatcher,
     logger: options.logger,
     shellEnv: options.runtimeShellEnv,
@@ -795,11 +804,7 @@ export async function createHostDaemonApp(
         source: "fetchProjectAttachment",
         request: () => serverClient.fetchProjectAttachment(args),
       }),
-    fetchSkillTree: (treeHash) =>
-      runSessionRequest({
-        source: "fetchSkillTree",
-        request: () => serverClient.fetchSkillTree(treeHash),
-      }),
+    skillTreeTransport,
     fetchPluginHostArtifact: (args) =>
       runSessionRequest({
         source: "fetchPluginHostArtifact",

apps/host-daemon/src/command-dispatch-support.ts

@@ -29,7 +29,7 @@ import { ensurePluginProcessDataDir } from "@bb/process-utils";
 import type { InteractiveResolveCommandInput } from "./interactive-request-registry.js";
 import { RuntimeManager, type RuntimeEntry } from "./runtime-manager.js";
 import type { FetchProjectAttachment } from "./project-attachments.js";
-import type { FetchSkillTree } from "./skill-trees.js";
+import type { SkillTreeTransport } from "./skill-trees.js";
 import type { HostDaemonLogger } from "./logger.js";
 import type { ServerMoveService } from "./server-move/service.js";
 import {
@@ -52,7 +52,7 @@ export interface CommandDispatchOptions {
   dataDir: string;
   logger: Pick<HostDaemonLogger, "debug" | "warn">;
   fetchProjectAttachment: FetchProjectAttachment;
-  fetchSkillTree?: FetchSkillTree;
+  skillTreeTransport?: SkillTreeTransport;
   fetchPluginHostArtifact?: FetchPluginHostArtifact;
   runtimeManager: RuntimeManager;
   eventSink: Pick<EventSink, "emit" | "flush">;

apps/host-daemon/src/command-handlers/install-global-skills.test.ts

@@ -3,45 +3,42 @@ import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { makeTempDir } from "@bb/test-helpers";
 import { describe, expect, it, vi } from "vitest";
-import type { HostDaemonSkillTree } from "@bb/host-daemon-contract";
+import { hashSkillTreeFiles } from "@bb/host-daemon-contract/skill-tree";
+import type { SkillTreeTransport } from "../skill-trees.js";
 import {
   installGlobalSkills,
   readGlobalSkillsStatus,
 } from "./install-global-skills.js";
 
 const logger = { debug: () => undefined, warn: () => undefined };
 
-function createTreePayload(name: string, body: string): HostDaemonSkillTree {
-  const entries = [
-    {
-      path: "SKILL.md",
-      mode: 0o644,
-      contentBase64: Buffer.from(
+function createTreePayload(name: string, body: string) {
+  const contents = new Map([
+    [
+      "SKILL.md",
+      Buffer.from(
         `---\nname: ${name}\ndescription: Use ${name} outside bb.\n---\n\n${body}\n`,
-      ).toString("base64"),
-    },
-    {
-      path: "references/usage.md",
-      mode: 0o644,
-      contentBase64: Buffer.from(`# ${name}\n`).toString("base64"),
-    },
-  ].sort((left, right) =>
-    left.path < right.path ? -1 : left.path > right.path ? 1 : 0,
-  );
-  const hash = createHash("sha256");
-  hash.update("bb-skill-tree-v1");
-  for (const entry of entries) {
-    const bytes = Buffer.from(entry.contentBase64, "base64");
-    hash.update("\0file\0");
-    hash.update(entry.path);
-    hash.update("\0");
-    hash.update(entry.mode.toString(8));
-    hash.update("\0");
-    hash.update(String(bytes.length));
-    hash.update("\0");
-    hash.update(bytes);
-  }
-  return { treeHash: hash.digest("hex"), entries };
+      ),
+    ],
+    ["references/usage.md", Buffer.from(`# ${name}\n`)],
+  ]);
+  const files = [...contents].map(([filePath, content]) => ({
+    path: filePath,
+    mode: 0o644,
+    sizeBytes: content.byteLength,
+    sha256: createHash("sha256").update(content).digest("hex"),
+  }));
+  const treeHash = hashSkillTreeFiles(files);
+  const skillTreeTransport: SkillTreeTransport = {
+    fetchManifest: async () => ({ treeHash, files }),
+    fetchContent: async (_treeHash, paths) =>
+      (async function* () {
+        for (const filePath of paths) {
+          yield contents.get(filePath) ?? Buffer.alloc(0);
+        }
+      })(),
+  };
+  return { skillTreeTransport, treeHash };
 }
 
 describe("install global skills", () => {
@@ -57,7 +54,12 @@ describe("install global skills", () => {
           { name: "bb-cli", treeHash: payload.treeHash, entryPath: "SKILL.md" },
         ],
       },
-      { dataDir, fetchSkillTree: async () => payload, homeDir, logger },
+      {
+        dataDir,
+        homeDir,
+        logger,
+        skillTreeTransport: payload.skillTreeTransport,
+      },
     );
 
     expect(result.installations.map((entry) => entry.path)).toEqual([
@@ -95,7 +97,12 @@ describe("install global skills", () => {
           { name: "bb-cli", treeHash: payload.treeHash, entryPath: "SKILL.md" },
         ],
       },
-      { dataDir, fetchSkillTree: async () => payload, homeDir, logger },
+      {
+        dataDir,
+        homeDir,
+        logger,
+        skillTreeTransport: payload.skillTreeTransport,
+      },
     );
 
     await expect(
@@ -115,9 +122,12 @@ describe("install global skills", () => {
     await mkdir(path.join(agentsRoot, "bb-cli"), { recursive: true });
     await writeFile(path.join(agentsRoot, "bb-cli", "SKILL.md"), "previous\n");
     const payload = createTreePayload("bb-cli", "never arrives");
-    const fetchSkillTree = vi.fn(async () => {
-      throw new Error("offline");
-    });
+    const skillTreeTransport: SkillTreeTransport = {
+      ...payload.skillTreeTransport,
+      fetchManifest: vi.fn(async () => {
+        throw new Error("offline");
+      }),
+    };
 
     await expect(
       installGlobalSkills(
@@ -131,7 +141,7 @@ describe("install global skills", () => {
             },
           ],
         },
-        { dataDir, fetchSkillTree, homeDir, logger },
+        { dataDir, homeDir, logger, skillTreeTransport },
       ),
     ).rejects.toThrow("offline");
 
@@ -165,9 +175,9 @@ describe("install global skills", () => {
 
       await installGlobalSkills(command, {
         dataDir,
-        fetchSkillTree: async () => payload,
         homeDir,
         logger,
+        skillTreeTransport: payload.skillTreeTransport,
       });
 
       const after = await readGlobalSkillsStatus(statusCommand, { homeDir });

apps/host-daemon/src/command-handlers/install-global-skills.ts

@@ -14,7 +14,7 @@ import {
   scheduleSkillStoreGc,
   type InjectedSkillsLogger,
 } from "../injected-skills.js";
-import type { FetchSkillTree } from "../skill-trees.js";
+import type { SkillTreeTransport } from "../skill-trees.js";
 
 const GLOBAL_SKILL_ROOT_SEGMENTS: readonly (readonly string[])[] = [
   [".agents", "skills"],
@@ -23,7 +23,7 @@ const GLOBAL_SKILL_ROOT_SEGMENTS: readonly (readonly string[])[] = [
 
 interface InstallGlobalSkillsOptions {
   dataDir: string;
-  fetchSkillTree?: FetchSkillTree;
+  skillTreeTransport?: SkillTreeTransport;
   homeDir?: string;
   logger: InjectedSkillsLogger;
 }
@@ -88,8 +88,8 @@ export async function installGlobalSkills(
   command: CommandOf<"host.install_global_skills">,
   options: InstallGlobalSkillsOptions,
 ): Promise<HostDaemonOnlineRpcResult<"host.install_global_skills">> {
-  const { fetchSkillTree } = options;
-  if (fetchSkillTree === undefined) {
+  const { skillTreeTransport } = options;
+  if (skillTreeTransport === undefined) {
     throw new CommandDispatchError(
       "skill_tree_transport_unavailable",
       "Skill tree fetch transport is unavailable",
@@ -101,7 +101,8 @@ export async function installGlobalSkills(
   for (const skill of command.skills) {
     const sourceRootPath = await ensureStoredSkillTree({
       dataDir: options.dataDir,
-      fetchSkillTree,
+      logger: options.logger,
+      skillTreeTransport,
       treeHash: skill.treeHash,
     });
     const skillFilePath = path.resolve(sourceRootPath, skill.entryPath);

apps/host-daemon/src/command-router.ts

@@ -66,7 +66,7 @@ export interface CommandRouterOptions {
   desktopBrowserBroker?: CommandDispatchOptions["desktopBrowserBroker"];
   dataDir: CommandDispatchOptions["dataDir"];
   fetchProjectAttachment: CommandDispatchOptions["fetchProjectAttachment"];
-  fetchSkillTree?: CommandDispatchOptions["fetchSkillTree"];
+  skillTreeTransport?: CommandDispatchOptions["skillTreeTransport"];
   fetchPluginHostArtifact?: CommandDispatchOptions["fetchPluginHostArtifact"];
   runtimeManager: RuntimeManager;
   eventSink: CommandDispatchOptions["eventSink"];
@@ -303,7 +303,7 @@ export class CommandRouter {
   private createDispatchOptions(): CommandDispatchOptions {
     return {
       fetchProjectAttachment: this.options.fetchProjectAttachment,
-      fetchSkillTree: this.options.fetchSkillTree,
+      skillTreeTransport: this.options.skillTreeTransport,
       fetchPluginHostArtifact: this.options.fetchPluginHostArtifact,
       runtimeManager: this.options.runtimeManager,
       desktopBrowserBroker: this.options.desktopBrowserBroker,