From 85e3b4d2cf9b2866f56d17cd735ae5f6fe77585c Mon Sep 17 00:00:00 2001 From: Ahmed Abushagur Date: Thu, 23 Apr 2026 23:24:09 -0700 Subject: [PATCH 1/3] feat(cli): add agent templates with spawn export and --repo flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the ability to share agent setups as templates: - `spawn export` — SSHes into a running spawn, scans installed MCP servers, CLI auths, and tools, generates a spawn.md recipe (no secrets), pushes to GitHub, and prints a shareable one-liner command - `spawn --repo user/repo` — clones the template repo onto the provisioned VM, reads spawn.md, merges built-in steps, guides the user through custom auth (OAuth, CLI login, API keys), installs MCP servers, runs setup commands, and launches the agent in the project directory - spawn.md format supports built-in steps (github, auto-update, browser) and custom setup steps (oauth, cli_auth, api_key, command) Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/package.json | 2 +- .../cli/src/__tests__/unknown-flags.test.ts | 1 + packages/cli/src/commands/export.ts | 378 +++++++++++ packages/cli/src/commands/help.ts | 8 +- packages/cli/src/commands/index.ts | 2 + packages/cli/src/flags.ts | 1 + packages/cli/src/index.ts | 23 + packages/cli/src/shared/orchestrate.ts | 47 +- packages/cli/src/shared/spawn-md.ts | 589 ++++++++++++++++++ 9 files changed, 1048 insertions(+), 3 deletions(-) create mode 100644 packages/cli/src/commands/export.ts create mode 100644 packages/cli/src/shared/spawn-md.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 467c18807..195411460 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openrouter/spawn", - "version": "0.27.1", + "version": "0.28.0", "type": "module", "bin": { "spawn": "cli.js" diff --git a/packages/cli/src/__tests__/unknown-flags.test.ts b/packages/cli/src/__tests__/unknown-flags.test.ts index a54db2a39..0c37f1119 100644 --- a/packages/cli/src/__tests__/unknown-flags.test.ts +++ b/packages/cli/src/__tests__/unknown-flags.test.ts @@ -227,6 +227,7 @@ describe("KNOWN_FLAGS completeness", () => { "-m", "--config", "--steps", + "--repo", "--fast", "--user", "-u", diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts new file mode 100644 index 000000000..c57438c5b --- /dev/null +++ b/packages/cli/src/commands/export.ts @@ -0,0 +1,378 @@ +// commands/export.ts — Export a running spawn's setup as a shareable template +// +// SSHes into a VM from spawn history, scans what's installed (MCP servers, +// CLI auths, tools), generates a spawn.md recipe (no secrets), pushes to +// GitHub, and prints a one-liner command for others to replicate the setup. + +import type { SpawnMdConfig } from "../shared/spawn-md.js"; + +import { execSync, spawnSync } from "node:child_process"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as p from "@clack/prompts"; +import { isString } from "@openrouter/spawn-shared"; +import pc from "picocolors"; +import * as v from "valibot"; +import { loadHistory } from "../history.js"; +import { validateConnectionIP, validateIdentifier, validateUsername } from "../security.js"; +import { parseJsonWith } from "../shared/parse.js"; +import { tryCatch } from "../shared/result.js"; +import { generateSpawnMd } from "../shared/spawn-md.js"; +import { SSH_BASE_OPTS } from "../shared/ssh.js"; +import { ensureSshKeys, getSshKeyOpts } from "../shared/ssh-keys.js"; +import { buildRecordLabel, buildRecordSubtitle } from "./list.js"; +import { handleCancel, isInteractiveTTY } from "./shared.js"; + +// Schema for parsing MCP server configs from Claude/Cursor settings +const McpEntrySchema = v.object({ + command: v.string(), + args: v.array(v.string()), + env: v.optional(v.record(v.string(), v.string())), +}); +const McpSettingsSchema = v.object({ + mcpServers: v.optional(v.record(v.string(), McpEntrySchema)), +}); + +/** Run a command on the remote VM and capture stdout */ +function sshCapture(ip: string, user: string, cmd: string, keyOpts: string[]): string { + const result = spawnSync( + "ssh", + [ + ...SSH_BASE_OPTS, + ...keyOpts, + `${user}@${ip}`, + "--", + cmd, + ], + { + encoding: "utf8", + timeout: 15_000, + stdio: [ + "pipe", + "pipe", + "pipe", + ], + }, + ); + return (result.stdout ?? "").trim(); +} + +/** Scan a remote VM for installed MCP servers, CLI auths, and tools */ +function scanVmSetup( + ip: string, + user: string, + agentKey: string, + keyOpts: string[], +): { + mcpServers: NonNullable; + steps: string[]; + setup: NonNullable; +} { + const mcpServers: NonNullable = []; + const steps: string[] = []; + const setup: NonNullable = []; + + // Detect MCP servers from Claude Code / Cursor config + if (agentKey === "claude" || agentKey === "cursor") { + const settingsPath = agentKey === "claude" ? "~/.claude/settings.json" : "~/.cursor/mcp.json"; + const raw = sshCapture(ip, user, `cat ${settingsPath} 2>/dev/null || echo '{}'`, keyOpts); + const parsed = parseJsonWith(raw, McpSettingsSchema); + if (parsed?.mcpServers) { + for (const [name, cfg] of Object.entries(parsed.mcpServers)) { + const entry: NonNullable[number] = { + name, + command: cfg.command, + args: cfg.args, + }; + if (cfg.env) { + // Replace actual values with ${NAME} placeholders — never export secrets + const envPlaceholders: Record = {}; + for (const k of Object.keys(cfg.env)) { + envPlaceholders[k] = `\${${k}}`; + } + entry.env = envPlaceholders; + } + mcpServers.push(entry); + } + } + } + + // Detect GitHub CLI auth + const ghStatus = sshCapture(ip, user, "gh auth status 2>&1 || true", keyOpts); + if (ghStatus.includes("Logged in")) { + steps.push("github"); + } + + // Detect common CLI tools that might need auth + const toolChecks = [ + { + cmd: "shopify", + name: "Shopify CLI", + authCmd: "shopify auth login", + }, + { + cmd: "vercel", + name: "Vercel CLI", + authCmd: "vercel login", + }, + { + cmd: "netlify", + name: "Netlify CLI", + authCmd: "netlify login", + }, + { + cmd: "firebase", + name: "Firebase CLI", + authCmd: "firebase login", + }, + { + cmd: "supabase", + name: "Supabase CLI", + authCmd: "supabase login", + }, + { + cmd: "stripe", + name: "Stripe CLI", + authCmd: "stripe login", + }, + ] as const; + + for (const tool of toolChecks) { + const which = sshCapture(ip, user, `which ${tool.cmd} 2>/dev/null || true`, keyOpts); + if (which) { + setup.push({ + type: "cli_auth", + name: tool.name, + command: tool.authCmd, + description: `Authenticate with ${tool.name}`, + }); + } + } + + // Always include auto-update + steps.push("auto-update"); + + return { + mcpServers, + steps, + setup, + }; +} + +/** Main export command */ +export async function cmdExport(): Promise { + if (!isInteractiveTTY()) { + console.error(pc.red("Error: spawn export requires an interactive terminal")); + process.exit(1); + } + + // 1. Check for gh CLI + const ghResult = tryCatch(() => + execSync("gh auth status 2>&1", { + encoding: "utf8", + }), + ); + if (!ghResult.ok) { + const ghWhich = tryCatch(() => + execSync("which gh 2>/dev/null", { + encoding: "utf8", + }), + ); + if (!ghWhich.ok) { + p.log.error("GitHub CLI (gh) is not installed."); + p.log.info(`Install it: ${pc.cyan("https://cli.github.com/")}`); + process.exit(1); + } + p.log.error("GitHub CLI is not authenticated."); + p.log.info(`Run: ${pc.cyan("gh auth login")}`); + process.exit(1); + } + + // 2. Show spawn history picker + const records = loadHistory().filter((r) => r.connection && !r.connection.deleted); + if (records.length === 0) { + p.log.error("No active spawns found. Start one first with 'spawn '."); + process.exit(1); + } + + const options = records.map((r) => ({ + value: r, + label: buildRecordLabel(r), + hint: buildRecordSubtitle(r), + })); + + const selected = await p.select({ + message: "Which spawn do you want to export?", + options, + }); + + if (p.isCancel(selected)) { + handleCancel(); + return; + } + + const record = selected; + const conn = record.connection; + if (!conn) { + p.log.error("Selected spawn has no connection info."); + return; + } + + // Validate connection fields + const validResult = tryCatch(() => { + validateIdentifier(record.agent, "Agent name"); + validateConnectionIP(conn.ip); + validateUsername(conn.user); + }); + if (!validResult.ok) { + p.log.error("Invalid connection data in spawn history."); + return; + } + + p.log.step(`Scanning ${pc.bold(record.agent)} on ${pc.bold(conn.ip)}...`); + + // 3. SSH in and scan the VM + const keys = await ensureSshKeys(); + const keyOpts = getSshKeyOpts(keys); + const { mcpServers, steps, setup } = scanVmSetup(conn.ip, conn.user, record.agent, keyOpts); + + // 4. Prompt for template details + const repoName = await p.text({ + message: "Repository name for this template:", + placeholder: record.name ?? `${record.agent}-template`, + validate: (val) => { + if (!val || val.trim() === "") { + return "Name is required"; + } + if (!/^[a-zA-Z0-9._-]+$/.test(val)) { + return "Only alphanumeric, dots, hyphens, underscores"; + } + return undefined; + }, + }); + + if (p.isCancel(repoName)) { + handleCancel(); + return; + } + + const description = await p.text({ + message: "Description (optional):", + placeholder: `${record.agent} agent template`, + }); + + if (p.isCancel(description)) { + handleCancel(); + return; + } + + // 5. Generate spawn.md + const config: SpawnMdConfig = { + name: repoName, + description: isString(description) ? description : undefined, + steps, + setup: setup.length > 0 ? setup : undefined, + mcp_servers: mcpServers.length > 0 ? mcpServers : undefined, + }; + + const spawnMdContent = generateSpawnMd(config, `# ${repoName}\n\n${isString(description) ? description : ""}`); + + // Show preview + p.log.info("Generated spawn.md:"); + console.error(pc.dim("─".repeat(40))); + console.error(pc.dim(spawnMdContent)); + console.error(pc.dim("─".repeat(40))); + + const confirm = await p.confirm({ + message: "Create GitHub repo and push?", + initialValue: true, + }); + + if (p.isCancel(confirm) || !confirm) { + handleCancel(); + return; + } + + // 6. Create temp dir, write spawn.md, init repo, push + const tmpDir = mkdtempSync(join(tmpdir(), "spawn-export-")); + + writeFileSync(join(tmpDir, "spawn.md"), spawnMdContent); + writeFileSync( + join(tmpDir, ".gitignore"), + [ + ".env", + ".env.*", + ".spawnrc", + "node_modules/", + "/etc/spawn/", + "", + ].join("\n"), + ); + + // Download project files from VM if they exist + p.log.step("Downloading project files from VM..."); + const hasProject = sshCapture(conn.ip, conn.user, "test -d ~/project && echo yes || echo no", keyOpts); + if (hasProject === "yes") { + const excludeFlags = [ + "--exclude=node_modules", + "--exclude=.git", + "--exclude=.env", + "--exclude=.env.*", + "--exclude=.spawnrc", + ]; + const rsyncResult = tryCatch(() => + execSync( + `rsync -az ${excludeFlags.join(" ")} -e "ssh ${SSH_BASE_OPTS.join(" ")} ${keyOpts.join(" ")}" ${conn.user}@${conn.ip}:~/project/ ${tmpDir}/`, + { + encoding: "utf8", + timeout: 60_000, + }, + ), + ); + if (!rsyncResult.ok) { + p.log.warn("Could not download project files — creating template with spawn.md only"); + } + } + + // 7. Create GitHub repo + p.log.step("Creating GitHub repository..."); + const repoResult = tryCatch(() => { + execSync(`cd ${tmpDir} && git init -q && git add -A && git commit -q -m "Initial template"`, { + encoding: "utf8", + }); + execSync(`cd ${tmpDir} && gh repo create ${repoName} --public --source=. --push`, { + encoding: "utf8", + stdio: [ + "pipe", + "pipe", + "inherit", + ], + }); + }); + if (!repoResult.ok) { + p.log.error("Failed to create GitHub repo."); + p.log.info(`You can manually push the template from: ${tmpDir}`); + return; + } + + // 8. Get the repo slug (user/repo) + let repoSlug = repoName; + const ghUserResult = tryCatch(() => + execSync("gh api user --jq .login", { + encoding: "utf8", + }).trim(), + ); + if (ghUserResult.ok && ghUserResult.data) { + repoSlug = `${ghUserResult.data}/${repoName}`; + } + + // 9. Print the shareable command + console.error(); + p.log.success("Template exported!"); + console.error(); + console.error(" Share this command to replicate your setup:"); + console.error(); + console.error(` ${pc.cyan(`spawn ${record.agent} ${record.cloud} --repo ${repoSlug}`)}`); + console.error(); +} diff --git a/packages/cli/src/commands/help.ts b/packages/cli/src/commands/help.ts index 9d819e797..ea2e0095d 100644 --- a/packages/cli/src/commands/help.ts +++ b/packages/cli/src/commands/help.ts @@ -21,6 +21,9 @@ function getHelpUsageSection(): string { Load all options from a JSON config file spawn --steps Comma-separated setup steps to enable + spawn --repo + Clone a template repo and apply spawn.md setup + spawn export Export a running spawn as a shareable template spawn Interactive cloud picker for agent spawn Show available agents for cloud spawn list Browse and rerun previous spawns (aliases: ls, history) @@ -85,7 +88,10 @@ function getHelpExamplesSection(): string { spawn list ${pc.dim("# Browse history and pick one to rerun")} spawn list codex ${pc.dim("# Filter history by agent name")} spawn last ${pc.dim("# Instantly rerun the most recent spawn")} - spawn matrix ${pc.dim("# See the full agent x cloud matrix")}`; + spawn matrix ${pc.dim("# See the full agent x cloud matrix")} + spawn claude sprite --repo user/my-template + ${pc.dim("# Clone template and auto-setup agent")} + spawn export ${pc.dim("# Export current spawn as a template")}`; } function getHelpAuthSection(): string { diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index bf8a35ca9..db281032c 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -2,6 +2,8 @@ // delete.ts — cmdDelete, cascadeDelete export { cascadeDelete, cmdDelete } from "./delete.js"; +// export.ts — cmdExport (agent template export) +export { cmdExport } from "./export.js"; // feedback.ts — cmdFeedback export { cmdFeedback } from "./feedback.js"; // fix.ts — cmdFix, fixSpawn, buildFixScript diff --git a/packages/cli/src/flags.ts b/packages/cli/src/flags.ts index 470f86cf8..c19524b76 100644 --- a/packages/cli/src/flags.ts +++ b/packages/cli/src/flags.ts @@ -35,6 +35,7 @@ export const KNOWN_FLAGS = new Set([ "-m", "--config", "--steps", + "--repo", "--fast", "--user", "-u", diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 1fc897ddd..679437617 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -12,6 +12,7 @@ import { cmdCloudInfo, cmdClouds, cmdDelete, + cmdExport, cmdFeedback, cmdFix, cmdHelp, @@ -128,6 +129,7 @@ function checkUnknownFlags(args: string[]): void { console.error(` ${pc.cyan("--reauth")} Force re-prompting for cloud credentials`); console.error(` ${pc.cyan("--config ")} Load config from JSON file`); console.error(` ${pc.cyan("--steps ")} Comma-separated setup steps to enable`); + console.error(` ${pc.cyan("--repo ")} Clone a template repo and apply spawn.md`); console.error(` ${pc.cyan("--beta tarball")} Use pre-built tarball for agent install (repeatable)`); console.error(` ${pc.cyan("--beta images")} Use pre-built DO marketplace images (faster boot)`); console.error(` ${pc.cyan("--beta parallel")} Parallelize server boot with setup prompts`); @@ -749,6 +751,14 @@ async function dispatchCommand( await cmdPullHistory(); return; } + if (cmd === "export") { + if (hasTrailingHelpFlag(filteredArgs)) { + cmdHelp(); + return; + } + await cmdExport(); + return; + } if (LIST_COMMANDS.has(cmd)) { // Handle "history export" subcommand if (cmd === "history" && filteredArgs[1] === "export") { @@ -1014,6 +1024,19 @@ async function main(): Promise { process.env.SPAWN_NAME = nameFlag; } + // Extract --repo flag — clone a template repo and apply spawn.md + const [repoFlag, repoFilteredArgs] = extractFlagValue( + filteredArgs, + [ + "--repo", + ], + 'spawn --repo "user/my-template"', + ); + filteredArgs.splice(0, filteredArgs.length, ...repoFilteredArgs); + if (repoFlag) { + process.env.SPAWN_REPO = repoFlag; + } + // Extract --zone / --region flag (maps to cloud-specific env vars) const [zoneFlag, zoneFilteredArgs] = extractFlagValue( filteredArgs, diff --git a/packages/cli/src/shared/orchestrate.ts b/packages/cli/src/shared/orchestrate.ts index 597063056..ab26ba37d 100644 --- a/packages/cli/src/shared/orchestrate.ts +++ b/packages/cli/src/shared/orchestrate.ts @@ -560,6 +560,33 @@ async function postInstall( spawnId: string, _options?: OrchestrationOptions, ): Promise { + // ── Repo clone + spawn.md (--repo mode) ──────────────────────────────── + // Clone early so spawn.md `steps` can merge into enabledSteps before + // the built-in step logic runs. + let spawnMdConfig: import("./spawn-md.js").SpawnMdConfig | null = null; + const repoSlug = process.env.SPAWN_REPO; + if (repoSlug && cloud.cloudName !== "local") { + // Validate slug format (user/repo, no path traversal) + if (!/^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/.test(repoSlug)) { + logWarn(`Invalid repo slug: ${repoSlug} — skipping repo clone`); + } else { + logStep("Cloning template repository..."); + const cloneResult = await asyncTryCatch(() => + cloud.runner.runServer(`git clone https://github.com/${repoSlug}.git ~/project 2>&1 || true`), + ); + if (!cloneResult.ok) { + logWarn("Repo clone failed — continuing without template"); + } else { + // Try to read spawn.md from the cloned repo + const { readRemoteSpawnMd } = await import("./spawn-md.js"); + spawnMdConfig = await readRemoteSpawnMd(cloud.runner); + if (spawnMdConfig) { + logInfo(`Template loaded: ${spawnMdConfig.name ?? repoSlug}`); + } + } + } + } + // Parse enabled setup steps let enabledSteps: Set | undefined; const stepsEnv = process.env.SPAWN_ENABLED_STEPS; @@ -577,6 +604,16 @@ async function postInstall( } } + // Merge spawn.md steps into enabledSteps + if (spawnMdConfig?.steps && spawnMdConfig.steps.length > 0) { + if (!enabledSteps) { + enabledSteps = new Set(); + } + for (const step of spawnMdConfig.steps) { + enabledSteps.add(step); + } + } + // Agent-specific configuration if (agent.configure) { const configResult = await asyncTryCatch(() => @@ -611,6 +648,12 @@ async function postInstall( await injectSpawnSkill(cloud.runner, agentName); } + // Apply spawn.md custom setup (after built-in steps, before pre-launch) + if (spawnMdConfig) { + const { applySpawnMdSetup } = await import("./spawn-md.js"); + await applySpawnMdSetup(cloud.runner, spawnMdConfig, agentName); + } + // Pre-launch hooks (retry loop) if (agent.preLaunch) { for (;;) { @@ -708,7 +751,9 @@ async function postInstall( logInfo(`Agent setup complete — ${agent.name} is ready on ${cloud.cloudLabel}`); process.stderr.write("\n"); - const launchCmd = agent.launchCmd(); + // When --repo is set, launch the agent inside the cloned project directory + const baseLaunchCmd = agent.launchCmd(); + const launchCmd = repoSlug ? `cd ~/project && ${baseLaunchCmd}` : baseLaunchCmd; saveLaunchCmd(launchCmd, spawnId); // In headless mode, provisioning is done — skip the interactive session. diff --git a/packages/cli/src/shared/spawn-md.ts b/packages/cli/src/shared/spawn-md.ts new file mode 100644 index 000000000..ad6368a4e --- /dev/null +++ b/packages/cli/src/shared/spawn-md.ts @@ -0,0 +1,589 @@ +// shared/spawn-md.ts — Parse, generate, and apply spawn.md template files +// +// spawn.md lives at the root of a user's repo and declares the "recipe" for +// setting up an agent: built-in steps, custom auth flows, MCP servers, and +// setup commands. It never contains actual secrets. + +import type { CloudRunner } from "./agent-setup.js"; + +import * as v from "valibot"; +import { asyncTryCatch, tryCatch } from "./result.js"; +import { logInfo, logStep, logWarn, openBrowser } from "./ui.js"; + +// ── YAML frontmatter parsing ─────────────────────────────────────────────── +// spawn.md uses a subset of YAML in the frontmatter (between --- delimiters). +// We parse it with a minimal hand-rolled parser to avoid adding a YAML dep. + +/** Split spawn.md content into { frontmatter, body } */ +function splitFrontmatter(content: string): { + frontmatter: string; + body: string; +} { + const trimmed = content.trimStart(); + if (!trimmed.startsWith("---")) { + return { + frontmatter: "", + body: content, + }; + } + const endIdx = trimmed.indexOf("\n---", 3); + if (endIdx === -1) { + return { + frontmatter: "", + body: content, + }; + } + const frontmatter = trimmed.slice(3, endIdx).trim(); + const body = trimmed.slice(endIdx + 4).trim(); + return { + frontmatter, + body, + }; +} + +function parseYamlScalar(s: string): string | number | boolean { + if (s === "true") { + return true; + } + if (s === "false") { + return false; + } + if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) { + return s.slice(1, -1); + } + const num = Number(s); + if (!Number.isNaN(num) && s !== "") { + return num; + } + return s; +} + +/** Helper to treat target as a record and set a key */ +function setOnRecord(target: Record | unknown[], key: string, val: unknown): void { + if (Array.isArray(target)) { + return; + } + target[key] = val; +} + +/** Helper to get from a record by key */ +function getFromRecord(target: Record | unknown[], key: string): unknown { + if (Array.isArray(target)) { + return undefined; + } + return target[key]; +} + +/** + * Minimal YAML-to-JSON parser for spawn.md frontmatter. + * Handles: scalars, arrays of scalars, arrays of objects, nested objects. + * Does NOT handle: anchors, tags, multi-line strings. Intentionally simple. + */ +function parseYamlFrontmatter(yaml: string): Record { + const lines = yaml.split("\n"); + const result: Record = {}; + + type Frame = { + indent: number; + target: Record | unknown[]; + key?: string; + }; + const stack: Frame[] = [ + { + indent: -1, + target: result, + }, + ]; + + const currentFrame = (): Frame => stack[stack.length - 1]; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trimStart(); + if (trimmed === "" || trimmed.startsWith("#")) { + continue; + } + + const indent = line.length - trimmed.length; + + // Pop stack to find the right nesting level + while (stack.length > 1 && indent <= currentFrame().indent) { + stack.pop(); + } + + // Array item: "- value" or "- key: value" + if (trimmed.startsWith("- ")) { + const itemContent = trimmed.slice(2).trim(); + const frame = currentFrame(); + let targetArray: unknown[] | null = null; + + if (Array.isArray(frame.target)) { + targetArray = frame.target; + } else if (frame.key) { + const existing = getFromRecord(frame.target, frame.key); + if (Array.isArray(existing)) { + targetArray = existing; + } + } + + if (!targetArray) { + continue; + } + + // Check if item is a key-value pair (object in array) + const colonIdx = itemContent.indexOf(":"); + if (colonIdx > 0 && !itemContent.startsWith("[") && !itemContent.startsWith('"')) { + const key = itemContent.slice(0, colonIdx).trim(); + const val = itemContent.slice(colonIdx + 1).trim(); + const obj: Record = {}; + obj[key] = parseYamlScalar(val); + targetArray.push(obj); + stack.push({ + indent: indent + 1, + target: obj, + }); + continue; + } + + // Flow sequence: [a, b, c] + if (itemContent.startsWith("[") && itemContent.endsWith("]")) { + const inner = itemContent.slice(1, -1); + targetArray.push(inner.split(",").map((s) => parseYamlScalar(s.trim()))); + continue; + } + + targetArray.push(parseYamlScalar(itemContent)); + continue; + } + + // Key-value pair: "key: value" + const colonIdx = trimmed.indexOf(":"); + if (colonIdx > 0) { + const key = trimmed.slice(0, colonIdx).trim(); + const rawVal = trimmed.slice(colonIdx + 1).trim(); + const frame = currentFrame(); + const target = frame.target; + + if (Array.isArray(target)) { + continue; + } + + if (rawVal === "" || rawVal === "|" || rawVal === ">") { + const nextLine = lines[i + 1]; + if (nextLine !== undefined) { + const nextTrimmed = nextLine.trimStart(); + if (nextTrimmed.startsWith("- ")) { + const arr: unknown[] = []; + setOnRecord(target, key, arr); + stack.push({ + indent, + target: arr, + key, + }); + continue; + } + const obj: Record = {}; + setOnRecord(target, key, obj); + stack.push({ + indent, + target: obj, + key, + }); + continue; + } + setOnRecord(target, key, ""); + continue; + } + + // Flow sequence: key: [a, b, c] + if (rawVal.startsWith("[") && rawVal.endsWith("]")) { + const inner = rawVal.slice(1, -1); + setOnRecord( + target, + key, + inner.split(",").map((s) => parseYamlScalar(s.trim())), + ); + continue; + } + + setOnRecord(target, key, parseYamlScalar(rawVal)); + } + } + + return result; +} + +// ── Valibot schemas ──────────────────────────────────────────────────────── + +const OAuthSetupSchema = v.object({ + type: v.literal("oauth"), + name: v.string(), + url: v.string(), + description: v.optional(v.string()), +}); + +const CliAuthSetupSchema = v.object({ + type: v.literal("cli_auth"), + name: v.string(), + command: v.string(), + description: v.optional(v.string()), +}); + +const ApiKeySetupSchema = v.object({ + type: v.literal("api_key"), + name: v.string(), + description: v.optional(v.string()), + guide_url: v.optional(v.string()), +}); + +const CommandSetupSchema = v.object({ + type: v.literal("command"), + name: v.optional(v.string()), + command: v.string(), + description: v.optional(v.string()), +}); + +const SetupStepSchema = v.union([ + OAuthSetupSchema, + CliAuthSetupSchema, + ApiKeySetupSchema, + CommandSetupSchema, +]); + +const McpServerEntrySchema = v.object({ + name: v.string(), + command: v.string(), + args: v.array(v.string()), + env: v.optional(v.record(v.string(), v.string())), +}); + +export const SpawnMdSchema = v.object({ + name: v.optional(v.string()), + description: v.optional(v.string()), + steps: v.optional(v.array(v.string())), + setup: v.optional(v.array(SetupStepSchema)), + mcp_servers: v.optional(v.array(McpServerEntrySchema)), + setup_commands: v.optional(v.array(v.string())), +}); + +export type SpawnMdConfig = v.InferOutput; +type SetupStep = v.InferOutput; +type McpServerEntry = v.InferOutput; + +// ── Parsing ──────────────────────────────────────────────────────────────── + +/** Parse spawn.md content into a typed config. Returns null on parse failure. */ +export function parseSpawnMd(content: string): SpawnMdConfig | null { + const { frontmatter } = splitFrontmatter(content); + if (!frontmatter) { + return null; + } + + const raw = parseYamlFrontmatter(frontmatter); + const result = v.safeParse(SpawnMdSchema, raw); + if (!result.success) { + logWarn("spawn.md has invalid frontmatter — ignoring"); + return null; + } + return result.output; +} + +// ── Generation ───────────────────────────────────────────────────────────── + +/** Generate spawn.md content from a config */ +export function generateSpawnMd(config: SpawnMdConfig, body?: string): string { + const lines: string[] = [ + "---", + ]; + + if (config.name) { + lines.push(`name: ${config.name}`); + } + if (config.description) { + lines.push(`description: ${config.description}`); + } + + if (config.steps && config.steps.length > 0) { + lines.push("steps:"); + for (const step of config.steps) { + lines.push(` - ${step}`); + } + } + + if (config.setup && config.setup.length > 0) { + lines.push("setup:"); + for (const step of config.setup) { + lines.push(` - type: ${step.type}`); + lines.push(` name: ${step.name ?? ""}`); + if ("url" in step) { + lines.push(` url: ${step.url}`); + } + if ("command" in step) { + lines.push(` command: ${step.command}`); + } + if (step.description) { + lines.push(` description: ${step.description}`); + } + if ("guide_url" in step && step.guide_url) { + lines.push(` guide_url: ${step.guide_url}`); + } + } + } + + if (config.mcp_servers && config.mcp_servers.length > 0) { + lines.push("mcp_servers:"); + for (const server of config.mcp_servers) { + lines.push(` - name: ${server.name}`); + lines.push(` command: ${server.command}`); + lines.push(` args: [${server.args.map((a) => `"${a}"`).join(", ")}]`); + if (server.env) { + lines.push(" env:"); + for (const [k, val] of Object.entries(server.env)) { + lines.push(` ${k}: "${val}"`); + } + } + } + } + + if (config.setup_commands && config.setup_commands.length > 0) { + lines.push("setup_commands:"); + for (const cmd of config.setup_commands) { + lines.push(` - ${cmd}`); + } + } + + lines.push("---"); + lines.push(""); + + if (body) { + lines.push(body); + } + + return lines.join("\n"); +} + +// ── Applying spawn.md on a VM ────────────────────────────────────────────── + +/** Read and parse spawn.md from a remote VM */ +export async function readRemoteSpawnMd(runner: CloudRunner): Promise { + const catResult = await captureCommand(runner, "cat ~/project/spawn.md 2>/dev/null"); + if (catResult) { + return parseSpawnMd(catResult); + } + return null; +} + +/** Run a command on the remote and capture its stdout */ +async function captureCommand(runner: CloudRunner, cmd: string): Promise { + const tmpFile = `/tmp/spawn-capture-${Date.now()}`; + const { readFileSync, unlinkSync } = await import("node:fs"); + const result = await asyncTryCatch(async () => { + await runner.runServer(`${cmd} > ${tmpFile} 2>/dev/null; true`); + await runner.downloadFile(tmpFile, tmpFile); + const content = readFileSync(tmpFile, "utf-8"); + const cleanupResult = tryCatch(() => unlinkSync(tmpFile)); + // ignore local cleanup failure + void cleanupResult; + await asyncTryCatch(() => runner.runServer(`rm -f ${tmpFile}`)); + return content || null; + }); + if (!result.ok) { + return null; + } + return result.data; +} + +/** + * Apply custom setup steps from spawn.md onto a running VM. + * Built-in `steps` (github, auto-update, etc.) are handled by the existing + * postInstall infrastructure — this function only handles the `setup` array, + * `mcp_servers`, and `setup_commands`. + */ +export async function applySpawnMdSetup(runner: CloudRunner, config: SpawnMdConfig, agentName: string): Promise { + if (config.setup && config.setup.length > 0) { + logStep("Running template setup steps..."); + for (const step of config.setup) { + await applySetupStep(runner, step); + } + } + + if (config.mcp_servers && config.mcp_servers.length > 0) { + logStep("Installing MCP servers from template..."); + await installMcpServersFromTemplate(runner, config.mcp_servers, agentName); + } + + if (config.setup_commands && config.setup_commands.length > 0) { + logStep("Running template setup commands..."); + for (const cmd of config.setup_commands) { + logInfo(` Running: ${cmd}`); + const cmdResult = await asyncTryCatch(() => runner.runServer(`cd ~/project 2>/dev/null; ${cmd}`)); + if (!cmdResult.ok) { + logWarn(` Setup command failed: ${cmd}`); + } + } + } +} + +async function applySetupStep(runner: CloudRunner, step: SetupStep): Promise { + switch (step.type) { + case "oauth": { + logInfo(` ${step.name}: Opening ${step.url}`); + if (step.description) { + logInfo(` ${step.description}`); + } + openBrowser(step.url); + logInfo(" Complete the OAuth flow in your browser, then press Enter to continue."); + await waitForEnter(); + break; + } + case "cli_auth": { + logInfo(` ${step.name}: Running ${step.command}`); + if (step.description) { + logInfo(` ${step.description}`); + } + const authResult = await asyncTryCatch(() => runner.runServer(step.command)); + if (authResult.ok) { + logInfo(` ${step.name} authenticated`); + } else { + logWarn(` ${step.name} auth failed — you can run it manually later: ${step.command}`); + } + break; + } + case "api_key": { + logInfo(` ${step.name}: API key required`); + if (step.description) { + logInfo(` ${step.description}`); + } + if (step.guide_url) { + logInfo(` Get your key: ${step.guide_url}`); + openBrowser(step.guide_url); + } + const value = await promptSecret(` Enter ${step.name}: `); + if (value) { + const escapedName = step.name.replace(/[^A-Z0-9_]/g, ""); + const b64Val = Buffer.from(value).toString("base64"); + await runner.runServer( + `mkdir -p /etc/spawn && printf 'export %s="%s"\\n' '${escapedName}' "$(echo '${b64Val}' | base64 -d)" >> /etc/spawn/secrets && chmod 600 /etc/spawn/secrets`, + ); + await runner.runServer( + `grep -q '/etc/spawn/secrets' ~/.bashrc 2>/dev/null || echo 'source /etc/spawn/secrets 2>/dev/null' >> ~/.bashrc`, + ); + logInfo(` ${step.name} saved`); + } else { + logWarn(` No value provided for ${step.name} — set it later in /etc/spawn/secrets`); + } + break; + } + case "command": { + const label = step.name ?? step.command; + logInfo(` Running: ${label}`); + if (step.description) { + logInfo(` ${step.description}`); + } + const runResult = await asyncTryCatch(() => runner.runServer(step.command)); + if (!runResult.ok) { + logWarn(` Command failed: ${step.command}`); + } + break; + } + } +} + +/** Install MCP servers from spawn.md template into agent config */ +async function installMcpServersFromTemplate( + runner: CloudRunner, + servers: McpServerEntry[], + agentName: string, +): Promise { + if (agentName === "claude" || agentName === "cursor") { + const mcpConfig: Record> = {}; + for (const server of servers) { + const entry: Record = { + command: server.command, + args: server.args, + }; + if (server.env) { + entry.env = server.env; + } + mcpConfig[server.name] = entry; + } + + const settingsPath = agentName === "claude" ? "~/.claude/settings.json" : "~/.cursor/mcp.json"; + const mcpJson = JSON.stringify(mcpConfig); + const mergeScript = ` + const fs = require('fs'); + const path = '${settingsPath}'.replace('~', process.env.HOME); + let settings = {}; + try { settings = JSON.parse(fs.readFileSync(path, 'utf-8')); } catch {} + settings.mcpServers = { ...settings.mcpServers, ...${mcpJson} }; + fs.mkdirSync(require('path').dirname(path), { recursive: true }); + fs.writeFileSync(path, JSON.stringify(settings, null, 2)); + `; + const b64 = Buffer.from(mergeScript).toString("base64"); + const installResult = await asyncTryCatch(() => runner.runServer(`echo '${b64}' | base64 -d | node -e "$(cat)"`)); + if (installResult.ok) { + logInfo(` Installed ${servers.length} MCP server${servers.length > 1 ? "s" : ""}`); + } else { + logWarn(" MCP server installation failed — configure manually"); + } + } else { + logInfo(` MCP server installation not yet supported for ${agentName} — skipping`); + } +} + +/** Wait for the user to press Enter */ +async function waitForEnter(): Promise { + return new Promise((resolve) => { + if (!process.stdin.isTTY) { + resolve(); + return; + } + const onData = (): void => { + process.stdin.removeListener("data", onData); + resolve(); + }; + process.stdin.once("data", onData); + }); +} + +/** Prompt for a secret value (no echo) */ +async function promptSecret(message: string): Promise { + process.stderr.write(message); + return new Promise((resolve) => { + if (!process.stdin.isTTY) { + resolve(""); + return; + } + let buf = ""; + const wasRaw = process.stdin.isRaw ?? false; + process.stdin.setRawMode(true); + process.stdin.resume(); + const onData = (data: Buffer): void => { + const ch = data.toString(); + if (ch === "\r" || ch === "\n") { + process.stdin.setRawMode(wasRaw); + process.stdin.pause(); + process.stdin.removeListener("data", onData); + process.stderr.write("\n"); + resolve(buf); + return; + } + if (ch === "\x03") { + process.stdin.setRawMode(wasRaw); + process.stdin.pause(); + process.stdin.removeListener("data", onData); + process.stderr.write("\n"); + resolve(""); + return; + } + if (ch === "\x7f" || ch === "\b") { + if (buf.length > 0) { + buf = buf.slice(0, -1); + } + return; + } + buf += ch; + }; + process.stdin.on("data", onData); + }); +} From 3aab1be4a8c8d865c67a3d700084613c252287e1 Mon Sep 17 00:00:00 2001 From: Ahmed Abushagur Date: Thu, 23 Apr 2026 23:27:19 -0700 Subject: [PATCH 2/3] fix(cli): move built-in steps from spawn.md to the CLI command Steps like github, auto-update, browser belong in the shareable --steps flag, not in spawn.md. spawn.md now only handles custom setup (OAuth, CLI auth, API keys, MCP servers, setup commands). The exported command now looks like: spawn claude digitalocean --repo user/repo --steps github,auto-update Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/src/commands/export.ts | 8 ++++---- packages/cli/src/shared/orchestrate.ts | 18 ++++-------------- packages/cli/src/shared/spawn-md.ts | 10 ++-------- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts index c57438c5b..3cf39fc12 100644 --- a/packages/cli/src/commands/export.ts +++ b/packages/cli/src/commands/export.ts @@ -267,11 +267,10 @@ export async function cmdExport(): Promise { return; } - // 5. Generate spawn.md + // 5. Generate spawn.md (steps go in the CLI command, not in spawn.md) const config: SpawnMdConfig = { name: repoName, description: isString(description) ? description : undefined, - steps, setup: setup.length > 0 ? setup : undefined, mcp_servers: mcpServers.length > 0 ? mcpServers : undefined, }; @@ -367,12 +366,13 @@ export async function cmdExport(): Promise { repoSlug = `${ghUserResult.data}/${repoName}`; } - // 9. Print the shareable command + // 9. Print the shareable command (steps baked into the command, not spawn.md) + const stepsArg = steps.length > 0 ? ` --steps ${steps.join(",")}` : ""; console.error(); p.log.success("Template exported!"); console.error(); console.error(" Share this command to replicate your setup:"); console.error(); - console.error(` ${pc.cyan(`spawn ${record.agent} ${record.cloud} --repo ${repoSlug}`)}`); + console.error(` ${pc.cyan(`spawn ${record.agent} ${record.cloud} --repo ${repoSlug}${stepsArg}`)}`); console.error(); } diff --git a/packages/cli/src/shared/orchestrate.ts b/packages/cli/src/shared/orchestrate.ts index ab26ba37d..9639c64bb 100644 --- a/packages/cli/src/shared/orchestrate.ts +++ b/packages/cli/src/shared/orchestrate.ts @@ -561,8 +561,9 @@ async function postInstall( _options?: OrchestrationOptions, ): Promise { // ── Repo clone + spawn.md (--repo mode) ──────────────────────────────── - // Clone early so spawn.md `steps` can merge into enabledSteps before - // the built-in step logic runs. + // Built-in steps (github, auto-update, etc.) come from the CLI --steps + // flag, not from spawn.md. spawn.md only handles custom setup (OAuth, + // MCP servers, setup commands). let spawnMdConfig: import("./spawn-md.js").SpawnMdConfig | null = null; const repoSlug = process.env.SPAWN_REPO; if (repoSlug && cloud.cloudName !== "local") { @@ -577,7 +578,6 @@ async function postInstall( if (!cloneResult.ok) { logWarn("Repo clone failed — continuing without template"); } else { - // Try to read spawn.md from the cloned repo const { readRemoteSpawnMd } = await import("./spawn-md.js"); spawnMdConfig = await readRemoteSpawnMd(cloud.runner); if (spawnMdConfig) { @@ -587,7 +587,7 @@ async function postInstall( } } - // Parse enabled setup steps + // Parse enabled setup steps (from --steps CLI flag) let enabledSteps: Set | undefined; const stepsEnv = process.env.SPAWN_ENABLED_STEPS; if (stepsEnv !== undefined) { @@ -604,16 +604,6 @@ async function postInstall( } } - // Merge spawn.md steps into enabledSteps - if (spawnMdConfig?.steps && spawnMdConfig.steps.length > 0) { - if (!enabledSteps) { - enabledSteps = new Set(); - } - for (const step of spawnMdConfig.steps) { - enabledSteps.add(step); - } - } - // Agent-specific configuration if (agent.configure) { const configResult = await asyncTryCatch(() => diff --git a/packages/cli/src/shared/spawn-md.ts b/packages/cli/src/shared/spawn-md.ts index ad6368a4e..e742b8559 100644 --- a/packages/cli/src/shared/spawn-md.ts +++ b/packages/cli/src/shared/spawn-md.ts @@ -260,7 +260,8 @@ const McpServerEntrySchema = v.object({ export const SpawnMdSchema = v.object({ name: v.optional(v.string()), description: v.optional(v.string()), - steps: v.optional(v.array(v.string())), + // Built-in steps (github, auto-update, etc.) go in the CLI --steps flag, + // not here. spawn.md only handles custom setup that Spawn doesn't know about. setup: v.optional(v.array(SetupStepSchema)), mcp_servers: v.optional(v.array(McpServerEntrySchema)), setup_commands: v.optional(v.array(v.string())), @@ -303,13 +304,6 @@ export function generateSpawnMd(config: SpawnMdConfig, body?: string): string { lines.push(`description: ${config.description}`); } - if (config.steps && config.steps.length > 0) { - lines.push("steps:"); - for (const step of config.steps) { - lines.push(` - ${step}`); - } - } - if (config.setup && config.setup.length > 0) { lines.push("setup:"); for (const step of config.setup) { From ed2e6dfee7bf33a14bb3f49f95c0679a966bee7c Mon Sep 17 00:00:00 2001 From: Ahmed Abushagur Date: Fri, 24 Apr 2026 22:53:08 -0700 Subject: [PATCH 3/3] feat(export): broader CLI/MCP scan + secret-aware publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `spawn export` now publishes ~/project as a real working-folder snapshot of a running spawn: - MCP scan reads from ~/.claude/settings.json, ~/.claude.json, ~/.cursor/mcp.json, ~/.codex/config.toml, and ~/.openclaw/openclaw.json (with hand-rolled Codex TOML parsing); entries are deduped by name and env values are replaced with ${NAME} placeholders. - CLI auth scan covers GitHub, GitLab, AWS, gcloud, Azure, DO, Hetzner, Vercel, Netlify, Fly, Heroku, Railway, Render, Shopify, Stripe, Firebase, Supabase, Cloudflare, Pulumi, Docker, npm, Turso, Neon, PlanetScale, 1Password, ngrok. One batched SSH probe replaces N round-trips. - rsync exclude list expanded to filter common secret/key files (.ssh, .aws, .gnupg, *.pem, *.key, terraform state, etc.) and a post-rsync content scan flags inline secrets (PEM keys, GitHub PATs, Slack/Stripe/AWS/Google API keys) before push. - Repo visibility prompt defaults to private. Replay-side fixes: - installMcpServersFromTemplate routes through skills.ts helpers (no more `node -e` injection); adds Codex TOML support. - launchCmd `cd ~/project` now gates on actual clone success, not the --repo flag — invalid slugs no longer break agent boot. - api_key var name regex no longer strips lowercase letters. Bumps CLI to 1.0.23. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/cli/package.json | 2 +- .../cli/src/__tests__/export-helpers.test.ts | 259 ++++ packages/cli/src/commands/export.ts | 1076 ++++++++++++++--- packages/cli/src/shared/orchestrate.ts | 13 +- packages/cli/src/shared/skills.ts | 66 +- packages/cli/src/shared/spawn-md.ts | 63 +- 6 files changed, 1306 insertions(+), 173 deletions(-) create mode 100644 packages/cli/src/__tests__/export-helpers.test.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index ede748d6c..38f81cd17 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openrouter/spawn", - "version": "1.0.22", + "version": "1.0.23", "type": "module", "bin": { "spawn": "cli.js" diff --git a/packages/cli/src/__tests__/export-helpers.test.ts b/packages/cli/src/__tests__/export-helpers.test.ts new file mode 100644 index 000000000..459a2fd46 --- /dev/null +++ b/packages/cli/src/__tests__/export-helpers.test.ts @@ -0,0 +1,259 @@ +// Unit tests for the pure helpers in commands/export.ts +// (CLI catalog, MCP scanners, secret content scanner). + +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { __testing } from "../commands/export.js"; + +const { + parseProbeOutput, + entriesFromCodexToml, + entriesFromJsonMcpServers, + entriesFromJsonRoot, + dedupeMcpServers, + buildSetupFromCliScan, + scanForSecrets, +} = __testing; + +describe("entriesFromJsonMcpServers", () => { + it("extracts mcpServers from a Claude/Cursor settings shape", () => { + const json = JSON.stringify({ + mcpServers: { + github: { + command: "gh-mcp", + args: [ + "serve", + ], + env: { + GH_TOKEN: "secret", + }, + }, + }, + }); + const out = entriesFromJsonMcpServers(json); + expect(out.github.command).toBe("gh-mcp"); + expect(out.github.args).toEqual([ + "serve", + ]); + expect(out.github.env).toEqual({ + GH_TOKEN: "secret", + }); + }); + + it("returns {} for malformed JSON", () => { + expect(entriesFromJsonMcpServers("not json")).toEqual({}); + }); +}); + +describe("entriesFromJsonRoot", () => { + it("accepts a root-level entries object", () => { + const json = JSON.stringify({ + svr: { + command: "x", + args: [], + }, + }); + expect(entriesFromJsonRoot(json).svr.command).toBe("x"); + }); + + it("falls back to mcpServers shape", () => { + const json = JSON.stringify({ + mcpServers: { + svr: { + command: "y", + args: [], + }, + }, + }); + expect(entriesFromJsonRoot(json).svr.command).toBe("y"); + }); +}); + +describe("entriesFromCodexToml", () => { + it("parses [mcp_servers.NAME] sections with env subsection", () => { + const toml = ` +[mcp_servers.fs] +command = "npx" +args = ["-y", "@modelcontextprotocol/server-filesystem", "/data"] +[mcp_servers.fs.env] +NODE_ENV = "production" + +[mcp_servers.weather] +command = "weather-mcp" +args = [] +`; + const out = entriesFromCodexToml(toml); + expect(out.fs.command).toBe("npx"); + expect(out.fs.args).toEqual([ + "-y", + "@modelcontextprotocol/server-filesystem", + "/data", + ]); + expect(out.fs.env).toEqual({ + NODE_ENV: "production", + }); + expect(out.weather.command).toBe("weather-mcp"); + expect(out.weather.args).toEqual([]); + }); + + it("ignores non-mcp_servers TOML sections", () => { + const toml = ` +model = "openai/gpt-5" +[model_providers.openrouter] +name = "OpenRouter" +[mcp_servers.foo] +command = "foo" +args = [] +`; + expect(Object.keys(entriesFromCodexToml(toml))).toEqual([ + "foo", + ]); + }); +}); + +describe("parseProbeOutput", () => { + const FRAME = "===SPAWN_EXPORT_FRAME==="; + it("splits CLI and MCP frames", () => { + const raw = `${FRAME} +CLI_BIN=gh +Logged in to github.com as alice +${FRAME} +CLI_BIN=vercel +alice +${FRAME} +MCP_PATH=/home/u/.claude/settings.json +MCP_FORMAT=json-mcpservers +{"mcpServers":{"x":{"command":"c","args":[]}}} +${FRAME}END +`; + const scan = parseProbeOutput(raw); + expect(scan.clis.get("gh")).toContain("Logged in"); + expect(scan.clis.get("vercel")).toContain("alice"); + expect(scan.mcps).toHaveLength(1); + expect(scan.mcps[0].format).toBe("json-mcpservers"); + }); + + it("ignores empty MCP files", () => { + const raw = `${FRAME} +MCP_PATH=/home/u/.claude/settings.json +MCP_FORMAT=json-mcpservers + +${FRAME}END +`; + expect(parseProbeOutput(raw).mcps).toEqual([]); + }); +}); + +describe("buildSetupFromCliScan", () => { + it("maps gh to the built-in github step", () => { + const clis = new Map([ + [ + "gh", + "Logged in to github.com", + ], + ]); + const { setup, steps } = buildSetupFromCliScan(clis); + expect(steps).toContain("github"); + expect(setup.find((s) => s.name === "GitHub CLI")).toBeUndefined(); + }); + + it("emits cli_auth steps for non-github authed CLIs", () => { + const clis = new Map([ + [ + "vercel", + "alice\n", + ], + [ + "stripe", + "test_mode_api_key = sk_test_xxx\n", + ], + ]); + const { setup } = buildSetupFromCliScan(clis); + const names = setup.map((s) => s.name); + expect(names).toContain("Vercel CLI"); + expect(names).toContain("Stripe CLI"); + }); + + it("ignores CLIs whose status output doesn't match auth markers", () => { + const clis = new Map([ + [ + "vercel", + "Error: Not authenticated\n", + ], + ]); + expect(buildSetupFromCliScan(clis).setup).toEqual([]); + }); +}); + +describe("dedupeMcpServers", () => { + it("dedupes by name and replaces env values with placeholders", () => { + const json = JSON.stringify({ + mcpServers: { + gh: { + command: "gh-mcp", + args: [], + env: { + GH_TOKEN: "secret", + }, + }, + }, + }); + const dup = JSON.stringify({ + mcpServers: { + gh: { + command: "different", + args: [], + }, + }, + }); + const out = dedupeMcpServers([ + { + path: "/a", + format: "json-mcpservers", + content: json, + }, + { + path: "/b", + format: "json-mcpservers", + content: dup, + }, + ]); + expect(out).toHaveLength(1); + expect(out[0].command).toBe("gh-mcp"); + expect(out[0].env).toEqual({ + GH_TOKEN: "${GH_TOKEN}", + }); + }); +}); + +describe("scanForSecrets", () => { + it("detects PEM private keys, AWS keys, and PATs", () => { + const dir = mkdtempSync(join(tmpdir(), "spawn-secret-test-")); + writeFileSync(join(dir, "harmless.md"), "hello world\n"); + writeFileSync(join(dir, "key.pem"), "-----BEGIN OPENSSH PRIVATE KEY-----\nfoo\n"); + writeFileSync(join(dir, "creds.txt"), "AWS_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE\n"); + writeFileSync(join(dir, "tokens.txt"), "GH=ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"); + const hits = scanForSecrets(dir); + rmSync(dir, { + recursive: true, + }); + const labels = hits.map((h) => `${h.relativePath}:${h.label}`).sort(); + expect(labels).toContain("creds.txt:AWS access key"); + expect(labels).toContain("key.pem:PEM private key"); + expect(labels).toContain("tokens.txt:GitHub PAT"); + expect(hits.find((h) => h.relativePath === "harmless.md")).toBeUndefined(); + }); + + it("skips known binary file extensions", () => { + const dir = mkdtempSync(join(tmpdir(), "spawn-secret-test-bin-")); + // PEM-looking content but in a .png file — should be skipped + writeFileSync(join(dir, "logo.png"), "-----BEGIN PRIVATE KEY-----\n"); + const hits = scanForSecrets(dir); + rmSync(dir, { + recursive: true, + }); + expect(hits).toEqual([]); + }); +}); diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts index 07569bfb1..9f344be73 100644 --- a/packages/cli/src/commands/export.ts +++ b/packages/cli/src/commands/export.ts @@ -1,15 +1,15 @@ // commands/export.ts — Export a running spawn's setup as a shareable template // // SSHes into a VM from spawn history, scans what's installed (MCP servers, -// CLI auths, tools), generates a spawn.md recipe (no secrets), pushes to -// GitHub, and prints a one-liner command for others to replicate the setup. +// CLI auths, tools), publishes ~/project to a public GitHub repo (filtering +// secrets), and prints a one-liner command for others to replicate the setup. import type { SpawnMdConfig } from "../shared/spawn-md.js"; import { execSync, spawnSync } from "node:child_process"; -import { mkdtempSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, statSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, relative } from "node:path"; import * as p from "@clack/prompts"; import { isString } from "@openrouter/spawn-shared"; import pc from "picocolors"; @@ -25,17 +25,523 @@ import { ensureSshKeys, getSshKeyOpts } from "../shared/ssh-keys.js"; import { buildRecordLabel, buildRecordSubtitle } from "./list.js"; import { handleCancel, isInteractiveTTY } from "./shared.js"; +// ── MCP scan sources ─────────────────────────────────────────────────────── +// Each spawned agent stores MCP server configs at a known path. We scan all of +// them regardless of agent so a Claude template can include MCPs the user added +// directly to Cursor, etc. +type McpFormat = "json-mcpservers" | "json-root" | "codex-toml"; + +interface McpSource { + path: string; + format: McpFormat; +} + +const MCP_SOURCES: McpSource[] = [ + { + path: "$HOME/.claude/settings.json", + format: "json-mcpservers", + }, + { + path: "$HOME/.claude.json", + format: "json-mcpservers", + }, + { + path: "$HOME/.cursor/mcp.json", + format: "json-mcpservers", + }, + { + path: "$HOME/.codex/config.toml", + format: "codex-toml", + }, + { + path: "$HOME/.openclaw/openclaw.json", + format: "json-root", + }, +]; + +// ── CLI auth catalog ─────────────────────────────────────────────────────── +// `bin` is what we probe with `command -v`. `statusCmd` runs on the VM and +// must exit 0 (or print one of `markers`) when the CLI is authed. `authCmd` +// is what the template recipient runs to re-auth. +interface CliProbe { + bin: string; + name: string; + statusCmd: string; + authCmd: string; + markers: RegExp[]; +} + +const CLI_PROBES: CliProbe[] = [ + // Source control / hosting + { + bin: "gh", + name: "GitHub CLI", + statusCmd: "gh auth status 2>&1", + authCmd: "gh auth login", + markers: [ + /Logged in/i, + ], + }, + { + bin: "glab", + name: "GitLab CLI", + statusCmd: "glab auth status 2>&1", + authCmd: "glab auth login", + markers: [ + /Logged in/i, + ], + }, + // Cloud providers + { + bin: "aws", + name: "AWS CLI", + statusCmd: "aws sts get-caller-identity 2>&1", + authCmd: "aws configure", + markers: [ + /"Account":/, + /"Arn":/, + ], + }, + { + bin: "gcloud", + name: "gcloud", + statusCmd: "gcloud auth list --filter=status:ACTIVE --format='value(account)' 2>/dev/null", + authCmd: "gcloud auth login", + markers: [ + /@/, + ], + }, + { + bin: "az", + name: "Azure CLI", + statusCmd: "az account show 2>/dev/null", + authCmd: "az login", + markers: [ + /"id":/, + /"tenantId":/, + ], + }, + { + bin: "doctl", + name: "DigitalOcean CLI", + statusCmd: "doctl account get 2>&1", + authCmd: "doctl auth init", + markers: [ + /active|Email/i, + ], + }, + { + bin: "hcloud", + name: "Hetzner Cloud CLI", + statusCmd: "hcloud context active 2>/dev/null", + authCmd: "hcloud context create default", + markers: [ + /^\S+/, + ], + }, + // PaaS + { + bin: "vercel", + name: "Vercel CLI", + statusCmd: "vercel whoami 2>&1", + authCmd: "vercel login", + markers: [ + /^[a-zA-Z0-9_-]+$/m, + ], + }, + { + bin: "netlify", + name: "Netlify CLI", + statusCmd: "netlify status 2>&1", + authCmd: "netlify login", + markers: [ + /Logged in|Email/i, + ], + }, + { + bin: "fly", + name: "Fly.io CLI", + statusCmd: "fly auth whoami 2>&1", + authCmd: "fly auth login", + markers: [ + /@/, + ], + }, + { + bin: "flyctl", + name: "Flyctl", + statusCmd: "flyctl auth whoami 2>&1", + authCmd: "flyctl auth login", + markers: [ + /@/, + ], + }, + { + bin: "heroku", + name: "Heroku CLI", + statusCmd: "heroku whoami 2>&1", + authCmd: "heroku login", + markers: [ + /@/, + ], + }, + { + bin: "railway", + name: "Railway CLI", + statusCmd: "railway whoami 2>&1", + authCmd: "railway login", + markers: [ + /@|Logged in/i, + ], + }, + { + bin: "render", + name: "Render CLI", + statusCmd: "render workspace current 2>&1", + authCmd: "render login", + markers: [ + /Workspace|Name/i, + ], + }, + // E-commerce / payments + { + bin: "shopify", + name: "Shopify CLI", + statusCmd: "shopify app info 2>&1", + authCmd: "shopify auth login", + markers: [ + /Logged in|Org|Partner/i, + ], + }, + { + bin: "stripe", + name: "Stripe CLI", + statusCmd: "stripe config --list 2>&1", + authCmd: "stripe login", + markers: [ + /test_mode_api_key|live_mode_api_key|account_id/, + ], + }, + // Backend-as-a-service + { + bin: "firebase", + name: "Firebase CLI", + statusCmd: "firebase login:list 2>&1", + authCmd: "firebase login", + markers: [ + /@/, + ], + }, + { + bin: "supabase", + name: "Supabase CLI", + statusCmd: "supabase projects list 2>&1", + authCmd: "supabase login", + markers: [ + /REFERENCE ID|^[a-z0-9]{20}/m, + ], + }, + // Cloudflare / edge + { + bin: "wrangler", + name: "Cloudflare Wrangler", + statusCmd: "wrangler whoami 2>&1", + authCmd: "wrangler login", + markers: [ + /@|associated with/i, + ], + }, + // IaC + { + bin: "pulumi", + name: "Pulumi CLI", + statusCmd: "pulumi whoami 2>&1", + authCmd: "pulumi login", + markers: [ + /^\S+/, + ], + }, + // Containers + { + bin: "docker", + name: "Docker Hub", + statusCmd: "docker info 2>/dev/null | grep -i Username", + authCmd: "docker login", + markers: [ + /Username:/i, + ], + }, + // Package registries + { + bin: "npm", + name: "npm registry", + statusCmd: "npm whoami 2>&1", + authCmd: "npm login", + markers: [ + /^[a-zA-Z0-9._-]+$/m, + ], + }, + // Database / data + { + bin: "turso", + name: "Turso CLI", + statusCmd: "turso auth whoami 2>&1", + authCmd: "turso auth login", + markers: [ + /^\S+/, + ], + }, + { + bin: "neonctl", + name: "Neon CLI", + statusCmd: "neonctl me 2>&1", + authCmd: "neonctl auth", + markers: [ + /email|name/i, + ], + }, + { + bin: "planetscale", + name: "PlanetScale CLI", + statusCmd: "planetscale auth check 2>&1", + authCmd: "planetscale auth login", + markers: [ + /Authenticated|@/i, + ], + }, + // 1Password (secrets) + { + bin: "op", + name: "1Password CLI", + statusCmd: "op whoami 2>&1", + authCmd: "op signin", + markers: [ + /URL|user_uuid/i, + ], + }, + // Tunnels + { + bin: "ngrok", + name: "ngrok", + statusCmd: "ngrok config check 2>&1", + authCmd: "ngrok config add-authtoken ", + markers: [ + /Valid configuration/i, + ], + }, +]; + +// ── Secret-pattern scanner (for content of files staged for publish) ─────── +// We err on the side of false positives: if anything matches, we make the user +// look at it before pushing. +const SECRET_PATTERNS: Array<{ + label: string; + re: RegExp; +}> = [ + { + label: "PEM private key", + re: /-----BEGIN [A-Z ]*PRIVATE KEY-----/, + }, + { + label: "OpenSSH private key", + re: /-----BEGIN OPENSSH PRIVATE KEY-----/, + }, + { + label: "GitHub PAT", + re: /\bghp_[A-Za-z0-9]{30,}\b|\bgithub_pat_[A-Za-z0-9_]{40,}\b/, + }, + { + label: "GitHub OAuth token", + re: /\bgho_[A-Za-z0-9]{30,}\b/, + }, + { + label: "OpenAI / Anthropic-style API key", + re: /\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/, + }, + { + label: "Slack token", + re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/, + }, + { + label: "AWS access key", + re: /\b(AKIA|ASIA)[A-Z0-9]{16}\b/, + }, + { + label: "Stripe secret key", + re: /\b(sk|rk)_(test|live)_[A-Za-z0-9]{24,}\b/, + }, + { + label: "Google API key", + re: /\bAIza[0-9A-Za-z_-]{35}\b/, + }, +]; + +// rsync exclude rules — written to an exclude-from file. Filename-based and +// extension-based; the post-rsync content scan catches inline secrets. +const RSYNC_EXCLUDES = [ + // Spawn / VCS / dependencies + ".git", + ".gitignore", + "node_modules", + ".spawnrc", + // Env / dotenv + ".env", + ".env.*", + "*.env", + ".direnv", + // SSH / GPG / netrc + ".ssh", + "ssh", + ".gnupg", + ".netrc", + "id_rsa*", + "id_dsa*", + "id_ecdsa*", + "id_ed25519*", + "*.pem", + "*.key", + "*.p12", + "*.pfx", + "*.keystore", + "*.jks", + // Cloud-provider creds + ".aws", + ".azure", + ".config/gcloud", + ".config/doctl", + ".config/hcloud", + ".config/fly", + ".config/flyctl", + ".kube", + ".docker/config.json", + "credentials.json", + "service-account*.json", + "*credentials*.json", + "gcp-key.json", + // Package-registry creds + ".npmrc", + ".yarnrc", + ".pypirc", + // Terraform / Pulumi state + ".terraform", + "terraform.tfstate*", + ".terraformrc", + ".pulumi", + // Build artifacts (large + uninteresting) + "dist", + "build", + ".next", + ".nuxt", + ".turbo", + ".cache", + "coverage", + "__pycache__", + "*.pyc", + ".venv", + "venv", + // Editors / OS noise + ".idea", + ".DS_Store", + "Thumbs.db", + "*.log", + "*.swp", +]; + // Schema for parsing MCP server configs from Claude/Cursor settings const McpEntrySchema = v.object({ command: v.string(), args: v.array(v.string()), env: v.optional(v.record(v.string(), v.string())), }); + +// JSON shapes we accept: either { mcpServers: {...} } or { ...directly... } const McpSettingsSchema = v.object({ mcpServers: v.optional(v.record(v.string(), McpEntrySchema)), }); +const McpRootSchema = v.record(v.string(), McpEntrySchema); + +// ── Remote scan ──────────────────────────────────────────────────────────── +// One SSH call probes everything. We emit framed sections separated by a +// distinctive marker so locally we can split on it without escaping JSON. +const FRAME = "===SPAWN_EXPORT_FRAME==="; + +function buildProbeScript(): string { + const cliBins = CLI_PROBES.map((c) => c.bin).join(" "); + // statusCmds keyed by bin via case statement (avoids quoting issues) + const cliCases = CLI_PROBES.map((c) => ` ${c.bin}) STATUS_CMD=${shellSqEscape(c.statusCmd)};;`).join("\n"); + const mcpReads = MCP_SOURCES.map( + (s) => + ` printf '%s\\nMCP_PATH=%s\\nMCP_FORMAT=%s\\n' "${FRAME}" "${s.path}" "${s.format}"\n cat "${s.path}" 2>/dev/null || true`, + ).join("\n"); -/** Run a command on the remote VM and capture stdout */ + return [ + "set +e", + "echo '" + FRAME + "'", + `for bin in ${cliBins}; do`, + ` if command -v "$bin" >/dev/null 2>&1; then`, + ` case "$bin" in`, + cliCases, + ` *) STATUS_CMD="echo present";;`, + " esac", + ` printf 'CLI_BIN=%s\\n' "$bin"`, + ` out=$(eval "$STATUS_CMD" 2>&1 || true)`, + ` printf '%s\\n' "$out" | head -c 4000`, + ` printf '\\n${FRAME}\\n'`, + " fi", + "done", + mcpReads, + `printf '\\n${FRAME}END\\n'`, + ].join("\n"); +} + +/** Single-quote-escape for a value going into a bash single-quoted literal. */ +function shellSqEscape(s: string): string { + return `'${s.replace(/'/g, "'\\''")}'`; +} + +interface RemoteScan { + clis: Map; + mcps: Array<{ + path: string; + format: McpFormat; + content: string; + }>; +} + +function parseProbeOutput(raw: string): RemoteScan { + const result: RemoteScan = { + clis: new Map(), + mcps: [], + }; + const sections = raw.split(`${FRAME}\n`).map((s) => s.replace(/\n?===SPAWN_EXPORT_FRAME===END\n?$/, "")); + for (const section of sections) { + if (!section.trim()) { + continue; + } + const cliMatch = section.match(/^CLI_BIN=([^\n]+)\n([\s\S]*)$/); + if (cliMatch) { + result.clis.set(cliMatch[1].trim(), cliMatch[2]); + continue; + } + const mcpMatch = section.match(/^MCP_PATH=([^\n]+)\nMCP_FORMAT=([^\n]+)\n([\s\S]*)$/); + if (mcpMatch) { + const content = mcpMatch[3]; + if (content.trim()) { + const fmt = mcpMatch[2].trim(); + if (fmt === "json-mcpservers" || fmt === "json-root" || fmt === "codex-toml") { + result.mcps.push({ + path: mcpMatch[1].trim(), + format: fmt, + content, + }); + } + } + } + } + return result; +} + +/** Run one SSH command on the VM and return stdout. */ function sshCapture(ip: string, user: string, cmd: string, keyOpts: string[]): string { const result = spawnSync( "ssh", @@ -48,7 +554,8 @@ function sshCapture(ip: string, user: string, cmd: string, keyOpts: string[]): s ], { encoding: "utf8", - timeout: 15_000, + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, stdio: [ "pipe", "pipe", @@ -56,112 +563,314 @@ function sshCapture(ip: string, user: string, cmd: string, keyOpts: string[]): s ], }, ); - return (result.stdout ?? "").trim(); + return result.stdout ?? ""; } -/** Scan a remote VM for installed MCP servers, CLI auths, and tools */ -function scanVmSetup( - ip: string, - user: string, - agentKey: string, - keyOpts: string[], -): { - mcpServers: NonNullable; - steps: string[]; - setup: NonNullable; -} { - const mcpServers: NonNullable = []; - const steps: string[] = []; - const setup: NonNullable = []; +// ── MCP parsing ──────────────────────────────────────────────────────────── +type GenericMcp = NonNullable[number]; + +function entriesFromJsonMcpServers(content: string): Record> { + const parsed = parseJsonWith(content, McpSettingsSchema); + return parsed?.mcpServers ?? {}; +} + +function entriesFromJsonRoot(content: string): Record> { + // Some configs put MCP entries at the root. Try both shapes. + const asRoot = parseJsonWith(content, McpRootSchema); + if (asRoot && Object.keys(asRoot).length > 0) { + return asRoot; + } + return entriesFromJsonMcpServers(content); +} - // Detect MCP servers from Claude Code / Cursor config - if (agentKey === "claude" || agentKey === "cursor") { - const settingsPath = agentKey === "claude" ? "~/.claude/settings.json" : "~/.cursor/mcp.json"; - const raw = sshCapture(ip, user, `cat ${settingsPath} 2>/dev/null || echo '{}'`, keyOpts); - const parsed = parseJsonWith(raw, McpSettingsSchema); - if (parsed?.mcpServers) { - for (const [name, cfg] of Object.entries(parsed.mcpServers)) { - const entry: NonNullable[number] = { - name, - command: cfg.command, - args: cfg.args, +/** + * Parse Codex-style TOML where MCP servers live in `[mcp_servers.NAME]` + * blocks. We only handle the subset Codex emits: command (string), + * args (string array, single-line), and a nested [mcp_servers.NAME.env] + * block with KEY = "VALUE" pairs. + */ +function entriesFromCodexToml(content: string): Record> { + const result: Record> = {}; + let currentName: string | null = null; + let currentEntry: { + command?: string; + args?: string[]; + env?: Record; + } | null = null; + let inEnv = false; + + const lines = content.split("\n"); + for (const rawLine of lines) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) { + continue; + } + const sectionMatch = line.match(/^\[mcp_servers\.([^.\]]+)(?:\.(env))?\]$/); + if (sectionMatch) { + // Flush previous entry if complete + if (currentName && currentEntry?.command && currentEntry.args) { + result[currentName] = { + command: currentEntry.command, + args: currentEntry.args, + ...(currentEntry.env + ? { + env: currentEntry.env, + } + : {}), }; - if (cfg.env) { - // Replace actual values with ${NAME} placeholders — never export secrets - const envPlaceholders: Record = {}; - for (const k of Object.keys(cfg.env)) { - envPlaceholders[k] = `\${${k}}`; - } - entry.env = envPlaceholders; - } - mcpServers.push(entry); + } + const newName = sectionMatch[1]; + if (sectionMatch[2] === "env" && currentName === newName) { + inEnv = true; + } else { + currentName = newName; + currentEntry = result[newName] + ? { + command: result[newName].command, + args: result[newName].args, + env: result[newName].env, + } + : {}; + inEnv = false; + } + continue; + } + if (line.startsWith("[")) { + currentName = null; + currentEntry = null; + inEnv = false; + continue; + } + const kv = line.match(/^(\w+)\s*=\s*(.+)$/); + if (!kv || !currentEntry) { + continue; + } + const key = kv[1]; + const val = kv[2].trim(); + if (inEnv) { + const sv = parseTomlString(val); + if (sv !== null) { + currentEntry.env = currentEntry.env ?? {}; + currentEntry.env[key] = sv; + } + continue; + } + if (key === "command") { + const sv = parseTomlString(val); + if (sv !== null) { + currentEntry.command = sv; + } + } else if (key === "args") { + const arr = parseTomlStringArray(val); + if (arr) { + currentEntry.args = arr; } } } + if (currentName && currentEntry?.command && currentEntry.args) { + result[currentName] = { + command: currentEntry.command, + args: currentEntry.args, + ...(currentEntry.env + ? { + env: currentEntry.env, + } + : {}), + }; + } + return result; +} - // Detect GitHub CLI auth - const ghStatus = sshCapture(ip, user, "gh auth status 2>&1 || true", keyOpts); - if (ghStatus.includes("Logged in")) { - steps.push("github"); +function parseTomlString(raw: string): string | null { + if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) { + return raw.slice(1, -1); } + return null; +} - // Detect common CLI tools that might need auth - const toolChecks = [ - { - cmd: "shopify", - name: "Shopify CLI", - authCmd: "shopify auth login", - }, - { - cmd: "vercel", - name: "Vercel CLI", - authCmd: "vercel login", - }, - { - cmd: "netlify", - name: "Netlify CLI", - authCmd: "netlify login", - }, - { - cmd: "firebase", - name: "Firebase CLI", - authCmd: "firebase login", - }, - { - cmd: "supabase", - name: "Supabase CLI", - authCmd: "supabase login", - }, - { - cmd: "stripe", - name: "Stripe CLI", - authCmd: "stripe login", - }, - ] as const; - - for (const tool of toolChecks) { - const which = sshCapture(ip, user, `which ${tool.cmd} 2>/dev/null || true`, keyOpts); - if (which) { - setup.push({ - type: "cli_auth", - name: tool.name, - command: tool.authCmd, - description: `Authenticate with ${tool.name}`, - }); +function parseTomlStringArray(raw: string): string[] | null { + if (!raw.startsWith("[") || !raw.endsWith("]")) { + return null; + } + const inner = raw.slice(1, -1).trim(); + if (!inner) { + return []; + } + // Naive split on commas outside quotes — sufficient for Codex output. + const items: string[] = []; + let buf = ""; + let inDq = false; + for (const ch of inner) { + if (ch === '"') { + inDq = !inDq; + buf += ch; + continue; + } + if (ch === "," && !inDq) { + items.push(buf.trim()); + buf = ""; + continue; } + buf += ch; + } + if (buf.trim()) { + items.push(buf.trim()); } + const parsed: string[] = []; + for (const item of items) { + const sv = parseTomlString(item); + if (sv === null) { + return null; + } + parsed.push(sv); + } + return parsed; +} - // Always include auto-update - steps.push("auto-update"); +function dedupeMcpServers(scans: RemoteScan["mcps"]): GenericMcp[] { + const byName = new Map(); + for (const src of scans) { + let entries: Record> = {}; + if (src.format === "json-mcpservers") { + entries = entriesFromJsonMcpServers(src.content); + } else if (src.format === "json-root") { + entries = entriesFromJsonRoot(src.content); + } else if (src.format === "codex-toml") { + entries = entriesFromCodexToml(src.content); + } + for (const [name, cfg] of Object.entries(entries)) { + if (byName.has(name)) { + continue; + } + const entry: GenericMcp = { + name, + command: cfg.command, + args: cfg.args, + }; + if (cfg.env) { + // Replace actual values with ${NAME} placeholders — never export secrets + const placeholders: Record = {}; + for (const k of Object.keys(cfg.env)) { + placeholders[k] = `\${${k}}`; + } + entry.env = placeholders; + } + byName.set(name, entry); + } + } + return Array.from(byName.values()); +} +// ── CLI status interpretation ────────────────────────────────────────────── +function buildSetupFromCliScan(clis: Map): { + setup: NonNullable; + steps: string[]; +} { + const setup: NonNullable = []; + const steps: string[] = []; + for (const probe of CLI_PROBES) { + const out = clis.get(probe.bin); + if (out === undefined) { + continue; + } + const authed = probe.markers.some((re) => re.test(out)); + if (!authed) { + continue; + } + if (probe.bin === "gh") { + // Spawn already has a built-in github step; prefer it over a custom auth flow. + steps.push("github"); + continue; + } + setup.push({ + type: "cli_auth", + name: probe.name, + command: probe.authCmd, + description: `Authenticate with ${probe.name}`, + }); + } return { - mcpServers, - steps, setup, + steps, }; } -/** Main export command */ +// ── Secret content scan over the staged repo ─────────────────────────────── +const TEXT_FILE_MAX = 1 * 1024 * 1024; // skip blobs > 1 MB + +function isLikelyTextFile(path: string): boolean { + const bin = + /\.(png|jpe?g|gif|webp|ico|pdf|zip|gz|tgz|tar|woff2?|ttf|otf|eot|mp[34]|wav|ogg|mov|webm|exe|dll|so|dylib|bin|class|jar|wasm)$/i; + return !bin.test(path); +} + +function walkFiles(root: string): string[] { + const out: string[] = []; + const stack: string[] = [ + root, + ]; + while (stack.length) { + const dir = stack.pop(); + if (dir === undefined) { + continue; + } + const entries = readdirSync(dir, { + withFileTypes: true, + }); + for (const e of entries) { + const full = join(dir, e.name); + if (e.isDirectory()) { + if (e.name === ".git") { + continue; + } + stack.push(full); + } else if (e.isFile()) { + out.push(full); + } + } + } + return out; +} + +interface SecretHit { + relativePath: string; + label: string; +} + +function scanForSecrets(rootDir: string): SecretHit[] { + const hits: SecretHit[] = []; + const files = walkFiles(rootDir); + for (const file of files) { + if (!isLikelyTextFile(file)) { + continue; + } + const sizeResult = tryCatch(() => statSync(file).size); + if (!sizeResult.ok || sizeResult.data > TEXT_FILE_MAX) { + continue; + } + const readResult = tryCatch(() => + readFileSync(file, { + encoding: "utf8", + }), + ); + if (!readResult.ok) { + continue; + } + const content = readResult.data; + for (const pat of SECRET_PATTERNS) { + if (pat.re.test(content)) { + hits.push({ + relativePath: relative(rootDir, file), + label: pat.label, + }); + break; + } + } + } + return hits; +} + +// ── Main command ─────────────────────────────────────────────────────────── export async function cmdExport(): Promise { if (!isInteractiveTTY()) { console.error(pc.red("Error: spawn export requires an interactive terminal")); @@ -234,12 +943,23 @@ export async function cmdExport(): Promise { return; } + // 3. SSH in and run the unified probe p.log.step(`Scanning ${pc.bold(record.agent)} on ${pc.bold(conn.ip)}...`); - - // 3. SSH in and scan the VM const keys = await ensureSshKeys(); const keyOpts = getSshKeyOpts(keys); - const { mcpServers, steps, setup } = scanVmSetup(conn.ip, conn.user, record.agent, keyOpts); + const probeOutput = sshCapture(conn.ip, conn.user, buildProbeScript(), keyOpts); + const scan = parseProbeOutput(probeOutput); + + const mcpServers = dedupeMcpServers(scan.mcps); + const { setup, steps } = buildSetupFromCliScan(scan.clis); + steps.push("auto-update"); + + if (mcpServers.length > 0) { + p.log.info(`MCP servers found: ${mcpServers.map((m) => m.name).join(", ")}`); + } + if (setup.length > 0) { + p.log.info(`Authenticated CLIs: ${setup.map((s) => s.name).join(", ")}`); + } // 4. Prompt for template details const repoName = await p.text({ @@ -271,6 +991,25 @@ export async function cmdExport(): Promise { return; } + const visibility = await p.select({ + message: "Repository visibility:", + options: [ + { + value: "private", + label: "Private (recommended)", + }, + { + value: "public", + label: "Public", + }, + ], + initialValue: "private", + }); + if (p.isCancel(visibility)) { + handleCancel(); + return; + } + // 5. Generate spawn.md (steps go in the CLI command, not in spawn.md) const config: SpawnMdConfig = { name: repoName, @@ -281,25 +1020,37 @@ export async function cmdExport(): Promise { const spawnMdContent = generateSpawnMd(config, `# ${repoName}\n\n${isString(description) ? description : ""}`); - // Show preview - p.log.info("Generated spawn.md:"); - console.error(pc.dim("─".repeat(40))); - console.error(pc.dim(spawnMdContent)); - console.error(pc.dim("─".repeat(40))); - - const confirm = await p.confirm({ - message: "Create GitHub repo and push?", - initialValue: true, - }); + // 6. Stage the working folder + spawn.md + const tmpDir = mkdtempSync(join(tmpdir(), "spawn-export-")); - if (p.isCancel(confirm) || !confirm) { - handleCancel(); - return; + // Download project files from VM if they exist + p.log.step("Downloading working folder from VM..."); + const hasProject = sshCapture(conn.ip, conn.user, "test -d ~/project && echo yes || echo no", keyOpts).trim(); + if (hasProject === "yes") { + const excludeFile = join(tmpDir, ".spawn-rsync-exclude"); + writeFileSync(excludeFile, RSYNC_EXCLUDES.join("\n")); + const sshArgs = [ + ...SSH_BASE_OPTS, + ...keyOpts, + ].join(" "); + const rsyncResult = tryCatch(() => + execSync( + `rsync -az --exclude-from='${excludeFile}' -e "ssh ${sshArgs}" '${conn.user}@${conn.ip}:~/project/' '${tmpDir}/'`, + { + encoding: "utf8", + timeout: 120_000, + }, + ), + ); + tryCatch(() => unlinkSync(excludeFile)); + if (!rsyncResult.ok) { + p.log.warn("Could not download project files — creating template with spawn.md only"); + } + } else { + p.log.info("No ~/project directory on VM — exporting setup only"); } - // 6. Create temp dir, write spawn.md, init repo, push - const tmpDir = mkdtempSync(join(tmpdir(), "spawn-export-")); - + // Write spawn.md AFTER rsync so it always wins over any existing one in the project. writeFileSync(join(tmpDir, "spawn.md"), spawnMdContent); writeFileSync( join(tmpDir, ".gitignore"), @@ -309,42 +1060,79 @@ export async function cmdExport(): Promise { ".spawnrc", "node_modules/", "/etc/spawn/", + "*.pem", + "*.key", + "id_rsa*", + "credentials.json", "", ].join("\n"), ); - // Download project files from VM if they exist - p.log.step("Downloading project files from VM..."); - const hasProject = sshCapture(conn.ip, conn.user, "test -d ~/project && echo yes || echo no", keyOpts); - if (hasProject === "yes") { - const excludeFlags = [ - "--exclude=node_modules", - "--exclude=.git", - "--exclude=.env", - "--exclude=.env.*", - "--exclude=.spawnrc", - ]; - const rsyncResult = tryCatch(() => - execSync( - `rsync -az ${excludeFlags.join(" ")} -e "ssh ${SSH_BASE_OPTS.join(" ")} ${keyOpts.join(" ")}" ${conn.user}@${conn.ip}:~/project/ ${tmpDir}/`, + // 7. Scan staged files for inline secrets + p.log.step("Scanning staged files for secrets..."); + const hits = scanForSecrets(tmpDir); + if (hits.length > 0) { + p.log.warn(`Found ${hits.length} file(s) that look like they contain secrets:`); + for (const hit of hits.slice(0, 20)) { + console.error(` ${pc.yellow("•")} ${pc.bold(hit.relativePath)} ${pc.dim(`(${hit.label})`)}`); + } + if (hits.length > 20) { + console.error(pc.dim(` ... and ${hits.length - 20} more`)); + } + const action = await p.select({ + message: "How do you want to handle these?", + options: [ { - encoding: "utf8", - timeout: 60_000, + value: "remove", + label: "Remove flagged files and continue", }, - ), - ); - if (!rsyncResult.ok) { - p.log.warn("Could not download project files — creating template with spawn.md only"); + { + value: "keep", + label: "Keep them (I checked, they're safe)", + }, + { + value: "abort", + label: "Abort export", + }, + ], + initialValue: "remove", + }); + if (p.isCancel(action) || action === "abort") { + handleCancel(); + return; + } + if (action === "remove") { + for (const hit of hits) { + tryCatch(() => unlinkSync(join(tmpDir, hit.relativePath))); + } + p.log.info(`Removed ${hits.length} flagged file(s)`); } } - // 7. Create GitHub repo + // 8. Show preview of generated spawn.md + p.log.info("Generated spawn.md:"); + console.error(pc.dim("─".repeat(40))); + console.error(pc.dim(spawnMdContent)); + console.error(pc.dim("─".repeat(40))); + + const confirm = await p.confirm({ + message: `Create ${visibility} GitHub repo "${repoName}" and push?`, + initialValue: true, + }); + + if (p.isCancel(confirm) || !confirm) { + handleCancel(); + return; + } + + // 9. Init git, commit, create repo, push p.log.step("Creating GitHub repository..."); + const visibilityFlag = visibility === "public" ? "--public" : "--private"; const repoResult = tryCatch(() => { execSync(`cd ${tmpDir} && git init -q && git add -A && git commit -q -m "Initial template"`, { encoding: "utf8", }); - execSync(`cd ${tmpDir} && gh repo create ${repoName} --public --source=. --push`, { + execSync(`cd ${tmpDir} && gh repo create ${repoName} ${visibilityFlag} --source=. --push`, { encoding: "utf8", stdio: [ "pipe", @@ -359,7 +1147,7 @@ export async function cmdExport(): Promise { return; } - // 8. Get the repo slug (user/repo) + // 10. Get the repo slug (user/repo) let repoSlug = repoName; const ghUserResult = tryCatch(() => execSync("gh api user --jq .login", { @@ -370,7 +1158,7 @@ export async function cmdExport(): Promise { repoSlug = `${ghUserResult.data}/${repoName}`; } - // 9. Print the shareable command (steps baked into the command, not spawn.md) + // 11. Print the shareable command (steps baked into the command, not spawn.md) const stepsArg = steps.length > 0 ? ` --steps ${steps.join(",")}` : ""; console.error(); p.log.success("Template exported!"); @@ -380,3 +1168,19 @@ export async function cmdExport(): Promise { console.error(` ${pc.cyan(`spawn ${record.agent} ${record.cloud} --repo ${repoSlug}${stepsArg}`)}`); console.error(); } + +// Internal exports for unit tests +export const __testing = { + parseProbeOutput, + entriesFromCodexToml, + entriesFromJsonMcpServers, + entriesFromJsonRoot, + dedupeMcpServers, + buildSetupFromCliScan, + scanForSecrets, + SECRET_PATTERNS, + RSYNC_EXCLUDES, + CLI_PROBES, + MCP_SOURCES, + buildProbeScript, +}; diff --git a/packages/cli/src/shared/orchestrate.ts b/packages/cli/src/shared/orchestrate.ts index c838a30e0..8373f6e8f 100644 --- a/packages/cli/src/shared/orchestrate.ts +++ b/packages/cli/src/shared/orchestrate.ts @@ -622,6 +622,7 @@ async function postInstall( // flag, not from spawn.md. spawn.md only handles custom setup (OAuth, // MCP servers, setup commands). let spawnMdConfig: import("./spawn-md.js").SpawnMdConfig | null = null; + let repoCloned = false; const repoSlug = process.env.SPAWN_REPO; if (repoSlug && cloud.cloudName !== "local") { // Validate slug format (user/repo, no path traversal) @@ -630,11 +631,12 @@ async function postInstall( } else { logStep("Cloning template repository..."); const cloneResult = await asyncTryCatch(() => - cloud.runner.runServer(`git clone https://github.com/${repoSlug}.git ~/project 2>&1 || true`), + cloud.runner.runServer(`git clone https://github.com/${repoSlug}.git ~/project`), ); if (!cloneResult.ok) { - logWarn("Repo clone failed — continuing without template"); + logWarn(`Repo clone failed (${getErrorMessage(cloneResult.error)}) — continuing without template`); } else { + repoCloned = true; const { readRemoteSpawnMd } = await import("./spawn-md.js"); spawnMdConfig = await readRemoteSpawnMd(cloud.runner); if (spawnMdConfig) { @@ -905,9 +907,12 @@ async function postInstall( headless: process.env.SPAWN_HEADLESS === "1", }); - // When --repo is set, launch the agent inside the cloned project directory + // When --repo cloned successfully, launch the agent inside the cloned + // project directory. Gate on the actual clone outcome rather than the flag + // so an invalid slug or clone failure doesn't leave the agent trying to cd + // into a non-existent dir. const baseLaunchCmd = agent.launchCmd(); - const launchCmd = repoSlug ? `cd ~/project && ${baseLaunchCmd}` : baseLaunchCmd; + const launchCmd = repoCloned ? `cd ~/project && ${baseLaunchCmd}` : baseLaunchCmd; saveLaunchCmd(launchCmd, spawnId); // In headless mode, provisioning is done — skip the interactive session. diff --git a/packages/cli/src/shared/skills.ts b/packages/cli/src/shared/skills.ts index 2884348aa..5c210f2e1 100644 --- a/packages/cli/src/shared/skills.ts +++ b/packages/cli/src/shared/skills.ts @@ -204,7 +204,10 @@ export async function installSkills( } /** Merge MCP servers into Claude Code's ~/.claude/settings.json. */ -async function installClaudeMcpServers(runner: CloudRunner, servers: Record): Promise { +export async function installClaudeMcpServers( + runner: CloudRunner, + servers: Record, +): Promise { const tmpLocal = join(getTmpDir(), `claude_settings_${Date.now()}.json`); const dlResult = await asyncTryCatch(() => runner.downloadFile("$HOME/.claude/settings.json", tmpLocal)); @@ -226,7 +229,10 @@ async function installClaudeMcpServers(runner: CloudRunner, servers: Record): Promise { +export async function installCursorMcpServers( + runner: CloudRunner, + servers: Record, +): Promise { const tmpLocal = join(getTmpDir(), `cursor_mcp_${Date.now()}.json`); const dlResult = await asyncTryCatch(() => runner.downloadFile("$HOME/.cursor/mcp.json", tmpLocal)); @@ -248,7 +254,7 @@ async function installCursorMcpServers(runner: CloudRunner, servers: Record, @@ -263,6 +269,60 @@ async function installGenericMcpServers( await uploadConfigFile(runner, config, `$HOME/.${agentName}/mcp.json`); } +/** + * Append MCP server entries to Codex's ~/.codex/config.toml under + * [mcp_servers.NAME] sections. Existing sections with the same name are + * left untouched (we don't try to merge mid-file); new ones are appended. + */ +export async function installCodexMcpServers( + runner: CloudRunner, + servers: Record, +): Promise { + const tmpLocal = join(getTmpDir(), `codex_config_${Date.now()}.toml`); + const dlResult = await asyncTryCatch(() => runner.downloadFile("$HOME/.codex/config.toml", tmpLocal)); + + let existing = ""; + if (dlResult.ok) { + const readResult = tryCatch(() => readFileSync(tmpLocal, "utf-8")); + if (readResult.ok) { + existing = readResult.data; + } + } + + const existingNames = new Set(); + for (const m of existing.matchAll(/^\[mcp_servers\.([^.\]]+)\]/gm)) { + existingNames.add(m[1]); + } + + const lines: string[] = []; + for (const [name, cfg] of Object.entries(servers)) { + if (existingNames.has(name)) { + continue; + } + lines.push(""); + lines.push(`[mcp_servers.${name}]`); + lines.push(`command = ${tomlString(cfg.command)}`); + lines.push(`args = [${cfg.args.map((a) => tomlString(a)).join(", ")}]`); + if (cfg.env) { + lines.push(`[mcp_servers.${name}.env]`); + for (const [k, val] of Object.entries(cfg.env)) { + lines.push(`${k} = ${tomlString(val)}`); + } + } + } + + if (lines.length === 0) { + return; + } + + const merged = `${existing.replace(/\n+$/, "")}\n${lines.join("\n")}\n`; + await uploadConfigFile(runner, merged, "$HOME/.codex/config.toml"); +} + +function tomlString(s: string): string { + return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} + /** Inject an instruction skill (SKILL.md) onto the remote VM. */ async function injectInstructionSkill( runner: CloudRunner, diff --git a/packages/cli/src/shared/spawn-md.ts b/packages/cli/src/shared/spawn-md.ts index e742b8559..1893b5073 100644 --- a/packages/cli/src/shared/spawn-md.ts +++ b/packages/cli/src/shared/spawn-md.ts @@ -454,7 +454,7 @@ async function applySetupStep(runner: CloudRunner, step: SetupStep): Promise> /etc/spawn/secrets && chmod 600 /etc/spawn/secrets`, @@ -489,39 +489,44 @@ async function installMcpServersFromTemplate( servers: McpServerEntry[], agentName: string, ): Promise { - if (agentName === "claude" || agentName === "cursor") { - const mcpConfig: Record> = {}; - for (const server of servers) { - const entry: Record = { - command: server.command, - args: server.args, - }; - if (server.env) { - entry.env = server.env; - } - mcpConfig[server.name] = entry; + const record: Record< + string, + { + command: string; + args: string[]; + env?: Record; } + > = {}; + for (const server of servers) { + record[server.name] = server.env + ? { + command: server.command, + args: server.args, + env: server.env, + } + : { + command: server.command, + args: server.args, + }; + } - const settingsPath = agentName === "claude" ? "~/.claude/settings.json" : "~/.cursor/mcp.json"; - const mcpJson = JSON.stringify(mcpConfig); - const mergeScript = ` - const fs = require('fs'); - const path = '${settingsPath}'.replace('~', process.env.HOME); - let settings = {}; - try { settings = JSON.parse(fs.readFileSync(path, 'utf-8')); } catch {} - settings.mcpServers = { ...settings.mcpServers, ...${mcpJson} }; - fs.mkdirSync(require('path').dirname(path), { recursive: true }); - fs.writeFileSync(path, JSON.stringify(settings, null, 2)); - `; - const b64 = Buffer.from(mergeScript).toString("base64"); - const installResult = await asyncTryCatch(() => runner.runServer(`echo '${b64}' | base64 -d | node -e "$(cat)"`)); - if (installResult.ok) { - logInfo(` Installed ${servers.length} MCP server${servers.length > 1 ? "s" : ""}`); + const { installClaudeMcpServers, installCursorMcpServers, installCodexMcpServers, installGenericMcpServers } = + await import("./skills.js"); + const installResult = await asyncTryCatch(async () => { + if (agentName === "claude") { + await installClaudeMcpServers(runner, record); + } else if (agentName === "cursor") { + await installCursorMcpServers(runner, record); + } else if (agentName === "codex") { + await installCodexMcpServers(runner, record); } else { - logWarn(" MCP server installation failed — configure manually"); + await installGenericMcpServers(runner, agentName, record); } + }); + if (installResult.ok) { + logInfo(` Installed ${servers.length} MCP server${servers.length > 1 ? "s" : ""}`); } else { - logInfo(` MCP server installation not yet supported for ${agentName} — skipping`); + logWarn(" MCP server installation failed — configure manually"); } }