diff --git a/packages/cli/package.json b/packages/cli/package.json index cde6a1905..38f81cd17 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openrouter/spawn", - "version": "1.0.21", + "version": "1.0.23", "type": "module", "bin": { "spawn": "cli.js" diff --git a/packages/cli/src/__tests__/export-helpers.test.ts b/packages/cli/src/__tests__/export-helpers.test.ts new file mode 100644 index 000000000..459a2fd46 --- /dev/null +++ b/packages/cli/src/__tests__/export-helpers.test.ts @@ -0,0 +1,259 @@ +// Unit tests for the pure helpers in commands/export.ts +// (CLI catalog, MCP scanners, secret content scanner). + +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { __testing } from "../commands/export.js"; + +const { + parseProbeOutput, + entriesFromCodexToml, + entriesFromJsonMcpServers, + entriesFromJsonRoot, + dedupeMcpServers, + buildSetupFromCliScan, + scanForSecrets, +} = __testing; + +describe("entriesFromJsonMcpServers", () => { + it("extracts mcpServers from a Claude/Cursor settings shape", () => { + const json = JSON.stringify({ + mcpServers: { + github: { + command: "gh-mcp", + args: [ + "serve", + ], + env: { + GH_TOKEN: "secret", + }, + }, + }, + }); + const out = entriesFromJsonMcpServers(json); + expect(out.github.command).toBe("gh-mcp"); + expect(out.github.args).toEqual([ + "serve", + ]); + expect(out.github.env).toEqual({ + GH_TOKEN: "secret", + }); + }); + + it("returns {} for malformed JSON", () => { + expect(entriesFromJsonMcpServers("not json")).toEqual({}); + }); +}); + +describe("entriesFromJsonRoot", () => { + it("accepts a root-level entries object", () => { + const json = JSON.stringify({ + svr: { + command: "x", + args: [], + }, + }); + expect(entriesFromJsonRoot(json).svr.command).toBe("x"); + }); + + it("falls back to mcpServers shape", () => { + const json = JSON.stringify({ + mcpServers: { + svr: { + command: "y", + args: [], + }, + }, + }); + expect(entriesFromJsonRoot(json).svr.command).toBe("y"); + }); +}); + +describe("entriesFromCodexToml", () => { + it("parses [mcp_servers.NAME] sections with env subsection", () => { + const toml = ` +[mcp_servers.fs] +command = "npx" +args = ["-y", "@modelcontextprotocol/server-filesystem", "/data"] +[mcp_servers.fs.env] +NODE_ENV = "production" + +[mcp_servers.weather] +command = "weather-mcp" +args = [] +`; + const out = entriesFromCodexToml(toml); + expect(out.fs.command).toBe("npx"); + expect(out.fs.args).toEqual([ + "-y", + "@modelcontextprotocol/server-filesystem", + "/data", + ]); + expect(out.fs.env).toEqual({ + NODE_ENV: "production", + }); + expect(out.weather.command).toBe("weather-mcp"); + expect(out.weather.args).toEqual([]); + }); + + it("ignores non-mcp_servers TOML sections", () => { + const toml = ` +model = "openai/gpt-5" +[model_providers.openrouter] +name = "OpenRouter" +[mcp_servers.foo] +command = "foo" +args = [] +`; + expect(Object.keys(entriesFromCodexToml(toml))).toEqual([ + "foo", + ]); + }); +}); + +describe("parseProbeOutput", () => { + const FRAME = "===SPAWN_EXPORT_FRAME==="; + it("splits CLI and MCP frames", () => { + const raw = `${FRAME} +CLI_BIN=gh +Logged in to github.com as alice +${FRAME} +CLI_BIN=vercel +alice +${FRAME} +MCP_PATH=/home/u/.claude/settings.json +MCP_FORMAT=json-mcpservers +{"mcpServers":{"x":{"command":"c","args":[]}}} +${FRAME}END +`; + const scan = parseProbeOutput(raw); + expect(scan.clis.get("gh")).toContain("Logged in"); + expect(scan.clis.get("vercel")).toContain("alice"); + expect(scan.mcps).toHaveLength(1); + expect(scan.mcps[0].format).toBe("json-mcpservers"); + }); + + it("ignores empty MCP files", () => { + const raw = `${FRAME} +MCP_PATH=/home/u/.claude/settings.json +MCP_FORMAT=json-mcpservers + +${FRAME}END +`; + expect(parseProbeOutput(raw).mcps).toEqual([]); + }); +}); + +describe("buildSetupFromCliScan", () => { + it("maps gh to the built-in github step", () => { + const clis = new Map([ + [ + "gh", + "Logged in to github.com", + ], + ]); + const { setup, steps } = buildSetupFromCliScan(clis); + expect(steps).toContain("github"); + expect(setup.find((s) => s.name === "GitHub CLI")).toBeUndefined(); + }); + + it("emits cli_auth steps for non-github authed CLIs", () => { + const clis = new Map([ + [ + "vercel", + "alice\n", + ], + [ + "stripe", + "test_mode_api_key = sk_test_xxx\n", + ], + ]); + const { setup } = buildSetupFromCliScan(clis); + const names = setup.map((s) => s.name); + expect(names).toContain("Vercel CLI"); + expect(names).toContain("Stripe CLI"); + }); + + it("ignores CLIs whose status output doesn't match auth markers", () => { + const clis = new Map([ + [ + "vercel", + "Error: Not authenticated\n", + ], + ]); + expect(buildSetupFromCliScan(clis).setup).toEqual([]); + }); +}); + +describe("dedupeMcpServers", () => { + it("dedupes by name and replaces env values with placeholders", () => { + const json = JSON.stringify({ + mcpServers: { + gh: { + command: "gh-mcp", + args: [], + env: { + GH_TOKEN: "secret", + }, + }, + }, + }); + const dup = JSON.stringify({ + mcpServers: { + gh: { + command: "different", + args: [], + }, + }, + }); + const out = dedupeMcpServers([ + { + path: "/a", + format: "json-mcpservers", + content: json, + }, + { + path: "/b", + format: "json-mcpservers", + content: dup, + }, + ]); + expect(out).toHaveLength(1); + expect(out[0].command).toBe("gh-mcp"); + expect(out[0].env).toEqual({ + GH_TOKEN: "${GH_TOKEN}", + }); + }); +}); + +describe("scanForSecrets", () => { + it("detects PEM private keys, AWS keys, and PATs", () => { + const dir = mkdtempSync(join(tmpdir(), "spawn-secret-test-")); + writeFileSync(join(dir, "harmless.md"), "hello world\n"); + writeFileSync(join(dir, "key.pem"), "-----BEGIN OPENSSH PRIVATE KEY-----\nfoo\n"); + writeFileSync(join(dir, "creds.txt"), "AWS_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE\n"); + writeFileSync(join(dir, "tokens.txt"), "GH=ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"); + const hits = scanForSecrets(dir); + rmSync(dir, { + recursive: true, + }); + const labels = hits.map((h) => `${h.relativePath}:${h.label}`).sort(); + expect(labels).toContain("creds.txt:AWS access key"); + expect(labels).toContain("key.pem:PEM private key"); + expect(labels).toContain("tokens.txt:GitHub PAT"); + expect(hits.find((h) => h.relativePath === "harmless.md")).toBeUndefined(); + }); + + it("skips known binary file extensions", () => { + const dir = mkdtempSync(join(tmpdir(), "spawn-secret-test-bin-")); + // PEM-looking content but in a .png file — should be skipped + writeFileSync(join(dir, "logo.png"), "-----BEGIN PRIVATE KEY-----\n"); + const hits = scanForSecrets(dir); + rmSync(dir, { + recursive: true, + }); + expect(hits).toEqual([]); + }); +}); diff --git a/packages/cli/src/__tests__/unknown-flags.test.ts b/packages/cli/src/__tests__/unknown-flags.test.ts index 169180b29..4f1a74632 100644 --- a/packages/cli/src/__tests__/unknown-flags.test.ts +++ b/packages/cli/src/__tests__/unknown-flags.test.ts @@ -227,6 +227,7 @@ describe("KNOWN_FLAGS completeness", () => { "-m", "--config", "--steps", + "--repo", "--fast", "--flat", "--user", diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts new file mode 100644 index 000000000..9f344be73 --- /dev/null +++ b/packages/cli/src/commands/export.ts @@ -0,0 +1,1186 @@ +// commands/export.ts — Export a running spawn's setup as a shareable template +// +// SSHes into a VM from spawn history, scans what's installed (MCP servers, +// CLI auths, tools), publishes ~/project to a public GitHub repo (filtering +// secrets), and prints a one-liner command for others to replicate the setup. + +import type { SpawnMdConfig } from "../shared/spawn-md.js"; + +import { execSync, spawnSync } from "node:child_process"; +import { mkdtempSync, readdirSync, readFileSync, statSync, unlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import * as p from "@clack/prompts"; +import { isString } from "@openrouter/spawn-shared"; +import pc from "picocolors"; +import * as v from "valibot"; +import { loadHistory } from "../history.js"; +import { loadManifest } from "../manifest.js"; +import { validateConnectionIP, validateIdentifier, validateUsername } from "../security.js"; +import { parseJsonWith } from "../shared/parse.js"; +import { asyncTryCatch, tryCatch } from "../shared/result.js"; +import { generateSpawnMd } from "../shared/spawn-md.js"; +import { SSH_BASE_OPTS } from "../shared/ssh.js"; +import { ensureSshKeys, getSshKeyOpts } from "../shared/ssh-keys.js"; +import { buildRecordLabel, buildRecordSubtitle } from "./list.js"; +import { handleCancel, isInteractiveTTY } from "./shared.js"; + +// ── MCP scan sources ─────────────────────────────────────────────────────── +// Each spawned agent stores MCP server configs at a known path. We scan all of +// them regardless of agent so a Claude template can include MCPs the user added +// directly to Cursor, etc. +type McpFormat = "json-mcpservers" | "json-root" | "codex-toml"; + +interface McpSource { + path: string; + format: McpFormat; +} + +const MCP_SOURCES: McpSource[] = [ + { + path: "$HOME/.claude/settings.json", + format: "json-mcpservers", + }, + { + path: "$HOME/.claude.json", + format: "json-mcpservers", + }, + { + path: "$HOME/.cursor/mcp.json", + format: "json-mcpservers", + }, + { + path: "$HOME/.codex/config.toml", + format: "codex-toml", + }, + { + path: "$HOME/.openclaw/openclaw.json", + format: "json-root", + }, +]; + +// ── CLI auth catalog ─────────────────────────────────────────────────────── +// `bin` is what we probe with `command -v`. `statusCmd` runs on the VM and +// must exit 0 (or print one of `markers`) when the CLI is authed. `authCmd` +// is what the template recipient runs to re-auth. +interface CliProbe { + bin: string; + name: string; + statusCmd: string; + authCmd: string; + markers: RegExp[]; +} + +const CLI_PROBES: CliProbe[] = [ + // Source control / hosting + { + bin: "gh", + name: "GitHub CLI", + statusCmd: "gh auth status 2>&1", + authCmd: "gh auth login", + markers: [ + /Logged in/i, + ], + }, + { + bin: "glab", + name: "GitLab CLI", + statusCmd: "glab auth status 2>&1", + authCmd: "glab auth login", + markers: [ + /Logged in/i, + ], + }, + // Cloud providers + { + bin: "aws", + name: "AWS CLI", + statusCmd: "aws sts get-caller-identity 2>&1", + authCmd: "aws configure", + markers: [ + /"Account":/, + /"Arn":/, + ], + }, + { + bin: "gcloud", + name: "gcloud", + statusCmd: "gcloud auth list --filter=status:ACTIVE --format='value(account)' 2>/dev/null", + authCmd: "gcloud auth login", + markers: [ + /@/, + ], + }, + { + bin: "az", + name: "Azure CLI", + statusCmd: "az account show 2>/dev/null", + authCmd: "az login", + markers: [ + /"id":/, + /"tenantId":/, + ], + }, + { + bin: "doctl", + name: "DigitalOcean CLI", + statusCmd: "doctl account get 2>&1", + authCmd: "doctl auth init", + markers: [ + /active|Email/i, + ], + }, + { + bin: "hcloud", + name: "Hetzner Cloud CLI", + statusCmd: "hcloud context active 2>/dev/null", + authCmd: "hcloud context create default", + markers: [ + /^\S+/, + ], + }, + // PaaS + { + bin: "vercel", + name: "Vercel CLI", + statusCmd: "vercel whoami 2>&1", + authCmd: "vercel login", + markers: [ + /^[a-zA-Z0-9_-]+$/m, + ], + }, + { + bin: "netlify", + name: "Netlify CLI", + statusCmd: "netlify status 2>&1", + authCmd: "netlify login", + markers: [ + /Logged in|Email/i, + ], + }, + { + bin: "fly", + name: "Fly.io CLI", + statusCmd: "fly auth whoami 2>&1", + authCmd: "fly auth login", + markers: [ + /@/, + ], + }, + { + bin: "flyctl", + name: "Flyctl", + statusCmd: "flyctl auth whoami 2>&1", + authCmd: "flyctl auth login", + markers: [ + /@/, + ], + }, + { + bin: "heroku", + name: "Heroku CLI", + statusCmd: "heroku whoami 2>&1", + authCmd: "heroku login", + markers: [ + /@/, + ], + }, + { + bin: "railway", + name: "Railway CLI", + statusCmd: "railway whoami 2>&1", + authCmd: "railway login", + markers: [ + /@|Logged in/i, + ], + }, + { + bin: "render", + name: "Render CLI", + statusCmd: "render workspace current 2>&1", + authCmd: "render login", + markers: [ + /Workspace|Name/i, + ], + }, + // E-commerce / payments + { + bin: "shopify", + name: "Shopify CLI", + statusCmd: "shopify app info 2>&1", + authCmd: "shopify auth login", + markers: [ + /Logged in|Org|Partner/i, + ], + }, + { + bin: "stripe", + name: "Stripe CLI", + statusCmd: "stripe config --list 2>&1", + authCmd: "stripe login", + markers: [ + /test_mode_api_key|live_mode_api_key|account_id/, + ], + }, + // Backend-as-a-service + { + bin: "firebase", + name: "Firebase CLI", + statusCmd: "firebase login:list 2>&1", + authCmd: "firebase login", + markers: [ + /@/, + ], + }, + { + bin: "supabase", + name: "Supabase CLI", + statusCmd: "supabase projects list 2>&1", + authCmd: "supabase login", + markers: [ + /REFERENCE ID|^[a-z0-9]{20}/m, + ], + }, + // Cloudflare / edge + { + bin: "wrangler", + name: "Cloudflare Wrangler", + statusCmd: "wrangler whoami 2>&1", + authCmd: "wrangler login", + markers: [ + /@|associated with/i, + ], + }, + // IaC + { + bin: "pulumi", + name: "Pulumi CLI", + statusCmd: "pulumi whoami 2>&1", + authCmd: "pulumi login", + markers: [ + /^\S+/, + ], + }, + // Containers + { + bin: "docker", + name: "Docker Hub", + statusCmd: "docker info 2>/dev/null | grep -i Username", + authCmd: "docker login", + markers: [ + /Username:/i, + ], + }, + // Package registries + { + bin: "npm", + name: "npm registry", + statusCmd: "npm whoami 2>&1", + authCmd: "npm login", + markers: [ + /^[a-zA-Z0-9._-]+$/m, + ], + }, + // Database / data + { + bin: "turso", + name: "Turso CLI", + statusCmd: "turso auth whoami 2>&1", + authCmd: "turso auth login", + markers: [ + /^\S+/, + ], + }, + { + bin: "neonctl", + name: "Neon CLI", + statusCmd: "neonctl me 2>&1", + authCmd: "neonctl auth", + markers: [ + /email|name/i, + ], + }, + { + bin: "planetscale", + name: "PlanetScale CLI", + statusCmd: "planetscale auth check 2>&1", + authCmd: "planetscale auth login", + markers: [ + /Authenticated|@/i, + ], + }, + // 1Password (secrets) + { + bin: "op", + name: "1Password CLI", + statusCmd: "op whoami 2>&1", + authCmd: "op signin", + markers: [ + /URL|user_uuid/i, + ], + }, + // Tunnels + { + bin: "ngrok", + name: "ngrok", + statusCmd: "ngrok config check 2>&1", + authCmd: "ngrok config add-authtoken ", + markers: [ + /Valid configuration/i, + ], + }, +]; + +// ── Secret-pattern scanner (for content of files staged for publish) ─────── +// We err on the side of false positives: if anything matches, we make the user +// look at it before pushing. +const SECRET_PATTERNS: Array<{ + label: string; + re: RegExp; +}> = [ + { + label: "PEM private key", + re: /-----BEGIN [A-Z ]*PRIVATE KEY-----/, + }, + { + label: "OpenSSH private key", + re: /-----BEGIN OPENSSH PRIVATE KEY-----/, + }, + { + label: "GitHub PAT", + re: /\bghp_[A-Za-z0-9]{30,}\b|\bgithub_pat_[A-Za-z0-9_]{40,}\b/, + }, + { + label: "GitHub OAuth token", + re: /\bgho_[A-Za-z0-9]{30,}\b/, + }, + { + label: "OpenAI / Anthropic-style API key", + re: /\bsk-(?:ant-)?[A-Za-z0-9_-]{20,}\b/, + }, + { + label: "Slack token", + re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/, + }, + { + label: "AWS access key", + re: /\b(AKIA|ASIA)[A-Z0-9]{16}\b/, + }, + { + label: "Stripe secret key", + re: /\b(sk|rk)_(test|live)_[A-Za-z0-9]{24,}\b/, + }, + { + label: "Google API key", + re: /\bAIza[0-9A-Za-z_-]{35}\b/, + }, +]; + +// rsync exclude rules — written to an exclude-from file. Filename-based and +// extension-based; the post-rsync content scan catches inline secrets. +const RSYNC_EXCLUDES = [ + // Spawn / VCS / dependencies + ".git", + ".gitignore", + "node_modules", + ".spawnrc", + // Env / dotenv + ".env", + ".env.*", + "*.env", + ".direnv", + // SSH / GPG / netrc + ".ssh", + "ssh", + ".gnupg", + ".netrc", + "id_rsa*", + "id_dsa*", + "id_ecdsa*", + "id_ed25519*", + "*.pem", + "*.key", + "*.p12", + "*.pfx", + "*.keystore", + "*.jks", + // Cloud-provider creds + ".aws", + ".azure", + ".config/gcloud", + ".config/doctl", + ".config/hcloud", + ".config/fly", + ".config/flyctl", + ".kube", + ".docker/config.json", + "credentials.json", + "service-account*.json", + "*credentials*.json", + "gcp-key.json", + // Package-registry creds + ".npmrc", + ".yarnrc", + ".pypirc", + // Terraform / Pulumi state + ".terraform", + "terraform.tfstate*", + ".terraformrc", + ".pulumi", + // Build artifacts (large + uninteresting) + "dist", + "build", + ".next", + ".nuxt", + ".turbo", + ".cache", + "coverage", + "__pycache__", + "*.pyc", + ".venv", + "venv", + // Editors / OS noise + ".idea", + ".DS_Store", + "Thumbs.db", + "*.log", + "*.swp", +]; + +// Schema for parsing MCP server configs from Claude/Cursor settings +const McpEntrySchema = v.object({ + command: v.string(), + args: v.array(v.string()), + env: v.optional(v.record(v.string(), v.string())), +}); + +// JSON shapes we accept: either { mcpServers: {...} } or { ...directly... } +const McpSettingsSchema = v.object({ + mcpServers: v.optional(v.record(v.string(), McpEntrySchema)), +}); +const McpRootSchema = v.record(v.string(), McpEntrySchema); + +// ── Remote scan ──────────────────────────────────────────────────────────── +// One SSH call probes everything. We emit framed sections separated by a +// distinctive marker so locally we can split on it without escaping JSON. +const FRAME = "===SPAWN_EXPORT_FRAME==="; + +function buildProbeScript(): string { + const cliBins = CLI_PROBES.map((c) => c.bin).join(" "); + // statusCmds keyed by bin via case statement (avoids quoting issues) + const cliCases = CLI_PROBES.map((c) => ` ${c.bin}) STATUS_CMD=${shellSqEscape(c.statusCmd)};;`).join("\n"); + const mcpReads = MCP_SOURCES.map( + (s) => + ` printf '%s\\nMCP_PATH=%s\\nMCP_FORMAT=%s\\n' "${FRAME}" "${s.path}" "${s.format}"\n cat "${s.path}" 2>/dev/null || true`, + ).join("\n"); + + return [ + "set +e", + "echo '" + FRAME + "'", + `for bin in ${cliBins}; do`, + ` if command -v "$bin" >/dev/null 2>&1; then`, + ` case "$bin" in`, + cliCases, + ` *) STATUS_CMD="echo present";;`, + " esac", + ` printf 'CLI_BIN=%s\\n' "$bin"`, + ` out=$(eval "$STATUS_CMD" 2>&1 || true)`, + ` printf '%s\\n' "$out" | head -c 4000`, + ` printf '\\n${FRAME}\\n'`, + " fi", + "done", + mcpReads, + `printf '\\n${FRAME}END\\n'`, + ].join("\n"); +} + +/** Single-quote-escape for a value going into a bash single-quoted literal. */ +function shellSqEscape(s: string): string { + return `'${s.replace(/'/g, "'\\''")}'`; +} + +interface RemoteScan { + clis: Map; + mcps: Array<{ + path: string; + format: McpFormat; + content: string; + }>; +} + +function parseProbeOutput(raw: string): RemoteScan { + const result: RemoteScan = { + clis: new Map(), + mcps: [], + }; + const sections = raw.split(`${FRAME}\n`).map((s) => s.replace(/\n?===SPAWN_EXPORT_FRAME===END\n?$/, "")); + for (const section of sections) { + if (!section.trim()) { + continue; + } + const cliMatch = section.match(/^CLI_BIN=([^\n]+)\n([\s\S]*)$/); + if (cliMatch) { + result.clis.set(cliMatch[1].trim(), cliMatch[2]); + continue; + } + const mcpMatch = section.match(/^MCP_PATH=([^\n]+)\nMCP_FORMAT=([^\n]+)\n([\s\S]*)$/); + if (mcpMatch) { + const content = mcpMatch[3]; + if (content.trim()) { + const fmt = mcpMatch[2].trim(); + if (fmt === "json-mcpservers" || fmt === "json-root" || fmt === "codex-toml") { + result.mcps.push({ + path: mcpMatch[1].trim(), + format: fmt, + content, + }); + } + } + } + } + return result; +} + +/** Run one SSH command on the VM and return stdout. */ +function sshCapture(ip: string, user: string, cmd: string, keyOpts: string[]): string { + const result = spawnSync( + "ssh", + [ + ...SSH_BASE_OPTS, + ...keyOpts, + `${user}@${ip}`, + "--", + cmd, + ], + { + encoding: "utf8", + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + stdio: [ + "pipe", + "pipe", + "pipe", + ], + }, + ); + return result.stdout ?? ""; +} + +// ── MCP parsing ──────────────────────────────────────────────────────────── +type GenericMcp = NonNullable[number]; + +function entriesFromJsonMcpServers(content: string): Record> { + const parsed = parseJsonWith(content, McpSettingsSchema); + return parsed?.mcpServers ?? {}; +} + +function entriesFromJsonRoot(content: string): Record> { + // Some configs put MCP entries at the root. Try both shapes. + const asRoot = parseJsonWith(content, McpRootSchema); + if (asRoot && Object.keys(asRoot).length > 0) { + return asRoot; + } + return entriesFromJsonMcpServers(content); +} + +/** + * Parse Codex-style TOML where MCP servers live in `[mcp_servers.NAME]` + * blocks. We only handle the subset Codex emits: command (string), + * args (string array, single-line), and a nested [mcp_servers.NAME.env] + * block with KEY = "VALUE" pairs. + */ +function entriesFromCodexToml(content: string): Record> { + const result: Record> = {}; + let currentName: string | null = null; + let currentEntry: { + command?: string; + args?: string[]; + env?: Record; + } | null = null; + let inEnv = false; + + const lines = content.split("\n"); + for (const rawLine of lines) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) { + continue; + } + const sectionMatch = line.match(/^\[mcp_servers\.([^.\]]+)(?:\.(env))?\]$/); + if (sectionMatch) { + // Flush previous entry if complete + if (currentName && currentEntry?.command && currentEntry.args) { + result[currentName] = { + command: currentEntry.command, + args: currentEntry.args, + ...(currentEntry.env + ? { + env: currentEntry.env, + } + : {}), + }; + } + const newName = sectionMatch[1]; + if (sectionMatch[2] === "env" && currentName === newName) { + inEnv = true; + } else { + currentName = newName; + currentEntry = result[newName] + ? { + command: result[newName].command, + args: result[newName].args, + env: result[newName].env, + } + : {}; + inEnv = false; + } + continue; + } + if (line.startsWith("[")) { + currentName = null; + currentEntry = null; + inEnv = false; + continue; + } + const kv = line.match(/^(\w+)\s*=\s*(.+)$/); + if (!kv || !currentEntry) { + continue; + } + const key = kv[1]; + const val = kv[2].trim(); + if (inEnv) { + const sv = parseTomlString(val); + if (sv !== null) { + currentEntry.env = currentEntry.env ?? {}; + currentEntry.env[key] = sv; + } + continue; + } + if (key === "command") { + const sv = parseTomlString(val); + if (sv !== null) { + currentEntry.command = sv; + } + } else if (key === "args") { + const arr = parseTomlStringArray(val); + if (arr) { + currentEntry.args = arr; + } + } + } + if (currentName && currentEntry?.command && currentEntry.args) { + result[currentName] = { + command: currentEntry.command, + args: currentEntry.args, + ...(currentEntry.env + ? { + env: currentEntry.env, + } + : {}), + }; + } + return result; +} + +function parseTomlString(raw: string): string | null { + if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) { + return raw.slice(1, -1); + } + return null; +} + +function parseTomlStringArray(raw: string): string[] | null { + if (!raw.startsWith("[") || !raw.endsWith("]")) { + return null; + } + const inner = raw.slice(1, -1).trim(); + if (!inner) { + return []; + } + // Naive split on commas outside quotes — sufficient for Codex output. + const items: string[] = []; + let buf = ""; + let inDq = false; + for (const ch of inner) { + if (ch === '"') { + inDq = !inDq; + buf += ch; + continue; + } + if (ch === "," && !inDq) { + items.push(buf.trim()); + buf = ""; + continue; + } + buf += ch; + } + if (buf.trim()) { + items.push(buf.trim()); + } + const parsed: string[] = []; + for (const item of items) { + const sv = parseTomlString(item); + if (sv === null) { + return null; + } + parsed.push(sv); + } + return parsed; +} + +function dedupeMcpServers(scans: RemoteScan["mcps"]): GenericMcp[] { + const byName = new Map(); + for (const src of scans) { + let entries: Record> = {}; + if (src.format === "json-mcpservers") { + entries = entriesFromJsonMcpServers(src.content); + } else if (src.format === "json-root") { + entries = entriesFromJsonRoot(src.content); + } else if (src.format === "codex-toml") { + entries = entriesFromCodexToml(src.content); + } + for (const [name, cfg] of Object.entries(entries)) { + if (byName.has(name)) { + continue; + } + const entry: GenericMcp = { + name, + command: cfg.command, + args: cfg.args, + }; + if (cfg.env) { + // Replace actual values with ${NAME} placeholders — never export secrets + const placeholders: Record = {}; + for (const k of Object.keys(cfg.env)) { + placeholders[k] = `\${${k}}`; + } + entry.env = placeholders; + } + byName.set(name, entry); + } + } + return Array.from(byName.values()); +} + +// ── CLI status interpretation ────────────────────────────────────────────── +function buildSetupFromCliScan(clis: Map): { + setup: NonNullable; + steps: string[]; +} { + const setup: NonNullable = []; + const steps: string[] = []; + for (const probe of CLI_PROBES) { + const out = clis.get(probe.bin); + if (out === undefined) { + continue; + } + const authed = probe.markers.some((re) => re.test(out)); + if (!authed) { + continue; + } + if (probe.bin === "gh") { + // Spawn already has a built-in github step; prefer it over a custom auth flow. + steps.push("github"); + continue; + } + setup.push({ + type: "cli_auth", + name: probe.name, + command: probe.authCmd, + description: `Authenticate with ${probe.name}`, + }); + } + return { + setup, + steps, + }; +} + +// ── Secret content scan over the staged repo ─────────────────────────────── +const TEXT_FILE_MAX = 1 * 1024 * 1024; // skip blobs > 1 MB + +function isLikelyTextFile(path: string): boolean { + const bin = + /\.(png|jpe?g|gif|webp|ico|pdf|zip|gz|tgz|tar|woff2?|ttf|otf|eot|mp[34]|wav|ogg|mov|webm|exe|dll|so|dylib|bin|class|jar|wasm)$/i; + return !bin.test(path); +} + +function walkFiles(root: string): string[] { + const out: string[] = []; + const stack: string[] = [ + root, + ]; + while (stack.length) { + const dir = stack.pop(); + if (dir === undefined) { + continue; + } + const entries = readdirSync(dir, { + withFileTypes: true, + }); + for (const e of entries) { + const full = join(dir, e.name); + if (e.isDirectory()) { + if (e.name === ".git") { + continue; + } + stack.push(full); + } else if (e.isFile()) { + out.push(full); + } + } + } + return out; +} + +interface SecretHit { + relativePath: string; + label: string; +} + +function scanForSecrets(rootDir: string): SecretHit[] { + const hits: SecretHit[] = []; + const files = walkFiles(rootDir); + for (const file of files) { + if (!isLikelyTextFile(file)) { + continue; + } + const sizeResult = tryCatch(() => statSync(file).size); + if (!sizeResult.ok || sizeResult.data > TEXT_FILE_MAX) { + continue; + } + const readResult = tryCatch(() => + readFileSync(file, { + encoding: "utf8", + }), + ); + if (!readResult.ok) { + continue; + } + const content = readResult.data; + for (const pat of SECRET_PATTERNS) { + if (pat.re.test(content)) { + hits.push({ + relativePath: relative(rootDir, file), + label: pat.label, + }); + break; + } + } + } + return hits; +} + +// ── Main command ─────────────────────────────────────────────────────────── +export async function cmdExport(): Promise { + if (!isInteractiveTTY()) { + console.error(pc.red("Error: spawn export requires an interactive terminal")); + process.exit(1); + } + + // 1. Check for gh CLI + const ghResult = tryCatch(() => + execSync("gh auth status 2>&1", { + encoding: "utf8", + }), + ); + if (!ghResult.ok) { + const ghWhich = tryCatch(() => + execSync("which gh 2>/dev/null", { + encoding: "utf8", + }), + ); + if (!ghWhich.ok) { + p.log.error("GitHub CLI (gh) is not installed."); + p.log.info(`Install it: ${pc.cyan("https://cli.github.com/")}`); + process.exit(1); + } + p.log.error("GitHub CLI is not authenticated."); + p.log.info(`Run: ${pc.cyan("gh auth login")}`); + process.exit(1); + } + + // 2. Show spawn history picker + const records = loadHistory().filter((r) => r.connection && !r.connection.deleted); + if (records.length === 0) { + p.log.error("No active spawns found. Start one first with 'spawn '."); + process.exit(1); + } + + const manifestResult = await asyncTryCatch(() => loadManifest()); + const manifest = manifestResult.ok ? manifestResult.data : null; + + const options = records.map((r) => ({ + value: r, + label: buildRecordLabel(r), + hint: buildRecordSubtitle(r, manifest), + })); + + const selected = await p.select({ + message: "Which spawn do you want to export?", + options, + }); + + if (p.isCancel(selected)) { + handleCancel(); + return; + } + + const record = selected; + const conn = record.connection; + if (!conn) { + p.log.error("Selected spawn has no connection info."); + return; + } + + // Validate connection fields + const validResult = tryCatch(() => { + validateIdentifier(record.agent, "Agent name"); + validateConnectionIP(conn.ip); + validateUsername(conn.user); + }); + if (!validResult.ok) { + p.log.error("Invalid connection data in spawn history."); + return; + } + + // 3. SSH in and run the unified probe + p.log.step(`Scanning ${pc.bold(record.agent)} on ${pc.bold(conn.ip)}...`); + const keys = await ensureSshKeys(); + const keyOpts = getSshKeyOpts(keys); + const probeOutput = sshCapture(conn.ip, conn.user, buildProbeScript(), keyOpts); + const scan = parseProbeOutput(probeOutput); + + const mcpServers = dedupeMcpServers(scan.mcps); + const { setup, steps } = buildSetupFromCliScan(scan.clis); + steps.push("auto-update"); + + if (mcpServers.length > 0) { + p.log.info(`MCP servers found: ${mcpServers.map((m) => m.name).join(", ")}`); + } + if (setup.length > 0) { + p.log.info(`Authenticated CLIs: ${setup.map((s) => s.name).join(", ")}`); + } + + // 4. Prompt for template details + const repoName = await p.text({ + message: "Repository name for this template:", + placeholder: record.name ?? `${record.agent}-template`, + validate: (val) => { + if (!val || val.trim() === "") { + return "Name is required"; + } + if (!/^[a-zA-Z0-9._-]+$/.test(val)) { + return "Only alphanumeric, dots, hyphens, underscores"; + } + return undefined; + }, + }); + + if (p.isCancel(repoName)) { + handleCancel(); + return; + } + + const description = await p.text({ + message: "Description (optional):", + placeholder: `${record.agent} agent template`, + }); + + if (p.isCancel(description)) { + handleCancel(); + return; + } + + const visibility = await p.select({ + message: "Repository visibility:", + options: [ + { + value: "private", + label: "Private (recommended)", + }, + { + value: "public", + label: "Public", + }, + ], + initialValue: "private", + }); + if (p.isCancel(visibility)) { + handleCancel(); + return; + } + + // 5. Generate spawn.md (steps go in the CLI command, not in spawn.md) + const config: SpawnMdConfig = { + name: repoName, + description: isString(description) ? description : undefined, + setup: setup.length > 0 ? setup : undefined, + mcp_servers: mcpServers.length > 0 ? mcpServers : undefined, + }; + + const spawnMdContent = generateSpawnMd(config, `# ${repoName}\n\n${isString(description) ? description : ""}`); + + // 6. Stage the working folder + spawn.md + const tmpDir = mkdtempSync(join(tmpdir(), "spawn-export-")); + + // Download project files from VM if they exist + p.log.step("Downloading working folder from VM..."); + const hasProject = sshCapture(conn.ip, conn.user, "test -d ~/project && echo yes || echo no", keyOpts).trim(); + if (hasProject === "yes") { + const excludeFile = join(tmpDir, ".spawn-rsync-exclude"); + writeFileSync(excludeFile, RSYNC_EXCLUDES.join("\n")); + const sshArgs = [ + ...SSH_BASE_OPTS, + ...keyOpts, + ].join(" "); + const rsyncResult = tryCatch(() => + execSync( + `rsync -az --exclude-from='${excludeFile}' -e "ssh ${sshArgs}" '${conn.user}@${conn.ip}:~/project/' '${tmpDir}/'`, + { + encoding: "utf8", + timeout: 120_000, + }, + ), + ); + tryCatch(() => unlinkSync(excludeFile)); + if (!rsyncResult.ok) { + p.log.warn("Could not download project files — creating template with spawn.md only"); + } + } else { + p.log.info("No ~/project directory on VM — exporting setup only"); + } + + // Write spawn.md AFTER rsync so it always wins over any existing one in the project. + writeFileSync(join(tmpDir, "spawn.md"), spawnMdContent); + writeFileSync( + join(tmpDir, ".gitignore"), + [ + ".env", + ".env.*", + ".spawnrc", + "node_modules/", + "/etc/spawn/", + "*.pem", + "*.key", + "id_rsa*", + "credentials.json", + "", + ].join("\n"), + ); + + // 7. Scan staged files for inline secrets + p.log.step("Scanning staged files for secrets..."); + const hits = scanForSecrets(tmpDir); + if (hits.length > 0) { + p.log.warn(`Found ${hits.length} file(s) that look like they contain secrets:`); + for (const hit of hits.slice(0, 20)) { + console.error(` ${pc.yellow("•")} ${pc.bold(hit.relativePath)} ${pc.dim(`(${hit.label})`)}`); + } + if (hits.length > 20) { + console.error(pc.dim(` ... and ${hits.length - 20} more`)); + } + const action = await p.select({ + message: "How do you want to handle these?", + options: [ + { + value: "remove", + label: "Remove flagged files and continue", + }, + { + value: "keep", + label: "Keep them (I checked, they're safe)", + }, + { + value: "abort", + label: "Abort export", + }, + ], + initialValue: "remove", + }); + if (p.isCancel(action) || action === "abort") { + handleCancel(); + return; + } + if (action === "remove") { + for (const hit of hits) { + tryCatch(() => unlinkSync(join(tmpDir, hit.relativePath))); + } + p.log.info(`Removed ${hits.length} flagged file(s)`); + } + } + + // 8. Show preview of generated spawn.md + p.log.info("Generated spawn.md:"); + console.error(pc.dim("─".repeat(40))); + console.error(pc.dim(spawnMdContent)); + console.error(pc.dim("─".repeat(40))); + + const confirm = await p.confirm({ + message: `Create ${visibility} GitHub repo "${repoName}" and push?`, + initialValue: true, + }); + + if (p.isCancel(confirm) || !confirm) { + handleCancel(); + return; + } + + // 9. Init git, commit, create repo, push + p.log.step("Creating GitHub repository..."); + const visibilityFlag = visibility === "public" ? "--public" : "--private"; + const repoResult = tryCatch(() => { + execSync(`cd ${tmpDir} && git init -q && git add -A && git commit -q -m "Initial template"`, { + encoding: "utf8", + }); + execSync(`cd ${tmpDir} && gh repo create ${repoName} ${visibilityFlag} --source=. --push`, { + encoding: "utf8", + stdio: [ + "pipe", + "pipe", + "inherit", + ], + }); + }); + if (!repoResult.ok) { + p.log.error("Failed to create GitHub repo."); + p.log.info(`You can manually push the template from: ${tmpDir}`); + return; + } + + // 10. Get the repo slug (user/repo) + let repoSlug = repoName; + const ghUserResult = tryCatch(() => + execSync("gh api user --jq .login", { + encoding: "utf8", + }).trim(), + ); + if (ghUserResult.ok && ghUserResult.data) { + repoSlug = `${ghUserResult.data}/${repoName}`; + } + + // 11. Print the shareable command (steps baked into the command, not spawn.md) + const stepsArg = steps.length > 0 ? ` --steps ${steps.join(",")}` : ""; + console.error(); + p.log.success("Template exported!"); + console.error(); + console.error(" Share this command to replicate your setup:"); + console.error(); + console.error(` ${pc.cyan(`spawn ${record.agent} ${record.cloud} --repo ${repoSlug}${stepsArg}`)}`); + console.error(); +} + +// Internal exports for unit tests +export const __testing = { + parseProbeOutput, + entriesFromCodexToml, + entriesFromJsonMcpServers, + entriesFromJsonRoot, + dedupeMcpServers, + buildSetupFromCliScan, + scanForSecrets, + SECRET_PATTERNS, + RSYNC_EXCLUDES, + CLI_PROBES, + MCP_SOURCES, + buildProbeScript, +}; diff --git a/packages/cli/src/commands/help.ts b/packages/cli/src/commands/help.ts index 067101a8e..4fef8490f 100644 --- a/packages/cli/src/commands/help.ts +++ b/packages/cli/src/commands/help.ts @@ -22,6 +22,9 @@ function getHelpUsageSection(): string { Load all options from a JSON config file spawn --steps Comma-separated setup steps to enable + spawn --repo + Clone a template repo and apply spawn.md setup + spawn export Export a running spawn as a shareable template spawn Interactive cloud picker for agent spawn Show available agents for cloud spawn list Browse and rerun previous spawns (aliases: ls, history) @@ -87,7 +90,10 @@ function getHelpExamplesSection(): string { spawn list ${pc.dim("# Browse history and pick one to rerun")} spawn list codex ${pc.dim("# Filter history by agent name")} spawn last ${pc.dim("# Instantly rerun the most recent spawn")} - spawn matrix ${pc.dim("# See the full agent x cloud matrix")}`; + spawn matrix ${pc.dim("# See the full agent x cloud matrix")} + spawn claude sprite --repo user/my-template + ${pc.dim("# Clone template and auto-setup agent")} + spawn export ${pc.dim("# Export current spawn as a template")}`; } function getHelpAuthSection(): string { diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 309a56362..9c80bb02c 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -2,6 +2,8 @@ // delete.ts — cmdDelete, cascadeDelete export { cascadeDelete, cmdDelete } from "./delete.js"; +// export.ts — cmdExport (agent template export) +export { cmdExport } from "./export.js"; // feedback.ts — cmdFeedback export { cmdFeedback } from "./feedback.js"; // fix.ts — cmdFix, fixSpawn diff --git a/packages/cli/src/flags.ts b/packages/cli/src/flags.ts index 71b46181b..e2be3de2c 100644 --- a/packages/cli/src/flags.ts +++ b/packages/cli/src/flags.ts @@ -35,6 +35,7 @@ export const KNOWN_FLAGS = new Set([ "-m", "--config", "--steps", + "--repo", "--fast", "--flat", "--user", diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 7234e5168..8f0a146e9 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -13,6 +13,7 @@ import { cmdCloudInfo, cmdClouds, cmdDelete, + cmdExport, cmdFeedback, cmdFix, cmdHelp, @@ -146,6 +147,7 @@ function checkUnknownFlags(args: string[]): void { console.error(` ${pc.cyan("--reauth")} Force re-prompting for cloud credentials`); console.error(` ${pc.cyan("--config ")} Load config from JSON file`); console.error(` ${pc.cyan("--steps ")} Comma-separated setup steps to enable`); + console.error(` ${pc.cyan("--repo ")} Clone a template repo and apply spawn.md`); console.error(` ${pc.cyan("--beta tarball")} Use pre-built tarball for agent install (repeatable)`); console.error(` ${pc.cyan("--beta images")} Use pre-built DO marketplace images (faster boot)`); console.error(` ${pc.cyan("--beta parallel")} Parallelize server boot with setup prompts`); @@ -778,6 +780,14 @@ async function dispatchCommand( await cmdPullHistory(); return; } + if (cmd === "export") { + if (hasTrailingHelpFlag(filteredArgs)) { + cmdHelp(); + return; + } + await cmdExport(); + return; + } if (LIST_COMMANDS.has(cmd)) { // Handle "history export" subcommand if (cmd === "history" && filteredArgs[1] === "export") { @@ -1047,6 +1057,19 @@ async function main(): Promise { process.env.SPAWN_NAME = nameFlag; } + // Extract --repo flag — clone a template repo and apply spawn.md + const [repoFlag, repoFilteredArgs] = extractFlagValue( + filteredArgs, + [ + "--repo", + ], + 'spawn --repo "user/my-template"', + ); + filteredArgs.splice(0, filteredArgs.length, ...repoFilteredArgs); + if (repoFlag) { + process.env.SPAWN_REPO = repoFlag; + } + // Extract --zone / --region flag (maps to cloud-specific env vars) const [zoneFlag, zoneFilteredArgs] = extractFlagValue( filteredArgs, diff --git a/packages/cli/src/shared/orchestrate.ts b/packages/cli/src/shared/orchestrate.ts index a10f69f01..8373f6e8f 100644 --- a/packages/cli/src/shared/orchestrate.ts +++ b/packages/cli/src/shared/orchestrate.ts @@ -617,7 +617,36 @@ async function postInstall( spawnId: string, _options?: OrchestrationOptions, ): Promise { - // Parse enabled setup steps + // ── Repo clone + spawn.md (--repo mode) ──────────────────────────────── + // Built-in steps (github, auto-update, etc.) come from the CLI --steps + // flag, not from spawn.md. spawn.md only handles custom setup (OAuth, + // MCP servers, setup commands). + let spawnMdConfig: import("./spawn-md.js").SpawnMdConfig | null = null; + let repoCloned = false; + const repoSlug = process.env.SPAWN_REPO; + if (repoSlug && cloud.cloudName !== "local") { + // Validate slug format (user/repo, no path traversal) + if (!/^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/.test(repoSlug)) { + logWarn(`Invalid repo slug: ${repoSlug} — skipping repo clone`); + } else { + logStep("Cloning template repository..."); + const cloneResult = await asyncTryCatch(() => + cloud.runner.runServer(`git clone https://github.com/${repoSlug}.git ~/project`), + ); + if (!cloneResult.ok) { + logWarn(`Repo clone failed (${getErrorMessage(cloneResult.error)}) — continuing without template`); + } else { + repoCloned = true; + const { readRemoteSpawnMd } = await import("./spawn-md.js"); + spawnMdConfig = await readRemoteSpawnMd(cloud.runner); + if (spawnMdConfig) { + logInfo(`Template loaded: ${spawnMdConfig.name ?? repoSlug}`); + } + } + } + } + + // Parse enabled setup steps (from --steps CLI flag) let enabledSteps: Set | undefined; const stepsEnv = process.env.SPAWN_ENABLED_STEPS; const isHeadless = process.env.SPAWN_HEADLESS === "1"; @@ -759,6 +788,12 @@ async function postInstall( } } + // Apply spawn.md custom setup (after built-in steps, before pre-launch) + if (spawnMdConfig) { + const { applySpawnMdSetup } = await import("./spawn-md.js"); + await applySpawnMdSetup(cloud.runner, spawnMdConfig, agentName); + } + // Pre-launch hooks (retry loop) if (agent.preLaunch) { for (;;) { @@ -872,7 +907,12 @@ async function postInstall( headless: process.env.SPAWN_HEADLESS === "1", }); - const launchCmd = agent.launchCmd(); + // When --repo cloned successfully, launch the agent inside the cloned + // project directory. Gate on the actual clone outcome rather than the flag + // so an invalid slug or clone failure doesn't leave the agent trying to cd + // into a non-existent dir. + const baseLaunchCmd = agent.launchCmd(); + const launchCmd = repoCloned ? `cd ~/project && ${baseLaunchCmd}` : baseLaunchCmd; saveLaunchCmd(launchCmd, spawnId); // In headless mode, provisioning is done — skip the interactive session. diff --git a/packages/cli/src/shared/skills.ts b/packages/cli/src/shared/skills.ts index 2884348aa..5c210f2e1 100644 --- a/packages/cli/src/shared/skills.ts +++ b/packages/cli/src/shared/skills.ts @@ -204,7 +204,10 @@ export async function installSkills( } /** Merge MCP servers into Claude Code's ~/.claude/settings.json. */ -async function installClaudeMcpServers(runner: CloudRunner, servers: Record): Promise { +export async function installClaudeMcpServers( + runner: CloudRunner, + servers: Record, +): Promise { const tmpLocal = join(getTmpDir(), `claude_settings_${Date.now()}.json`); const dlResult = await asyncTryCatch(() => runner.downloadFile("$HOME/.claude/settings.json", tmpLocal)); @@ -226,7 +229,10 @@ async function installClaudeMcpServers(runner: CloudRunner, servers: Record): Promise { +export async function installCursorMcpServers( + runner: CloudRunner, + servers: Record, +): Promise { const tmpLocal = join(getTmpDir(), `cursor_mcp_${Date.now()}.json`); const dlResult = await asyncTryCatch(() => runner.downloadFile("$HOME/.cursor/mcp.json", tmpLocal)); @@ -248,7 +254,7 @@ async function installCursorMcpServers(runner: CloudRunner, servers: Record, @@ -263,6 +269,60 @@ async function installGenericMcpServers( await uploadConfigFile(runner, config, `$HOME/.${agentName}/mcp.json`); } +/** + * Append MCP server entries to Codex's ~/.codex/config.toml under + * [mcp_servers.NAME] sections. Existing sections with the same name are + * left untouched (we don't try to merge mid-file); new ones are appended. + */ +export async function installCodexMcpServers( + runner: CloudRunner, + servers: Record, +): Promise { + const tmpLocal = join(getTmpDir(), `codex_config_${Date.now()}.toml`); + const dlResult = await asyncTryCatch(() => runner.downloadFile("$HOME/.codex/config.toml", tmpLocal)); + + let existing = ""; + if (dlResult.ok) { + const readResult = tryCatch(() => readFileSync(tmpLocal, "utf-8")); + if (readResult.ok) { + existing = readResult.data; + } + } + + const existingNames = new Set(); + for (const m of existing.matchAll(/^\[mcp_servers\.([^.\]]+)\]/gm)) { + existingNames.add(m[1]); + } + + const lines: string[] = []; + for (const [name, cfg] of Object.entries(servers)) { + if (existingNames.has(name)) { + continue; + } + lines.push(""); + lines.push(`[mcp_servers.${name}]`); + lines.push(`command = ${tomlString(cfg.command)}`); + lines.push(`args = [${cfg.args.map((a) => tomlString(a)).join(", ")}]`); + if (cfg.env) { + lines.push(`[mcp_servers.${name}.env]`); + for (const [k, val] of Object.entries(cfg.env)) { + lines.push(`${k} = ${tomlString(val)}`); + } + } + } + + if (lines.length === 0) { + return; + } + + const merged = `${existing.replace(/\n+$/, "")}\n${lines.join("\n")}\n`; + await uploadConfigFile(runner, merged, "$HOME/.codex/config.toml"); +} + +function tomlString(s: string): string { + return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} + /** Inject an instruction skill (SKILL.md) onto the remote VM. */ async function injectInstructionSkill( runner: CloudRunner, diff --git a/packages/cli/src/shared/spawn-md.ts b/packages/cli/src/shared/spawn-md.ts new file mode 100644 index 000000000..1893b5073 --- /dev/null +++ b/packages/cli/src/shared/spawn-md.ts @@ -0,0 +1,588 @@ +// shared/spawn-md.ts — Parse, generate, and apply spawn.md template files +// +// spawn.md lives at the root of a user's repo and declares the "recipe" for +// setting up an agent: built-in steps, custom auth flows, MCP servers, and +// setup commands. It never contains actual secrets. + +import type { CloudRunner } from "./agent-setup.js"; + +import * as v from "valibot"; +import { asyncTryCatch, tryCatch } from "./result.js"; +import { logInfo, logStep, logWarn, openBrowser } from "./ui.js"; + +// ── YAML frontmatter parsing ─────────────────────────────────────────────── +// spawn.md uses a subset of YAML in the frontmatter (between --- delimiters). +// We parse it with a minimal hand-rolled parser to avoid adding a YAML dep. + +/** Split spawn.md content into { frontmatter, body } */ +function splitFrontmatter(content: string): { + frontmatter: string; + body: string; +} { + const trimmed = content.trimStart(); + if (!trimmed.startsWith("---")) { + return { + frontmatter: "", + body: content, + }; + } + const endIdx = trimmed.indexOf("\n---", 3); + if (endIdx === -1) { + return { + frontmatter: "", + body: content, + }; + } + const frontmatter = trimmed.slice(3, endIdx).trim(); + const body = trimmed.slice(endIdx + 4).trim(); + return { + frontmatter, + body, + }; +} + +function parseYamlScalar(s: string): string | number | boolean { + if (s === "true") { + return true; + } + if (s === "false") { + return false; + } + if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) { + return s.slice(1, -1); + } + const num = Number(s); + if (!Number.isNaN(num) && s !== "") { + return num; + } + return s; +} + +/** Helper to treat target as a record and set a key */ +function setOnRecord(target: Record | unknown[], key: string, val: unknown): void { + if (Array.isArray(target)) { + return; + } + target[key] = val; +} + +/** Helper to get from a record by key */ +function getFromRecord(target: Record | unknown[], key: string): unknown { + if (Array.isArray(target)) { + return undefined; + } + return target[key]; +} + +/** + * Minimal YAML-to-JSON parser for spawn.md frontmatter. + * Handles: scalars, arrays of scalars, arrays of objects, nested objects. + * Does NOT handle: anchors, tags, multi-line strings. Intentionally simple. + */ +function parseYamlFrontmatter(yaml: string): Record { + const lines = yaml.split("\n"); + const result: Record = {}; + + type Frame = { + indent: number; + target: Record | unknown[]; + key?: string; + }; + const stack: Frame[] = [ + { + indent: -1, + target: result, + }, + ]; + + const currentFrame = (): Frame => stack[stack.length - 1]; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trimStart(); + if (trimmed === "" || trimmed.startsWith("#")) { + continue; + } + + const indent = line.length - trimmed.length; + + // Pop stack to find the right nesting level + while (stack.length > 1 && indent <= currentFrame().indent) { + stack.pop(); + } + + // Array item: "- value" or "- key: value" + if (trimmed.startsWith("- ")) { + const itemContent = trimmed.slice(2).trim(); + const frame = currentFrame(); + let targetArray: unknown[] | null = null; + + if (Array.isArray(frame.target)) { + targetArray = frame.target; + } else if (frame.key) { + const existing = getFromRecord(frame.target, frame.key); + if (Array.isArray(existing)) { + targetArray = existing; + } + } + + if (!targetArray) { + continue; + } + + // Check if item is a key-value pair (object in array) + const colonIdx = itemContent.indexOf(":"); + if (colonIdx > 0 && !itemContent.startsWith("[") && !itemContent.startsWith('"')) { + const key = itemContent.slice(0, colonIdx).trim(); + const val = itemContent.slice(colonIdx + 1).trim(); + const obj: Record = {}; + obj[key] = parseYamlScalar(val); + targetArray.push(obj); + stack.push({ + indent: indent + 1, + target: obj, + }); + continue; + } + + // Flow sequence: [a, b, c] + if (itemContent.startsWith("[") && itemContent.endsWith("]")) { + const inner = itemContent.slice(1, -1); + targetArray.push(inner.split(",").map((s) => parseYamlScalar(s.trim()))); + continue; + } + + targetArray.push(parseYamlScalar(itemContent)); + continue; + } + + // Key-value pair: "key: value" + const colonIdx = trimmed.indexOf(":"); + if (colonIdx > 0) { + const key = trimmed.slice(0, colonIdx).trim(); + const rawVal = trimmed.slice(colonIdx + 1).trim(); + const frame = currentFrame(); + const target = frame.target; + + if (Array.isArray(target)) { + continue; + } + + if (rawVal === "" || rawVal === "|" || rawVal === ">") { + const nextLine = lines[i + 1]; + if (nextLine !== undefined) { + const nextTrimmed = nextLine.trimStart(); + if (nextTrimmed.startsWith("- ")) { + const arr: unknown[] = []; + setOnRecord(target, key, arr); + stack.push({ + indent, + target: arr, + key, + }); + continue; + } + const obj: Record = {}; + setOnRecord(target, key, obj); + stack.push({ + indent, + target: obj, + key, + }); + continue; + } + setOnRecord(target, key, ""); + continue; + } + + // Flow sequence: key: [a, b, c] + if (rawVal.startsWith("[") && rawVal.endsWith("]")) { + const inner = rawVal.slice(1, -1); + setOnRecord( + target, + key, + inner.split(",").map((s) => parseYamlScalar(s.trim())), + ); + continue; + } + + setOnRecord(target, key, parseYamlScalar(rawVal)); + } + } + + return result; +} + +// ── Valibot schemas ──────────────────────────────────────────────────────── + +const OAuthSetupSchema = v.object({ + type: v.literal("oauth"), + name: v.string(), + url: v.string(), + description: v.optional(v.string()), +}); + +const CliAuthSetupSchema = v.object({ + type: v.literal("cli_auth"), + name: v.string(), + command: v.string(), + description: v.optional(v.string()), +}); + +const ApiKeySetupSchema = v.object({ + type: v.literal("api_key"), + name: v.string(), + description: v.optional(v.string()), + guide_url: v.optional(v.string()), +}); + +const CommandSetupSchema = v.object({ + type: v.literal("command"), + name: v.optional(v.string()), + command: v.string(), + description: v.optional(v.string()), +}); + +const SetupStepSchema = v.union([ + OAuthSetupSchema, + CliAuthSetupSchema, + ApiKeySetupSchema, + CommandSetupSchema, +]); + +const McpServerEntrySchema = v.object({ + name: v.string(), + command: v.string(), + args: v.array(v.string()), + env: v.optional(v.record(v.string(), v.string())), +}); + +export const SpawnMdSchema = v.object({ + name: v.optional(v.string()), + description: v.optional(v.string()), + // Built-in steps (github, auto-update, etc.) go in the CLI --steps flag, + // not here. spawn.md only handles custom setup that Spawn doesn't know about. + setup: v.optional(v.array(SetupStepSchema)), + mcp_servers: v.optional(v.array(McpServerEntrySchema)), + setup_commands: v.optional(v.array(v.string())), +}); + +export type SpawnMdConfig = v.InferOutput; +type SetupStep = v.InferOutput; +type McpServerEntry = v.InferOutput; + +// ── Parsing ──────────────────────────────────────────────────────────────── + +/** Parse spawn.md content into a typed config. Returns null on parse failure. */ +export function parseSpawnMd(content: string): SpawnMdConfig | null { + const { frontmatter } = splitFrontmatter(content); + if (!frontmatter) { + return null; + } + + const raw = parseYamlFrontmatter(frontmatter); + const result = v.safeParse(SpawnMdSchema, raw); + if (!result.success) { + logWarn("spawn.md has invalid frontmatter — ignoring"); + return null; + } + return result.output; +} + +// ── Generation ───────────────────────────────────────────────────────────── + +/** Generate spawn.md content from a config */ +export function generateSpawnMd(config: SpawnMdConfig, body?: string): string { + const lines: string[] = [ + "---", + ]; + + if (config.name) { + lines.push(`name: ${config.name}`); + } + if (config.description) { + lines.push(`description: ${config.description}`); + } + + if (config.setup && config.setup.length > 0) { + lines.push("setup:"); + for (const step of config.setup) { + lines.push(` - type: ${step.type}`); + lines.push(` name: ${step.name ?? ""}`); + if ("url" in step) { + lines.push(` url: ${step.url}`); + } + if ("command" in step) { + lines.push(` command: ${step.command}`); + } + if (step.description) { + lines.push(` description: ${step.description}`); + } + if ("guide_url" in step && step.guide_url) { + lines.push(` guide_url: ${step.guide_url}`); + } + } + } + + if (config.mcp_servers && config.mcp_servers.length > 0) { + lines.push("mcp_servers:"); + for (const server of config.mcp_servers) { + lines.push(` - name: ${server.name}`); + lines.push(` command: ${server.command}`); + lines.push(` args: [${server.args.map((a) => `"${a}"`).join(", ")}]`); + if (server.env) { + lines.push(" env:"); + for (const [k, val] of Object.entries(server.env)) { + lines.push(` ${k}: "${val}"`); + } + } + } + } + + if (config.setup_commands && config.setup_commands.length > 0) { + lines.push("setup_commands:"); + for (const cmd of config.setup_commands) { + lines.push(` - ${cmd}`); + } + } + + lines.push("---"); + lines.push(""); + + if (body) { + lines.push(body); + } + + return lines.join("\n"); +} + +// ── Applying spawn.md on a VM ────────────────────────────────────────────── + +/** Read and parse spawn.md from a remote VM */ +export async function readRemoteSpawnMd(runner: CloudRunner): Promise { + const catResult = await captureCommand(runner, "cat ~/project/spawn.md 2>/dev/null"); + if (catResult) { + return parseSpawnMd(catResult); + } + return null; +} + +/** Run a command on the remote and capture its stdout */ +async function captureCommand(runner: CloudRunner, cmd: string): Promise { + const tmpFile = `/tmp/spawn-capture-${Date.now()}`; + const { readFileSync, unlinkSync } = await import("node:fs"); + const result = await asyncTryCatch(async () => { + await runner.runServer(`${cmd} > ${tmpFile} 2>/dev/null; true`); + await runner.downloadFile(tmpFile, tmpFile); + const content = readFileSync(tmpFile, "utf-8"); + const cleanupResult = tryCatch(() => unlinkSync(tmpFile)); + // ignore local cleanup failure + void cleanupResult; + await asyncTryCatch(() => runner.runServer(`rm -f ${tmpFile}`)); + return content || null; + }); + if (!result.ok) { + return null; + } + return result.data; +} + +/** + * Apply custom setup steps from spawn.md onto a running VM. + * Built-in `steps` (github, auto-update, etc.) are handled by the existing + * postInstall infrastructure — this function only handles the `setup` array, + * `mcp_servers`, and `setup_commands`. + */ +export async function applySpawnMdSetup(runner: CloudRunner, config: SpawnMdConfig, agentName: string): Promise { + if (config.setup && config.setup.length > 0) { + logStep("Running template setup steps..."); + for (const step of config.setup) { + await applySetupStep(runner, step); + } + } + + if (config.mcp_servers && config.mcp_servers.length > 0) { + logStep("Installing MCP servers from template..."); + await installMcpServersFromTemplate(runner, config.mcp_servers, agentName); + } + + if (config.setup_commands && config.setup_commands.length > 0) { + logStep("Running template setup commands..."); + for (const cmd of config.setup_commands) { + logInfo(` Running: ${cmd}`); + const cmdResult = await asyncTryCatch(() => runner.runServer(`cd ~/project 2>/dev/null; ${cmd}`)); + if (!cmdResult.ok) { + logWarn(` Setup command failed: ${cmd}`); + } + } + } +} + +async function applySetupStep(runner: CloudRunner, step: SetupStep): Promise { + switch (step.type) { + case "oauth": { + logInfo(` ${step.name}: Opening ${step.url}`); + if (step.description) { + logInfo(` ${step.description}`); + } + openBrowser(step.url); + logInfo(" Complete the OAuth flow in your browser, then press Enter to continue."); + await waitForEnter(); + break; + } + case "cli_auth": { + logInfo(` ${step.name}: Running ${step.command}`); + if (step.description) { + logInfo(` ${step.description}`); + } + const authResult = await asyncTryCatch(() => runner.runServer(step.command)); + if (authResult.ok) { + logInfo(` ${step.name} authenticated`); + } else { + logWarn(` ${step.name} auth failed — you can run it manually later: ${step.command}`); + } + break; + } + case "api_key": { + logInfo(` ${step.name}: API key required`); + if (step.description) { + logInfo(` ${step.description}`); + } + if (step.guide_url) { + logInfo(` Get your key: ${step.guide_url}`); + openBrowser(step.guide_url); + } + const value = await promptSecret(` Enter ${step.name}: `); + if (value) { + const escapedName = step.name.replace(/[^A-Za-z0-9_]/g, ""); + const b64Val = Buffer.from(value).toString("base64"); + await runner.runServer( + `mkdir -p /etc/spawn && printf 'export %s="%s"\\n' '${escapedName}' "$(echo '${b64Val}' | base64 -d)" >> /etc/spawn/secrets && chmod 600 /etc/spawn/secrets`, + ); + await runner.runServer( + `grep -q '/etc/spawn/secrets' ~/.bashrc 2>/dev/null || echo 'source /etc/spawn/secrets 2>/dev/null' >> ~/.bashrc`, + ); + logInfo(` ${step.name} saved`); + } else { + logWarn(` No value provided for ${step.name} — set it later in /etc/spawn/secrets`); + } + break; + } + case "command": { + const label = step.name ?? step.command; + logInfo(` Running: ${label}`); + if (step.description) { + logInfo(` ${step.description}`); + } + const runResult = await asyncTryCatch(() => runner.runServer(step.command)); + if (!runResult.ok) { + logWarn(` Command failed: ${step.command}`); + } + break; + } + } +} + +/** Install MCP servers from spawn.md template into agent config */ +async function installMcpServersFromTemplate( + runner: CloudRunner, + servers: McpServerEntry[], + agentName: string, +): Promise { + const record: Record< + string, + { + command: string; + args: string[]; + env?: Record; + } + > = {}; + for (const server of servers) { + record[server.name] = server.env + ? { + command: server.command, + args: server.args, + env: server.env, + } + : { + command: server.command, + args: server.args, + }; + } + + const { installClaudeMcpServers, installCursorMcpServers, installCodexMcpServers, installGenericMcpServers } = + await import("./skills.js"); + const installResult = await asyncTryCatch(async () => { + if (agentName === "claude") { + await installClaudeMcpServers(runner, record); + } else if (agentName === "cursor") { + await installCursorMcpServers(runner, record); + } else if (agentName === "codex") { + await installCodexMcpServers(runner, record); + } else { + await installGenericMcpServers(runner, agentName, record); + } + }); + if (installResult.ok) { + logInfo(` Installed ${servers.length} MCP server${servers.length > 1 ? "s" : ""}`); + } else { + logWarn(" MCP server installation failed — configure manually"); + } +} + +/** Wait for the user to press Enter */ +async function waitForEnter(): Promise { + return new Promise((resolve) => { + if (!process.stdin.isTTY) { + resolve(); + return; + } + const onData = (): void => { + process.stdin.removeListener("data", onData); + resolve(); + }; + process.stdin.once("data", onData); + }); +} + +/** Prompt for a secret value (no echo) */ +async function promptSecret(message: string): Promise { + process.stderr.write(message); + return new Promise((resolve) => { + if (!process.stdin.isTTY) { + resolve(""); + return; + } + let buf = ""; + const wasRaw = process.stdin.isRaw ?? false; + process.stdin.setRawMode(true); + process.stdin.resume(); + const onData = (data: Buffer): void => { + const ch = data.toString(); + if (ch === "\r" || ch === "\n") { + process.stdin.setRawMode(wasRaw); + process.stdin.pause(); + process.stdin.removeListener("data", onData); + process.stderr.write("\n"); + resolve(buf); + return; + } + if (ch === "\x03") { + process.stdin.setRawMode(wasRaw); + process.stdin.pause(); + process.stdin.removeListener("data", onData); + process.stderr.write("\n"); + resolve(""); + return; + } + if (ch === "\x7f" || ch === "\b") { + if (buf.length > 0) { + buf = buf.slice(0, -1); + } + return; + } + buf += ch; + }; + process.stdin.on("data", onData); + }); +}