From a4f8979dd6775782d6de5a3af5f7e6583a34fb88 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 May 2026 14:22:38 +0100 Subject: [PATCH 1/7] fix gh sync --- packages/cli/src/deploy/handler.ts | 2 +- packages/cli/src/pull/handler.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index 63bfaae66..52d399c01 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -55,7 +55,7 @@ async function deployHandler( ...options, force: true, endpoint: config.endpoint, - apiKey: config.apiKey ?? undefined, + // Do not pass explicit api key here - it breaks GH sync }, logger ); diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts index dadfc04a1..d42bb1200 100644 --- a/packages/cli/src/pull/handler.ts +++ b/packages/cli/src/pull/handler.ts @@ -42,8 +42,8 @@ async function pullHandler(options: PullOptions, logger: Logger) { project: options.projectId, force: true, endpoint: config.endpoint, - apiKey: config.apiKey ?? undefined, createCredentials: false, + // Do not pass explicit api key here - it breaks GH sync }, logger ); From 6191ee20d6554694ac201428be688ff8c0fe51d3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 May 2026 15:15:23 +0100 Subject: [PATCH 2/7] fix endpoint argument tracing and a little refactor to the v2 pull proxy --- packages/cli/src/deploy/handler.ts | 3 +- packages/cli/src/projects/fetch.ts | 7 ++-- packages/cli/src/pull/command.ts | 2 ++ packages/cli/src/pull/handler.ts | 50 +++++++++++++++----------- packages/cli/test/pull/handler.test.ts | 44 ++++++++++++++++++++++- 5 files changed, 82 insertions(+), 24 deletions(-) diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index 52d399c01..d71419db3 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -34,6 +34,7 @@ async function deployHandler( } try { + console.log({ options }); const config = mergeOverrides(await getConfig(options.configPath), options); const v2ConfigPath = path.join( @@ -55,7 +56,7 @@ async function deployHandler( ...options, force: true, endpoint: config.endpoint, - // Do not pass explicit api key here - it breaks GH sync + apiKey: config.apiKey ?? undefined, }, logger ); diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index b45ade597..e87f0b3fa 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -93,7 +93,7 @@ const fetchV1 = async (options: FetchOptions, logger: Logger) => { const config = loadAppAuthConfig(options, logger); const { data } = await fetchProject( - options.endpoint ?? localProject?.openfn?.endpoint!, + config.endpoint ?? localProject?.openfn?.endpoint!, config.apiKey, localProject?.uuid ?? options.project!, logger @@ -265,7 +265,10 @@ export async function fetchRemoteProject( ); } - const projectEndpoint = localProject?.openfn?.endpoint ?? config.endpoint; + // TODO this resolution is pretty awkward. The problem is we don't + // know if config.endpoint comes from an env var or explicit option + const projectEndpoint = + options.endpoint ?? localProject?.openfn?.endpoint ?? config.endpoint; const { data } = await fetchProject( projectEndpoint, diff --git a/packages/cli/src/pull/command.ts b/packages/cli/src/pull/command.ts index 314164623..dd5dd8034 100644 --- a/packages/cli/src/pull/command.ts +++ b/packages/cli/src/pull/command.ts @@ -8,8 +8,10 @@ import * as po from '../projects/options'; export type PullOptions = Required< Pick< Opts & POpts, + | 'apiKey' | 'beta' | 'command' + | 'endpoint' | 'log' | 'logJson' | 'statePath' diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts index d42bb1200..9a376789a 100644 --- a/packages/cli/src/pull/handler.ts +++ b/packages/cli/src/pull/handler.ts @@ -26,27 +26,9 @@ async function pullHandler(options: PullOptions, logger: Logger) { options.workspace || process.cwd(), 'openfn.yaml' ); + if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) { - // default endpoint to one from openfn.yaml - const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8')); - if (!config.endpoint && v2config?.project?.endpoint) { - config.endpoint = v2config.project.endpoint; - } - - logger.always( - 'Detected openfn.yaml file - switching to v2 pull (openfn project pull). Set PREFER_LEGACY_SYNC to disable this.' - ); - return beta( - { - ...options, - project: options.projectId, - force: true, - endpoint: config.endpoint, - createCredentials: false, - // Do not pass explicit api key here - it breaks GH sync - }, - logger - ); + return redirectTov2(v2ConfigPath, options, config, logger); } if (process.env['OPENFN_API_KEY']) { @@ -179,4 +161,32 @@ function pickFirst(...args: (T | null | undefined)[]): T { return args.find((arg) => arg !== undefined && arg !== null) as T; } +const redirectTov2 = async ( + v2ConfigPath: string, + options: PullOptions, + config: DeployConfig, + logger: Logger +) => { + logger.always( + 'Detected openfn.yaml file - switching to v2 pull (openfn project pull). Set PREFER_LEGACY_SYNC to disable this.' + ); + + // default endpoint to one from openfn.yaml + const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8')); + + const endpoint = + options.endpoint ?? v2config?.project?.endpoint ?? config.endpoint; + return beta( + { + ...options, + endpoint, + project: options.projectId, + force: true, + apiKey: options.apiKey ?? config.apiKey ?? undefined, + createCredentials: false, + }, + logger + ); +}; + export default pullHandler; diff --git a/packages/cli/test/pull/handler.test.ts b/packages/cli/test/pull/handler.test.ts index 472b62371..7ac152850 100644 --- a/packages/cli/test/pull/handler.test.ts +++ b/packages/cli/test/pull/handler.test.ts @@ -36,7 +36,7 @@ test.before(() => { .persist(); }); -const options: PullOptions = { +const options: Partial = { beta: false, command: 'pull', configPath: '/tmp/config.json', @@ -65,6 +65,48 @@ project: } ); +test.serial( + 'openfn yaml endpoint preferred to config.json endpoint', + async (t) => { + const logger = createMockLogger('', { level: 'debug' }); + mockfs({ + ['/tmp/config.json']: `{"apiKey": "123", "endpoint": "DOES_NOT_EXIST"}`, + ['/tmp/openfn.yaml']: ` +project: + endpoint: ${ENDPOINT}`, + }); + + // This would throw if the config.json endpoint is used + await pullHandler(options, logger); + + t.pass(); + } +); + +test.serial( + 'CLI endpoint preferred to openfn.yaml and config.json endpoints', + async (t) => { + const logger = createMockLogger('', { level: 'debug' }); + mockfs({ + ['/tmp/config.json']: `{"apiKey": "123", "endpoint": "DOES_NOT_EXIST"}`, + ['/tmp/openfn.yaml']: ` +project: + endpoint: INVALID_URL`, + }); + + // This would throw if any config endpoint is used + await pullHandler( + { + ...options, + endpoint: ENDPOINT, + } as any, + logger + ); + + t.pass(); + } +); + test.serial('does not create credentials.yaml when redirecting', async (t) => { const logger = createMockLogger('', { level: 'debug' }); mockfs({ From 5bfff638c61ba0fea4954f227b941aaa68ac9177 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 May 2026 15:31:14 +0100 Subject: [PATCH 3/7] refactor deploy function --- packages/cli/src/deploy/handler.ts | 46 ++++++++++++++++++------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index d71419db3..a3ea04ae7 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -42,24 +42,7 @@ async function deployHandler( 'openfn.yaml' ); if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) { - // default endpoint to one from openfn.yaml - const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8')); - if (!config.endpoint && v2config?.project?.endpoint) { - config.endpoint = v2config.project.endpoint; - } - - logger.always( - 'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.' - ); - return beta.handler( - { - ...options, - force: true, - endpoint: config.endpoint, - apiKey: config.apiKey ?? undefined, - }, - logger - ); + return redirectTov2(v2ConfigPath, options, config, logger); } if (options.confirm === false) { @@ -124,4 +107,31 @@ function pickFirst(...args: (T | null | undefined)[]): T { return args.find((arg) => arg !== undefined && arg !== null) as T; } +const redirectTov2 = async ( + v2ConfigPath: string, + options: DeployOptions, + config: DeployConfig, + logger: Logger +) => { + logger.always( + 'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.' + ); + + // default endpoint to one from openfn.yaml + const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8')); + if (!config.endpoint && v2config?.project?.endpoint) { + config.endpoint = v2config.project.endpoint; + } + + return beta.handler( + { + ...options, + force: true, + endpoint: config.endpoint, + apiKey: config.apiKey ?? undefined, + }, + logger + ); +}; + export default deployHandler; From be1c588c2b628ba875cdf30cd3bd5b96f8f60569 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 May 2026 15:51:07 +0100 Subject: [PATCH 4/7] integration tests for deploy --- integration-tests/cli/test/deploy.test.ts | 73 ++++++++++++++ integration-tests/cli/test/deploy.v2.test.ts | 95 +------------------ .../cli/test/fixtures/projects.ts | 92 ++++++++++++++++++ 3 files changed, 166 insertions(+), 94 deletions(-) create mode 100644 integration-tests/cli/test/fixtures/projects.ts diff --git a/integration-tests/cli/test/deploy.test.ts b/integration-tests/cli/test/deploy.test.ts index 54227d485..607cd6b82 100644 --- a/integration-tests/cli/test/deploy.test.ts +++ b/integration-tests/cli/test/deploy.test.ts @@ -5,6 +5,7 @@ import run from '../src/run'; import createLightningServer from '@openfn/lightning-mock'; import { extractLogs, assertLog } from '../src/util'; import { rimraf } from 'rimraf'; +import { makeProject } from './fixtures/projects'; let server: any; const port = 8967; @@ -196,6 +197,78 @@ test.serial('pull a project', async (t) => { t.is(workflow.version_history.length, 1); }); +test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => { + const projectId = 'redirect-test-1'; + server.addProject(makeProject(projectId) as any); + + // create an empty openfn.yaml to trigger the v1 -> v2 redirect + await fs.writeFile(path.join(tmpDir, 'openfn.yaml'), ''); + + const bootstrap = await run( + `openfn pull ${projectId} --workspace ${tmpDir} --log-json -l debug` + ); + t.falsy(bootstrap.stderr); + assertLog(t, extractLogs(bootstrap.stdout), /Detected openfn.yaml file/i); + + const yaml = await fs.readFile(path.join(tmpDir, 'openfn.yaml'), 'utf8'); + t.regex(yaml, new RegExp(`uuid\\: ${projectId}`)); + + const workflowYaml = await fs.readFile( + path.join(tmpDir, 'workflows/my-workflow/my-workflow.yaml'), + 'utf8' + ); + t.regex(workflowYaml, /id: my-workflow/); + t.regex(workflowYaml, /name: My Workflow/); + t.regex(workflowYaml, /expression: \.\/my-job\.js/); + + const stepJs = await fs.readFile( + path.join(tmpDir, 'workflows/my-workflow/my-job.js'), + 'utf8' + ); + t.is(stepJs, 'fn(s => s)'); + + // simulate a remote change + const remoteProject = server.state.projects[projectId]; + const wf = Object.values(remoteProject.workflows as any).find( + (w: any) => w.id === 'my-workflow-1' + ) as any; + server.updateWorkflow(projectId, { + ...wf, + jobs: Object.values(wf.jobs ?? {}).map((j: any) => + j.id === 'my-job-1' + ? { ...j, body: 'fn(s => ({ ...s, remote: true }))' } + : j + ), + }); + + // v1 pull -> should redirect to v2 because openfn.yaml exists + const pullResult = await run( + `openfn pull ${projectId} --workspace ${tmpDir} --log-json -l debug` + ); + t.falsy(pullResult.stderr); + assertLog(t, extractLogs(pullResult.stdout), /Detected openfn.yaml file/i); + + const exprPath = path.join(tmpDir, 'workflows/my-workflow/my-job.js'); + t.regex(await fs.readFile(exprPath, 'utf8'), /remote: true/); + + // make a local change + await fs.writeFile(exprPath, 'fn(s => ({ ...s, local: true }))'); + + // v1 deploy -> should redirect to v2 + const { stdout, stderr } = await run( + `openfn deploy --workspace ${tmpDir} --no-confirm --log-json -l debug` + ); + t.falsy(stderr); + assertLog(t, extractLogs(stdout), /Detected openfn.yaml file/i); + + // confirm the local change made it to the server + const serverProj = server.state.projects[projectId]; + t.regex( + serverProj.workflows['my-workflow-1'].jobs['my-job'].body, + /local: true/ + ); +}); + test.serial('deploy then pull, changes one workflow, deploy', async (t) => { t.is(Object.keys(server.state.projects).length, 0); diff --git a/integration-tests/cli/test/deploy.v2.test.ts b/integration-tests/cli/test/deploy.v2.test.ts index de33bbc65..c69fa39f4 100644 --- a/integration-tests/cli/test/deploy.v2.test.ts +++ b/integration-tests/cli/test/deploy.v2.test.ts @@ -8,6 +8,7 @@ import createLightningServer, { import Project from '@openfn/project'; import { extractLogs, assertLog } from '../src/util'; import { rimraf } from 'rimraf'; +import { makeProject, makeMultiProject } from './fixtures/projects'; let server: ReturnType; @@ -16,100 +17,6 @@ const endpoint = `http://localhost:${port}`; const tmpDir = path.resolve('tmp/deploy-v2'); -const makeProject = (id: string) => ({ - id, - name: 'test-project', - workflows: [ - { - id: 'my-workflow-1', - name: 'My Workflow', - jobs: [ - { - id: 'my-job-1', - name: 'My Job', - body: 'fn(s => s)', - adaptor: '@openfn/language-common@latest', - project_credential_id: null, - }, - ], - triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }], - edges: [ - { - id: 'my-edge-1', - condition_type: 'always', - source_trigger_id: 'my-trigger-1', - target_job_id: 'my-job-1', - enabled: true, - }, - ], - lock_version: 1, - deleted_at: null, - }, - ], - project_credentials: [], - collections: [], -}); - -// A two-workflow project for isolation/divergence tests -const makeMultiProject = (id: string): any => ({ - id, - name: 'test-project', - workflows: [ - { - id: 'my-workflow-1', - name: 'My Workflow', - jobs: [ - { - id: 'my-job-1', - name: 'My Job', - body: 'fn(s => s)', - adaptor: '@openfn/language-common@latest', - project_credential_id: null, - }, - ], - triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }], - edges: [ - { - id: 'my-edge-1', - condition_type: 'always', - source_trigger_id: 'my-trigger-1', - target_job_id: 'my-job-1', - enabled: true, - }, - ], - lock_version: 1, - deleted_at: null, - }, - { - id: 'another-workflow-1', - name: 'Another Workflow', - jobs: [ - { - id: 'another-job-1', - name: 'Another Job', - body: "get('http://example.com')", - adaptor: '@openfn/language-http@latest', - project_credential_id: null, - }, - ], - triggers: [{ id: 'another-trigger-1', type: 'webhook', enabled: true }], - edges: [ - { - id: 'another-edge-1', - condition_type: 'always', - source_trigger_id: 'another-trigger-1', - target_job_id: 'another-job-1', - enabled: true, - }, - ], - lock_version: 1, - deleted_at: null, - }, - ], - project_credentials: [], - collections: [], -}); - test.before(async () => { server = await createLightningServer({ port }); diff --git a/integration-tests/cli/test/fixtures/projects.ts b/integration-tests/cli/test/fixtures/projects.ts new file mode 100644 index 000000000..9bb85f2b9 --- /dev/null +++ b/integration-tests/cli/test/fixtures/projects.ts @@ -0,0 +1,92 @@ +export const makeProject = (id: string) => ({ + id, + name: 'test-project', + workflows: [ + { + id: 'my-workflow-1', + name: 'My Workflow', + jobs: [ + { + id: 'my-job-1', + name: 'My Job', + body: 'fn(s => s)', + adaptor: '@openfn/language-common@latest', + project_credential_id: null, + }, + ], + triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }], + edges: [ + { + id: 'my-edge-1', + condition_type: 'always', + source_trigger_id: 'my-trigger-1', + target_job_id: 'my-job-1', + enabled: true, + }, + ], + lock_version: 1, + deleted_at: null, + }, + ], + project_credentials: [], + collections: [], +}); + +export const makeMultiProject = (id: string): any => ({ + id, + name: 'test-project', + workflows: [ + { + id: 'my-workflow-1', + name: 'My Workflow', + jobs: [ + { + id: 'my-job-1', + name: 'My Job', + body: 'fn(s => s)', + adaptor: '@openfn/language-common@latest', + project_credential_id: null, + }, + ], + triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }], + edges: [ + { + id: 'my-edge-1', + condition_type: 'always', + source_trigger_id: 'my-trigger-1', + target_job_id: 'my-job-1', + enabled: true, + }, + ], + lock_version: 1, + deleted_at: null, + }, + { + id: 'another-workflow-1', + name: 'Another Workflow', + jobs: [ + { + id: 'another-job-1', + name: 'Another Job', + body: "get('http://example.com')", + adaptor: '@openfn/language-http@latest', + project_credential_id: null, + }, + ], + triggers: [{ id: 'another-trigger-1', type: 'webhook', enabled: true }], + edges: [ + { + id: 'another-edge-1', + condition_type: 'always', + source_trigger_id: 'another-trigger-1', + target_job_id: 'another-job-1', + enabled: true, + }, + ], + lock_version: 1, + deleted_at: null, + }, + ], + project_credentials: [], + collections: [], +}); From 8ebd5c94c2c9bf83409a66b8897f100e84949b00 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 May 2026 15:51:54 +0100 Subject: [PATCH 5/7] changeset --- .changeset/slimy-trams-bet.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/slimy-trams-bet.md diff --git a/.changeset/slimy-trams-bet.md b/.changeset/slimy-trams-bet.md new file mode 100644 index 000000000..af53bf59b --- /dev/null +++ b/.changeset/slimy-trams-bet.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': patch +--- + +Fix an issue where pull and deploy do not track the endpoint argument properly when redirecting to v2 From 4ce4c11f8686d4af0486f97cfe61e2595de43464 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 May 2026 15:55:39 +0100 Subject: [PATCH 6/7] remove log --- packages/cli/src/deploy/handler.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index a3ea04ae7..b448a6210 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -34,7 +34,6 @@ async function deployHandler( } try { - console.log({ options }); const config = mergeOverrides(await getConfig(options.configPath), options); const v2ConfigPath = path.join( From a7e1cd3f80f6713ed5eed8f77b84aaf207275df7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 May 2026 16:05:06 +0100 Subject: [PATCH 7/7] version --- .changeset/slimy-trams-bet.md | 5 ----- packages/cli/CHANGELOG.md | 6 ++++++ packages/cli/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/slimy-trams-bet.md diff --git a/.changeset/slimy-trams-bet.md b/.changeset/slimy-trams-bet.md deleted file mode 100644 index af53bf59b..000000000 --- a/.changeset/slimy-trams-bet.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/cli': patch ---- - -Fix an issue where pull and deploy do not track the endpoint argument properly when redirecting to v2 diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index b6f77d9e6..5432f762f 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/cli +## 1.36.2 + +### Patch Changes + +- 8ebd5c9: Fix an issue where pull and deploy do not track the endpoint argument properly when redirecting to v2 + ## 1.36.1 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 00007d2db..d588d5661 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.36.1", + "version": "1.36.2", "description": "CLI devtools for the OpenFn toolchain", "engines": { "node": ">=18",