diff --git a/integration-tests/cli/test/deploy.test.ts b/integration-tests/cli/test/deploy.test.ts index 54227d485..607cd6b82 100644 --- a/integration-tests/cli/test/deploy.test.ts +++ b/integration-tests/cli/test/deploy.test.ts @@ -5,6 +5,7 @@ import run from '../src/run'; import createLightningServer from '@openfn/lightning-mock'; import { extractLogs, assertLog } from '../src/util'; import { rimraf } from 'rimraf'; +import { makeProject } from './fixtures/projects'; let server: any; const port = 8967; @@ -196,6 +197,78 @@ test.serial('pull a project', async (t) => { t.is(workflow.version_history.length, 1); }); +test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => { + const projectId = 'redirect-test-1'; + server.addProject(makeProject(projectId) as any); + + // create an empty openfn.yaml to trigger the v1 -> v2 redirect + await fs.writeFile(path.join(tmpDir, 'openfn.yaml'), ''); + + const bootstrap = await run( + `openfn pull ${projectId} --workspace ${tmpDir} --log-json -l debug` + ); + t.falsy(bootstrap.stderr); + assertLog(t, extractLogs(bootstrap.stdout), /Detected openfn.yaml file/i); + + const yaml = await fs.readFile(path.join(tmpDir, 'openfn.yaml'), 'utf8'); + t.regex(yaml, new RegExp(`uuid\\: ${projectId}`)); + + const workflowYaml = await fs.readFile( + path.join(tmpDir, 'workflows/my-workflow/my-workflow.yaml'), + 'utf8' + ); + t.regex(workflowYaml, /id: my-workflow/); + t.regex(workflowYaml, /name: My Workflow/); + t.regex(workflowYaml, /expression: \.\/my-job\.js/); + + const stepJs = await fs.readFile( + path.join(tmpDir, 'workflows/my-workflow/my-job.js'), + 'utf8' + ); + t.is(stepJs, 'fn(s => s)'); + + // simulate a remote change + const remoteProject = server.state.projects[projectId]; + const wf = Object.values(remoteProject.workflows as any).find( + (w: any) => w.id === 'my-workflow-1' + ) as any; + server.updateWorkflow(projectId, { + ...wf, + jobs: Object.values(wf.jobs ?? {}).map((j: any) => + j.id === 'my-job-1' + ? { ...j, body: 'fn(s => ({ ...s, remote: true }))' } + : j + ), + }); + + // v1 pull -> should redirect to v2 because openfn.yaml exists + const pullResult = await run( + `openfn pull ${projectId} --workspace ${tmpDir} --log-json -l debug` + ); + t.falsy(pullResult.stderr); + assertLog(t, extractLogs(pullResult.stdout), /Detected openfn.yaml file/i); + + const exprPath = path.join(tmpDir, 'workflows/my-workflow/my-job.js'); + t.regex(await fs.readFile(exprPath, 'utf8'), /remote: true/); + + // make a local change + await fs.writeFile(exprPath, 'fn(s => ({ ...s, local: true }))'); + + // v1 deploy -> should redirect to v2 + const { stdout, stderr } = await run( + `openfn deploy --workspace ${tmpDir} --no-confirm --log-json -l debug` + ); + t.falsy(stderr); + assertLog(t, extractLogs(stdout), /Detected openfn.yaml file/i); + + // confirm the local change made it to the server + const serverProj = server.state.projects[projectId]; + t.regex( + serverProj.workflows['my-workflow-1'].jobs['my-job'].body, + /local: true/ + ); +}); + test.serial('deploy then pull, changes one workflow, deploy', async (t) => { t.is(Object.keys(server.state.projects).length, 0); diff --git a/integration-tests/cli/test/deploy.v2.test.ts b/integration-tests/cli/test/deploy.v2.test.ts index de33bbc65..c69fa39f4 100644 --- a/integration-tests/cli/test/deploy.v2.test.ts +++ b/integration-tests/cli/test/deploy.v2.test.ts @@ -8,6 +8,7 @@ import createLightningServer, { import Project from '@openfn/project'; import { extractLogs, assertLog } from '../src/util'; import { rimraf } from 'rimraf'; +import { makeProject, makeMultiProject } from './fixtures/projects'; let server: ReturnType; @@ -16,100 +17,6 @@ const endpoint = `http://localhost:${port}`; const tmpDir = path.resolve('tmp/deploy-v2'); -const makeProject = (id: string) => ({ - id, - name: 'test-project', - workflows: [ - { - id: 'my-workflow-1', - name: 'My Workflow', - jobs: [ - { - id: 'my-job-1', - name: 'My Job', - body: 'fn(s => s)', - adaptor: '@openfn/language-common@latest', - project_credential_id: null, - }, - ], - triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }], - edges: [ - { - id: 'my-edge-1', - condition_type: 'always', - source_trigger_id: 'my-trigger-1', - target_job_id: 'my-job-1', - enabled: true, - }, - ], - lock_version: 1, - deleted_at: null, - }, - ], - project_credentials: [], - collections: [], -}); - -// A two-workflow project for isolation/divergence tests -const makeMultiProject = (id: string): any => ({ - id, - name: 'test-project', - workflows: [ - { - id: 'my-workflow-1', - name: 'My Workflow', - jobs: [ - { - id: 'my-job-1', - name: 'My Job', - body: 'fn(s => s)', - adaptor: '@openfn/language-common@latest', - project_credential_id: null, - }, - ], - triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }], - edges: [ - { - id: 'my-edge-1', - condition_type: 'always', - source_trigger_id: 'my-trigger-1', - target_job_id: 'my-job-1', - enabled: true, - }, - ], - lock_version: 1, - deleted_at: null, - }, - { - id: 'another-workflow-1', - name: 'Another Workflow', - jobs: [ - { - id: 'another-job-1', - name: 'Another Job', - body: "get('http://example.com')", - adaptor: '@openfn/language-http@latest', - project_credential_id: null, - }, - ], - triggers: [{ id: 'another-trigger-1', type: 'webhook', enabled: true }], - edges: [ - { - id: 'another-edge-1', - condition_type: 'always', - source_trigger_id: 'another-trigger-1', - target_job_id: 'another-job-1', - enabled: true, - }, - ], - lock_version: 1, - deleted_at: null, - }, - ], - project_credentials: [], - collections: [], -}); - test.before(async () => { server = await createLightningServer({ port }); diff --git a/integration-tests/cli/test/fixtures/projects.ts b/integration-tests/cli/test/fixtures/projects.ts new file mode 100644 index 000000000..9bb85f2b9 --- /dev/null +++ b/integration-tests/cli/test/fixtures/projects.ts @@ -0,0 +1,92 @@ +export const makeProject = (id: string) => ({ + id, + name: 'test-project', + workflows: [ + { + id: 'my-workflow-1', + name: 'My Workflow', + jobs: [ + { + id: 'my-job-1', + name: 'My Job', + body: 'fn(s => s)', + adaptor: '@openfn/language-common@latest', + project_credential_id: null, + }, + ], + triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }], + edges: [ + { + id: 'my-edge-1', + condition_type: 'always', + source_trigger_id: 'my-trigger-1', + target_job_id: 'my-job-1', + enabled: true, + }, + ], + lock_version: 1, + deleted_at: null, + }, + ], + project_credentials: [], + collections: [], +}); + +export const makeMultiProject = (id: string): any => ({ + id, + name: 'test-project', + workflows: [ + { + id: 'my-workflow-1', + name: 'My Workflow', + jobs: [ + { + id: 'my-job-1', + name: 'My Job', + body: 'fn(s => s)', + adaptor: '@openfn/language-common@latest', + project_credential_id: null, + }, + ], + triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }], + edges: [ + { + id: 'my-edge-1', + condition_type: 'always', + source_trigger_id: 'my-trigger-1', + target_job_id: 'my-job-1', + enabled: true, + }, + ], + lock_version: 1, + deleted_at: null, + }, + { + id: 'another-workflow-1', + name: 'Another Workflow', + jobs: [ + { + id: 'another-job-1', + name: 'Another Job', + body: "get('http://example.com')", + adaptor: '@openfn/language-http@latest', + project_credential_id: null, + }, + ], + triggers: [{ id: 'another-trigger-1', type: 'webhook', enabled: true }], + edges: [ + { + id: 'another-edge-1', + condition_type: 'always', + source_trigger_id: 'another-trigger-1', + target_job_id: 'another-job-1', + enabled: true, + }, + ], + lock_version: 1, + deleted_at: null, + }, + ], + project_credentials: [], + collections: [], +}); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index b6f77d9e6..5432f762f 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/cli +## 1.36.2 + +### Patch Changes + +- 8ebd5c9: Fix an issue where pull and deploy do not track the endpoint argument properly when redirecting to v2 + ## 1.36.1 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 00007d2db..d588d5661 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.36.1", + "version": "1.36.2", "description": "CLI devtools for the OpenFn toolchain", "engines": { "node": ">=18", diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index 63bfaae66..b448a6210 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -41,24 +41,7 @@ async function deployHandler( 'openfn.yaml' ); if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) { - // default endpoint to one from openfn.yaml - const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8')); - if (!config.endpoint && v2config?.project?.endpoint) { - config.endpoint = v2config.project.endpoint; - } - - logger.always( - 'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.' - ); - return beta.handler( - { - ...options, - force: true, - endpoint: config.endpoint, - apiKey: config.apiKey ?? undefined, - }, - logger - ); + return redirectTov2(v2ConfigPath, options, config, logger); } if (options.confirm === false) { @@ -123,4 +106,31 @@ function pickFirst(...args: (T | null | undefined)[]): T { return args.find((arg) => arg !== undefined && arg !== null) as T; } +const redirectTov2 = async ( + v2ConfigPath: string, + options: DeployOptions, + config: DeployConfig, + logger: Logger +) => { + logger.always( + 'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.' + ); + + // default endpoint to one from openfn.yaml + const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8')); + if (!config.endpoint && v2config?.project?.endpoint) { + config.endpoint = v2config.project.endpoint; + } + + return beta.handler( + { + ...options, + force: true, + endpoint: config.endpoint, + apiKey: config.apiKey ?? undefined, + }, + logger + ); +}; + export default deployHandler; diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index b45ade597..e87f0b3fa 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -93,7 +93,7 @@ const fetchV1 = async (options: FetchOptions, logger: Logger) => { const config = loadAppAuthConfig(options, logger); const { data } = await fetchProject( - options.endpoint ?? localProject?.openfn?.endpoint!, + config.endpoint ?? localProject?.openfn?.endpoint!, config.apiKey, localProject?.uuid ?? options.project!, logger @@ -265,7 +265,10 @@ export async function fetchRemoteProject( ); } - const projectEndpoint = localProject?.openfn?.endpoint ?? config.endpoint; + // TODO this resolution is pretty awkward. The problem is we don't + // know if config.endpoint comes from an env var or explicit option + const projectEndpoint = + options.endpoint ?? localProject?.openfn?.endpoint ?? config.endpoint; const { data } = await fetchProject( projectEndpoint, diff --git a/packages/cli/src/pull/command.ts b/packages/cli/src/pull/command.ts index 314164623..dd5dd8034 100644 --- a/packages/cli/src/pull/command.ts +++ b/packages/cli/src/pull/command.ts @@ -8,8 +8,10 @@ import * as po from '../projects/options'; export type PullOptions = Required< Pick< Opts & POpts, + | 'apiKey' | 'beta' | 'command' + | 'endpoint' | 'log' | 'logJson' | 'statePath' diff --git a/packages/cli/src/pull/handler.ts b/packages/cli/src/pull/handler.ts index dadfc04a1..9a376789a 100644 --- a/packages/cli/src/pull/handler.ts +++ b/packages/cli/src/pull/handler.ts @@ -26,27 +26,9 @@ async function pullHandler(options: PullOptions, logger: Logger) { options.workspace || process.cwd(), 'openfn.yaml' ); + if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) { - // default endpoint to one from openfn.yaml - const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8')); - if (!config.endpoint && v2config?.project?.endpoint) { - config.endpoint = v2config.project.endpoint; - } - - logger.always( - 'Detected openfn.yaml file - switching to v2 pull (openfn project pull). Set PREFER_LEGACY_SYNC to disable this.' - ); - return beta( - { - ...options, - project: options.projectId, - force: true, - endpoint: config.endpoint, - apiKey: config.apiKey ?? undefined, - createCredentials: false, - }, - logger - ); + return redirectTov2(v2ConfigPath, options, config, logger); } if (process.env['OPENFN_API_KEY']) { @@ -179,4 +161,32 @@ function pickFirst(...args: (T | null | undefined)[]): T { return args.find((arg) => arg !== undefined && arg !== null) as T; } +const redirectTov2 = async ( + v2ConfigPath: string, + options: PullOptions, + config: DeployConfig, + logger: Logger +) => { + logger.always( + 'Detected openfn.yaml file - switching to v2 pull (openfn project pull). Set PREFER_LEGACY_SYNC to disable this.' + ); + + // default endpoint to one from openfn.yaml + const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8')); + + const endpoint = + options.endpoint ?? v2config?.project?.endpoint ?? config.endpoint; + return beta( + { + ...options, + endpoint, + project: options.projectId, + force: true, + apiKey: options.apiKey ?? config.apiKey ?? undefined, + createCredentials: false, + }, + logger + ); +}; + export default pullHandler; diff --git a/packages/cli/test/pull/handler.test.ts b/packages/cli/test/pull/handler.test.ts index 472b62371..7ac152850 100644 --- a/packages/cli/test/pull/handler.test.ts +++ b/packages/cli/test/pull/handler.test.ts @@ -36,7 +36,7 @@ test.before(() => { .persist(); }); -const options: PullOptions = { +const options: Partial = { beta: false, command: 'pull', configPath: '/tmp/config.json', @@ -65,6 +65,48 @@ project: } ); +test.serial( + 'openfn yaml endpoint preferred to config.json endpoint', + async (t) => { + const logger = createMockLogger('', { level: 'debug' }); + mockfs({ + ['/tmp/config.json']: `{"apiKey": "123", "endpoint": "DOES_NOT_EXIST"}`, + ['/tmp/openfn.yaml']: ` +project: + endpoint: ${ENDPOINT}`, + }); + + // This would throw if the config.json endpoint is used + await pullHandler(options, logger); + + t.pass(); + } +); + +test.serial( + 'CLI endpoint preferred to openfn.yaml and config.json endpoints', + async (t) => { + const logger = createMockLogger('', { level: 'debug' }); + mockfs({ + ['/tmp/config.json']: `{"apiKey": "123", "endpoint": "DOES_NOT_EXIST"}`, + ['/tmp/openfn.yaml']: ` +project: + endpoint: INVALID_URL`, + }); + + // This would throw if any config endpoint is used + await pullHandler( + { + ...options, + endpoint: ENDPOINT, + } as any, + logger + ); + + t.pass(); + } +); + test.serial('does not create credentials.yaml when redirecting', async (t) => { const logger = createMockLogger('', { level: 'debug' }); mockfs({