Fix v2 sync

integration-tests/cli/test/deploy.test.ts

@@ -5,6 +5,7 @@ import run from '../src/run';
 import createLightningServer from '@openfn/lightning-mock';
 import { extractLogs, assertLog } from '../src/util';
 import { rimraf } from 'rimraf';
+import { makeProject } from './fixtures/projects';
 
 let server: any;
 const port = 8967;
@@ -196,6 +197,78 @@ test.serial('pull a project', async (t) => {
   t.is(workflow.version_history.length, 1);
 });
 
+test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => {
+  const projectId = 'redirect-test-1';
+  server.addProject(makeProject(projectId) as any);
+
+  // create an empty openfn.yaml to trigger the v1 -> v2 redirect
+  await fs.writeFile(path.join(tmpDir, 'openfn.yaml'), '');
+
+  const bootstrap = await run(
+    `openfn pull ${projectId} --workspace ${tmpDir} --log-json -l debug`
+  );
+  t.falsy(bootstrap.stderr);
+  assertLog(t, extractLogs(bootstrap.stdout), /Detected openfn.yaml file/i);
+
+  const yaml = await fs.readFile(path.join(tmpDir, 'openfn.yaml'), 'utf8');
+  t.regex(yaml, new RegExp(`uuid\\: ${projectId}`));
+
+  const workflowYaml = await fs.readFile(
+    path.join(tmpDir, 'workflows/my-workflow/my-workflow.yaml'),
+    'utf8'
+  );
+  t.regex(workflowYaml, /id: my-workflow/);
+  t.regex(workflowYaml, /name: My Workflow/);
+  t.regex(workflowYaml, /expression: \.\/my-job\.js/);
+
+  const stepJs = await fs.readFile(
+    path.join(tmpDir, 'workflows/my-workflow/my-job.js'),
+    'utf8'
+  );
+  t.is(stepJs, 'fn(s => s)');
+
+  // simulate a remote change
+  const remoteProject = server.state.projects[projectId];
+  const wf = Object.values(remoteProject.workflows as any).find(
+    (w: any) => w.id === 'my-workflow-1'
+  ) as any;
+  server.updateWorkflow(projectId, {
+    ...wf,
+    jobs: Object.values(wf.jobs ?? {}).map((j: any) =>
+      j.id === 'my-job-1'
+        ? { ...j, body: 'fn(s => ({ ...s, remote: true }))' }
+        : j
+    ),
+  });
+
+  // v1 pull -> should redirect to v2 because openfn.yaml exists
+  const pullResult = await run(
+    `openfn pull ${projectId} --workspace ${tmpDir} --log-json -l debug`
+  );
+  t.falsy(pullResult.stderr);
+  assertLog(t, extractLogs(pullResult.stdout), /Detected openfn.yaml file/i);
+
+  const exprPath = path.join(tmpDir, 'workflows/my-workflow/my-job.js');
+  t.regex(await fs.readFile(exprPath, 'utf8'), /remote: true/);
+
+  // make a local change
+  await fs.writeFile(exprPath, 'fn(s => ({ ...s, local: true }))');
+
+  // v1 deploy -> should redirect to v2
+  const { stdout, stderr } = await run(
+    `openfn deploy --workspace ${tmpDir} --no-confirm --log-json -l debug`
+  );
+  t.falsy(stderr);
+  assertLog(t, extractLogs(stdout), /Detected openfn.yaml file/i);
+
+  // confirm the local change made it to the server
+  const serverProj = server.state.projects[projectId];
+  t.regex(
+    serverProj.workflows['my-workflow-1'].jobs['my-job'].body,
+    /local: true/
+  );
+});
+
 test.serial('deploy then pull, changes one workflow, deploy', async (t) => {
   t.is(Object.keys(server.state.projects).length, 0);
 

integration-tests/cli/test/deploy.v2.test.ts

@@ -8,6 +8,7 @@ import createLightningServer, {
 import Project from '@openfn/project';
 import { extractLogs, assertLog } from '../src/util';
 import { rimraf } from 'rimraf';
+import { makeProject, makeMultiProject } from './fixtures/projects';
 
 let server: ReturnType<typeof createLightningServer>;
 
@@ -16,100 +17,6 @@ const endpoint = `http://localhost:${port}`;
 
 const tmpDir = path.resolve('tmp/deploy-v2');
 
-const makeProject = (id: string) => ({
-  id,
-  name: 'test-project',
-  workflows: [
-    {
-      id: 'my-workflow-1',
-      name: 'My Workflow',
-      jobs: [
-        {
-          id: 'my-job-1',
-          name: 'My Job',
-          body: 'fn(s => s)',
-          adaptor: '@openfn/language-common@latest',
-          project_credential_id: null,
-        },
-      ],
-      triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }],
-      edges: [
-        {
-          id: 'my-edge-1',
-          condition_type: 'always',
-          source_trigger_id: 'my-trigger-1',
-          target_job_id: 'my-job-1',
-          enabled: true,
-        },
-      ],
-      lock_version: 1,
-      deleted_at: null,
-    },
-  ],
-  project_credentials: [],
-  collections: [],
-});
-
-// A two-workflow project for isolation/divergence tests
-const makeMultiProject = (id: string): any => ({
-  id,
-  name: 'test-project',
-  workflows: [
-    {
-      id: 'my-workflow-1',
-      name: 'My Workflow',
-      jobs: [
-        {
-          id: 'my-job-1',
-          name: 'My Job',
-          body: 'fn(s => s)',
-          adaptor: '@openfn/language-common@latest',
-          project_credential_id: null,
-        },
-      ],
-      triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }],
-      edges: [
-        {
-          id: 'my-edge-1',
-          condition_type: 'always',
-          source_trigger_id: 'my-trigger-1',
-          target_job_id: 'my-job-1',
-          enabled: true,
-        },
-      ],
-      lock_version: 1,
-      deleted_at: null,
-    },
-    {
-      id: 'another-workflow-1',
-      name: 'Another Workflow',
-      jobs: [
-        {
-          id: 'another-job-1',
-          name: 'Another Job',
-          body: "get('http://example.com')",
-          adaptor: '@openfn/language-http@latest',
-          project_credential_id: null,
-        },
-      ],
-      triggers: [{ id: 'another-trigger-1', type: 'webhook', enabled: true }],
-      edges: [
-        {
-          id: 'another-edge-1',
-          condition_type: 'always',
-          source_trigger_id: 'another-trigger-1',
-          target_job_id: 'another-job-1',
-          enabled: true,
-        },
-      ],
-      lock_version: 1,
-      deleted_at: null,
-    },
-  ],
-  project_credentials: [],
-  collections: [],
-});
-
 test.before(async () => {
   server = await createLightningServer({ port });
 

integration-tests/cli/test/fixtures/projects.ts

@@ -0,0 +1,92 @@
+export const makeProject = (id: string) => ({
+  id,
+  name: 'test-project',
+  workflows: [
+    {
+      id: 'my-workflow-1',
+      name: 'My Workflow',
+      jobs: [
+        {
+          id: 'my-job-1',
+          name: 'My Job',
+          body: 'fn(s => s)',
+          adaptor: '@openfn/language-common@latest',
+          project_credential_id: null,
+        },
+      ],
+      triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }],
+      edges: [
+        {
+          id: 'my-edge-1',
+          condition_type: 'always',
+          source_trigger_id: 'my-trigger-1',
+          target_job_id: 'my-job-1',
+          enabled: true,
+        },
+      ],
+      lock_version: 1,
+      deleted_at: null,
+    },
+  ],
+  project_credentials: [],
+  collections: [],
+});
+
+export const makeMultiProject = (id: string): any => ({
+  id,
+  name: 'test-project',
+  workflows: [
+    {
+      id: 'my-workflow-1',
+      name: 'My Workflow',
+      jobs: [
+        {
+          id: 'my-job-1',
+          name: 'My Job',
+          body: 'fn(s => s)',
+          adaptor: '@openfn/language-common@latest',
+          project_credential_id: null,
+        },
+      ],
+      triggers: [{ id: 'my-trigger-1', type: 'webhook', enabled: true }],
+      edges: [
+        {
+          id: 'my-edge-1',
+          condition_type: 'always',
+          source_trigger_id: 'my-trigger-1',
+          target_job_id: 'my-job-1',
+          enabled: true,
+        },
+      ],
+      lock_version: 1,
+      deleted_at: null,
+    },
+    {
+      id: 'another-workflow-1',
+      name: 'Another Workflow',
+      jobs: [
+        {
+          id: 'another-job-1',
+          name: 'Another Job',
+          body: "get('http://example.com')",
+          adaptor: '@openfn/language-http@latest',
+          project_credential_id: null,
+        },
+      ],
+      triggers: [{ id: 'another-trigger-1', type: 'webhook', enabled: true }],
+      edges: [
+        {
+          id: 'another-edge-1',
+          condition_type: 'always',
+          source_trigger_id: 'another-trigger-1',
+          target_job_id: 'another-job-1',
+          enabled: true,
+        },
+      ],
+      lock_version: 1,
+      deleted_at: null,
+    },
+  ],
+  project_credentials: [],
+  collections: [],
+});

packages/cli/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/cli
 
+## 1.36.2
+
+### Patch Changes
+
+- 8ebd5c9: Fix an issue where pull and deploy do not track the endpoint argument properly when redirecting to v2
+
 ## 1.36.1
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.36.1",
+  "version": "1.36.2",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/deploy/handler.ts

@@ -41,24 +41,7 @@ async function deployHandler(
       'openfn.yaml'
     );
     if (!process.env.PREFER_LEGACY_SYNC && (await fileExists(v2ConfigPath))) {
-      // default endpoint to one from openfn.yaml
-      const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8'));
-      if (!config.endpoint && v2config?.project?.endpoint) {
-        config.endpoint = v2config.project.endpoint;
-      }
-
-      logger.always(
-        'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.'
-      );
-      return beta.handler(
-        {
-          ...options,
-          force: true,
-          endpoint: config.endpoint,
-          apiKey: config.apiKey ?? undefined,
-        },
-        logger
-      );
+      return redirectTov2(v2ConfigPath, options, config, logger);
     }
 
     if (options.confirm === false) {
@@ -123,4 +106,31 @@ function pickFirst<T>(...args: (T | null | undefined)[]): T {
   return args.find((arg) => arg !== undefined && arg !== null) as T;
 }
 
+const redirectTov2 = async (
+  v2ConfigPath: string,
+  options: DeployOptions,
+  config: DeployConfig,
+  logger: Logger
+) => {
+  logger.always(
+    'Detected openfn.yaml file - switching to v2 deploy (openfn project deploy). Set PREFER_LEGACY_SYNC to disable this.'
+  );
+
+  // default endpoint to one from openfn.yaml
+  const v2config = yamlToJson(await fs.readFile(v2ConfigPath, 'utf-8'));
+  if (!config.endpoint && v2config?.project?.endpoint) {
+    config.endpoint = v2config.project.endpoint;
+  }
+
+  return beta.handler(
+    {
+      ...options,
+      force: true,
+      endpoint: config.endpoint,
+      apiKey: config.apiKey ?? undefined,
+    },
+    logger
+  );
+};
+
 export default deployHandler;

packages/cli/src/projects/fetch.ts

@@ -93,7 +93,7 @@ const fetchV1 = async (options: FetchOptions, logger: Logger) => {
   const config = loadAppAuthConfig(options, logger);
 
   const { data } = await fetchProject(
-    options.endpoint ?? localProject?.openfn?.endpoint!,
+    config.endpoint ?? localProject?.openfn?.endpoint!,
     config.apiKey,
     localProject?.uuid ?? options.project!,
     logger
@@ -265,7 +265,10 @@ export async function fetchRemoteProject(
     );
   }
 
-  const projectEndpoint = localProject?.openfn?.endpoint ?? config.endpoint;
+  // TODO this resolution is pretty awkward. The problem is we don't
+  // know if config.endpoint comes from an env var or explicit option
+  const projectEndpoint =
+    options.endpoint ?? localProject?.openfn?.endpoint ?? config.endpoint;
 
   const { data } = await fetchProject(
     projectEndpoint,

packages/cli/src/pull/command.ts

@@ -8,8 +8,10 @@ import * as po from '../projects/options';
 export type PullOptions = Required<
   Pick<
     Opts & POpts,
+    | 'apiKey'
     | 'beta'
     | 'command'
+    | 'endpoint'
     | 'log'
     | 'logJson'
     | 'statePath'