CLI: add --workflows flag to merge and deploy

.changeset/few-cobras-pick.md

@@ -0,0 +1,7 @@
+---
+'@openfn/cli': minor
+---
+
+Allow users to specifiy which workflows to deploy or merge by passing `-w`.
+
+NOTE: the `-w` alias has been repurposed from `--workspace` to `--workflow`. This may affect your local development environment. If so, just expand `-w` to `--workspace`.

packages/cli/src/projects/deploy.ts

@@ -1,5 +1,9 @@
 import yargs from 'yargs';
-import Project, { versionsEqual, Workspace } from '@openfn/project';
+import Project, {
+  MergeProjectOptions,
+  versionsEqual,
+  Workspace,
+} from '@openfn/project';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
 
@@ -42,6 +46,7 @@ export type DeployOptions = Pick<
   name?: string;
   alias?: string;
   jsonDiff?: boolean;
+  workflow?: string[];
 };
 
 const options = [
@@ -53,6 +58,7 @@ const options = [
   o2.name,
   o2.alias,
   o2.jsonDiff,
+  o2.workflow,
 
   // general options
   o.apiKey,
@@ -170,14 +176,29 @@ const syncProjects = async (
     // this will actually happen later
   }
 
-  const locallyChangedWorkflows = await findLocallyChangedWorkflows(
-    ws,
-    localProject
-  );
+  let mergeCandidates: string[];
+  if (options.workflow?.length) {
+    const missing = options.workflow.filter(
+      (id) => !localProject.workflows.some((w) => w.id === id)
+    );
+    if (missing.length) {
+      throw new Error(
+        `The following workflows were not found in local project ${
+          localProject.id
+        }: ${missing.join(', ')}`
+      );
+    }
+    logger.info(
+      `--workflow passed: forcing deploy of ${options.workflow.join(', ')}`
+    );
+    mergeCandidates = options.workflow;
+  } else {
+    mergeCandidates = await findLocallyChangedWorkflows(ws, localProject);
+  }
 
   // TODO: what if remote diff and the version checked disagree for some reason?
-  const diffs = locallyChangedWorkflows.length
-    ? remoteProject.diff(localProject, locallyChangedWorkflows)
+  const diffs = mergeCandidates.length
+    ? remoteProject.diff(localProject, mergeCandidates)
     : [];
 
   if (!diffs.length) {
@@ -203,7 +224,7 @@ const syncProjects = async (
     const divergentWorkflows = hasRemoteDiverged(
       localProject,
       remoteProject!,
-      locallyChangedWorkflows
+      mergeCandidates
     );
     if (divergentWorkflows) {
       logger.warn(
@@ -231,16 +252,24 @@ const syncProjects = async (
   }
 
   logger.info('Merging changes into remote project');
-  // TODO I would like to log which workflows are being updated
-  const merged = Project.merge(localProject, remoteProject!, {
+  const mergeOptions: MergeProjectOptions = {
     // If pushing the same project, we use a replace strategy
     // Otherwise, use the sandbox strategy to preserve UUIDs
     mode: localProject.uuid === remoteProject.uuid ? 'replace' : 'sandbox',
     force: true,
-    onlyUpdated: true,
-  });
+  };
+  if (options.workflow?.length) {
+    // If --workflow is passed, force-include exactly the listed workflows via workflowMappings
+    mergeOptions.workflowMappings = Object.fromEntries(
+      options.workflow.map((id) => [id, id])
+    );
+  } else {
+    // Otherwise only merge locally updated workflows
+    mergeOptions.onlyUpdated = true;
+  }
+  const merged = Project.merge(localProject, remoteProject!, mergeOptions);
 
-  return { merged, remoteProject, locallyChangedWorkflows };
+  return { merged, remoteProject, locallyChangedWorkflows: mergeCandidates };
 };
 
 export async function handler(options: DeployOptions, logger: Logger) {

packages/cli/src/projects/merge.ts

@@ -17,11 +17,12 @@ export type MergeOptions = Required<
     'command' | 'project' | 'workspace' | 'removeUnmapped' | 'workflowMappings'
   >
 > &
-  Pick<Opts, 'log' | 'force' | 'outputPath'> & { base?: string };
+  Pick<Opts, 'log' | 'force' | 'outputPath' | 'workflow'> & { base?: string };
 
 const options = [
   po.removeUnmapped,
   po.workflowMappings,
+  po.workflow,
   po.workspace,
   o.log,
   // custom output because we don't want defaults or anything
@@ -109,6 +110,29 @@ export const handler = async (options: MergeOptions, logger: Logger) => {
     logger.error('The checked out project has no id');
     return;
   }
+
+  let workflowMappings = options.workflowMappings;
+  if (options.workflow?.length) {
+    if (workflowMappings && Object.keys(workflowMappings).length) {
+      logger.error(
+        '--workflow and --workflow-mappings are mutually exclusive'
+      );
+      return;
+    }
+    const missing = options.workflow.filter(
+      (id) => !sourceProject.workflows.some((w) => w.id === id)
+    );
+    if (missing.length) {
+      logger.error(
+        `The following workflows were not found in source project ${sourceProject.id}: ${missing.join(', ')}`
+      );
+      return;
+    }
+    workflowMappings = Object.fromEntries(
+      options.workflow.map((id) => [id, id])
+    );
+  }
+
   const finalPath =
     options.outputPath ?? workspace.getProjectPath(targetProject.id);
   if (!finalPath) {
@@ -117,7 +141,7 @@ export const handler = async (options: MergeOptions, logger: Logger) => {
   }
   const final = Project.merge(sourceProject, targetProject, {
     removeUnmapped: options.removeUnmapped,
-    workflowMappings: options.workflowMappings,
+    workflowMappings,
     force: options.force,
   });
 

packages/cli/src/projects/options.ts

@@ -8,6 +8,7 @@ export type Opts = BaseOpts & {
   workspace?: string;
   removeUnmapped?: boolean | undefined;
   workflowMappings?: Record<string, string> | undefined;
+  workflow?: string[];
   project?: string;
   format?: 'yaml' | 'json' | 'state';
   clean?: boolean;
@@ -86,6 +87,22 @@ export const workflowMappings: CLIOption = {
   },
 };
 
+export const workflow: CLIOption = {
+  name: 'workflow',
+  yargs: {
+    alias: ['w'],
+    array: true,
+    description:
+      'Restrict merge/deploy to the given workflow ids. Pass multiple times to include multiple workflows. Listed workflows are force-included from the source and will overwrite the target/remote even if unchanged locally. Mutually exclusive with --workflow-mappings.',
+  },
+  ensure: (opts: any) => {
+    if (opts.workflow?.length) {
+      opts.workflow = Array.from(new Set(opts.workflow));
+    }
+    delete opts.w;
+  },
+};
+
 // We declare a new output path here, overriding the default cli one,
 // because default rules are different
 export const outputPath: CLIOption = {
@@ -100,7 +117,6 @@ export const outputPath: CLIOption = {
 export const workspace: CLIOption = {
   name: 'workspace',
   yargs: {
-    alias: ['w'],
     description: 'Path to the project workspace (ie, path to openfn.yaml)',
   },
   ensure: (opts: any) => {

packages/cli/test/projects/deploy.test.ts

@@ -11,7 +11,13 @@ import {
   hasRemoteDiverged,
 } from '../../src/projects/deploy';
 import { printRichDiff } from '../../src/projects/diff';
-import { myProject_yaml, myProject_v1, UUID } from './fixtures';
+import {
+  myProject_yaml,
+  myProject_v1,
+  UUID,
+  two_workflows_yaml as twowfs,
+  TWO_WORKFLOWS_UUID,
+} from './fixtures';
 import { checkout } from '../../src/projects';
 
 let server: any;
@@ -21,6 +27,7 @@ const ENDPOINT = `http://localhost:${port}`;
 
 // quick fix to the fixture yaml, otherwise the deploy code kicks off
 const projectYaml = myProject_yaml.replace('https://app.openfn.org', ENDPOINT);
+const two_workflows_yaml = twowfs.replace('https://app.openfn.org', ENDPOINT);
 
 const mockFs = (paths: Record<string, string>) => {
   const pnpm = path.resolve('../../node_modules/.pnpm');
@@ -194,6 +201,125 @@ test.serial(
   }
 );
 
+test.serial(
+  'Passing --workflow only updates the requested workflows',
+  async (t) => {
+    await server.addProject(two_workflows_yaml);
+    await setup(two_workflows_yaml);
+
+    // Change both workflows locally
+    await writeFile('/ws/workflows/workflow-a/job-a.js', 'modifiedA()');
+    await writeFile('/ws/workflows/workflow-b/job-b.js', 'modifiedB()');
+
+    await deploy(
+      {
+        endpoint: ENDPOINT,
+        apiKey: 'test-api-key',
+        workspace: '/ws',
+        confirm: false,
+        workflow: ['workflow-a'],
+      } as any,
+      logger
+    );
+
+    const remoteProject = server.state.projects[TWO_WORKFLOWS_UUID];
+    t.is(
+      remoteProject.workflows['workflow-a'].jobs['job-a'].body,
+      'modifiedA()'
+    );
+    t.is(remoteProject.workflows['workflow-b'].jobs['job-b'].body, 'fn()');
+  }
+);
+
+test.serial(
+  '--workflow errors when an id is not in the local project',
+  async (t) => {
+    await setup(projectYaml);
+
+    await t.throwsAsync(
+      () =>
+        deploy(
+          {
+            endpoint: ENDPOINT,
+            apiKey: 'test-api-key',
+            workspace: '/ws',
+            confirm: false,
+            workflow: ['nope-not-a-real-workflow'],
+          } as any,
+          logger
+        ),
+      { message: /nope-not-a-real-workflow/ }
+    );
+  }
+);
+
+// TODO check this langauge and behaviour
+test.serial(
+  '--workflow force-pushes a locally-unchanged workflow over a diverged remote when --force is set',
+  async (t) => {
+    t.truthy(server.state.projects[UUID]);
+    await setup(projectYaml);
+
+    // Local is unchanged. Modify remote so it diverges.
+    const modified = JSON.parse(
+      JSON.stringify(server.state.projects[UUID].workflows['my-workflow'])
+    );
+    modified.jobs['transform-data'].body = 'each()';
+    server.updateWorkflow(UUID, modified);
+
+    // Without --workflow, change-detection would say "nothing to deploy"
+    // With --workflow + --force, we should revert the remote to local
+    await deploy(
+      {
+        endpoint: ENDPOINT,
+        apiKey: 'test-api-key',
+        workspace: '/ws',
+        confirm: false,
+        workflow: ['my-workflow'],
+        force: true,
+      } as any,
+      logger
+    );
+
+    t.truthy(logger._find('success', /Updated project at/));
+
+    // The remote should have been overwritten with the local body
+    const transformData =
+      server.state.projects[UUID].workflows['my-workflow'].jobs[
+        'transform-data'
+      ];
+    t.is(transformData.body, 'fn()');
+  }
+);
+
+test.serial(
+  '--workflow still errors on divergence without --force',
+  async (t) => {
+    await setup(projectYaml);
+
+    const modified = JSON.parse(
+      JSON.stringify(server.state.projects[UUID].workflows['my-workflow'])
+    );
+    modified.jobs['transform-data'].body = 'each()';
+    server.updateWorkflow(UUID, modified);
+
+    await t.throwsAsync(
+      () =>
+        deploy(
+          {
+            endpoint: ENDPOINT,
+            apiKey: 'test-api-key',
+            workspace: '/ws',
+            confirm: false,
+            workflow: ['my-workflow'],
+          } as any,
+          logger
+        ),
+      { message: /PROJECTS_DIVERGED/ }
+    );
+  }
+);
+
 test('printRichDiff: should report no changes for identical projects', (t) => {
   const wf = generateWorkflow('@id a trigger-x');