Improve error handling for ilastik command execution

- Capture stdout and stderr when commands fail to show actual error messages
- Detect line continuations (backslashes) to properly run multiline commands in shell
- Better error messages showing both stdout and stderr output
- This will help debug ilastik failures that were previously hidden by --redirect_output

Author: jkim810

imc/scripts/predict.py

@@ -70,6 +70,8 @@ def predict_with_ilastik(
     """
     Use a trained ilastik model to classify pixels in an IMC image.
     """
+    # Don't redirect output if not quiet, so errors are visible
+    # Even when quiet, we might want to see errors, so we'll capture stderr separately
     quiet_arg = "\n        --redirect_output /dev/null \\" if quiet else ""
     cmd = f"""{ilastik_sh} \\
         --headless \\
@@ -79,7 +81,8 @@ def predict_with_ilastik(
         """
     # Shell expansion of input files won't happen in subprocess call
     cmd += " ".join([x.replace_(" ", r"\ ").as_posix() for x in tiff_files])
-    return run_shell_command(cmd, quiet=True)
+    # Don't use quiet=True so we can see errors even when redirecting ilastik output
+    return run_shell_command(cmd, quiet=quiet)
 
 
 def get_ilastik(lib_dir: Path, version: str = "1.3.3post2") -> Path:

imc/utils.py

@@ -347,7 +347,9 @@ def run_shell_command(cmd: str, dry_run: bool = False, quiet: bool = False) -> i
     # and needs a command to be called prior such as conda activate, etc
     symbol = any(x in cmd for x in ["&", "&&", "|"])
     source = cmd.startswith("source")
-    shell = bool(symbol or source)
+    # Commands with line continuations (backslashes) need shell processing
+    has_line_continuation = "\\" in cmd and "\n" in cmd
+    shell = bool(symbol or source or has_line_continuation)
     if not quiet:
         print(
             "Running command:\n",
@@ -358,19 +360,35 @@ def run_shell_command(cmd: str, dry_run: bool = False, quiet: bool = False) -> i
         if shell:
             if not quiet:
                 print("Running command in shell.")
-            code = subprocess.call(cmd, shell=shell)
+            # Capture stderr to show actual error messages when command fails
+            result = subprocess.run(cmd, shell=shell, capture_output=True, text=True)
+            code = result.returncode
+            stdout_output = result.stdout
+            stderr_output = result.stderr
         else:
             # Allow spaces in file names
             c = re.findall(r"\S+", cmd.replace(r"\ ", "__space__").replace("\\\n", ""))
             c = [x.replace("__space__", " ") for x in c]
-            code = subprocess.call(c, shell=shell)
+            # Capture stderr to show actual error messages when command fails
+            result = subprocess.run(c, shell=shell, capture_output=True, text=True)
+            code = result.returncode
+            stdout_output = result.stdout
+            stderr_output = result.stderr
         if code != 0:
-            error_msg = "Process for command below failed with exit code {}.\nCommand:\n{}\nTerminating pipeline.\n".format(
+            error_msg = "Process for command below failed with exit code {}.\nCommand:\n{}\n".format(
                 code,
                 textwrap.dedent(cmd)
             )
+            if stdout_output:
+                error_msg += f"\nStdout:\n{stdout_output}\n"
+            if stderr_output:
+                error_msg += f"\nStderr:\n{stderr_output}\n"
+            error_msg += "Terminating pipeline.\n"
             print(error_msg)
             sys.exit(code)
+        elif not quiet and stdout_output:
+            # Print output if not quiet and there's output
+            print(stdout_output)
         if not shell:
             pass
             # usage = resource.getrusage(resource.RUSAGE_SELF)