From f3d8362979ab0ca9b4c61f5581285ce60caaa63e Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Mon, 30 Mar 2026 18:24:18 +0200 Subject: [PATCH 1/5] Added new testing framework for linting tests Co-authored-by: Claude Opus 4.6 (1M context) Signed-off-by: Ole Herman Schumacher Elgesem --- tests/lint/001_hello_world.cf | 5 ++++ tests/lint/002_ifvarclass.output.txt | 7 +++++ tests/lint/002_ifvarclass.x.cf | 6 ++++ tests/run-lint-tests.sh | 45 ++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 tests/lint/001_hello_world.cf create mode 100644 tests/lint/002_ifvarclass.output.txt create mode 100644 tests/lint/002_ifvarclass.x.cf create mode 100644 tests/run-lint-tests.sh diff --git a/tests/lint/001_hello_world.cf b/tests/lint/001_hello_world.cf new file mode 100644 index 0000000..49d6483 --- /dev/null +++ b/tests/lint/001_hello_world.cf @@ -0,0 +1,5 @@ +bundle agent main +{ + reports: + "Hello, world!"; +} diff --git a/tests/lint/002_ifvarclass.output.txt b/tests/lint/002_ifvarclass.output.txt new file mode 100644 index 0000000..af068bf --- /dev/null +++ b/tests/lint/002_ifvarclass.output.txt @@ -0,0 +1,7 @@ + + "Hello, CFEngine" + ifvarclass => "cfengine"; + ^--------^ +Deprecation: Use 'if' instead of 'ifvarclass' at tests/lint/002_ifvarclass.x.cf:5:7 +FAIL: tests/lint/002_ifvarclass.x.cf (1 errors) +Failure, 1 errors in total. diff --git a/tests/lint/002_ifvarclass.x.cf b/tests/lint/002_ifvarclass.x.cf new file mode 100644 index 0000000..984f715 --- /dev/null +++ b/tests/lint/002_ifvarclass.x.cf @@ -0,0 +1,6 @@ +bundle agent main +{ + reports: + "Hello, CFEngine" + ifvarclass => "cfengine"; +} diff --git a/tests/run-lint-tests.sh b/tests/run-lint-tests.sh new file mode 100644 index 0000000..ced0b37 --- /dev/null +++ b/tests/run-lint-tests.sh @@ -0,0 +1,45 @@ +#/usr/bin/env bash + +set -e +# set -x + +echo "These tests expect cfengine CLI to be installed globally or in venv" + +echo "Looking for CFEngine CLI:" +which cfengine + +echo "Check that test files are in expected location:" +ls -al tests/lint/*.cf + +mkdir -p tmp + +echo "Run lint tests:" +for file in tests/lint/*.cf; do + if echo "$file" | grep -q '\.x\.cf$'; then + # File ends with .x.cf, we expect it to: + # - Fail (non-zero exit code) + # - Output the correct error message + + expected="$(echo $file | sed 's/\.x\.cf$/.output.txt/')" + if [ ! -f "$expected" ]; then + echo "FAIL: Missing expected output file: $expected" + exit 1 + fi + output="tmp/$(basename $file .x.cf).lint-output.txt" + if cfengine lint "$file" > "$output" 2>&1; then + echo "FAIL: $file - expected lint failure but got success" + exit 1 + fi + diff -u "$expected" "$output" + echo "OK (expected failure): $file" + else + # Expect success + if ! cfengine lint "$file"; then + echo "FAIL: $file - expected lint success but got failure" + exit 1 + fi + echo "OK: $file" + fi +done + +echo "All lint tests successful!" From a88b7ef80067c79534868f5c4d3a5c473d1bd731 Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Mon, 30 Mar 2026 18:30:15 +0200 Subject: [PATCH 2/5] Moved unit tests into subdirectory Signed-off-by: Ole Herman Schumacher Elgesem --- tests/{ => unit}/__init__.py | 0 tests/{ => unit}/test_deps.py | 0 tests/{ => unit}/test_paths.py | 0 tests/{ => unit}/test_utils.py | 0 tests/{ => unit}/test_version.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename tests/{ => unit}/__init__.py (100%) rename tests/{ => unit}/test_deps.py (100%) rename tests/{ => unit}/test_paths.py (100%) rename tests/{ => unit}/test_utils.py (100%) rename tests/{ => unit}/test_version.py (100%) diff --git a/tests/__init__.py b/tests/unit/__init__.py similarity index 100% rename from tests/__init__.py rename to tests/unit/__init__.py diff --git a/tests/test_deps.py b/tests/unit/test_deps.py similarity index 100% rename from tests/test_deps.py rename to tests/unit/test_deps.py diff --git a/tests/test_paths.py b/tests/unit/test_paths.py similarity index 100% rename from tests/test_paths.py rename to tests/unit/test_paths.py diff --git a/tests/test_utils.py b/tests/unit/test_utils.py similarity index 100% rename from tests/test_utils.py rename to tests/unit/test_utils.py diff --git a/tests/test_version.py b/tests/unit/test_version.py similarity index 100% rename from tests/test_version.py rename to tests/unit/test_version.py From 52e633508d54ac0e779b7538a451ed50e44710bf Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Mon, 30 Mar 2026 18:42:01 +0200 Subject: [PATCH 3/5] Added linting tests to GH Actions Signed-off-by: Ole Herman Schumacher Elgesem --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b63caf8..17144e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,3 +47,7 @@ jobs: run: | source .venv/bin/activate bash tests/run-format-tests.sh + - name: Lint tests + run: | + source .venv/bin/activate + bash tests/run-lint-tests.sh From a4c314f1b0a8f8e2d4e35888bb271e4b0d69b0c2 Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Mon, 30 Mar 2026 18:42:43 +0200 Subject: [PATCH 4/5] Added REAME about the different tests Co-authored-by: Claude Opus 4.6 (1M context) Signed-off-by: Ole Herman Schumacher Elgesem --- tests/README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..df5855e --- /dev/null +++ b/tests/README.md @@ -0,0 +1,58 @@ +# Tests + +There are 4 types of tests: unit, shell, format, and lint. +All test scripts should be run from the repository root. + +## Unit tests + +Python unit tests using pytest, located in `tests/unit/` as `test_*.py` files. + +```bash +uv run pytest +``` + +To add a new test, create a `tests/unit/test_.py` file with test functions prefixed `test_`. + +## Shell tests + +Bash scripts in `tests/shell/` that exercise the CLI end-to-end. +Each script is a standalone test that runs commands and relies on `set -e` to fail on errors. + +```bash +bash tests/run-shell-tests.sh +``` + +To add a new test, create a `tests/shell/-.sh` file. + +## Format tests + +Tests for `cfengine format`, located in `tests/format/`. +Each test is a pair of files: + +- `.input.cf` - the unformatted input +- `.expected.cf` - the expected formatted output + +The test runner formats each input file and diffs against the expected output. + +```bash +bash tests/run-format-tests.sh +``` + +To add a new test, create both an `.input.cf` and `.expected.cf` file with matching names. + +## Lint tests + +Tests for `cfengine lint`, located in `tests/lint/`. +Each test is a `.cf` file. +The file extension determines the expected outcome: + +- `.cf` - expected to pass linting (exit code 0) +- `.x.cf` - expected to fail linting (non-zero exit code), must have a corresponding `.output.txt` file containing the expected error output + +```bash +bash tests/run-lint-tests.sh +``` + +To add a passing test, create a `tests/lint/_.cf` file. +To add a failing test, create both a `tests/lint/_.x.cf` file and a `tests/lint/_.output.txt` file with the expected lint output. +The output file must match exactly, including the relative file path (e.g. `tests/lint/_.x.cf`). From f64432e29ef3d3e14f4e0b201f65055c5a3caa6e Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Mon, 30 Mar 2026 18:53:23 +0200 Subject: [PATCH 5/5] Added more linting tests Co-authored-by: Claude Opus 4.6 (1M context) Signed-off-by: Ole Herman Schumacher Elgesem --- tests/lint/003_deprecated_promise_type.cf | 5 +++++ tests/lint/003_deprecated_promise_type.output.txt | 7 +++++++ tests/lint/003_deprecated_promise_type.x.cf | 5 +++++ tests/lint/004_bundle_name_lowercase.cf | 5 +++++ tests/lint/004_bundle_name_lowercase.output.txt | 6 ++++++ tests/lint/004_bundle_name_lowercase.x.cf | 5 +++++ tests/lint/005_bundle_type.cf | 5 +++++ tests/lint/005_bundle_type.output.txt | 6 ++++++ tests/lint/005_bundle_type.x.cf | 5 +++++ tests/lint/006_syntax_error.cf | 5 +++++ tests/lint/006_syntax_error.output.txt | 7 +++++++ tests/lint/006_syntax_error.x.cf | 5 +++++ tests/lint/007_empty_file.output.txt | 3 +++ tests/lint/007_empty_file.x.cf | 0 14 files changed, 69 insertions(+) create mode 100644 tests/lint/003_deprecated_promise_type.cf create mode 100644 tests/lint/003_deprecated_promise_type.output.txt create mode 100644 tests/lint/003_deprecated_promise_type.x.cf create mode 100644 tests/lint/004_bundle_name_lowercase.cf create mode 100644 tests/lint/004_bundle_name_lowercase.output.txt create mode 100644 tests/lint/004_bundle_name_lowercase.x.cf create mode 100644 tests/lint/005_bundle_type.cf create mode 100644 tests/lint/005_bundle_type.output.txt create mode 100644 tests/lint/005_bundle_type.x.cf create mode 100644 tests/lint/006_syntax_error.cf create mode 100644 tests/lint/006_syntax_error.output.txt create mode 100644 tests/lint/006_syntax_error.x.cf create mode 100644 tests/lint/007_empty_file.output.txt create mode 100644 tests/lint/007_empty_file.x.cf diff --git a/tests/lint/003_deprecated_promise_type.cf b/tests/lint/003_deprecated_promise_type.cf new file mode 100644 index 0000000..bc55af2 --- /dev/null +++ b/tests/lint/003_deprecated_promise_type.cf @@ -0,0 +1,5 @@ +bundle agent main +{ + vars: + "x" string => "value"; +} diff --git a/tests/lint/003_deprecated_promise_type.output.txt b/tests/lint/003_deprecated_promise_type.output.txt new file mode 100644 index 0000000..94f2fbd --- /dev/null +++ b/tests/lint/003_deprecated_promise_type.output.txt @@ -0,0 +1,7 @@ + +{ + defaults: + ^-------^ +Deprecation: Promise type 'defaults' is deprecated at tests/lint/003_deprecated_promise_type.x.cf:3:3 +FAIL: tests/lint/003_deprecated_promise_type.x.cf (1 errors) +Failure, 1 errors in total. diff --git a/tests/lint/003_deprecated_promise_type.x.cf b/tests/lint/003_deprecated_promise_type.x.cf new file mode 100644 index 0000000..b845f35 --- /dev/null +++ b/tests/lint/003_deprecated_promise_type.x.cf @@ -0,0 +1,5 @@ +bundle agent main +{ + defaults: + "x" string => "value"; +} diff --git a/tests/lint/004_bundle_name_lowercase.cf b/tests/lint/004_bundle_name_lowercase.cf new file mode 100644 index 0000000..2f876b1 --- /dev/null +++ b/tests/lint/004_bundle_name_lowercase.cf @@ -0,0 +1,5 @@ +bundle agent my_bundle +{ + reports: + "Hello"; +} diff --git a/tests/lint/004_bundle_name_lowercase.output.txt b/tests/lint/004_bundle_name_lowercase.output.txt new file mode 100644 index 0000000..7e5e3f5 --- /dev/null +++ b/tests/lint/004_bundle_name_lowercase.output.txt @@ -0,0 +1,6 @@ + +bundle agent MyBundle + ^------^ +Convention: Bundle name should be lowercase at tests/lint/004_bundle_name_lowercase.x.cf:1:14 +FAIL: tests/lint/004_bundle_name_lowercase.x.cf (1 errors) +Failure, 1 errors in total. diff --git a/tests/lint/004_bundle_name_lowercase.x.cf b/tests/lint/004_bundle_name_lowercase.x.cf new file mode 100644 index 0000000..46d3776 --- /dev/null +++ b/tests/lint/004_bundle_name_lowercase.x.cf @@ -0,0 +1,5 @@ +bundle agent MyBundle +{ + reports: + "Hello"; +} diff --git a/tests/lint/005_bundle_type.cf b/tests/lint/005_bundle_type.cf new file mode 100644 index 0000000..cff6057 --- /dev/null +++ b/tests/lint/005_bundle_type.cf @@ -0,0 +1,5 @@ +bundle common my_bundle +{ + vars: + "x" string => "value"; +} diff --git a/tests/lint/005_bundle_type.output.txt b/tests/lint/005_bundle_type.output.txt new file mode 100644 index 0000000..a1f4412 --- /dev/null +++ b/tests/lint/005_bundle_type.output.txt @@ -0,0 +1,6 @@ + +bundle notavalidtype my_bundle + ^-----------^ +Error: Bundle type must be one of (agent, common, monitor, server, edit_line, edit_xml), not 'notavalidtype' at tests/lint/005_bundle_type.x.cf:1:8 +FAIL: tests/lint/005_bundle_type.x.cf (1 errors) +Failure, 1 errors in total. diff --git a/tests/lint/005_bundle_type.x.cf b/tests/lint/005_bundle_type.x.cf new file mode 100644 index 0000000..3503de6 --- /dev/null +++ b/tests/lint/005_bundle_type.x.cf @@ -0,0 +1,5 @@ +bundle notavalidtype my_bundle +{ + reports: + "Hello"; +} diff --git a/tests/lint/006_syntax_error.cf b/tests/lint/006_syntax_error.cf new file mode 100644 index 0000000..d83815d --- /dev/null +++ b/tests/lint/006_syntax_error.cf @@ -0,0 +1,5 @@ +bundle agent main +{ + reports: + "Valid syntax"; +} diff --git a/tests/lint/006_syntax_error.output.txt b/tests/lint/006_syntax_error.output.txt new file mode 100644 index 0000000..5e55fa8 --- /dev/null +++ b/tests/lint/006_syntax_error.output.txt @@ -0,0 +1,7 @@ + +{ + reports + ^-----^ +Error: Syntax error at tests/lint/006_syntax_error.x.cf:3:3 +FAIL: tests/lint/006_syntax_error.x.cf (1 errors) +Failure, 1 errors in total. diff --git a/tests/lint/006_syntax_error.x.cf b/tests/lint/006_syntax_error.x.cf new file mode 100644 index 0000000..2915caa --- /dev/null +++ b/tests/lint/006_syntax_error.x.cf @@ -0,0 +1,5 @@ +bundle agent main +{ + reports + "Missing colon after promise type"; +} diff --git a/tests/lint/007_empty_file.output.txt b/tests/lint/007_empty_file.output.txt new file mode 100644 index 0000000..f4ab399 --- /dev/null +++ b/tests/lint/007_empty_file.output.txt @@ -0,0 +1,3 @@ +Error: Empty policy file 'tests/lint/007_empty_file.x.cf' +FAIL: tests/lint/007_empty_file.x.cf (1 errors) +Failure, 1 errors in total. diff --git a/tests/lint/007_empty_file.x.cf b/tests/lint/007_empty_file.x.cf new file mode 100644 index 0000000..e69de29