Skip to content

Commit 6d94bcb

Browse files
pdapda
committed
preflight: show Test Engine failures from the build
1 parent eb0b3df commit 6d94bcb

7 files changed

Lines changed: 275 additions & 7 deletions

File tree

‎cmd/preflight/preflight.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,12 @@ func (c *PreflightCmd) Run(kongCtx *kong.Context, globals cli.GlobalFlags) error
147147
return renderStatusError{err: err}
148148
}
149149
return nil
150-
})
150+
}, watch.WithTestTracking(func(newFailures []buildkite.BuildTest) error {
151+
if err := renderer.renderTestFailures(newFailures); err != nil {
152+
return renderStatusError{err: err}
153+
}
154+
return nil
155+
}))
151156
if err != nil {
152157
var renderErr renderStatusError
153158
if errors.As(err, &renderErr) {

‎cmd/preflight/render.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
"github.com/buildkite/cli/v3/internal/build/watch"
1111
internalpreflight "github.com/buildkite/cli/v3/internal/preflight"
12+
buildkite "github.com/buildkite/go-buildkite/v4"
1213
)
1314

1415
const maxTTYRunningJobs = 10
@@ -17,6 +18,7 @@ type renderer interface {
1718
appendSnapshotLine(string)
1819
setSnapshot(*internalpreflight.SnapshotResult)
1920
renderStatus(watch.BuildStatus, string) error
21+
renderTestFailures([]buildkite.BuildTest) error
2022
flush()
2123
renderFinalFailures(Result, watch.FailedJobs)
2224
}
@@ -92,6 +94,14 @@ func (r *ttyRenderer) renderStatus(status watch.BuildStatus, buildState string)
9294
return nil
9395
}
9496

97+
func (r *ttyRenderer) renderTestFailures(tests []buildkite.BuildTest) error {
98+
for _, t := range tests {
99+
line := formatTestFailureLine(t)
100+
r.failedRegion.AppendLine(line)
101+
}
102+
return nil
103+
}
104+
95105
func (r *ttyRenderer) flush() {
96106
r.screen.Flush()
97107
}
@@ -122,6 +132,15 @@ func (r *plainRenderer) setSnapshot(result *internalpreflight.SnapshotResult) {
122132
}
123133
}
124134

135+
func (r *plainRenderer) renderTestFailures(tests []buildkite.BuildTest) error {
136+
for _, t := range tests {
137+
if _, err := fmt.Fprintf(r.stdout, "%s\n", formatTestFailureLine(t)); err != nil {
138+
return err
139+
}
140+
}
141+
return nil
142+
}
143+
125144
func (r *plainRenderer) renderStatus(status watch.BuildStatus, buildState string) error {
126145
var presenter jobPresenter = plainJobPresenter{pipeline: r.pipeline, buildNumber: r.buildNumber}
127146
for _, failed := range status.NewlyFailed {
@@ -206,6 +225,32 @@ func jobLogCommand(pipeline string, buildNumber int, jobID string) string {
206225
return fmt.Sprintf("bk job log -b %d -p %s %s", buildNumber, pipeline, jobID)
207226
}
208227

228+
func formatTestFailureLine(t buildkite.BuildTest) string {
229+
name := t.Name
230+
if t.Scope != "" {
231+
name = t.Scope + " " + name
232+
}
233+
line := fmt.Sprintf(" \033[31m✗\033[0m \033[33mtest:\033[0m %s", name)
234+
if t.Location != "" {
235+
line += fmt.Sprintf(" \033[2m%s\033[0m", t.Location)
236+
}
237+
if t.LatestFail == nil {
238+
return line
239+
}
240+
if t.LatestFail.FailureReason != "" {
241+
line += fmt.Sprintf("\n \033[2m%s\033[0m", t.LatestFail.FailureReason)
242+
}
243+
for _, fe := range t.LatestFail.FailureExpanded {
244+
for _, exp := range fe.Expanded {
245+
line += fmt.Sprintf("\n \033[2m%s\033[0m", exp)
246+
}
247+
for _, bt := range fe.Backtrace {
248+
line += fmt.Sprintf("\n \033[2m%s\033[0m", bt)
249+
}
250+
}
251+
return line
252+
}
253+
209254
func formatSummaryLine(s watch.JobSummary) string {
210255
var parts []string
211256
if s.Passed > 0 {

‎go.mod‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.25.0
55
require (
66
github.com/alecthomas/kong v1.14.0
77
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
8-
github.com/buildkite/go-buildkite/v4 v4.17.0
8+
github.com/buildkite/go-buildkite/v4 v4.18.0
99
github.com/buildkite/roko v1.4.0
1010
github.com/go-git/go-git/v5 v5.17.1
1111
github.com/goccy/go-yaml v1.19.2
@@ -19,7 +19,6 @@ require (
1919
)
2020

2121
require (
22-
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
2322
github.com/agnivade/levenshtein v1.2.1 // indirect
2423
github.com/alexflint/go-arg v1.5.1 // indirect
2524
github.com/alexflint/go-scalar v1.2.0 // indirect

‎go.sum‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
77
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
88
github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
99
github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
10-
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
11-
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
1210
github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
1311
github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=
1412
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
@@ -33,8 +31,8 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN
3331
github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
3432
github.com/bradleyjkemp/cupaloy/v2 v2.6.0 h1:knToPYa2xtfg42U3I6punFEjaGFKWQRXJwj0JTv4mTs=
3533
github.com/bradleyjkemp/cupaloy/v2 v2.6.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
36-
github.com/buildkite/go-buildkite/v4 v4.17.0 h1:OYy9PG5A15K4Up2dkZgvXP7esAqzQskA0VGXvciRUNQ=
37-
github.com/buildkite/go-buildkite/v4 v4.17.0/go.mod h1:8+7GiWBKwEPAWoZnRU/kpNCt46j1iVH8kFMMbD4YDfc=
34+
github.com/buildkite/go-buildkite/v4 v4.18.0 h1:L0zypZi+jo9jqqcsrXSk5Jkyn3Hsdi2fwo5g7ltDbtU=
35+
github.com/buildkite/go-buildkite/v4 v4.18.0/go.mod h1:8+7GiWBKwEPAWoZnRU/kpNCt46j1iVH8kFMMbD4YDfc=
3836
github.com/buildkite/roko v1.4.0 h1:DxixoCdpNqxu4/1lXrXbfsKbJSd7r1qoxtef/TT2J80=
3937
github.com/buildkite/roko v1.4.0/go.mod h1:0vbODqUFEcVf4v2xVXRfZZRsqJVsCCHTG/TBRByGK4E=
4038
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=

‎internal/build/watch/test_tracker.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package watch
2+
3+
import (
4+
buildkite "github.com/buildkite/go-buildkite/v4"
5+
)
6+
7+
// TestTracker tracks which test failure executions have already been reported,
8+
// so that each failure is only surfaced once across polling iterations.
9+
type TestTracker struct {
10+
seen map[string]bool // keyed by latest_fail execution ID
11+
}
12+
13+
// NewTestTracker creates a new TestTracker.
14+
func NewTestTracker() *TestTracker {
15+
return &TestTracker{
16+
seen: make(map[string]bool),
17+
}
18+
}
19+
20+
// Update processes a list of build tests and returns only those with
21+
// a LatestFail execution that has not been seen before.
22+
func (t *TestTracker) Update(tests []buildkite.BuildTest) []buildkite.BuildTest {
23+
var newFailures []buildkite.BuildTest
24+
for _, test := range tests {
25+
if test.LatestFail == nil {
26+
continue
27+
}
28+
if t.seen[test.LatestFail.ID] {
29+
continue
30+
}
31+
t.seen[test.LatestFail.ID] = true
32+
newFailures = append(newFailures, test)
33+
}
34+
return newFailures
35+
}

‎internal/build/watch/test_tracker_test.go‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package watch
2+
3+
import (
4+
"testing"
5+
6+
buildkite "github.com/buildkite/go-buildkite/v4"
7+
)
8+
9+
func TestTestTracker_Update(t *testing.T) {
10+
t.Run("reports new failures", func(t *testing.T) {
11+
tracker := NewTestTracker()
12+
tests := []buildkite.BuildTest{
13+
{
14+
ID: "test-1",
15+
Name: "flaky test",
16+
LatestFail: &buildkite.BuildTestLatestFail{
17+
ID: "exec-1",
18+
FailureReason: "expected 3, got 2",
19+
},
20+
},
21+
}
22+
23+
newFailures := tracker.Update(tests)
24+
if len(newFailures) != 1 {
25+
t.Fatalf("expected 1 new failure, got %d", len(newFailures))
26+
}
27+
if newFailures[0].Name != "flaky test" {
28+
t.Errorf("expected 'flaky test', got %q", newFailures[0].Name)
29+
}
30+
})
31+
32+
t.Run("does not re-report same execution", func(t *testing.T) {
33+
tracker := NewTestTracker()
34+
tests := []buildkite.BuildTest{
35+
{
36+
ID: "test-1",
37+
Name: "flaky test",
38+
LatestFail: &buildkite.BuildTestLatestFail{
39+
ID: "exec-1",
40+
FailureReason: "expected 3, got 2",
41+
},
42+
},
43+
}
44+
45+
tracker.Update(tests)
46+
newFailures := tracker.Update(tests)
47+
if len(newFailures) != 0 {
48+
t.Errorf("expected 0 new failures on second poll, got %d", len(newFailures))
49+
}
50+
})
51+
52+
t.Run("reports new execution for same test", func(t *testing.T) {
53+
tracker := NewTestTracker()
54+
tracker.Update([]buildkite.BuildTest{
55+
{
56+
ID: "test-1",
57+
Name: "flaky test",
58+
LatestFail: &buildkite.BuildTestLatestFail{
59+
ID: "exec-1",
60+
FailureReason: "first failure",
61+
},
62+
},
63+
})
64+
65+
newFailures := tracker.Update([]buildkite.BuildTest{
66+
{
67+
ID: "test-1",
68+
Name: "flaky test",
69+
LatestFail: &buildkite.BuildTestLatestFail{
70+
ID: "exec-2",
71+
FailureReason: "second failure",
72+
},
73+
},
74+
})
75+
76+
if len(newFailures) != 1 {
77+
t.Fatalf("expected 1 new failure, got %d", len(newFailures))
78+
}
79+
if newFailures[0].LatestFail.ID != "exec-2" {
80+
t.Errorf("expected exec-2, got %s", newFailures[0].LatestFail.ID)
81+
}
82+
})
83+
84+
t.Run("skips tests without latest_fail", func(t *testing.T) {
85+
tracker := NewTestTracker()
86+
tests := []buildkite.BuildTest{
87+
{ID: "test-1", Name: "passing test"},
88+
{
89+
ID: "test-2",
90+
Name: "failing test",
91+
LatestFail: &buildkite.BuildTestLatestFail{
92+
ID: "exec-1",
93+
FailureReason: "boom",
94+
},
95+
},
96+
}
97+
98+
newFailures := tracker.Update(tests)
99+
if len(newFailures) != 1 {
100+
t.Fatalf("expected 1 new failure, got %d", len(newFailures))
101+
}
102+
if newFailures[0].ID != "test-2" {
103+
t.Errorf("expected test-2, got %s", newFailures[0].ID)
104+
}
105+
})
106+
107+
t.Run("handles multiple new failures at once", func(t *testing.T) {
108+
tracker := NewTestTracker()
109+
tests := []buildkite.BuildTest{
110+
{
111+
ID: "test-1",
112+
LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-1"},
113+
},
114+
{
115+
ID: "test-2",
116+
LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-2"},
117+
},
118+
{
119+
ID: "test-3",
120+
LatestFail: &buildkite.BuildTestLatestFail{ID: "exec-3"},
121+
},
122+
}
123+
124+
newFailures := tracker.Update(tests)
125+
if len(newFailures) != 3 {
126+
t.Fatalf("expected 3 new failures, got %d", len(newFailures))
127+
}
128+
})
129+
}

‎internal/build/watch/watch.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,25 @@ const (
2222
// Returning an error aborts the watch loop and propagates that error to the caller.
2323
type StatusFunc func(b buildkite.Build) error
2424

25+
// TestStatusFunc is called with newly-seen failed test executions on each poll.
26+
// Returning an error aborts the watch loop.
27+
type TestStatusFunc func(newFailures []buildkite.BuildTest) error
28+
29+
// WatchOpt configures optional WatchBuild behavior.
30+
type WatchOpt func(*watchConfig)
31+
32+
type watchConfig struct {
33+
onTestStatus TestStatusFunc
34+
}
35+
36+
// WithTestTracking enables polling BuildTests.List for failed tests on each
37+
// iteration, calling onTestStatus with any newly-seen failures.
38+
func WithTestTracking(fn TestStatusFunc) WatchOpt {
39+
return func(c *watchConfig) {
40+
c.onTestStatus = fn
41+
}
42+
}
43+
2544
// WatchBuild polls a build until it reaches a terminal state (FinishedAt != nil).
2645
// It calls onStatus after each successful poll so callers can render progress.
2746
func WatchBuild(
@@ -31,7 +50,18 @@ func WatchBuild(
3150
buildNumber int,
3251
interval time.Duration,
3352
onStatus StatusFunc,
53+
opts ...WatchOpt,
3454
) (buildkite.Build, error) {
55+
cfg := &watchConfig{}
56+
for _, opt := range opts {
57+
opt(cfg)
58+
}
59+
60+
var testTracker *TestTracker
61+
if cfg.onTestStatus != nil {
62+
testTracker = NewTestTracker()
63+
}
64+
3565
var (
3666
consecutiveErrors int
3767
lastBuild buildkite.Build
@@ -60,6 +90,12 @@ func WatchBuild(
6090
}
6191
}
6292

93+
if testTracker != nil && b.ID != "" {
94+
if err := pollTestFailures(ctx, client, org, b.ID, testTracker, cfg.onTestStatus); err != nil {
95+
return b, err
96+
}
97+
}
98+
6399
if b.FinishedAt != nil || buildstate.IsTerminal(buildstate.State(b.State)) {
64100
return b, nil
65101
}
@@ -72,3 +108,24 @@ func WatchBuild(
72108
}
73109
}
74110
}
111+
112+
func pollTestFailures(ctx context.Context, client *buildkite.Client, org, buildID string, tracker *TestTracker, onTestStatus TestStatusFunc) error {
113+
reqCtx, cancel := context.WithTimeout(ctx, DefaultRequestTimeout)
114+
defer cancel()
115+
116+
tests, _, err := client.BuildTests.List(reqCtx, org, buildID, &buildkite.BuildTestsListOptions{
117+
Result: "^failed",
118+
State: "enabled",
119+
Include: "latest_fail",
120+
})
121+
if err != nil {
122+
// Test data may not be available yet; don't treat as fatal.
123+
return nil
124+
}
125+
126+
newFailures := tracker.Update(tests)
127+
if len(newFailures) > 0 {
128+
return onTestStatus(newFailures)
129+
}
130+
return nil
131+
}

0 commit comments

Comments
 (0)