Skip to content

Commit 39b5652

Browse files
0x416e746f6e0x416e746f6e
committed
feat: implement chaos for flashblocks
1 parent e209bda commit 39b5652

14 files changed

Lines changed: 502 additions & 198 deletions

File tree

‎Makefile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@ serve:
2626
--authrpc-listen-address 127.0.0.1:18651 \
2727
--authrpc-log-requests \
2828
--authrpc-log-responses \
29-
--authrpc-max-request-size 150 \
30-
--authrpc-max-response-size 1150 \
29+
--flashblocks-backend ws://127.0.0.1:1111 \
30+
--flashblocks-enabled \
31+
--flashblocks-listen-address 127.0.0.1:11111 \
32+
--flashblocks-log-messages \
3133
--rpc-backend http://127.0.0.1:8645 \
3234
--rpc-enabled \
3335
--rpc-listen-address 127.0.0.1:18645 \
3436
--rpc-log-requests \
3537
--rpc-log-responses
36-
--rpc-max-request-size 150 \
37-
--rpc-max-response-size 1150 \

‎cmd/serve.go‎

Lines changed: 105 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,59 @@ func CommandServe(cfg *config.Config) *cli.Command {
4545
Value: time.Second,
4646
},
4747

48+
&cli.BoolFlag{ // --chaos-xxx-enabled
49+
Category: strings.ToUpper(categoryChaos),
50+
Destination: &cfg.Chaos.Enabled,
51+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos+"_"+category) + "_ENABLED"},
52+
Name: categoryChaos + "-" + category + "-enabled",
53+
Usage: "whether " + category + " proxy should be injecting artificial error conditions",
54+
Value: false,
55+
},
56+
57+
&cli.Float64Flag{ // --chaos-xxx-injected-http-error-probability
58+
Category: strings.ToUpper(categoryChaos),
59+
Destination: &cfg.Chaos.InjectedHttpErrorProbability,
60+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos+"_"+category) + "_INJECTED_HTTP_ERROR_PROBABILITY"},
61+
Name: categoryChaos + "-" + category + "-injected-http-error-probability",
62+
Usage: "probability in `percent` at which to randomly inject http errors into responses processed by " + category + " proxy",
63+
Value: 0,
64+
},
65+
66+
&cli.Float64Flag{ // --chaos-xxx-injected-jrpc-error-probability
67+
Category: strings.ToUpper(categoryChaos),
68+
Destination: &cfg.Chaos.InjectedJrpcErrorProbability,
69+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos+"_"+category) + "_INJECTED_JRPC_ERROR_PROBABILITY"},
70+
Name: categoryChaos + "-" + category + "-injected-jrpc-error-probability",
71+
Usage: "probability in `percent` at which to randomly inject jrpc errors into responses processed by " + category + " proxy",
72+
Value: 0,
73+
},
74+
75+
&cli.Float64Flag{ // --chaos-xxx-injected-invalid-jrpc-response-probability
76+
Category: strings.ToUpper(categoryChaos),
77+
Destination: &cfg.Chaos.InjectedInvalidJrpcResponseProbability,
78+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos+"_"+category) + "_INJECTED_INVALID_JRPC_RESPONSE_PROBABILITY"},
79+
Name: categoryChaos + "-" + category + "-injected-invalid-jrpc-response-probability",
80+
Usage: "probability in `percent` at which to randomly inject invalid jrpc into responses processed by " + category + " proxy",
81+
Value: 0,
82+
},
83+
84+
&cli.DurationFlag{ // --chaos-xxx-min-injected-latency
85+
Category: strings.ToUpper(categoryChaos),
86+
Destination: &cfg.Chaos.MinInjectedLatency,
87+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos+"_"+category) + "_MIN_INJECTED_LATENCY"},
88+
Name: categoryChaos + "-" + category + "-min-injected-latency",
89+
Usage: "min `latency` to enforce on every response processed by " + category + " proxy",
90+
Value: 100 * time.Millisecond,
91+
},
92+
93+
&cli.DurationFlag{ // --chaos-xxx-max-injected-latency
94+
Category: strings.ToUpper(categoryChaos),
95+
Destination: &cfg.Chaos.MaxInjectedLatency,
96+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos+"_"+category) + "_MAX_INJECTED_LATENCY"},
97+
Name: categoryChaos + "-" + category + "-max-injected-latency",
98+
Usage: "max `latency` to randomly enforce on every response processed by " + category + " proxy",
99+
},
100+
48101
&cli.DurationFlag{ // --xxx-client-idle-connection-timeout
49102
Category: strings.ToUpper(category),
50103
Destination: &cfg.ClientIdleConnectionTimeout,
@@ -268,78 +321,76 @@ func CommandServe(cfg *config.Config) *cli.Command {
268321
},
269322
)
270323

271-
chaosFlags := []cli.Flag{ // --chaos-xxx
272-
&cli.BoolFlag{ // --chaos-enabled
273-
Category: strings.ToUpper(categoryChaos),
274-
Destination: &cfg.Chaos.Enabled,
275-
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_ENABLED"},
276-
Name: categoryChaos + "-enabled",
277-
Usage: "whether bproxy should be injecting artificial error conditions",
278-
Value: false,
324+
flashblocksFlags := []cli.Flag{ // --flashblocks-xxx
325+
&cli.StringFlag{ // --flashblocks-backend
326+
Category: strings.ToUpper(categoryFlashblocks),
327+
Destination: &cfg.Flashblocks.BackendURL,
328+
EnvVars: []string{envPrefix + strings.ToUpper(categoryFlashblocks) + "_BACKEND"},
329+
Name: categoryFlashblocks + "-backend",
330+
Usage: "`url` of flashblocks backend",
331+
Value: "ws://127.0.0.1:11111",
279332
},
280333

281-
&cli.Float64Flag{ // --chaos-injected-http-error-probability
282-
Category: strings.ToUpper(categoryChaos),
283-
Destination: &cfg.Chaos.InjectedHttpErrorProbability,
284-
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_INJECTED_HTTP_ERROR_PROBABILITY"},
285-
Name: categoryChaos + "-injected-http-error-probability",
286-
Usage: "probability in `percent` at which to randomly inject http errors into proxied responses",
287-
Value: 20,
334+
&cli.DurationFlag{ // --flashblocks-backward-timeout
335+
Category: strings.ToUpper(categoryFlashblocks),
336+
Destination: &cfg.Flashblocks.BackwardTimeout,
337+
EnvVars: []string{envPrefix + strings.ToUpper(categoryFlashblocks) + "_BACKWARD_TIMEOUT"},
338+
Name: categoryFlashblocks + "-backward-timeout",
339+
Usage: "max `duration` for flashblocks frontend reads and backend writes (0s means no timeout)",
340+
Value: 0,
288341
},
289342

290-
&cli.Float64Flag{ // --chaos-injected-jrpc-error-probability
343+
&cli.Float64Flag{ // --chaos-flashblocks-dropped-message-probability
291344
Category: strings.ToUpper(categoryChaos),
292-
Destination: &cfg.Chaos.InjectedJrpcErrorProbability,
293-
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_INJECTED_JRPC_ERROR_PROBABILITY"},
294-
Name: categoryChaos + "-injected-jrpc-error-probability",
295-
Usage: "probability in `percent` at which to randomly inject jrpc errors into proxied responses",
296-
Value: 20,
345+
Destination: &cfg.Flashblocks.Chaos.DroppedMessageProbability,
346+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_FLASHBLOCKS_DROPPED_MESSAGE_PROBABILITY"},
347+
Name: categoryChaos + "-flashblocks-dropped-message-probability",
348+
Usage: "probability in `percent` at which to randomly drop messages processed by flashblocks proxy",
349+
Value: 0,
297350
},
298351

299-
&cli.Float64Flag{ // --chaos-injected-invalid-jrpc-response-probability
352+
&cli.BoolFlag{ // --chaos-flashblocks-enabled
300353
Category: strings.ToUpper(categoryChaos),
301-
Destination: &cfg.Chaos.InjectedInvalidJrpcResponseProbability,
302-
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_INJECTED_INVALID_JRPC_RESPONSE_PROBABILITY"},
303-
Name: categoryChaos + "-injected-invalid-jrpc-response-probability",
304-
Usage: "probability in `percent` at which to randomly inject invalid jrpc into proxied responses",
305-
Value: 20,
354+
Destination: &cfg.Flashblocks.Chaos.Enabled,
355+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_FLASHBLOCKS_ENABLED"},
356+
Name: categoryChaos + "-flashblocks-enabled",
357+
Usage: "whether flashblocks proxy should be injecting artificial error conditions",
358+
Value: false,
306359
},
307360

308-
&cli.DurationFlag{ // --chaos-min-injected-latency
361+
&cli.Float64Flag{ // --chaos-flashblocks-injected-invalid-flashblock-payload-probability
309362
Category: strings.ToUpper(categoryChaos),
310-
Destination: &cfg.Chaos.MinInjectedLatency,
311-
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_MIN_INJECTED_LATENCY"},
312-
Name: categoryChaos + "-min-injected-latency",
313-
Usage: "min `latency` to enforce on every proxied response",
314-
Value: 50 * time.Millisecond,
363+
Destination: &cfg.Flashblocks.Chaos.InjectedInvalidFlashblockPayloadProbability,
364+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_FLASHBLOCKS_INJECTED_INVALID_FLASHBLOCK_PAYLOAD_PROBABILITY"},
365+
Name: categoryChaos + "-flashblocks-injected-invalid-flashblock-payload-probability",
366+
Usage: "probability in `percent` at which to randomly inject an invalid flashblock",
367+
Value: 0,
315368
},
316369

317-
&cli.DurationFlag{ // --chaos-max-injected-latency
370+
&cli.Float64Flag{ // --chaos-flashblocks-injected-malformed-json-message-probability
318371
Category: strings.ToUpper(categoryChaos),
319-
Destination: &cfg.Chaos.MaxInjectedLatency,
320-
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_MAX_INJECTED_LATENCY"},
321-
Name: categoryChaos + "-max-injected-latency",
322-
Usage: "max `latency` to randomly enforce on every proxied response",
372+
Destination: &cfg.Flashblocks.Chaos.InjectedMalformedJsonMessageProbability,
373+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_FLASHBLOCKS_INJECTED_MALFORMED_JSON_MESSAGE_PROBABILITY"},
374+
Name: categoryChaos + "-flashblocks-injected-malformed-json-message-probability",
375+
Usage: "probability in `percent` at which to randomly inject a malformed json message",
376+
Value: 0,
323377
},
324-
}
325378

326-
flashblocksFlags := []cli.Flag{ // --flashblocks-xxx
327-
&cli.StringFlag{ // --flashblocks-backend
328-
Category: strings.ToUpper(categoryFlashblocks),
329-
Destination: &cfg.Flashblocks.BackendURL,
330-
EnvVars: []string{envPrefix + strings.ToUpper(categoryFlashblocks) + "_BACKEND"},
331-
Name: categoryFlashblocks + "-backend",
332-
Usage: "`url` of flashblocks backend",
333-
Value: "ws://127.0.0.1:11111",
379+
&cli.DurationFlag{ // --chaos-flashblocks-min-injected-latency
380+
Category: strings.ToUpper(categoryChaos),
381+
Destination: &cfg.Flashblocks.Chaos.MinInjectedLatency,
382+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_FLASHBLOCKS_MIN_INJECTED_LATENCY"},
383+
Name: categoryChaos + "-flashblocks-min-injected-latency",
384+
Usage: "min `latency` to enforce on every response processed by flashblocks proxy",
385+
Value: 100 * time.Millisecond,
334386
},
335387

336-
&cli.DurationFlag{ // --flashblocks-backward-timeout
337-
Category: strings.ToUpper(categoryFlashblocks),
338-
Destination: &cfg.Flashblocks.BackwardTimeout,
339-
EnvVars: []string{envPrefix + strings.ToUpper(categoryFlashblocks) + "_BACKWARD_TIMEOUT"},
340-
Name: categoryFlashblocks + "-backward-timeout",
341-
Usage: "max `duration` for flashblocks frontend reads and backend writes (0s means no timeout)",
342-
Value: 0,
388+
&cli.DurationFlag{ // --chaos-flashblocks-max-injected-latency
389+
Category: strings.ToUpper(categoryChaos),
390+
Destination: &cfg.Flashblocks.Chaos.MaxInjectedLatency,
391+
EnvVars: []string{envPrefix + strings.ToUpper(categoryChaos) + "_FLASHBLOCKS_MAX_INJECTED_LATENCY"},
392+
Name: categoryChaos + "-flashblocks-max-injected-latency",
393+
Usage: "max `latency` to randomly enforce on every response processed by flashblocks proxy",
343394
},
344395

345396
&cli.DurationFlag{ // --flashblocks-control-timeout
@@ -489,7 +540,6 @@ func CommandServe(cfg *config.Config) *cli.Command {
489540
authrpcFlags,
490541
flashblocksFlags,
491542
rpcFlags,
492-
chaosFlags,
493543
metricsFlags,
494544
)
495545

‎config/chaos.go‎

Lines changed: 88 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"github.com/flashbots/bproxy/utils"
99
)
1010

11-
type Chaos struct {
11+
type ChaosHttp struct {
1212
Enabled bool `yaml:"enabled"`
1313

1414
MinInjectedLatency time.Duration `yaml:"min_injected_latency"`
@@ -18,15 +18,28 @@ type Chaos struct {
1818
InjectedInvalidJrpcResponseProbability float64 `yaml:"injected_invalid_jrpc_response_probability"`
1919
}
2020

21+
type ChaosWebsocket struct {
22+
Enabled bool `yaml:"enabled"`
23+
24+
MinInjectedLatency time.Duration `yaml:"min_injected_latency"`
25+
MaxInjectedLatency time.Duration `yaml:"max_injected_latency"`
26+
DroppedMessageProbability float64 `yaml:"dropped_message_probability"`
27+
InjectedInvalidFlashblockPayloadProbability float64 `yaml:"injected_invalid_flashblock_payload_probability"`
28+
InjectedMalformedJsonMessageProbability float64 `yaml:"injected_invalid_malformed_json_message_probability"`
29+
}
30+
2131
var (
22-
errChaosInvalidMinInjectedLatency = errors.New("invalid min injected latency")
23-
errChaosInvalidMaxInjectedLatency = errors.New("invalid max injected latency")
24-
errChaosInvalidInjectedHttpErrorProbability = errors.New("injected http error probability must be in [0, 100] range")
25-
errChaosInvalidInjectedJrpcErrorProbability = errors.New("injected jrpc error probability must be in [0, 100] range")
26-
errChaosInvalidInjectedInvalidJrpcResponseProbability = errors.New("injected invalid jrpc error probability must be in [0, 100] range")
32+
errChaosInvalidDroppedMessageProbability = errors.New("dropped message probability must be in [0, 100] range")
33+
errChaosInvalidInjectedHttpErrorProbability = errors.New("injected http error probability must be in [0, 100] range")
34+
errChaosInvalidInjectedInvalidFlashblockProbability = errors.New("dropped message probability must be in [0, 100] range")
35+
errChaosInvalidInjectedInvalidJrpcResponseProbability = errors.New("injected invalid jrpc error probability must be in [0, 100] range")
36+
errChaosInvalidInjectedJrpcErrorProbability = errors.New("injected jrpc error probability must be in [0, 100] range")
37+
errChaosInvalidInjectedMalformedJsonMessageProbability = errors.New("dropped message probability must be in [0, 100] range")
38+
errChaosInvalidMaxInjectedLatency = errors.New("invalid max injected latency")
39+
errChaosInvalidMinInjectedLatency = errors.New("invalid min injected latency")
2740
)
2841

29-
func (cfg *Chaos) Validate() error {
42+
func (cfg *ChaosHttp) Validate() error {
3043
errs := make([]error, 0)
3144

3245
{ // min injected latency
@@ -89,5 +102,73 @@ func (cfg *Chaos) Validate() error {
89102
))
90103
}
91104
}
105+
106+
return utils.FlattenErrors(errs)
107+
}
108+
109+
func (cfg *ChaosWebsocket) Validate() error {
110+
errs := make([]error, 0)
111+
112+
{ // min injected latency
113+
if cfg.MinInjectedLatency < 0 {
114+
errs = append(errs, fmt.Errorf("%w: can not be negative: %s",
115+
errChaosInvalidMinInjectedLatency,
116+
cfg.MaxInjectedLatency.String(),
117+
))
118+
}
119+
if cfg.MinInjectedLatency > time.Minute {
120+
errs = append(errs, fmt.Errorf("%w: can not be more than 1 minute: %s",
121+
errChaosInvalidMinInjectedLatency,
122+
cfg.MaxInjectedLatency.String(),
123+
))
124+
}
125+
126+
if cfg.MaxInjectedLatency == 0 {
127+
cfg.MaxInjectedLatency = cfg.MinInjectedLatency
128+
}
129+
}
130+
131+
{ // max injected latency
132+
if cfg.MaxInjectedLatency < 0 {
133+
errs = append(errs, fmt.Errorf("%w: can not be negative: %s",
134+
errChaosInvalidMaxInjectedLatency,
135+
cfg.MaxInjectedLatency.String(),
136+
))
137+
}
138+
if cfg.MaxInjectedLatency > time.Minute {
139+
errs = append(errs, fmt.Errorf("%w: can not be more than 1 minute: %s",
140+
errChaosInvalidMaxInjectedLatency,
141+
cfg.MaxInjectedLatency.String(),
142+
))
143+
}
144+
}
145+
146+
{ // dropped message probability
147+
if cfg.DroppedMessageProbability < 0 || cfg.DroppedMessageProbability > 100 {
148+
errs = append(errs, fmt.Errorf("%w: %f",
149+
errChaosInvalidDroppedMessageProbability,
150+
cfg.DroppedMessageProbability,
151+
))
152+
}
153+
}
154+
155+
{ // injected invalid flashblock payload probability
156+
if cfg.InjectedInvalidFlashblockPayloadProbability < 0 || cfg.InjectedInvalidFlashblockPayloadProbability > 100 {
157+
errs = append(errs, fmt.Errorf("%w: %f",
158+
errChaosInvalidInjectedInvalidFlashblockProbability,
159+
cfg.InjectedInvalidFlashblockPayloadProbability,
160+
))
161+
}
162+
}
163+
164+
{ // injected malformed json message probability
165+
if cfg.InjectedMalformedJsonMessageProbability < 0 || cfg.InjectedMalformedJsonMessageProbability > 100 {
166+
errs = append(errs, fmt.Errorf("%w: %f",
167+
errChaosInvalidInjectedMalformedJsonMessageProbability,
168+
cfg.InjectedMalformedJsonMessageProbability,
169+
))
170+
}
171+
}
172+
92173
return utils.FlattenErrors(errs)
93174
}

‎config/config.go‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,35 @@ import (
66
)
77

88
type Config struct {
9-
Chaos *Chaos `yaml:"chaos"`
10-
Log *Log `yaml:"log"`
11-
129
Authrpc *Authrpc `yaml:"authrpc"`
1310
Flashblocks *Flashblocks `yaml:"flashblocks"`
1411
Rpc *Rpc `yaml:"rpc"`
1512

13+
Log *Log `yaml:"log"`
1614
Metrics *Metrics `yaml:"metrics"`
17-
18-
Version string `yaml:"-"`
15+
Version string `yaml:"-"`
1916
}
2017

2118
func New(version string) *Config {
2219
return &Config{
23-
Chaos: &Chaos{},
24-
Log: &Log{},
25-
26-
Authrpc: &Authrpc{HttpProxy: &HttpProxy{Healthcheck: &Healthcheck{}}},
27-
Flashblocks: &Flashblocks{WebsocketProxy: &WebsocketProxy{Healthcheck: &Healthcheck{}}},
28-
Rpc: &Rpc{HttpProxy: &HttpProxy{Healthcheck: &Healthcheck{}}},
29-
20+
Log: &Log{},
3021
Metrics: &Metrics{},
3122
Version: version,
23+
24+
Authrpc: &Authrpc{HttpProxy: &HttpProxy{
25+
Chaos: &ChaosHttp{},
26+
Healthcheck: &Healthcheck{}},
27+
},
28+
29+
Flashblocks: &Flashblocks{WebsocketProxy: &WebsocketProxy{
30+
Chaos: &ChaosWebsocket{},
31+
Healthcheck: &Healthcheck{},
32+
}},
33+
34+
Rpc: &Rpc{HttpProxy: &HttpProxy{
35+
Chaos: &ChaosHttp{},
36+
Healthcheck: &Healthcheck{},
37+
}},
3238
}
3339
}
3440

‎config/http_proxy.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
type HttpProxy struct {
1717
BackendTimeout time.Duration `yaml:"backend_timeout"`
1818
BackendURL string `yaml:"backend_url"`
19+
Chaos *ChaosHttp `yaml:"chaos"`
1920
ClientIdleConnectionTimeout time.Duration `yaml:"client_idle_connection_timeout"`
2021
Enabled bool `yaml:"enabled"`
2122
ExtraMirroredJrpcMethods []string `yaml:"extra_mirrored_jrpc_methods"`

0 commit comments

Comments
 (0)