Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion .github/workflows/dwarf-perf-regression.yml
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,7 @@ jobs:
compare_target() {
local target="$1"
local safe_target="$2"
local enforce_query_metrics="$3"
local base_result="$DWARF_PERF_RESULTS_DIR/base-${target}.json"
local head_result="$DWARF_PERF_RESULTS_DIR/head-${target}.json"
local summary_result="$DWARF_PERF_RESULTS_DIR/summary-${safe_target}.md"
Expand All @@ -271,6 +272,13 @@ jobs:
--result-file "$compare_result"
)

if [[ "$enforce_query_metrics" != "true" ]]; then
args+=(
--skip-query-metrics
--skip-query-reason "Source-line query uses the same query-hotspot corpus for every parse target and is enforced only for parse-stress."
)
fi

if [[ "$REPORT_ONLY" == "true" ]]; then
args+=(
--report-only
Expand Down Expand Up @@ -309,8 +317,12 @@ jobs:

for target in "${parse_targets[@]}"; do
safe_target=${target//-/_}
enforce_query_metrics=false
if [[ "$target" == "parse-stress" ]]; then
enforce_query_metrics=true
fi
if [[ -f "$DWARF_PERF_RESULTS_DIR/base-${target}.json" ]]; then
compare_target "$target" "$safe_target"
compare_target "$target" "$safe_target" "$enforce_query_metrics"
target_exit=$?
if [[ "$target_exit" -ne 0 ]]; then
overall_exit="$target_exit"
Expand Down
34 changes: 32 additions & 2 deletions scripts/dwarf-perf/compare_baseline.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class SupplementalMetricComparison:
status: str


RULES = [
PARSE_RULES = [
MetricRule(
name="fast_parse_p50",
json_path="parse_benchmark.metrics_ms.p50",
Expand All @@ -70,6 +70,9 @@ class SupplementalMetricComparison:
severity="fail",
description="Fast parse p95",
),
]

QUERY_RULES = [
MetricRule(
name="source_line_query_p50",
json_path="query_benchmark.metrics_ms.p50",
Expand Down Expand Up @@ -155,6 +158,20 @@ def parse_args() -> argparse.Namespace:
default="",
help="Optional note explaining why the comparison is report-only",
)
parser.add_argument(
"--skip-query-metrics",
action="store_true",
help=(
"Skip source-line query metrics for this comparison. The query "
"benchmark is independent of the parse target and should only be "
"enforced once per base/head comparison set."
),
)
parser.add_argument(
"--skip-query-reason",
default="",
help="Optional note explaining why source-line query metrics were skipped.",
)
return parser.parse_args()


Expand Down Expand Up @@ -344,6 +361,7 @@ def build_console_report(
head_label: str,
report_only: bool,
report_only_reason: str,
query_skip_reason: str,
overall_status: str,
) -> str:
mode = "report-only" if report_only else "enforced"
Expand All @@ -366,6 +384,8 @@ def build_console_report(
)
if report_only_reason:
lines.append(f" note: {report_only_reason}")
if query_skip_reason:
lines.append(f" query metrics: skipped ({query_skip_reason})")
return "\n".join(lines)


Expand Down Expand Up @@ -395,6 +415,7 @@ def build_summary(
head_label: str,
report_only: bool,
report_only_reason: str,
query_skip_reason: str,
overall_status: str,
) -> str:
mode = "report-only" if report_only else "enforced"
Expand All @@ -410,6 +431,8 @@ def build_summary(

if report_only_reason:
lines.append(f"- Note: {report_only_reason}")
if query_skip_reason:
lines.append(f"- Query metrics: skipped. {query_skip_reason}")

lines.extend(
[
Expand Down Expand Up @@ -441,15 +464,18 @@ def main() -> int:
base_data = load_json(args.base)
head_data = load_json(args.head)

metric_rules = PARSE_RULES if args.skip_query_metrics else PARSE_RULES + QUERY_RULES
metrics = [
compare_metric(base_data, head_data, rule, args.report_only) for rule in RULES
compare_metric(base_data, head_data, rule, args.report_only)
for rule in metric_rules
]
index_phase_metrics = [
compare_supplemental_metric(base_data, head_data, rule)
for rule in INDEX_PHASE_RULES
]
overall_status = determine_overall_status(metrics, args.report_only)
parse_target_name = resolve_parse_target_name(base_data, head_data)
query_skip_reason = args.skip_query_reason if args.skip_query_metrics else ""

summary = build_summary(
parse_target_name,
Expand All @@ -459,6 +485,7 @@ def main() -> int:
args.head_label,
args.report_only,
args.report_only_reason,
query_skip_reason,
overall_status,
)
Path(args.summary_file).write_text(summary)
Expand All @@ -467,6 +494,8 @@ def main() -> int:
"schema_version": 2,
"mode": "report-only" if args.report_only else "enforced",
"report_only_reason": args.report_only_reason,
"query_metrics_skipped": args.skip_query_metrics,
"query_skip_reason": query_skip_reason,
"parse_target": parse_target_name,
"base": {"label": args.base_label, "path": args.base},
"head": {"label": args.head_label, "path": args.head},
Expand All @@ -487,6 +516,7 @@ def main() -> int:
args.head_label,
args.report_only,
args.report_only_reason,
query_skip_reason,
overall_status,
)
)
Expand Down
Loading