From 7d78360cdd8c942eea5c36016083ba4a21ea57d9 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Wed, 25 Mar 2026 14:29:25 -0700 Subject: [PATCH 1/6] chore: update turbostat Signed-off-by: Harper, Jason M --- internal/extract/turbostat.go | 12 ++++++------ tools/Makefile | 18 ++++++++++-------- tools/build.Dockerfile | 1 + 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/internal/extract/turbostat.go b/internal/extract/turbostat.go index 9aa98e3d..f2dc439b 100644 --- a/internal/extract/turbostat.go +++ b/internal/extract/turbostat.go @@ -62,12 +62,12 @@ func parseTurbostatOutput(output string) ([]map[string]string, error) { if len(headers) == 0 { continue } - if len(fields) != len(headers) { - continue + if len(fields) > len(headers) { + return nil, fmt.Errorf("more turbostat row values than headers: %d values, %d headers", len(fields), len(headers)) } - row := make(map[string]string, len(headers)) - for i, h := range headers { - row[h] = fields[i] + row := make(map[string]string, len(fields)) + for i := range fields { + row[headers[i]] = fields[i] // this assumes any missing fields are at the end and will be empty string, which is consistent with turbostat output } if timeParsed && interval > 0 { row["timestamp"] = timestamp.Format("15:04:05") @@ -360,7 +360,7 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([] } func isPackageRow(row map[string]string) bool { - if val, ok := row["Package"]; ok && val != "-" { + if val, ok := row["Package"]; ok && val != "-" && row["Core"] == "0" { return true } return false diff --git a/tools/Makefile b/tools/Makefile index 48aba635..6ab7e7c3 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -25,7 +25,7 @@ default: tools-x86_64 .PHONY: sysstat sysstat-aarch64 sysstat-repo .PHONY: tsc turbostat -tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat +tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc mkdir -p bin/x86_64 cp -R async-profiler bin/x86_64/ cp avx-turbo/avx-turbo bin/x86_64/ @@ -50,7 +50,6 @@ tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw cp sysstat/sar bin/x86_64/ cp sysstat/sadc bin/x86_64/ cp tsc/tsc bin/x86_64/ - cp linux_turbostat/tools/power/x86/turbostat/turbostat bin/x86_64/ cd bin/x86_64 && strip --strip-unneeded * || true tools-aarch64: async-profiler-aarch64 dmidecode-aarch64 ethtool-aarch64 fio-aarch64 ipmitool-aarch64 lshw-aarch64 lspci-aarch64 spectre-meltdown-checker sshpass-aarch64 stackcollapse-perf-aarch64 stress-ng-aarch64 sysstat-aarch64 @@ -348,7 +347,7 @@ else cd processwatch && git fetch --tags && git checkout $(PROCESSWATCH_VERSION) endif cd processwatch && ./build.sh - mkdir -p bin + mkdir -p bin/x86_64 cp processwatch/processwatch bin/x86_64/ strip --strip-unneeded bin/x86_64/processwatch @@ -437,14 +436,18 @@ endif tsc: cd tsc && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -TURBOSTAT_VERSION := 6.9.12 +# turbostat version 2025.12.02 is in kernel 6.19 +TURBOSTAT_LINUX_VERSION := 6.19.9 turbostat: ifeq ("$(wildcard linux_turbostat)","") - wget $(WGET_OPTS) https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$(TURBOSTAT_VERSION).tar.xz - tar -xf linux-$(TURBOSTAT_VERSION).tar.xz && mv linux-$(TURBOSTAT_VERSION)/ linux_turbostat/ + wget $(WGET_OPTS) https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$(TURBOSTAT_LINUX_VERSION).tar.xz + tar -xf linux-$(TURBOSTAT_LINUX_VERSION).tar.xz && mv linux-$(TURBOSTAT_LINUX_VERSION)/ linux_turbostat/ endif sed -i '/_Static_assert/d' linux_turbostat/tools/power/x86/turbostat/turbostat.c cd linux_turbostat/tools/power/x86/turbostat && make + mkdir -p bin/x86_64 + cp linux_turbostat/tools/power/x86/turbostat/turbostat bin/x86_64/ + strip --strip-unneeded bin/x86_64/turbostat reset: cd async-profiler @@ -462,7 +465,6 @@ reset: cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc - cd linux_turbostat/tools/power/x86/turbostat && make clean # libs are not directly used in the build but are required in the oss archive file because some of the tools are statically linked # glibc 2.27 was the version used in Ubuntu 18.04 which is the base docker image we used to build the tools @@ -475,5 +477,5 @@ libcrypt.tar.gz: libs: glibc.tar.gz zlib.tar.gz libcrypt.tar.gz oss-source: reset libs - tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ perf-archive/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc.tar.gz zlib.tar.gz libcrypt.tar.gz + tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ perf-archive/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ glibc.tar.gz zlib.tar.gz libcrypt.tar.gz md5sum oss_source.tgz > oss_source.tgz.md5 diff --git a/tools/build.Dockerfile b/tools/build.Dockerfile index 6fe43370..a838b930 100644 --- a/tools/build.Dockerfile +++ b/tools/build.Dockerfile @@ -167,6 +167,7 @@ ADD . /workdir WORKDIR /workdir RUN make perf RUN make processwatch +RUN make turbostat FROM scratch AS output COPY --from=builder workdir/bin /bin From 9f573c67b2f55a4b566e0e18dd6dedbfdd0cb2f4 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Wed, 25 Mar 2026 15:08:32 -0700 Subject: [PATCH 2/6] fix unit test Signed-off-by: Harper, Jason M --- internal/extract/turbostat_test.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/internal/extract/turbostat_test.go b/internal/extract/turbostat_test.go index 228f504f..b13d0be5 100644 --- a/internal/extract/turbostat_test.go +++ b/internal/extract/turbostat_test.go @@ -443,14 +443,11 @@ func TestTurbostatPackageRows(t *testing.T) { wantErr bool }{ { - name: "package rows with UncoreMHz, PKGTmp, PkgWatt", + name: "package rows with UncoreMHz, PKGTmp, PkgWatt (sparse rows)", turbostatOutput: turbostatOutput, fieldNames: []string{"UncMHz", "PkgTmp", "PkgWatt"}, - want: [][][]string{ - {{"15:04:05", "2350", "57", "223.53"}, {"15:04:07", "2400", "59", "229.53"}, {"15:04:09", "2400", "57", "223.53"}}, - {{"15:04:05", "2300", "53", "208.40"}, {"15:04:07", "2400", "55", "218.40"}, {"15:04:09", "2400", "53", "208.40"}}, - }, - wantErr: false, + want: nil, + wantErr: true, }, { name: "Typical output, two packages, one field", @@ -590,8 +587,8 @@ func TestTurbostatPackageRowsByRegexMatch(t *testing.T) { turbostatOutput: turbostatOutput, fieldRegexs: []*regexp.Regexp{regexp.MustCompile(`^\w+Watt$`)}, want: [][][]string{ - {{"timestamp", "PkgWatt", "RAMWatt"}, {"15:04:05", "223.53", "7.38"}, {"15:04:07", "229.53", "7.38"}, {"15:04:09", "223.53", "7.38"}}, - {{"timestamp", "PkgWatt", "RAMWatt"}, {"15:04:05", "208.40", "16.83"}, {"15:04:07", "218.40", "16.83"}, {"15:04:09", "208.40", "16.83"}}, + {{"timestamp", "PkgWatt", "RAMWatt"}, {"15:04:05", "223.53", "7.38"}, {"15:04:05", "", ""}, {"15:04:07", "229.53", "7.38"}, {"15:04:07", "", ""}, {"15:04:09", "223.53", "7.38"}, {"15:04:09", "", ""}}, + {{"timestamp", "PkgWatt", "RAMWatt"}, {"15:04:05", "208.40", "16.83"}, {"15:04:05", "", ""}, {"15:04:07", "218.40", "16.83"}, {"15:04:07", "", ""}, {"15:04:09", "208.40", "16.83"}, {"15:04:09", "", ""}}, }, wantErr: false, }, From c7552fe27a8350ca0fad483429999ae987b8d449 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Wed, 25 Mar 2026 15:42:11 -0700 Subject: [PATCH 3/6] add fallback repo for libaio Signed-off-by: Harper, Jason M --- tools/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/Makefile b/tools/Makefile index 6ab7e7c3..414305ec 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -189,9 +189,12 @@ else endif LIBAIO_VERSION := libaio-0.3.113 +PRIMARY_REPO="https://github.com/littledan/linux-libaio.git" +FALLBACK_REPO="https://github.com/yugabyte/libaio.git" libaio-aarch64: ifeq ("$(wildcard libaio-aarch64)","") - git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) https://pagure.io/libaio libaio-aarch64 + git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) $(PRIMARY_REPO) libaio-aarch64 || \ + git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) $(FALLBACK_REPO) libaio-aarch64 else cd libaio-aarch64 && git fetch --tags && git checkout $(LIBAIO_VERSION) endif From 8d3bf880cf3da79bbe0908c3af84bc51a4261266 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Thu, 26 Mar 2026 11:05:46 -0700 Subject: [PATCH 4/6] package row fix Signed-off-by: Harper, Jason M --- cmd/telemetry/telemetry_tables.go | 30 +++++++++++++-------- internal/extract/turbostat.go | 45 ++++++++++++++++++------------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/cmd/telemetry/telemetry_tables.go b/cmd/telemetry/telemetry_tables.go index 1a673620..cd3af355 100644 --- a/cmd/telemetry/telemetry_tables.go +++ b/cmd/telemetry/telemetry_tables.go @@ -387,7 +387,7 @@ func powerTelemetryTableValues(outputs map[string]script.ScriptOutput) []table.F // dynamically build fields from the header row of the first package // header format: ["timestamp", "PkgWatt", "RAMWatt", ...] // fields will be: "Time", then for each package, one field per matched watt metric - // e.g., "Package 0 PkgWatt", "Package 0 RAMWatt", "Package 1 PkgWatt", ... + // e.g., "Pkg 0 PkgWatt", "Pkg 0 RAMWatt", "Pkg 1 PkgWatt", ... header := packageRows[0][0] fields := []table.Field{{Name: "Time"}} for i, pkgRows := range packageRows { @@ -395,7 +395,7 @@ func powerTelemetryTableValues(outputs map[string]script.ScriptOutput) []table.F continue } for _, fieldName := range header[1:] { - fields = append(fields, table.Field{Name: fmt.Sprintf("Package %d %s", i, fieldName)}) + fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)}) } } numMetrics := len(header) - 1 // number of matched watt fields per package @@ -458,15 +458,13 @@ func frequencyTelemetryTableValues(outputs map[string]script.ScriptOutput) []tab slog.Warn(err.Error()) return []table.Field{} } - packageRows, err := extract.TurbostatPackageRows(outputs[script.TurbostatTelemetryScriptName].Stdout, []string{"UncMHz"}) + // UncMHz, UMHz0.0, UMHz1.0, etc. are the uncore frequency fields in turbostat output, but they aren't always present, so we look for any field that matches the regex + uncoreRegex := regexp.MustCompile(`^(UncMHz|UMHz\d+\.\d+)$`) + packageRows, err := extract.TurbostatPackageRowsByRegexMatch(outputs[script.TurbostatTelemetryScriptName].Stdout, []*regexp.Regexp{uncoreRegex}) if err != nil { // not an error, just means no package rows (uncore frequency) slog.Warn(err.Error()) } - // add the package rows to the fields - for i := range packageRows { - fields = append(fields, table.Field{Name: fmt.Sprintf("Uncore Package %d", i)}) - } // for each platform row for i := range platformRows { // append the timestamp to the fields @@ -474,12 +472,22 @@ func frequencyTelemetryTableValues(outputs map[string]script.ScriptOutput) []tab // append the core frequency values to the fields fields[1].Values = append(fields[1].Values, platformRows[i][1]) // Core frequency } - // for each package + // dynamically build fields for uncore frequencies based on the package rows we found that match the uncore frequency regex + for i := range packageRows { + // the first package row contains the field names, so use that to build the fields + for _, fieldName := range packageRows[i][0][1:] { // skip timestamp field + fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)}) + } + } + numUncoreMetrics := (len(fields) - 2) / len(packageRows) // number of uncore frequency fields per package, after time and core frequency fields + // append the uncore frequency values to the fields for i := range packageRows { - // traverse the rows - for _, row := range packageRows[i] { + for _, row := range packageRows[i][1:] { // skip header row // append the package frequency to the fields - fields[i+2].Values = append(fields[i+2].Values, row[1]) // Package frequency + // uncore frequency fields start after time and core frequency fields + for j := range row[1:] { // skip timestamp field + fields[2+i*numUncoreMetrics+j].Values = append(fields[2+i*numUncoreMetrics+j].Values, row[j+1]) + } } } if len(fields[0].Values) == 0 { diff --git a/internal/extract/turbostat.go b/internal/extract/turbostat.go index f2dc439b..ec02f3bc 100644 --- a/internal/extract/turbostat.go +++ b/internal/extract/turbostat.go @@ -219,9 +219,16 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs if len(rows) == 0 { return nil, fmt.Errorf("no rows found in turbostat output") } + // filter all rows down to only package rows + rows, err = extractPackageRows(rows) + if err != nil { + return nil, fmt.Errorf("unable to extract package rows: %w", err) + } + if len(rows) == 0 { + return nil, fmt.Errorf("no package rows found in turbostat output") + } // Build our list of matched field names from the first package row var matchedFields []string - foundPackageRow := false for _, row := range rows { if _, ok := row["Package"]; !ok { if row["CPU"] == "0" { @@ -230,10 +237,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs continue } } - if !isPackageRow(row) { - continue - } - foundPackageRow = true for field := range row { for _, re := range fieldRegexs { if re.MatchString(field) { @@ -246,9 +249,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs } break // only need the first package row to discover fields } - if !foundPackageRow { - return nil, fmt.Errorf("no package rows found in turbostat output") - } if len(matchedFields) == 0 { return nil, fmt.Errorf("no fields matched the provided regexes in turbostat output") } @@ -268,9 +268,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs continue } } - if !isPackageRow(row) { - continue - } rowValues := make([]string, len(matchedFields)+1) rowValues[0] = row["timestamp"] for i, field := range matchedFields { @@ -322,6 +319,14 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([] if len(rows) == 0 { return nil, fmt.Errorf("no package rows found in turbostat output") } + // filter all rows down to only package rows + rows, err = extractPackageRows(rows) + if err != nil { + return nil, fmt.Errorf("unable to extract package rows: %w", err) + } + if len(rows) == 0 { + return nil, fmt.Errorf("no package rows found in turbostat output") + } var packageRows [][][]string for _, row := range rows { if _, ok := row["Package"]; !ok { @@ -331,9 +336,6 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([] continue } } - if !isPackageRow(row) { - continue - } rowValues := make([]string, len(fieldNames)+1) rowValues[0] = row["timestamp"] for i, fieldName := range fieldNames { @@ -359,11 +361,18 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([] return packageRows, nil } -func isPackageRow(row map[string]string) bool { - if val, ok := row["Package"]; ok && val != "-" && row["Core"] == "0" { - return true +func extractPackageRows(rows []map[string]string) ([]map[string]string, error) { + var packageRows []map[string]string + for i, row := range rows { + if val, ok := row["Package"]; ok && val != "-" && row["Core"] == "0" { + if i > 0 && rows[i-1]["Package"] == val && rows[i-1]["Core"] == "0" { + // This is the hyperthread associated with the package row, skip it + continue + } + packageRows = append(packageRows, row) + } } - return false + return packageRows, nil } // MaxTotalPackagePowerFromOutput calculates the maximum total package power from the turbostat output. From eca7fbf43657e15b46d3baf3f626bb44c744ef63 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Thu, 26 Mar 2026 11:15:49 -0700 Subject: [PATCH 5/6] update unit test Signed-off-by: Harper, Jason M --- internal/extract/turbostat_test.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/internal/extract/turbostat_test.go b/internal/extract/turbostat_test.go index b13d0be5..49662a25 100644 --- a/internal/extract/turbostat_test.go +++ b/internal/extract/turbostat_test.go @@ -442,13 +442,6 @@ func TestTurbostatPackageRows(t *testing.T) { want [][][]string wantErr bool }{ - { - name: "package rows with UncoreMHz, PKGTmp, PkgWatt (sparse rows)", - turbostatOutput: turbostatOutput, - fieldNames: []string{"UncMHz", "PkgTmp", "PkgWatt"}, - want: nil, - wantErr: true, - }, { name: "Typical output, two packages, one field", turbostatOutput: ` @@ -587,8 +580,8 @@ func TestTurbostatPackageRowsByRegexMatch(t *testing.T) { turbostatOutput: turbostatOutput, fieldRegexs: []*regexp.Regexp{regexp.MustCompile(`^\w+Watt$`)}, want: [][][]string{ - {{"timestamp", "PkgWatt", "RAMWatt"}, {"15:04:05", "223.53", "7.38"}, {"15:04:05", "", ""}, {"15:04:07", "229.53", "7.38"}, {"15:04:07", "", ""}, {"15:04:09", "223.53", "7.38"}, {"15:04:09", "", ""}}, - {{"timestamp", "PkgWatt", "RAMWatt"}, {"15:04:05", "208.40", "16.83"}, {"15:04:05", "", ""}, {"15:04:07", "218.40", "16.83"}, {"15:04:07", "", ""}, {"15:04:09", "208.40", "16.83"}, {"15:04:09", "", ""}}, + {{"timestamp", "PkgWatt", "RAMWatt"}, {"15:04:05", "223.53", "7.38"}, {"15:04:07", "229.53", "7.38"}, {"15:04:09", "223.53", "7.38"}}, + {{"timestamp", "PkgWatt", "RAMWatt"}, {"15:04:05", "208.40", "16.83"}, {"15:04:07", "218.40", "16.83"}, {"15:04:09", "208.40", "16.83"}}, }, wantErr: false, }, From 3a34800651073b7dad8dd69ec5e2ec5c0be75808 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Thu, 26 Mar 2026 12:04:23 -0700 Subject: [PATCH 6/6] handle no uncore freqs Signed-off-by: Harper, Jason M --- cmd/telemetry/telemetry_tables.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/telemetry/telemetry_tables.go b/cmd/telemetry/telemetry_tables.go index cd3af355..fa0d2821 100644 --- a/cmd/telemetry/telemetry_tables.go +++ b/cmd/telemetry/telemetry_tables.go @@ -479,14 +479,16 @@ func frequencyTelemetryTableValues(outputs map[string]script.ScriptOutput) []tab fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)}) } } - numUncoreMetrics := (len(fields) - 2) / len(packageRows) // number of uncore frequency fields per package, after time and core frequency fields - // append the uncore frequency values to the fields - for i := range packageRows { - for _, row := range packageRows[i][1:] { // skip header row - // append the package frequency to the fields - // uncore frequency fields start after time and core frequency fields - for j := range row[1:] { // skip timestamp field - fields[2+i*numUncoreMetrics+j].Values = append(fields[2+i*numUncoreMetrics+j].Values, row[j+1]) + if len(packageRows) > 0 { + numUncoreMetrics := (len(fields) - 2) / len(packageRows) // number of uncore frequency fields per package, after time and core frequency fields + // append the uncore frequency values to the fields + for i := range packageRows { + for _, row := range packageRows[i][1:] { // skip header row + // append the package frequency to the fields + // uncore frequency fields start after time and core frequency fields + for j := range row[1:] { // skip timestamp field + fields[2+i*numUncoreMetrics+j].Values = append(fields[2+i*numUncoreMetrics+j].Values, row[j+1]) + } } } }