diff --git a/cmd/telemetry/telemetry_tables.go b/cmd/telemetry/telemetry_tables.go index 1a673620..fa0d2821 100644 --- a/cmd/telemetry/telemetry_tables.go +++ b/cmd/telemetry/telemetry_tables.go @@ -387,7 +387,7 @@ func powerTelemetryTableValues(outputs map[string]script.ScriptOutput) []table.F // dynamically build fields from the header row of the first package // header format: ["timestamp", "PkgWatt", "RAMWatt", ...] // fields will be: "Time", then for each package, one field per matched watt metric - // e.g., "Package 0 PkgWatt", "Package 0 RAMWatt", "Package 1 PkgWatt", ... + // e.g., "Pkg 0 PkgWatt", "Pkg 0 RAMWatt", "Pkg 1 PkgWatt", ... header := packageRows[0][0] fields := []table.Field{{Name: "Time"}} for i, pkgRows := range packageRows { @@ -395,7 +395,7 @@ func powerTelemetryTableValues(outputs map[string]script.ScriptOutput) []table.F continue } for _, fieldName := range header[1:] { - fields = append(fields, table.Field{Name: fmt.Sprintf("Package %d %s", i, fieldName)}) + fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)}) } } numMetrics := len(header) - 1 // number of matched watt fields per package @@ -458,15 +458,13 @@ func frequencyTelemetryTableValues(outputs map[string]script.ScriptOutput) []tab slog.Warn(err.Error()) return []table.Field{} } - packageRows, err := extract.TurbostatPackageRows(outputs[script.TurbostatTelemetryScriptName].Stdout, []string{"UncMHz"}) + // UncMHz, UMHz0.0, UMHz1.0, etc. are the uncore frequency fields in turbostat output, but they aren't always present, so we look for any field that matches the regex + uncoreRegex := regexp.MustCompile(`^(UncMHz|UMHz\d+\.\d+)$`) + packageRows, err := extract.TurbostatPackageRowsByRegexMatch(outputs[script.TurbostatTelemetryScriptName].Stdout, []*regexp.Regexp{uncoreRegex}) if err != nil { // not an error, just means no package rows (uncore frequency) slog.Warn(err.Error()) } - // add the package rows to the fields - for i := range packageRows { - fields = append(fields, table.Field{Name: fmt.Sprintf("Uncore Package %d", i)}) - } // for each platform row for i := range platformRows { // append the timestamp to the fields @@ -474,12 +472,24 @@ func frequencyTelemetryTableValues(outputs map[string]script.ScriptOutput) []tab // append the core frequency values to the fields fields[1].Values = append(fields[1].Values, platformRows[i][1]) // Core frequency } - // for each package + // dynamically build fields for uncore frequencies based on the package rows we found that match the uncore frequency regex for i := range packageRows { - // traverse the rows - for _, row := range packageRows[i] { - // append the package frequency to the fields - fields[i+2].Values = append(fields[i+2].Values, row[1]) // Package frequency + // the first package row contains the field names, so use that to build the fields + for _, fieldName := range packageRows[i][0][1:] { // skip timestamp field + fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)}) + } + } + if len(packageRows) > 0 { + numUncoreMetrics := (len(fields) - 2) / len(packageRows) // number of uncore frequency fields per package, after time and core frequency fields + // append the uncore frequency values to the fields + for i := range packageRows { + for _, row := range packageRows[i][1:] { // skip header row + // append the package frequency to the fields + // uncore frequency fields start after time and core frequency fields + for j := range row[1:] { // skip timestamp field + fields[2+i*numUncoreMetrics+j].Values = append(fields[2+i*numUncoreMetrics+j].Values, row[j+1]) + } + } } } if len(fields[0].Values) == 0 { diff --git a/internal/extract/turbostat.go b/internal/extract/turbostat.go index 9aa98e3d..ec02f3bc 100644 --- a/internal/extract/turbostat.go +++ b/internal/extract/turbostat.go @@ -62,12 +62,12 @@ func parseTurbostatOutput(output string) ([]map[string]string, error) { if len(headers) == 0 { continue } - if len(fields) != len(headers) { - continue + if len(fields) > len(headers) { + return nil, fmt.Errorf("more turbostat row values than headers: %d values, %d headers", len(fields), len(headers)) } - row := make(map[string]string, len(headers)) - for i, h := range headers { - row[h] = fields[i] + row := make(map[string]string, len(fields)) + for i := range fields { + row[headers[i]] = fields[i] // this assumes any missing fields are at the end and will be empty string, which is consistent with turbostat output } if timeParsed && interval > 0 { row["timestamp"] = timestamp.Format("15:04:05") @@ -219,9 +219,16 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs if len(rows) == 0 { return nil, fmt.Errorf("no rows found in turbostat output") } + // filter all rows down to only package rows + rows, err = extractPackageRows(rows) + if err != nil { + return nil, fmt.Errorf("unable to extract package rows: %w", err) + } + if len(rows) == 0 { + return nil, fmt.Errorf("no package rows found in turbostat output") + } // Build our list of matched field names from the first package row var matchedFields []string - foundPackageRow := false for _, row := range rows { if _, ok := row["Package"]; !ok { if row["CPU"] == "0" { @@ -230,10 +237,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs continue } } - if !isPackageRow(row) { - continue - } - foundPackageRow = true for field := range row { for _, re := range fieldRegexs { if re.MatchString(field) { @@ -246,9 +249,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs } break // only need the first package row to discover fields } - if !foundPackageRow { - return nil, fmt.Errorf("no package rows found in turbostat output") - } if len(matchedFields) == 0 { return nil, fmt.Errorf("no fields matched the provided regexes in turbostat output") } @@ -268,9 +268,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs continue } } - if !isPackageRow(row) { - continue - } rowValues := make([]string, len(matchedFields)+1) rowValues[0] = row["timestamp"] for i, field := range matchedFields { @@ -322,6 +319,14 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([] if len(rows) == 0 { return nil, fmt.Errorf("no package rows found in turbostat output") } + // filter all rows down to only package rows + rows, err = extractPackageRows(rows) + if err != nil { + return nil, fmt.Errorf("unable to extract package rows: %w", err) + } + if len(rows) == 0 { + return nil, fmt.Errorf("no package rows found in turbostat output") + } var packageRows [][][]string for _, row := range rows { if _, ok := row["Package"]; !ok { @@ -331,9 +336,6 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([] continue } } - if !isPackageRow(row) { - continue - } rowValues := make([]string, len(fieldNames)+1) rowValues[0] = row["timestamp"] for i, fieldName := range fieldNames { @@ -359,11 +361,18 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([] return packageRows, nil } -func isPackageRow(row map[string]string) bool { - if val, ok := row["Package"]; ok && val != "-" { - return true +func extractPackageRows(rows []map[string]string) ([]map[string]string, error) { + var packageRows []map[string]string + for i, row := range rows { + if val, ok := row["Package"]; ok && val != "-" && row["Core"] == "0" { + if i > 0 && rows[i-1]["Package"] == val && rows[i-1]["Core"] == "0" { + // This is the hyperthread associated with the package row, skip it + continue + } + packageRows = append(packageRows, row) + } } - return false + return packageRows, nil } // MaxTotalPackagePowerFromOutput calculates the maximum total package power from the turbostat output. diff --git a/internal/extract/turbostat_test.go b/internal/extract/turbostat_test.go index 228f504f..49662a25 100644 --- a/internal/extract/turbostat_test.go +++ b/internal/extract/turbostat_test.go @@ -442,16 +442,6 @@ func TestTurbostatPackageRows(t *testing.T) { want [][][]string wantErr bool }{ - { - name: "package rows with UncoreMHz, PKGTmp, PkgWatt", - turbostatOutput: turbostatOutput, - fieldNames: []string{"UncMHz", "PkgTmp", "PkgWatt"}, - want: [][][]string{ - {{"15:04:05", "2350", "57", "223.53"}, {"15:04:07", "2400", "59", "229.53"}, {"15:04:09", "2400", "57", "223.53"}}, - {{"15:04:05", "2300", "53", "208.40"}, {"15:04:07", "2400", "55", "218.40"}, {"15:04:09", "2400", "53", "208.40"}}, - }, - wantErr: false, - }, { name: "Typical output, two packages, one field", turbostatOutput: ` diff --git a/tools/Makefile b/tools/Makefile index 48aba635..414305ec 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -25,7 +25,7 @@ default: tools-x86_64 .PHONY: sysstat sysstat-aarch64 sysstat-repo .PHONY: tsc turbostat -tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat +tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc mkdir -p bin/x86_64 cp -R async-profiler bin/x86_64/ cp avx-turbo/avx-turbo bin/x86_64/ @@ -50,7 +50,6 @@ tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw cp sysstat/sar bin/x86_64/ cp sysstat/sadc bin/x86_64/ cp tsc/tsc bin/x86_64/ - cp linux_turbostat/tools/power/x86/turbostat/turbostat bin/x86_64/ cd bin/x86_64 && strip --strip-unneeded * || true tools-aarch64: async-profiler-aarch64 dmidecode-aarch64 ethtool-aarch64 fio-aarch64 ipmitool-aarch64 lshw-aarch64 lspci-aarch64 spectre-meltdown-checker sshpass-aarch64 stackcollapse-perf-aarch64 stress-ng-aarch64 sysstat-aarch64 @@ -190,9 +189,12 @@ else endif LIBAIO_VERSION := libaio-0.3.113 +PRIMARY_REPO="https://github.com/littledan/linux-libaio.git" +FALLBACK_REPO="https://github.com/yugabyte/libaio.git" libaio-aarch64: ifeq ("$(wildcard libaio-aarch64)","") - git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) https://pagure.io/libaio libaio-aarch64 + git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) $(PRIMARY_REPO) libaio-aarch64 || \ + git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) $(FALLBACK_REPO) libaio-aarch64 else cd libaio-aarch64 && git fetch --tags && git checkout $(LIBAIO_VERSION) endif @@ -348,7 +350,7 @@ else cd processwatch && git fetch --tags && git checkout $(PROCESSWATCH_VERSION) endif cd processwatch && ./build.sh - mkdir -p bin + mkdir -p bin/x86_64 cp processwatch/processwatch bin/x86_64/ strip --strip-unneeded bin/x86_64/processwatch @@ -437,14 +439,18 @@ endif tsc: cd tsc && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -TURBOSTAT_VERSION := 6.9.12 +# turbostat version 2025.12.02 is in kernel 6.19 +TURBOSTAT_LINUX_VERSION := 6.19.9 turbostat: ifeq ("$(wildcard linux_turbostat)","") - wget $(WGET_OPTS) https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$(TURBOSTAT_VERSION).tar.xz - tar -xf linux-$(TURBOSTAT_VERSION).tar.xz && mv linux-$(TURBOSTAT_VERSION)/ linux_turbostat/ + wget $(WGET_OPTS) https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$(TURBOSTAT_LINUX_VERSION).tar.xz + tar -xf linux-$(TURBOSTAT_LINUX_VERSION).tar.xz && mv linux-$(TURBOSTAT_LINUX_VERSION)/ linux_turbostat/ endif sed -i '/_Static_assert/d' linux_turbostat/tools/power/x86/turbostat/turbostat.c cd linux_turbostat/tools/power/x86/turbostat && make + mkdir -p bin/x86_64 + cp linux_turbostat/tools/power/x86/turbostat/turbostat bin/x86_64/ + strip --strip-unneeded bin/x86_64/turbostat reset: cd async-profiler @@ -462,7 +468,6 @@ reset: cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc - cd linux_turbostat/tools/power/x86/turbostat && make clean # libs are not directly used in the build but are required in the oss archive file because some of the tools are statically linked # glibc 2.27 was the version used in Ubuntu 18.04 which is the base docker image we used to build the tools @@ -475,5 +480,5 @@ libcrypt.tar.gz: libs: glibc.tar.gz zlib.tar.gz libcrypt.tar.gz oss-source: reset libs - tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ perf-archive/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc.tar.gz zlib.tar.gz libcrypt.tar.gz + tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ perf-archive/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ glibc.tar.gz zlib.tar.gz libcrypt.tar.gz md5sum oss_source.tgz > oss_source.tgz.md5 diff --git a/tools/build.Dockerfile b/tools/build.Dockerfile index 6fe43370..a838b930 100644 --- a/tools/build.Dockerfile +++ b/tools/build.Dockerfile @@ -167,6 +167,7 @@ ADD . /workdir WORKDIR /workdir RUN make perf RUN make processwatch +RUN make turbostat FROM scratch AS output COPY --from=builder workdir/bin /bin