Skip to content

Commit 5207d18

Browse files
authored
chore: update turbostat (#672)
* chore: update turbostat Signed-off-by: Harper, Jason M <jason.m.harper@intel.com> * fix unit test Signed-off-by: Harper, Jason M <jason.m.harper@intel.com> * add fallback repo for libaio Signed-off-by: Harper, Jason M <jason.m.harper@intel.com> * package row fix Signed-off-by: Harper, Jason M <jason.m.harper@intel.com> * update unit test Signed-off-by: Harper, Jason M <jason.m.harper@intel.com> * handle no uncore freqs Signed-off-by: Harper, Jason M <jason.m.harper@intel.com> --------- Signed-off-by: Harper, Jason M <jason.m.harper@intel.com>
1 parent ebfb257 commit 5207d18

5 files changed

Lines changed: 69 additions & 54 deletions

File tree

cmd/telemetry/telemetry_tables.go

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -387,15 +387,15 @@ func powerTelemetryTableValues(outputs map[string]script.ScriptOutput) []table.F
387387
// dynamically build fields from the header row of the first package
388388
// header format: ["timestamp", "PkgWatt", "RAMWatt", ...]
389389
// fields will be: "Time", then for each package, one field per matched watt metric
390-
// e.g., "Package 0 PkgWatt", "Package 0 RAMWatt", "Package 1 PkgWatt", ...
390+
// e.g., "Pkg 0 PkgWatt", "Pkg 0 RAMWatt", "Pkg 1 PkgWatt", ...
391391
header := packageRows[0][0]
392392
fields := []table.Field{{Name: "Time"}}
393393
for i, pkgRows := range packageRows {
394394
if len(pkgRows) == 0 {
395395
continue
396396
}
397397
for _, fieldName := range header[1:] {
398-
fields = append(fields, table.Field{Name: fmt.Sprintf("Package %d %s", i, fieldName)})
398+
fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)})
399399
}
400400
}
401401
numMetrics := len(header) - 1 // number of matched watt fields per package
@@ -458,28 +458,38 @@ func frequencyTelemetryTableValues(outputs map[string]script.ScriptOutput) []tab
458458
slog.Warn(err.Error())
459459
return []table.Field{}
460460
}
461-
packageRows, err := extract.TurbostatPackageRows(outputs[script.TurbostatTelemetryScriptName].Stdout, []string{"UncMHz"})
461+
// UncMHz, UMHz0.0, UMHz1.0, etc. are the uncore frequency fields in turbostat output, but they aren't always present, so we look for any field that matches the regex
462+
uncoreRegex := regexp.MustCompile(`^(UncMHz|UMHz\d+\.\d+)$`)
463+
packageRows, err := extract.TurbostatPackageRowsByRegexMatch(outputs[script.TurbostatTelemetryScriptName].Stdout, []*regexp.Regexp{uncoreRegex})
462464
if err != nil {
463465
// not an error, just means no package rows (uncore frequency)
464466
slog.Warn(err.Error())
465467
}
466-
// add the package rows to the fields
467-
for i := range packageRows {
468-
fields = append(fields, table.Field{Name: fmt.Sprintf("Uncore Package %d", i)})
469-
}
470468
// for each platform row
471469
for i := range platformRows {
472470
// append the timestamp to the fields
473471
fields[0].Values = append(fields[0].Values, platformRows[i][0]) // Timestamp
474472
// append the core frequency values to the fields
475473
fields[1].Values = append(fields[1].Values, platformRows[i][1]) // Core frequency
476474
}
477-
// for each package
475+
// dynamically build fields for uncore frequencies based on the package rows we found that match the uncore frequency regex
478476
for i := range packageRows {
479-
// traverse the rows
480-
for _, row := range packageRows[i] {
481-
// append the package frequency to the fields
482-
fields[i+2].Values = append(fields[i+2].Values, row[1]) // Package frequency
477+
// the first package row contains the field names, so use that to build the fields
478+
for _, fieldName := range packageRows[i][0][1:] { // skip timestamp field
479+
fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)})
480+
}
481+
}
482+
if len(packageRows) > 0 {
483+
numUncoreMetrics := (len(fields) - 2) / len(packageRows) // number of uncore frequency fields per package, after time and core frequency fields
484+
// append the uncore frequency values to the fields
485+
for i := range packageRows {
486+
for _, row := range packageRows[i][1:] { // skip header row
487+
// append the package frequency to the fields
488+
// uncore frequency fields start after time and core frequency fields
489+
for j := range row[1:] { // skip timestamp field
490+
fields[2+i*numUncoreMetrics+j].Values = append(fields[2+i*numUncoreMetrics+j].Values, row[j+1])
491+
}
492+
}
483493
}
484494
}
485495
if len(fields[0].Values) == 0 {

internal/extract/turbostat.go

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,12 @@ func parseTurbostatOutput(output string) ([]map[string]string, error) {
6262
if len(headers) == 0 {
6363
continue
6464
}
65-
if len(fields) != len(headers) {
66-
continue
65+
if len(fields) > len(headers) {
66+
return nil, fmt.Errorf("more turbostat row values than headers: %d values, %d headers", len(fields), len(headers))
6767
}
68-
row := make(map[string]string, len(headers))
69-
for i, h := range headers {
70-
row[h] = fields[i]
68+
row := make(map[string]string, len(fields))
69+
for i := range fields {
70+
row[headers[i]] = fields[i] // this assumes any missing fields are at the end and will be empty string, which is consistent with turbostat output
7171
}
7272
if timeParsed && interval > 0 {
7373
row["timestamp"] = timestamp.Format("15:04:05")
@@ -219,9 +219,16 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs
219219
if len(rows) == 0 {
220220
return nil, fmt.Errorf("no rows found in turbostat output")
221221
}
222+
// filter all rows down to only package rows
223+
rows, err = extractPackageRows(rows)
224+
if err != nil {
225+
return nil, fmt.Errorf("unable to extract package rows: %w", err)
226+
}
227+
if len(rows) == 0 {
228+
return nil, fmt.Errorf("no package rows found in turbostat output")
229+
}
222230
// Build our list of matched field names from the first package row
223231
var matchedFields []string
224-
foundPackageRow := false
225232
for _, row := range rows {
226233
if _, ok := row["Package"]; !ok {
227234
if row["CPU"] == "0" {
@@ -230,10 +237,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs
230237
continue
231238
}
232239
}
233-
if !isPackageRow(row) {
234-
continue
235-
}
236-
foundPackageRow = true
237240
for field := range row {
238241
for _, re := range fieldRegexs {
239242
if re.MatchString(field) {
@@ -246,9 +249,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs
246249
}
247250
break // only need the first package row to discover fields
248251
}
249-
if !foundPackageRow {
250-
return nil, fmt.Errorf("no package rows found in turbostat output")
251-
}
252252
if len(matchedFields) == 0 {
253253
return nil, fmt.Errorf("no fields matched the provided regexes in turbostat output")
254254
}
@@ -268,9 +268,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs
268268
continue
269269
}
270270
}
271-
if !isPackageRow(row) {
272-
continue
273-
}
274271
rowValues := make([]string, len(matchedFields)+1)
275272
rowValues[0] = row["timestamp"]
276273
for i, field := range matchedFields {
@@ -322,6 +319,14 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([]
322319
if len(rows) == 0 {
323320
return nil, fmt.Errorf("no package rows found in turbostat output")
324321
}
322+
// filter all rows down to only package rows
323+
rows, err = extractPackageRows(rows)
324+
if err != nil {
325+
return nil, fmt.Errorf("unable to extract package rows: %w", err)
326+
}
327+
if len(rows) == 0 {
328+
return nil, fmt.Errorf("no package rows found in turbostat output")
329+
}
325330
var packageRows [][][]string
326331
for _, row := range rows {
327332
if _, ok := row["Package"]; !ok {
@@ -331,9 +336,6 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([]
331336
continue
332337
}
333338
}
334-
if !isPackageRow(row) {
335-
continue
336-
}
337339
rowValues := make([]string, len(fieldNames)+1)
338340
rowValues[0] = row["timestamp"]
339341
for i, fieldName := range fieldNames {
@@ -359,11 +361,18 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([]
359361
return packageRows, nil
360362
}
361363

362-
func isPackageRow(row map[string]string) bool {
363-
if val, ok := row["Package"]; ok && val != "-" {
364-
return true
364+
func extractPackageRows(rows []map[string]string) ([]map[string]string, error) {
365+
var packageRows []map[string]string
366+
for i, row := range rows {
367+
if val, ok := row["Package"]; ok && val != "-" && row["Core"] == "0" {
368+
if i > 0 && rows[i-1]["Package"] == val && rows[i-1]["Core"] == "0" {
369+
// This is the hyperthread associated with the package row, skip it
370+
continue
371+
}
372+
packageRows = append(packageRows, row)
373+
}
365374
}
366-
return false
375+
return packageRows, nil
367376
}
368377

369378
// MaxTotalPackagePowerFromOutput calculates the maximum total package power from the turbostat output.

internal/extract/turbostat_test.go

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -442,16 +442,6 @@ func TestTurbostatPackageRows(t *testing.T) {
442442
want [][][]string
443443
wantErr bool
444444
}{
445-
{
446-
name: "package rows with UncoreMHz, PKGTmp, PkgWatt",
447-
turbostatOutput: turbostatOutput,
448-
fieldNames: []string{"UncMHz", "PkgTmp", "PkgWatt"},
449-
want: [][][]string{
450-
{{"15:04:05", "2350", "57", "223.53"}, {"15:04:07", "2400", "59", "229.53"}, {"15:04:09", "2400", "57", "223.53"}},
451-
{{"15:04:05", "2300", "53", "208.40"}, {"15:04:07", "2400", "55", "218.40"}, {"15:04:09", "2400", "53", "208.40"}},
452-
},
453-
wantErr: false,
454-
},
455445
{
456446
name: "Typical output, two packages, one field",
457447
turbostatOutput: `

tools/Makefile

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ default: tools-x86_64
2525
.PHONY: sysstat sysstat-aarch64 sysstat-repo
2626
.PHONY: tsc turbostat
2727

28-
tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat
28+
tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc
2929
mkdir -p bin/x86_64
3030
cp -R async-profiler bin/x86_64/
3131
cp avx-turbo/avx-turbo bin/x86_64/
@@ -50,7 +50,6 @@ tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw
5050
cp sysstat/sar bin/x86_64/
5151
cp sysstat/sadc bin/x86_64/
5252
cp tsc/tsc bin/x86_64/
53-
cp linux_turbostat/tools/power/x86/turbostat/turbostat bin/x86_64/
5453
cd bin/x86_64 && strip --strip-unneeded * || true
5554

5655
tools-aarch64: async-profiler-aarch64 dmidecode-aarch64 ethtool-aarch64 fio-aarch64 ipmitool-aarch64 lshw-aarch64 lspci-aarch64 spectre-meltdown-checker sshpass-aarch64 stackcollapse-perf-aarch64 stress-ng-aarch64 sysstat-aarch64
@@ -190,9 +189,12 @@ else
190189
endif
191190

192191
LIBAIO_VERSION := libaio-0.3.113
192+
PRIMARY_REPO="https://github.com/littledan/linux-libaio.git"
193+
FALLBACK_REPO="https://github.com/yugabyte/libaio.git"
193194
libaio-aarch64:
194195
ifeq ("$(wildcard libaio-aarch64)","")
195-
git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) https://pagure.io/libaio libaio-aarch64
196+
git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) $(PRIMARY_REPO) libaio-aarch64 || \
197+
git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) $(FALLBACK_REPO) libaio-aarch64
196198
else
197199
cd libaio-aarch64 && git fetch --tags && git checkout $(LIBAIO_VERSION)
198200
endif
@@ -348,7 +350,7 @@ else
348350
cd processwatch && git fetch --tags && git checkout $(PROCESSWATCH_VERSION)
349351
endif
350352
cd processwatch && ./build.sh
351-
mkdir -p bin
353+
mkdir -p bin/x86_64
352354
cp processwatch/processwatch bin/x86_64/
353355
strip --strip-unneeded bin/x86_64/processwatch
354356

@@ -437,14 +439,18 @@ endif
437439
tsc:
438440
cd tsc && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build
439441

440-
TURBOSTAT_VERSION := 6.9.12
442+
# turbostat version 2025.12.02 is in kernel 6.19
443+
TURBOSTAT_LINUX_VERSION := 6.19.9
441444
turbostat:
442445
ifeq ("$(wildcard linux_turbostat)","")
443-
wget $(WGET_OPTS) https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$(TURBOSTAT_VERSION).tar.xz
444-
tar -xf linux-$(TURBOSTAT_VERSION).tar.xz && mv linux-$(TURBOSTAT_VERSION)/ linux_turbostat/
446+
wget $(WGET_OPTS) https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$(TURBOSTAT_LINUX_VERSION).tar.xz
447+
tar -xf linux-$(TURBOSTAT_LINUX_VERSION).tar.xz && mv linux-$(TURBOSTAT_LINUX_VERSION)/ linux_turbostat/
445448
endif
446449
sed -i '/_Static_assert/d' linux_turbostat/tools/power/x86/turbostat/turbostat.c
447450
cd linux_turbostat/tools/power/x86/turbostat && make
451+
mkdir -p bin/x86_64
452+
cp linux_turbostat/tools/power/x86/turbostat/turbostat bin/x86_64/
453+
strip --strip-unneeded bin/x86_64/turbostat
448454

449455
reset:
450456
cd async-profiler
@@ -462,7 +468,6 @@ reset:
462468
cd stress-ng && git clean -fdx && git reset --hard
463469
cd sysstat && git clean -fdx && git reset --hard
464470
cd tsc && rm -f tsc
465-
cd linux_turbostat/tools/power/x86/turbostat && make clean
466471

467472
# libs are not directly used in the build but are required in the oss archive file because some of the tools are statically linked
468473
# glibc 2.27 was the version used in Ubuntu 18.04 which is the base docker image we used to build the tools
@@ -475,5 +480,5 @@ libcrypt.tar.gz:
475480
libs: glibc.tar.gz zlib.tar.gz libcrypt.tar.gz
476481

477482
oss-source: reset libs
478-
tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ perf-archive/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc.tar.gz zlib.tar.gz libcrypt.tar.gz
483+
tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ perf-archive/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ glibc.tar.gz zlib.tar.gz libcrypt.tar.gz
479484
md5sum oss_source.tgz > oss_source.tgz.md5

tools/build.Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ ADD . /workdir
167167
WORKDIR /workdir
168168
RUN make perf
169169
RUN make processwatch
170+
RUN make turbostat
170171

171172
FROM scratch AS output
172173
COPY --from=builder workdir/bin /bin

0 commit comments

Comments
 (0)