Skip to content

Bdn update test 0.16.0 nightly.20260324.479 #5200

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
LoopedBard3 wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

Bdn update test 0.16.0 nightly.20260324.479 #5200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7c2ea71
Update BenchmarkDotNet to 0.16.0-nightly.20260324.479 for verificatio…
LoopedBard3 Mar 26, 2026
0d1bb35
Update BDN.Extensions harness for BDN 0.16.0 async API changes
LoopedBard3 Mar 27, 2026
d1ae1e2
Add Microsoft.Bcl.AsyncInterfaces package reference for netstandard2.0
LoopedBard3 Mar 27, 2026
ed282d8
Fix BDN 0.16.0 async validators to avoid deadlock with BDN Synchroniz…
LoopedBard3 Apr 1, 2026
ad69833
Replace custom SyncAsyncEnumerable with System.Linq.AsyncEnumerable.T…
LoopedBard3 Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion eng/Versions.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<MicrosoftNETILLinkPackageVersion>10.0.0-rc.1.25555.107</MicrosoftNETILLinkPackageVersion>
<SystemThreadingChannelsPackageVersion>10.0.0-rc.1.25555.107</SystemThreadingChannelsPackageVersion>
<MicrosoftExtensionsLoggingPackageVersion>10.0.0-rc.1.25555.107</MicrosoftExtensionsLoggingPackageVersion>
<BenchmarkDotNetVersion>0.16.0-nightly.20260320.467</BenchmarkDotNetVersion>
<BenchmarkDotNetVersion>0.16.0-nightly.20260324.479</BenchmarkDotNetVersion>
Copy link
Copy Markdown

@timcassell timcassell Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't it be 0.16.0-nightly.20260410.495?

Copy link
Copy Markdown
Member Author

@LoopedBard3 LoopedBard3 Apr 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is currently a WIP draft that I had been using to test a new update verification flow. We are planning to start another test of the verification flow with the latest nightly, I uploaded this mostly for easy viewing of the changes made to get 20260324.479 working.

<MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>10.0.0-rc.1.25555.107</MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>
<MicrosoftDotNetXHarnessCLIVersion>11.0.0-prerelease.26169.1</MicrosoftDotNetXHarnessCLIVersion>
</PropertyGroup>
Expand Down
9 changes: 9 additions & 0 deletions src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,22 @@
<TargetFramework>netstandard2.0</TargetFramework>
<Nullable>enable</Nullable>
<ProduceReferenceAssembly>true</ProduceReferenceAssembly>
<NoWarn>$(NoWarn);NU1510</NoWarn>
</PropertyGroup>
<PropertyGroup Condition="'$(BenchmarkDotNetSources)' != ''">
<_BenchmarkDotNetSourcesN>$([MSBuild]::NormalizeDirectory('$(BenchmarkDotNetSources)'))</_BenchmarkDotNetSourcesN>
</PropertyGroup>
<ItemGroup Condition="'$(BenchmarkDotNetSources)' == ''">
<PackageReference Include="BenchmarkDotNet" Version="$(BenchmarkDotNetVersion)" />
<PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="$(BenchmarkDotNetVersion)" />
<PackageReference Include="System.Linq.AsyncEnumerable" Version="10.0.5">
<PrivateAssets>none</PrivateAssets>
<Publish>true</Publish>
</PackageReference>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.5">
<PrivateAssets>none</PrivateAssets>
<Publish>true</Publish>
</PackageReference>
</ItemGroup>
<ItemGroup Condition="'$(BenchmarkDotNetSources)' != ''">
<ProjectReference Include="$(_BenchmarkDotNetSourcesN)src\BenchmarkDotNet\BenchmarkDotNet.csproj" SetTargetFramework="TargetFramework=netstandard2.0" />
Expand Down
15 changes: 8 additions & 7 deletions src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,16 @@ public class MandatoryCategoryValidator : IValidator

public MandatoryCategoryValidator(ImmutableHashSet<string> categories) => _mandatoryCategories = categories;

public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
=> validationParameters.Benchmarks
public IAsyncEnumerable<ValidationError> ValidateAsync(ValidationParameters validationParameters)
{
return validationParameters.Benchmarks
.Where(benchmark => !benchmark.Descriptor.Categories.Any(category => _mandatoryCategories.Contains(category)))
.Select(benchmark => benchmark.Descriptor.GetFilterName())
.Distinct()
.Select(benchmarkId =>
new ValidationError(
isCritical: TreatsWarningsAsErrors,
$"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]")
);
.Select(benchmarkId => new ValidationError(
isCritical: TreatsWarningsAsErrors,
$"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]"))
.ToAsyncEnumerable();
}
}
}
15 changes: 8 additions & 7 deletions src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,17 @@ public class NoWasmValidator : IValidator

public NoWasmValidator(string noWasmCategory) => _noWasmCategory = noWasmCategory;

public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
=> validationParameters.Benchmarks
public IAsyncEnumerable<ValidationError> ValidateAsync(ValidationParameters validationParameters)
{
return validationParameters.Benchmarks
.Where(benchmark => IsAsyncMethod(benchmark.Descriptor.WorkloadMethod) && !benchmark.Descriptor.Categories.Any(category => category.Equals(_noWasmCategory, StringComparison.Ordinal)))
.Select(benchmark => benchmark.Descriptor.GetFilterName())
.Distinct()
.Select(benchmarkId =>
new ValidationError(
isCritical: TreatsWarningsAsErrors,
$"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]")
);
.Select(benchmarkId => new ValidationError(
isCritical: TreatsWarningsAsErrors,
$"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]"))
.ToAsyncEnumerable();
}

private bool IsAsyncMethod(MethodInfo workloadMethod)
{
Expand Down
20 changes: 14 additions & 6 deletions src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,25 @@
using Reporting;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace BenchmarkDotNet.Extensions
{
public class PerfLabExporter : ExporterBase
/// <summary>
/// Exports benchmark results to JSON format for PerfLab ingestion.
/// </summary>
public class PerfLabExporter : IExporter
{
protected override string FileExtension => "json";
protected override string FileCaption => "perf-lab-report";
public string Name => "PerfLab";

public PerfLabExporter()
public ValueTask ExportAsync(Summary summary, ILogger logger, CancellationToken cancellationToken)
{
Export(summary, logger);
return default;
}

public override void ExportToLog(Summary summary, ILogger logger)
public void Export(Summary summary, ILogger logger)
{
var reporter = new Reporter();

Expand Down Expand Up @@ -132,7 +138,9 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit

var jsonOutput = reporter.GetJson();
if (jsonOutput is not null)
logger.WriteLine(jsonOutput);
{
Console.WriteLine(jsonOutput);
}
}
}
}
14 changes: 7 additions & 7 deletions src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,17 @@ public class TooManyTestCasesValidator : IValidator

public bool TreatsWarningsAsErrors => true;

public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
public IAsyncEnumerable<ValidationError> ValidateAsync(ValidationParameters validationParameters)
{
var byDescriptor = validationParameters.Benchmarks
return validationParameters.Benchmarks
.Where(benchmark => !SkipValidation(benchmark.Descriptor.WorkloadMethod))
.GroupBy(benchmark => (benchmark.Descriptor, benchmark.Job)); // descriptor = type + method

return byDescriptor.Where(benchmarkCase => benchmarkCase.Count() > Limit).Select(group =>
new ValidationError(
.GroupBy(benchmark => (benchmark.Descriptor, benchmark.Job))
.Where(group => group.Count() > Limit)
.Select(group => new ValidationError(
isCritical: true,
message: $"{group.Key.Descriptor.Type.Name}.{group.Key.Descriptor.WorkloadMethod.Name} has {group.Count()} test cases. It MUST NOT have more than {Limit} test cases. We don't have inifinite amount of time to run all the benchmarks!!",
benchmarkCase: group.First()));
benchmarkCase: group.First()))
.ToAsyncEnumerable();
}

private static bool SkipValidation(MemberInfo member)
Expand Down
11 changes: 7 additions & 4 deletions src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ public class UniqueArgumentsValidator : IValidator
{
public bool TreatsWarningsAsErrors => true;

public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
=> validationParameters.Benchmarks
public IAsyncEnumerable<ValidationError> ValidateAsync(ValidationParameters validationParameters)
{
return validationParameters.Benchmarks
.Where(benchmark => benchmark.HasArguments || benchmark.HasParameters)
.GroupBy(benchmark => (benchmark.Descriptor.Type, benchmark.Descriptor.WorkloadMethod, benchmark.Job))
.Where(sameBenchmark =>
Expand All @@ -25,7 +26,9 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara

return numberOfTestCases != numberOfUniqueTestCases;
})
.Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First()));
.Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First()))
.ToAsyncEnumerable();
}

private class BenchmarkArgumentsComparer : IEqualityComparer<BenchmarkCase>
{
Expand All @@ -34,7 +37,7 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y)
if (FullNameProvider.GetBenchmarkName(x).Equals(FullNameProvider.GetBenchmarkName(y), System.StringComparison.Ordinal))
return true;

return Enumerable.SequenceEqual(
return System.Linq.Enumerable.SequenceEqual(
x.Parameters.Items.Select(argument => argument.Value),
y.Parameters.Items.Select(argument => argument.Value));
}
Expand Down
Loading