From fccd1f24e672ffa2fa4593f9ec2b6750ea655ebb Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 13 May 2026 12:07:32 +0100 Subject: [PATCH 1/6] feat: introduce instrumentation report visualisation protocol Define a common protocol for instrumentation report visualisations with a rendering interface. The idea is to produce a single HTML instrumentation report with various different visualisations (which themselves could be graphs, images, text, etc.) which can be switched between. The rendered outputs themselves are just embedded HTML fragments (or `None` on failure). This should be simple enough whilst allowing the level of extensibility and flexibility that is needed for the current level of instrumentation reporting. Signed-off-by: Alex Jones --- src/dvsim/instrumentation/report/__init__.py | 12 +++++ src/dvsim/instrumentation/report/base.py | 49 ++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 src/dvsim/instrumentation/report/__init__.py create mode 100644 src/dvsim/instrumentation/report/base.py diff --git a/src/dvsim/instrumentation/report/__init__.py b/src/dvsim/instrumentation/report/__init__.py new file mode 100644 index 00000000..87e54362 --- /dev/null +++ b/src/dvsim/instrumentation/report/__init__.py @@ -0,0 +1,12 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +"""DVSim Scheduler Instrumentation report.""" + +from dvsim.instrumentation.report.base import InstrumentationVisualizer, RenderProfile + +__all__ = ( + "InstrumentationVisualizer", + "RenderProfile", +) diff --git a/src/dvsim/instrumentation/report/base.py b/src/dvsim/instrumentation/report/base.py new file mode 100644 index 00000000..3582c668 --- /dev/null +++ b/src/dvsim/instrumentation/report/base.py @@ -0,0 +1,49 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +"""DVSim scheduler instrumentation reporting & visualizations.""" + +from enum import Enum +from typing import Protocol + +from typing_extensions import Self + +from dvsim.instrumentation import InstrumentationResults +from dvsim.logging import log + +__all__ = ( + "InstrumentationVisualizer", + "RenderProfile", +) + + +class RenderProfile(Enum): + """Levels of visualization rendering detail, which impact report size & responsiveness.""" + + NORMAL = "normal" + HIGH = "high" + FULL = "full" + + +class InstrumentationVisualizer(Protocol): + """Builder & renderer for HTML instrumentation visualizations.""" + + # A short name / title of the visualization, used in the HTML report navigation tab + title: str + + def render(self, results: InstrumentationResults) -> str | None: + """Render a visualization from the instrumentation results as a HTML fragment. + + If the required data is not provide in the instrumentation results (e.g. not enough + data, or not the correct type of data recorded), or the visualization should not be + generated, this can also optionally return `None`. + + """ + ... + + @classmethod + def for_profile(cls, profile: RenderProfile) -> Self: + """Create a visualizer instance configured for a given rendering profile (if supported).""" + log.debug("Render profile %s not used by visualization '%s'", profile.name, cls.title) + return cls() From 9579045d7062477243b89d41f27ff2f675e862d5 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 13 May 2026 12:13:19 +0100 Subject: [PATCH 2/6] feat: add instrumentation report visualisation registry Add a registry (like the existing registry/factory for the different instrumentation types) that we can use to add new instrumentation visualisations. In the same manner that we currently support plugins to extend the instrumentation functionality of DVSim, this will allow users extending DVSim to hook in and add their own custom instrumentation visualisations as well. Signed-off-by: Alex Jones --- src/dvsim/instrumentation/report/__init__.py | 2 + src/dvsim/instrumentation/report/registry.py | 47 ++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 src/dvsim/instrumentation/report/registry.py diff --git a/src/dvsim/instrumentation/report/__init__.py b/src/dvsim/instrumentation/report/__init__.py index 87e54362..abf41b82 100644 --- a/src/dvsim/instrumentation/report/__init__.py +++ b/src/dvsim/instrumentation/report/__init__.py @@ -5,8 +5,10 @@ """DVSim Scheduler Instrumentation report.""" from dvsim.instrumentation.report.base import InstrumentationVisualizer, RenderProfile +from dvsim.instrumentation.report.registry import ReportVisualizationRegistry __all__ = ( "InstrumentationVisualizer", "RenderProfile", + "ReportVisualizationRegistry", ) diff --git a/src/dvsim/instrumentation/report/registry.py b/src/dvsim/instrumentation/report/registry.py new file mode 100644 index 00000000..548f807a --- /dev/null +++ b/src/dvsim/instrumentation/report/registry.py @@ -0,0 +1,47 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +"""DVSim scheduler instrumentation report visualization registry.""" + +from typing import ClassVar + +from dvsim.instrumentation.report.base import InstrumentationVisualizer, RenderProfile + +__all__ = ("ReportVisualizationRegistry",) + + +class ReportVisualizationRegistry: + """Registry for scheduler instrumentation visualizer classes.""" + + _registry: ClassVar[dict[str, type[InstrumentationVisualizer]]] = {} + + @classmethod + def register(cls, vis_cls: type[InstrumentationVisualizer]) -> None: + """Register a new instrumentation visualization type.""" + cls._registry[vis_cls.title] = vis_cls + + @classmethod + def clear(cls) -> None: + """Clear any registered instrumentation visualization types.""" + cls._registry.clear() + + @classmethod + def registered(cls) -> dict[str, type[InstrumentationVisualizer]]: + """Get the current state of the registered instrumentation types.""" + return cls._registry.copy() + + @classmethod + def create(cls, profile: RenderProfile | None = None) -> list[InstrumentationVisualizer]: + """Create instances of registered visualization types for a given (optional) profile. + + Args: + profile: The rendering profile (level of detail) to target, if provided. + + Returns: + A list of InstrumentationVisualizer implementations created for the given profile. + + """ + if profile is None: + return [vis_cls() for vis_cls in cls._registry.values()] + return [vis_cls.for_profile(profile) for vis_cls in cls._registry.values()] From ba6352af263b95699b78f604a98745e21c184ce0 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 13 May 2026 12:24:45 +0100 Subject: [PATCH 3/6] feat: add head block to the HTML report wrapper template This will allow reports to add custom content to the HTML head if desired for e.g. styling purposes. Signed-off-by: Alex Jones --- src/dvsim/templates/reports/wrapper.html | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dvsim/templates/reports/wrapper.html b/src/dvsim/templates/reports/wrapper.html index 3f84d392..0e98b5a8 100644 --- a/src/dvsim/templates/reports/wrapper.html +++ b/src/dvsim/templates/reports/wrapper.html @@ -16,6 +16,7 @@ + {% block head_content %}{% endblock %} From 62da0ab1c7fcbdb60b9d9f5b6f26bb95672218bf Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Thu, 30 Apr 2026 16:38:57 +0100 Subject: [PATCH 4/6] build: add plotly dependency for instrumentation graph visualisations Signed-off-by: Alex Jones --- pyproject.toml | 1 + uv.lock | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d1c4e1d5..c089fa46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "gitpython>=3.1.45", "hjson>=3.1.0", "jinja2>=3.1.6", + "plotly>=6.7.0", "psutil>=7.2.2", "pydantic>=2.9.2", "pyyaml>=6.0.2", diff --git a/uv.lock b/uv.lock index 8a481817..5784a873 100644 --- a/uv.lock +++ b/uv.lock @@ -374,6 +374,7 @@ dependencies = [ { name = "gitpython" }, { name = "hjson" }, { name = "jinja2" }, + { name = "plotly" }, { name = "psutil" }, { name = "pydantic" }, { name = "pyyaml" }, @@ -465,6 +466,7 @@ requires-dist = [ { name = "hjson", specifier = ">=3.1.0" }, { name = "ipython", marker = "extra == 'debug'", specifier = ">=8.18.1" }, { name = "jinja2", specifier = ">=3.1.6" }, + { name = "plotly", specifier = ">=6.7.0" }, { name = "psutil", specifier = ">=7.2.2" }, { name = "pydantic", specifier = ">=2.9.2" }, { name = "pyhamcrest", marker = "extra == 'test'", specifier = ">=2.1.0" }, @@ -828,6 +830,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "narwhals" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/f3/257adc69a71011b4c8cda321b00f02c5bf1980ae38ffd05a58d9632d4de8/narwhals-2.20.0.tar.gz", hash = "sha256:c10994975fa7dc5a68c2cffcddbd5908fc8ebb2d463c5bab085309c0ee1f551e", size = 627848, upload-time = "2026-04-20T12:11:45.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/69/f24d3d1c38ad69e256138b4ec2452a8c7cf66be49dc214771ae99dd4f0a0/narwhals-2.20.0-py3-none-any.whl", hash = "sha256:16e750ea5507d4ba6e8d03455b5f93a535e0405976561baea235bca5dc9f475d", size = 449373, upload-time = "2026-04-20T12:11:43.596Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -867,6 +878,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, ] +[[package]] +name = "plotly" +version = "6.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "narwhals" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/7f/0f100df1172aadf88a929a9dbb902656b0880ba4b960fe5224867159d8f4/plotly-6.7.0.tar.gz", hash = "sha256:45eea0ff27e2a23ccd62776f77eb43aa1ca03df4192b76036e380bb479b892c6", size = 6911286, upload-time = "2026-04-09T20:36:45.738Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/ad/cba91b3bcf04073e4d1655a5c1710ef3f457f56f7d1b79dcc3d72f4dd912/plotly-6.7.0-py3-none-any.whl", hash = "sha256:ac8aca1c25c663a59b5b9140a549264a5badde2e057d79b8c772ae2920e32ff0", size = 9898444, upload-time = "2026-04-09T20:36:39.812Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" From 8474b146cf9a61aa687c16b891913ca7e1cc2b3c Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 13 May 2026 12:29:47 +0100 Subject: [PATCH 5/6] feat: add instrumentation report rendering from template This commit introduces the instrumentation report template itself, and a function `render_html_report` for rendering the instrumentation report with a list of given instrumentation report visualization implementations. Some key things to note: - We use `plotly.offline` to get the version of the minified plotly.js that is packaged with the plotly Python package. Like vendored static files this will still work in an airgapped environment, but this way it should remain in sync even when the plotly dependency has its version updated. - We only include plotly JS (which is around 4MB minified) as a dependency when we actually have renders that might use it. - For now we include all visualizations as different tabs on the same page. This is to keep things simple as the goal is to keep all the report logic relatively small and self-contained, but this could be expanded upon in the future if it is deemed too restrictive of an interface. Signed-off-by: Alex Jones --- src/dvsim/instrumentation/report/__init__.py | 7 +- src/dvsim/instrumentation/report/base.py | 83 +++++++++++++++++++ .../reports/instrumentation_report.html | 73 ++++++++++++++++ 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 src/dvsim/templates/reports/instrumentation_report.html diff --git a/src/dvsim/instrumentation/report/__init__.py b/src/dvsim/instrumentation/report/__init__.py index abf41b82..624c1e31 100644 --- a/src/dvsim/instrumentation/report/__init__.py +++ b/src/dvsim/instrumentation/report/__init__.py @@ -4,11 +4,16 @@ """DVSim Scheduler Instrumentation report.""" -from dvsim.instrumentation.report.base import InstrumentationVisualizer, RenderProfile +from dvsim.instrumentation.report.base import ( + InstrumentationVisualizer, + RenderProfile, + render_html_report, +) from dvsim.instrumentation.report.registry import ReportVisualizationRegistry __all__ = ( "InstrumentationVisualizer", "RenderProfile", "ReportVisualizationRegistry", + "render_html_report", ) diff --git a/src/dvsim/instrumentation/report/base.py b/src/dvsim/instrumentation/report/base.py index 3582c668..3425da8e 100644 --- a/src/dvsim/instrumentation/report/base.py +++ b/src/dvsim/instrumentation/report/base.py @@ -4,17 +4,23 @@ """DVSim scheduler instrumentation reporting & visualizations.""" +from collections.abc import Sequence from enum import Enum +from pathlib import Path from typing import Protocol +import plotly.offline from typing_extensions import Self from dvsim.instrumentation import InstrumentationResults from dvsim.logging import log +from dvsim.report.artifacts import ReportArtifacts, render_static_content +from dvsim.templates.render import render_template __all__ = ( "InstrumentationVisualizer", "RenderProfile", + "render_html_report", ) @@ -47,3 +53,80 @@ def for_profile(cls, profile: RenderProfile) -> Self: """Create a visualizer instance configured for a given rendering profile (if supported).""" log.debug("Render profile %s not used by visualization '%s'", profile.name, cls.title) return cls() + + +def render_html_report( + results: InstrumentationResults, + *, + visualizations: Sequence[InstrumentationVisualizer] | None = None, + outdir: Path | None = None, + json_path: Path | None = None, +) -> ReportArtifacts: + """Render a HTML instrumentation report for some results & visualizations. + + Args: + results: The instrumentation results to generate a report from. + visualizations: The list of visualizations (if any) to display in the report. + outdir: The optional directory to write the 'metrics.html' report to, if desired. + json_path: Optional path to the 'metrics.json' file. + + Returns: + The generated file contents for the report - 'metrics.html' and static CSS/JS content. + + """ + log.info("Rendering instrumentation HTML report...") + + visualizations = visualizations or [] + renders: list[tuple[InstrumentationVisualizer, str]] = [] + for i, vis in enumerate(visualizations, start=1): + log.debug( + "Attempting to render instrumentation visualization: %s [%d/%d]", + vis.title, + i, + len(visualizations), + ) + render = vis.render(results) + if render is not None: + log.info("Rendered instrumentation visualization: %s", vis.title) + renders.append((vis, render)) + + metrics_json_path = json_path + if metrics_json_path and outdir and metrics_json_path.is_relative_to(outdir): + metrics_json_path = metrics_json_path.relative_to(outdir) + if outdir is not None: + outdir.mkdir(parents=True, exist_ok=True) + + artifacts = {} + + # Render the visualizations to a single metrics.html file + artifacts["metrics.html"] = render_template( + path="reports/instrumentation_report.html", + data={"renders": renders, "metrics_json": metrics_json_path}, + ) + if outdir is not None: + report_path = outdir / "metrics.html" + report_path.write_text(artifacts["metrics.html"]) + log.info("HTML instrumentation report written to %s", report_path) + + # Render static content needed for the report + artifacts.update( + render_static_content( + static_files=[ + "css/style.css", + "css/bootstrap.min.css", + "js/bootstrap.bundle.min.js", + "js/htmx.min.js", + ], + outdir=outdir, + ) + ) + + # Render static plotly.js separately. We generate the static minified JS from the plotly + # library itself to make sure we are using the correct version. + if renders: + plotly_js_path = "js/plotly.min.js" + artifacts[plotly_js_path] = plotly.offline.get_plotlyjs() + if outdir is not None: + (outdir / plotly_js_path).write_text(artifacts[plotly_js_path]) + + return artifacts diff --git a/src/dvsim/templates/reports/instrumentation_report.html b/src/dvsim/templates/reports/instrumentation_report.html new file mode 100644 index 00000000..7747a285 --- /dev/null +++ b/src/dvsim/templates/reports/instrumentation_report.html @@ -0,0 +1,73 @@ + +{% extends "reports/wrapper.html" %} +{% block head_content %} +{% if renders %} + +{% endif %} +{% endblock %} +{% block content %} +
+
+
+

Instrumentation report

+
+ {% if metrics_json %} +
+ JSON +
+ {% endif %} +
+ +
+
+ {% if not renders %} +

No instrumentation visualizations matched the available report data.

+ {% else %} +
    + {% for vis, _ in renders %} + {% set class = "nav-link" + (" active" if loop.index0 == 0 else "") %} + {% set slug = vis.title.lower() | replace("_", "-") | replace (" ", "-") %} + + {% endfor %} +
+
+ {% for vis, fragment in renders %} + {% set class = "tab-pane fade" + (" show active" if loop.index0 == 0 else "") %} + {% set slug = vis.title.lower() | replace("_", "-") | replace (" ", "-") %} +
+
+ {{ fragment | safe }} +
+
+ {% endfor %} +
+ {% endif %} +
+
+
+ + +{% endblock %} From 8fa788387c6b80a3a645878360cb7b7001996886 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 13 May 2026 13:59:56 +0100 Subject: [PATCH 6/6] feat: add HTML instrumentation report generation Whenever a DVSim scheduler run completes and we flush the JSON instrumentation report, add additional logic to create the HTML instrumentation report as well - under the same report directory that is currently being used for the existing simulation HTML report outputs. It is not 100% clear whether this is the best place to integrate the instrumentation reporting yet. It seems like it might be more sensible to do it alongside the simulation reporting to keep common functionality grouped together, but this then means that we do not get the same benefits for other flows (e.g. linting, formal) that do not have their own custom HTML reports. So for now, we keep this in the generic scheduler running logic. Signed-off-by: Alex Jones --- src/dvsim/instrumentation/__init__.py | 11 ++++++- src/dvsim/instrumentation/runtime.py | 42 ++++++++++++++++++++++++++- src/dvsim/scheduler/runner.py | 2 +- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/dvsim/instrumentation/__init__.py b/src/dvsim/instrumentation/__init__.py index 1f13f8d2..7f6fd9e0 100644 --- a/src/dvsim/instrumentation/__init__.py +++ b/src/dvsim/instrumentation/__init__.py @@ -12,7 +12,14 @@ JobMetrics, SchedulerMetrics, ) -from dvsim.instrumentation.runtime import flush, get, set_instrumentation, set_report_path +from dvsim.instrumentation.runtime import ( + flush, + gen_html_report, + get, + get_report, + set_instrumentation, + set_report_path, +) __all__ = ( "InstrumentationAggregator", @@ -23,7 +30,9 @@ "SchedulerInstrumentation", "SchedulerMetrics", "flush", + "gen_html_report", "get", + "get_report", "set_instrumentation", "set_report_path", ) diff --git a/src/dvsim/instrumentation/runtime.py b/src/dvsim/instrumentation/runtime.py index d7447e6b..2ee25ef6 100644 --- a/src/dvsim/instrumentation/runtime.py +++ b/src/dvsim/instrumentation/runtime.py @@ -2,15 +2,21 @@ # Licensed under the Apache License, Version 2.0, see LICENSE for details. # SPDX-License-Identifier: Apache-2.0 -"""DVSim scheduler instrumentation for timing-related information.""" +"""DVSim scheduler instrumentation runtime implementation.""" from pathlib import Path from dvsim.instrumentation.base import InstrumentationAggregator, InstrumentationResults +from dvsim.instrumentation.report import ( + RenderProfile, + ReportVisualizationRegistry, + render_html_report, +) from dvsim.logging import log __all__ = ( "flush", + "gen_html_report", "get", "get_report", "set_instrumentation", @@ -19,7 +25,10 @@ class _Runtime: + """Runtime singleton for the configured scheduler instrumentation.""" + def __init__(self) -> None: + """Construct a Runtime for the scheduler instrumentation.""" self.instrumentation: InstrumentationAggregator | None = None self.report_path: Path | None = None self.report: InstrumentationResults | None = None @@ -71,3 +80,34 @@ def flush() -> InstrumentationResults | None: def get_report() -> InstrumentationResults | None: """Get the latest flushed instrumentation report contents, if any exist.""" return _runtime.report + + +def gen_html_report( + results: InstrumentationResults, + *, + profile: RenderProfile | None = None, + outdir: Path | None = None, + json_path: Path | None = None, +) -> None: + """Generate a HTML report of the instrumentation results with rendered visualizations. + + Args: + results: The instrumentation results to render a HTML report for. + profile: Optional rendering profile to customize level of detail vs. report optimization. + outdir: The path to the directory to write the generated HTML report files to. If not + provided, this defaults to the parent directory of the configured report path. + json_path: The path to the metrics.json file. If not provided, this defaults to the + configured report path (if it exists). + + """ + if outdir is None: + if _runtime.report_path is None: + return + outdir = _runtime.report_path.parent + + if json_path is None: + json_path = _runtime.report_path + + log.debug("HTML instrumentation report will be written to %s", outdir) + visualizations = ReportVisualizationRegistry.create(profile) + render_html_report(results, visualizations=visualizations, outdir=outdir, json_path=json_path) diff --git a/src/dvsim/scheduler/runner.py b/src/dvsim/scheduler/runner.py index 3fc5876a..587f5c8e 100644 --- a/src/dvsim/scheduler/runner.py +++ b/src/dvsim/scheduler/runner.py @@ -141,6 +141,6 @@ async def run_scheduler( # Finalize instrumentation if inst is not None: inst.stop() - instrumentation.flush() + instrumentation.gen_html_report(instrumentation.flush()) return results