Add HTML instrumentation report types, registry & template generation

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "gitpython>=3.1.45",
     "hjson>=3.1.0",
     "jinja2>=3.1.6",
+    "plotly>=6.7.0",
     "psutil>=7.2.2",
     "pydantic>=2.9.2",
     "pyyaml>=6.0.2",

src/dvsim/instrumentation/__init__.py

@@ -12,7 +12,14 @@
     JobMetrics,
     SchedulerMetrics,
 )
-from dvsim.instrumentation.runtime import flush, get, set_instrumentation, set_report_path
+from dvsim.instrumentation.runtime import (
+    flush,
+    gen_html_report,
+    get,
+    get_report,
+    set_instrumentation,
+    set_report_path,
+)
 
 __all__ = (
     "InstrumentationAggregator",
@@ -23,7 +30,9 @@
     "SchedulerInstrumentation",
     "SchedulerMetrics",
     "flush",
+    "gen_html_report",
     "get",
+    "get_report",
     "set_instrumentation",
     "set_report_path",
 )

src/dvsim/instrumentation/report/__init__.py

@@ -0,0 +1,19 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+"""DVSim Scheduler Instrumentation report."""
+
+from dvsim.instrumentation.report.base import (
+    InstrumentationVisualizer,
+    RenderProfile,
+    render_html_report,
+)
+from dvsim.instrumentation.report.registry import ReportVisualizationRegistry
+
+__all__ = (
+    "InstrumentationVisualizer",
+    "RenderProfile",
+    "ReportVisualizationRegistry",
+    "render_html_report",
+)

src/dvsim/instrumentation/report/base.py

@@ -0,0 +1,132 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+"""DVSim scheduler instrumentation reporting & visualizations."""
+
+from collections.abc import Sequence
+from enum import Enum
+from pathlib import Path
+from typing import Protocol
+
+import plotly.offline
+from typing_extensions import Self
+
+from dvsim.instrumentation import InstrumentationResults
+from dvsim.logging import log
+from dvsim.report.artifacts import ReportArtifacts, render_static_content
+from dvsim.templates.render import render_template
+
+__all__ = (
+    "InstrumentationVisualizer",
+    "RenderProfile",
+    "render_html_report",
+)
+
+
+class RenderProfile(Enum):
+    """Levels of visualization rendering detail, which impact report size & responsiveness."""
+
+    NORMAL = "normal"
+    HIGH = "high"
+    FULL = "full"
+
+
+class InstrumentationVisualizer(Protocol):
+    """Builder & renderer for HTML instrumentation visualizations."""
+
+    # A short name / title of the visualization, used in the HTML report navigation tab
+    title: str
+
+    def render(self, results: InstrumentationResults) -> str | None:
+        """Render a visualization from the instrumentation results as a HTML fragment.
+
+        If the required data is not provide in the instrumentation results (e.g. not enough
+        data, or not the correct type of data recorded), or the visualization should not be
+        generated, this can also optionally return `None`.
+
+        """
+        ...
+
+    @classmethod
+    def for_profile(cls, profile: RenderProfile) -> Self:
+        """Create a visualizer instance configured for a given rendering profile (if supported)."""
+        log.debug("Render profile %s not used by visualization '%s'", profile.name, cls.title)
+        return cls()
+
+
+def render_html_report(
+    results: InstrumentationResults,
+    *,
+    visualizations: Sequence[InstrumentationVisualizer] | None = None,
+    outdir: Path | None = None,
+    json_path: Path | None = None,
+) -> ReportArtifacts:
+    """Render a HTML instrumentation report for some results & visualizations.
+
+    Args:
+        results: The instrumentation results to generate a report from.
+        visualizations: The list of visualizations (if any) to display in the report.
+        outdir: The optional directory to write the 'metrics.html' report to, if desired.
+        json_path: Optional path to the 'metrics.json' file.
+
+    Returns:
+        The generated file contents for the report - 'metrics.html' and static CSS/JS content.
+
+    """
+    log.info("Rendering instrumentation HTML report...")
+
+    visualizations = visualizations or []
+    renders: list[tuple[InstrumentationVisualizer, str]] = []
+    for i, vis in enumerate(visualizations, start=1):
+        log.debug(
+            "Attempting to render instrumentation visualization: %s [%d/%d]",
+            vis.title,
+            i,
+            len(visualizations),
+        )
+        render = vis.render(results)
+        if render is not None:
+            log.info("Rendered instrumentation visualization: %s", vis.title)
+            renders.append((vis, render))
+
+    metrics_json_path = json_path
+    if metrics_json_path and outdir and metrics_json_path.is_relative_to(outdir):
+        metrics_json_path = metrics_json_path.relative_to(outdir)
+    if outdir is not None:
+        outdir.mkdir(parents=True, exist_ok=True)
+
+    artifacts = {}
+
+    # Render the visualizations to a single metrics.html file
+    artifacts["metrics.html"] = render_template(
+        path="reports/instrumentation_report.html",
+        data={"renders": renders, "metrics_json": metrics_json_path},
+    )
+    if outdir is not None:
+        report_path = outdir / "metrics.html"
+        report_path.write_text(artifacts["metrics.html"])
+        log.info("HTML instrumentation report written to %s", report_path)
+
+    # Render static content needed for the report
+    artifacts.update(
+        render_static_content(
+            static_files=[
+                "css/style.css",
+                "css/bootstrap.min.css",
+                "js/bootstrap.bundle.min.js",
+                "js/htmx.min.js",
+            ],
+            outdir=outdir,
+        )
+    )
+
+    # Render static plotly.js separately. We generate the static minified JS from the plotly
+    # library itself to make sure we are using the correct version.
+    if renders:
+        plotly_js_path = "js/plotly.min.js"
+        artifacts[plotly_js_path] = plotly.offline.get_plotlyjs()
+        if outdir is not None:
+            (outdir / plotly_js_path).write_text(artifacts[plotly_js_path])
+
+    return artifacts

src/dvsim/instrumentation/report/registry.py

@@ -0,0 +1,47 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+"""DVSim scheduler instrumentation report visualization registry."""
+
+from typing import ClassVar
+
+from dvsim.instrumentation.report.base import InstrumentationVisualizer, RenderProfile
+
+__all__ = ("ReportVisualizationRegistry",)
+
+
+class ReportVisualizationRegistry:
+    """Registry for scheduler instrumentation visualizer classes."""
+
+    _registry: ClassVar[dict[str, type[InstrumentationVisualizer]]] = {}
+
+    @classmethod
+    def register(cls, vis_cls: type[InstrumentationVisualizer]) -> None:
+        """Register a new instrumentation visualization type."""
+        cls._registry[vis_cls.title] = vis_cls
+
+    @classmethod
+    def clear(cls) -> None:
+        """Clear any registered instrumentation visualization types."""
+        cls._registry.clear()
+
+    @classmethod
+    def registered(cls) -> dict[str, type[InstrumentationVisualizer]]:
+        """Get the current state of the registered instrumentation types."""
+        return cls._registry.copy()
+
+    @classmethod
+    def create(cls, profile: RenderProfile | None = None) -> list[InstrumentationVisualizer]:
+        """Create instances of registered visualization types for a given (optional) profile.
+
+        Args:
+            profile: The rendering profile (level of detail) to target, if provided.
+
+        Returns:
+            A list of InstrumentationVisualizer implementations created for the given profile.
+
+        """
+        if profile is None:
+            return [vis_cls() for vis_cls in cls._registry.values()]
+        return [vis_cls.for_profile(profile) for vis_cls in cls._registry.values()]

src/dvsim/instrumentation/runtime.py

@@ -2,15 +2,21 @@
 # Licensed under the Apache License, Version 2.0, see LICENSE for details.
 # SPDX-License-Identifier: Apache-2.0
 
-"""DVSim scheduler instrumentation for timing-related information."""
+"""DVSim scheduler instrumentation runtime implementation."""
 
 from pathlib import Path
 
 from dvsim.instrumentation.base import InstrumentationAggregator, InstrumentationResults
+from dvsim.instrumentation.report import (
+    RenderProfile,
+    ReportVisualizationRegistry,
+    render_html_report,
+)
 from dvsim.logging import log
 
 __all__ = (
     "flush",
+    "gen_html_report",
     "get",
     "get_report",
     "set_instrumentation",
@@ -19,7 +25,10 @@
 
 
 class _Runtime:
+    """Runtime singleton for the configured scheduler instrumentation."""
+
     def __init__(self) -> None:
+        """Construct a Runtime for the scheduler instrumentation."""
         self.instrumentation: InstrumentationAggregator | None = None
         self.report_path: Path | None = None
         self.report: InstrumentationResults | None = None
@@ -71,3 +80,34 @@ def flush() -> InstrumentationResults | None:
 def get_report() -> InstrumentationResults | None:
     """Get the latest flushed instrumentation report contents, if any exist."""
     return _runtime.report
+
+
+def gen_html_report(
+    results: InstrumentationResults,
+    *,
+    profile: RenderProfile | None = None,
+    outdir: Path | None = None,
+    json_path: Path | None = None,
+) -> None:
+    """Generate a HTML report of the instrumentation results with rendered visualizations.
+
+    Args:
+        results: The instrumentation results to render a HTML report for.
+        profile: Optional rendering profile to customize level of detail vs. report optimization.
+        outdir: The path to the directory to write the generated HTML report files to. If not
+          provided, this defaults to the parent directory of the configured report path.
+        json_path: The path to the metrics.json file. If not provided, this defaults to the
+          configured report path (if it exists).
+
+    """
+    if outdir is None:
+        if _runtime.report_path is None:
+            return
+        outdir = _runtime.report_path.parent
+
+    if json_path is None:
+        json_path = _runtime.report_path
+
+    log.debug("HTML instrumentation report will be written to %s", outdir)
+    visualizations = ReportVisualizationRegistry.create(profile)
+    render_html_report(results, visualizations=visualizations, outdir=outdir, json_path=json_path)

src/dvsim/scheduler/runner.py

@@ -141,6 +141,6 @@ async def run_scheduler(
     # Finalize instrumentation
     if inst is not None:
         inst.stop()
-        instrumentation.flush()
+        instrumentation.gen_html_report(instrumentation.flush())
 
     return results

src/dvsim/templates/reports/instrumentation_report.html

@@ -0,0 +1,73 @@
+<!--
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+-->
+{% extends "reports/wrapper.html" %}
+{% block head_content %}
+{% if renders %}
+<script src="js/plotly.min.js"></script>
+{% endif %}
+{% endblock %}
+{% block content %}
+<div class="container-md">
+    <div class="row py-3 justify-content-between">
+        <div class="col">
+            <h2>Instrumentation report</h2>
+        </div>
+        {% if metrics_json %}
+        <div class="col-auto">
+            <a class="badge text-bg-secondary link-underline link-underline-opacity-0"
+                style="font-size: 15px;" href="{{ metrics_json }}">JSON</a>
+        </div>
+        {% endif %}
+    </div>
+
+    <div class="row py-3">
+        <div class="col">
+            {% if not renders %}
+            <p padding="2 rem">No instrumentation visualizations matched the available report data.</p>
+            {% else %}
+            <ul class="nav nav-underline" style="flex-wrap: nowrap; white-space: nowrap; overflow-x: auto;
+                                                 overflow-y: hidden;">
+                {% for vis, _ in renders %}
+                {% set class = "nav-link" + (" active" if loop.index0 == 0 else "") %}
+                {% set slug = vis.title.lower() | replace("_", "-") | replace (" ", "-") %}
+                <li class="nav-item" role="presentation">
+                    <button class="{{ class }}" data-bs-toggle="tab" data-bs-target="#{{ slug }}-pane" 
+                            type="button" role="tab">
+                        {{ vis.title }}
+                    </button>
+                </li>
+                {% endfor %}
+            </ul>
+            <div class="tab-content border rounded-bottom">
+                {% for vis, fragment in renders %}
+                {% set class = "tab-pane fade" + (" show active" if loop.index0 == 0 else "") %}
+                {% set slug = vis.title.lower() | replace("_", "-") | replace (" ", "-") %}
+                <div class="{{ class }}" id="{{ slug }}-pane" role="tabpanel">
+                    <div class="p-3">
+                        {{ fragment | safe }}
+                    </div>
+                </div>
+                {% endfor %}
+            </div>
+            {% endif %}
+        </div>
+    </div>
+</div>
+
+<script>
+    /* Plotly renders charts inside hidden tab panes before they are visible, causing
+    Plotly to measure the container width as zero/partial, setting the size accordingly.
+    We must call Plotly.relayout on each chart when its tab is shown to fix this. */
+    document.querySelectorAll('[data-bs-toggle="tab"]').forEach(tab => {
+        tab.addEventListener('shown.bs.tab', () => {
+            document.querySelectorAll('.tab-pane .plotly-graph-div').forEach(elem => {
+                Plotly.relayout(elem, { autosize: true });
+            });
+        });
+    });
+</script>
+{% endblock %}

src/dvsim/templates/reports/wrapper.html

@@ -16,6 +16,7 @@
         </script>
         <link href="css/bootstrap.min.css" rel="stylesheet">
         <link href="css/style.css" rel="stylesheet">
+        {% block head_content %}{% endblock %}
     </head>
     <body>