From 693bd561efd634531ac3fe72df5b0c9680feea3a Mon Sep 17 00:00:00 2001 From: James McCorrie Date: Tue, 18 Nov 2025 17:47:53 +0000 Subject: [PATCH] feat: summary report more dashboard like Introduce a CoverageMetrics model instead of the dict. Create a new method on the tool plugins to convert from eda tool specific coverage fields to generic names that match the opentitan-site dashboard fields. Update the report summary and block report templates to use the new model. Add the extra fields in the summary report to make it more like the dashboard (contain the same information). Signed-off-by: James McCorrie --- src/dvsim/flow/sim.py | 8 ++- src/dvsim/report/data.py | 71 ++++++++++++++++++- src/dvsim/templates/reports/block_report.html | 38 +++++++--- .../templates/reports/summary_report.html | 63 +++++++++++----- src/dvsim/tool/sim.py | 17 ++++- src/dvsim/tool/vcs.py | 31 +++++++- src/dvsim/tool/xcelium.py | 31 +++++++- 7 files changed, 227 insertions(+), 32 deletions(-) diff --git a/src/dvsim/flow/sim.py b/src/dvsim/flow/sim.py index 3f8a7f70..b2a54ff7 100644 --- a/src/dvsim/flow/sim.py +++ b/src/dvsim/flow/sim.py @@ -29,6 +29,7 @@ from dvsim.sim_results import SimResults from dvsim.test import Test from dvsim.testplan import Testplan +from dvsim.tool.utils import get_sim_tool_plugin from dvsim.utils import TS_FORMAT, rm_path # This affects the bucketizer failure report. @@ -684,6 +685,7 @@ def make_test_result(tr) -> TestResult | None: # --- Coverage --- coverage: dict[str, float | None] = {} + coverage_model = None if self.cov_report_deploy: for k, v in self.cov_report_deploy.cov_results_dict.items(): try: @@ -691,13 +693,17 @@ def make_test_result(tr) -> TestResult | None: except (ValueError, TypeError, AttributeError): coverage[k.lower()] = None + coverage_model = get_sim_tool_plugin(self.tool).get_coverage_metrics( + raw_metrics=coverage, + ) + # --- Final result --- return FlowResults( block=block, tool=tool, timestamp=timestamp, stages=stages, - coverage=coverage, + coverage=coverage_model, passed=total_passed, total=total_runs, percent=100.0 * total_passed / total_runs if total_runs else 0.0, diff --git a/src/dvsim/report/data.py b/src/dvsim/report/data.py index 76230df6..e6e48e9e 100644 --- a/src/dvsim/report/data.py +++ b/src/dvsim/report/data.py @@ -95,6 +95,75 @@ class TestStage(BaseModel): """Percentage test pass rate.""" +class CodeCoverageMetrics(BaseModel): + """CodeCoverage metrics.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + block: float | None + """Block Coverage (%) - did this part of the code execute?""" + line_statement: float | None + """Line/Statement Coverage (%) - did this part of the code execute?""" + branch: float | None + """Branch Coverage (%) - did this if/case take all paths?""" + condition_expression: float | None + """Condition/Expression Coverage (%) - did the logic evaluate to 0 & 1?""" + toggle: float | None + """Toggle Coverage (%) - did the signal wiggle?""" + fsm: float | None + """FSM Coverage (%) - did the state machine transition?""" + + @property + def average(self) -> float | None: + """Average code coverage (%).""" + all_cov = [ + c + for c in [ + self.line_statement, + self.branch, + self.condition_expression, + self.toggle, + self.fsm, + ] + if c is not None + ] + + if len(all_cov) == 0: + return None + + return sum(all_cov) / len(all_cov) + + +class CoverageMetrics(BaseModel): + """Coverage metrics.""" + + code: CodeCoverageMetrics | None + """Code Coverage.""" + assertion: float | None + """Assertion Coverage.""" + functional: float | None + """Functional coverage.""" + + @property + def average(self) -> float | None: + """Average code coverage (%) or None if there is no coverage.""" + code = self.code.average if self.code is not None else None + all_cov = [ + c + for c in [ + code, + self.assertion, + self.functional, + ] + if c is not None + ] + + if len(all_cov) == 0: + return None + + return sum(all_cov) / len(all_cov) + + class FlowResults(BaseModel): """Flow results data.""" @@ -109,7 +178,7 @@ class FlowResults(BaseModel): stages: Mapping[str, TestStage] """Results per test stage.""" - coverage: Mapping[str, float | None] + coverage: CoverageMetrics | None """Coverage metrics.""" passed: int diff --git a/src/dvsim/templates/reports/block_report.html b/src/dvsim/templates/reports/block_report.html index 2754cc85..64638a24 100644 --- a/src/dvsim/templates/reports/block_report.html +++ b/src/dvsim/templates/reports/block_report.html @@ -91,21 +91,39 @@

Simulation Results: {{ block.name }}

+{% macro coverage_stat(cov, kind, label) %} + {% if cov and cov|attr(kind) is not none %} + {% set value = cov|attr(kind) %} +
+
    +
  • + {{ label }} +
    • +
  • {{ "%.2f"|format(value) }} %
    • +
+
+ {% endif %} +{% endmacro %} + + {% if coverage %}
Coverage statistics
- {% for name, val in coverage.items() %} -
-
    -
  • - {{ name }} -
    • -
  • {{ val }}
    • -
-
- {% endfor %} + {{ coverage_stat(coverage, "average", "Total") }} + + {% set code = coverage.code %} + {{ coverage_stat(code, "average", "code") }} + {{ coverage_stat(coverage, "assertion", "assert") }} + {{ coverage_stat(coverage, "functional", "func") }} + + {{ coverage_stat(code, "block", "block") }} + {{ coverage_stat(code, "line_statement", "line") }} + {{ coverage_stat(code, "branch", "branch") }} + {{ coverage_stat(code, "condition_expression", "cond") }} + {{ coverage_stat(code, "toggle", "toggle") }} + {{ coverage_stat(code, "fsm", "FSM") }}
diff --git a/src/dvsim/templates/reports/summary_report.html b/src/dvsim/templates/reports/summary_report.html index 8054a406..3ec93d34 100644 --- a/src/dvsim/templates/reports/summary_report.html +++ b/src/dvsim/templates/reports/summary_report.html @@ -88,16 +88,44 @@

Simulation Results: {{ top.name }}

+{% macro coverage_cell(cov, kind) %} + {% if cov and cov|attr(kind) is not none %} + {% set value = cov|attr(kind) %} + {{ "%.2f"|format(value) }} + {% else %} + - + {% endif %} +{% endmacro %} +
- - +
+ - + + + + + + + - + + + + + + + + + + + + + + @@ -114,19 +142,20 @@

Simulation Results: {{ top.name }}

- + {% set cov = flow.coverage %} + {{ coverage_cell(cov, "average") }} + + {% set code = cov|attr("code") %} + {{ coverage_cell(code, "average") }} + {{ coverage_cell(cov, "functional") }} + {{ coverage_cell(cov, "assertion") }} + + {{ coverage_cell(code, "block") }} + {{ coverage_cell(code, "line_statement") }} + {{ coverage_cell(code, "branch") }} + {{ coverage_cell(code, "condition_expression") }} + {{ coverage_cell(code, "toggle") }} + {{ coverage_cell(code, "fsm") }} {% endfor %} diff --git a/src/dvsim/tool/sim.py b/src/dvsim/tool/sim.py index 49249e55..2807dbbe 100644 --- a/src/dvsim/tool/sim.py +++ b/src/dvsim/tool/sim.py @@ -4,10 +4,12 @@ """EDA simulation tool interface.""" -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Protocol, runtime_checkable +from dvsim.report.data import CoverageMetrics + __all__ = ("SimTool",) @@ -67,3 +69,16 @@ def get_simulated_time(log_text: Sequence[str]) -> tuple[float, str]: """ ... + + @staticmethod + def get_coverage_metrics(raw_metrics: Mapping[str, float | None] | None) -> CoverageMetrics: + """Get a CoverageMetrics model from raw coverage data. + + Args: + raw_metrics: raw coverage metrics as parsed from the tool. + + Returns: + CoverageMetrics model. + + """ + ... diff --git a/src/dvsim/tool/vcs.py b/src/dvsim/tool/vcs.py index 6f84ede6..b40cc6ec 100644 --- a/src/dvsim/tool/vcs.py +++ b/src/dvsim/tool/vcs.py @@ -5,9 +5,11 @@ """EDA tool plugin providing VCS support to DVSim.""" import re -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from pathlib import Path +from dvsim.report.data import CodeCoverageMetrics, CoverageMetrics + __all__ = ("VCS",) @@ -103,3 +105,30 @@ def get_simulated_time(log_text: Sequence[str]) -> tuple[float, str]: msg = "Simulated time not found in the log." raise RuntimeError(msg) + + @staticmethod + def get_coverage_metrics(raw_metrics: Mapping[str, float | None] | None) -> CoverageMetrics: + """Get a CoverageMetrics model from raw coverage data. + + Args: + raw_metrics: raw coverage metrics as parsed from the tool. + + Returns: + CoverageMetrics model. + + """ + if raw_metrics is None: + return CoverageMetrics(code=None, assertion=None, functional=None) + + return CoverageMetrics( + functional=raw_metrics.get("group"), + assertion=raw_metrics.get("assert"), + code=CodeCoverageMetrics( + block=None, + line_statement=raw_metrics.get("line"), + branch=raw_metrics.get("branch"), + condition_expression=raw_metrics.get("cond"), + toggle=raw_metrics.get("toggle"), + fsm=raw_metrics.get("fsm"), + ), + ) diff --git a/src/dvsim/tool/xcelium.py b/src/dvsim/tool/xcelium.py index 2df28b6d..b5e62ad7 100644 --- a/src/dvsim/tool/xcelium.py +++ b/src/dvsim/tool/xcelium.py @@ -6,9 +6,11 @@ import re from collections import OrderedDict -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from pathlib import Path +from dvsim.report.data import CodeCoverageMetrics, CoverageMetrics + __all__ = ("Xcelium",) @@ -128,3 +130,30 @@ def get_simulated_time(log_text: Sequence[str]) -> tuple[float, str]: msg = "Simulated time not found in the log." raise RuntimeError(msg) + + @staticmethod + def get_coverage_metrics(raw_metrics: Mapping[str, float | None] | None) -> CoverageMetrics: + """Get a CoverageMetrics model from raw coverage data. + + Args: + raw_metrics: raw coverage metrics as parsed from the tool. + + Returns: + CoverageMetrics model. + + """ + if raw_metrics is None: + return CoverageMetrics(code=None, assertion=None, functional=None) + + return CoverageMetrics( + functional=raw_metrics.get("covergroup"), + assertion=raw_metrics.get("assertion"), + code=CodeCoverageMetrics( + block=raw_metrics.get("block"), + line_statement=raw_metrics.get("statement"), + branch=raw_metrics.get("branch"), + condition_expression=raw_metrics.get("cond"), + toggle=raw_metrics.get("toggle"), + fsm=raw_metrics.get("fsm"), + ), + )
BlockBlockTestsCoverage SummaryCode Coverage
Pass Total %CovOverallCodeFunctionalAssertionBlockLineBranchConditionToggleFSM
{{ "%.2f" | format(flow.percent) }} - {% if flow.coverage and flow.coverage.get('score') is not none %} - {{ "%.2f" | format(flow.coverage.score) }} - {% else %} - - - {% endif %} -