From 272653ec97fe89f0a31cd28c826b04bfc26c5210 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Tue, 22 Sep 2026 19:06:50 +0200 Subject: [PATCH 01/17] feat: implement SCORE tool management workflow --- docs/how-to/index.rst | 1 + docs/how-to/perform_tool_verification.md | 1166 +++++++++++++++++ .../internals/requirements/classification.rst | 527 ++++++++ docs/internals/requirements/index.rst | 6 + docs/internals/requirements/qualification.rst | 176 +++ .../checks/tool_qualification.py | 351 +++++ src/extensions/score_metamodel/metamodel.yaml | 47 + .../rst/architecture/architecture_tests.rst | 1 + .../tests/rst/graph/test_metamodel_graph.rst | 34 + .../rst/graph/test_tool_qualification.rst | 81 ++ .../tests/test_metamodel_load.py | 49 + .../tests/test_tool_qualification.py | 333 +++++ .../tests/test_traceability_metrics.py | 10 +- .../test_source_code_link_integration.py | 9 + .../tests/test_testlink.py | 4 + .../tests/test_xml_parser.py | 4 + .../tool_qualification_report.need | 213 +++ .../_expected/needs_json/needs.json | 36 + .../_expected/data_bundle_needs/needs.json | 36 + .../isolated_source_bundle_needs/needs.json | 36 + .../_expected/needs_json/needs.json | 36 + .../_expected/needs_json/needs.json | 36 + .../_expected/needs_local.json | 36 + .../_expected/needs_json/needs.json | 36 + .../component/_expected/needs_local.json | 36 + .../_expected/needs_json/needs.json | 36 + .../_expected/needs_local.json | 36 + .../_expected/needs_json/needs.json | 36 + .../consumer/_expected/needs_json/needs.json | 36 + .../producer/_expected/needs_json/needs.json | 36 + 30 files changed, 3478 insertions(+), 2 deletions(-) create mode 100644 docs/how-to/perform_tool_verification.md create mode 100644 docs/internals/requirements/classification.rst create mode 100644 docs/internals/requirements/qualification.rst create mode 100644 src/extensions/score_metamodel/checks/tool_qualification.py create mode 100644 src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst create mode 100644 src/extensions/score_metamodel/tests/test_tool_qualification.py create mode 100644 src/needs_templates/tool_qualification_report.need diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst index 8f78a58cf..5d35f18fb 100644 --- a/docs/how-to/index.rst +++ b/docs/how-to/index.rst @@ -31,5 +31,6 @@ Here you find practical guides on how to use docs-as-code. dashboards_and_quality_gates source_to_doc_links test_to_doc_links + perform_tool_verification generated_docs add_extensions diff --git a/docs/how-to/perform_tool_verification.md b/docs/how-to/perform_tool_verification.md new file mode 100644 index 000000000..29435aa78 --- /dev/null +++ b/docs/how-to/perform_tool_verification.md @@ -0,0 +1,1166 @@ + + +# How to Apply the SCORE Tool Management Process + +This guide explains how to execute the +[Eclipse SCORE Tool Management Process](https://eclipse-score.github.io/process_description/main/process_areas/tool_management/tool_management_workflow.html) +using SCORE Docs-as-Code. + +The goal is **not** to reproduce ISO 26262 or define a separate qualification +process. SCORE already defines the process. This guide explains how to represent +and execute that process with the available S-CORE Docs-as-Code model. + +```{note} +This guide focuses on the safety-related evaluation and qualification aspects +of SCORE Tool Management. Security-related evaluation remains unsolved for now. +``` + +--- + +## Mental model + +The important distinction is between: + +- **Tool Verification Report (`doc_tool`)** — records the SCORE tool-management + state and the overall evaluation result. +- **Tool requirements (`tool_req`)** — what the project relies on the tool to do. +- **Tool use cases (`tool_usecase`)** — the usage context in which the project + relies on those requirements. +- **Potential tool malfunctions (`potential_tool_malfunction`)** — ways in which + the expected tool behaviour can fail. +- **Safety measures** — measures active during intended tool usage that prevent + or detect a potential malfunction. +- **Testcases (`testcase`)** — verification evidence for requirements. For tool + qualification, these verify the relevant `tool_req` needs. + +```{mermaid} +flowchart LR + UC["tool_usecase
Tool Use Case"] + PM["potential_tool_malfunction
Potential Tool Malfunction"] + + STKH["stkh_req
Stakeholder Requirement"] + TR["tool_req
Tool Requirement"] + TC["testcase
Qualification Evidence"] + + UC -->|"realizes
(optional)"| STKH + UC -->|"realized_by"| TR + + PM -->|"violates"| UC + PM -->|"violates"| STKH + PM -->|"violates"| TR + + TC -->|"fully_verifies /
partially_verifies"| TR + TR -->|"realizes
(optional)"| STKH + + style UC fill:#E1D5E7 + style PM fill:#F8CECC + style STKH fill:#DAE8FC + style TR fill:#F5F5F5 +``` + +The two analysis views answer different questions: + +| View | Question | +|---|---| +| **Evaluation / classification** | How can our intended use of the tool fail, and would the intended usage detect or prevent that failure? | +| **Qualification** | Do we have sufficient evidence that the specific tool version satisfies the tool requirements we rely on? | + +Qualification evidence and detection sufficiency are separate decisions; see +[Qualification is not a detection measure](#qualification-is-not-a-detection-measure). + +--- + +## External vs. self-developed tools + +The evaluation is based on **how the project uses the tool**, not on who +developed it. + +Use cases, potential malfunctions, safety impact, and detection are therefore +evaluated the same way for external and self-developed tools. + +The differences matter mainly when defining requirements and qualification +evidence. + +| Topic | External tool | Self-developed tool | +|---|---|---| +| Version | Pin the exact external version and relevant configuration | Identify the exact internal release/version | +| `tool_req` | Define the behaviour **our project relies on** | Reuse existing tool requirements where suitable | +| Implementation | Usually treated largely as a black box | Architecture and implementation may be available | +| Existing tests | Vendor/upstream tests can be supporting information | Existing development tests may already provide qualification evidence | +| Qualification | Usually validate our `tool_req` against the configured external tool | Reuse suitable requirements-based tests where possible | +| Corrective action | Upgrade, configure, constrain usage, add detection, or replace | The tool implementation itself can also be changed | + +Do not duplicate requirements or tests solely because a self-developed tool is +now managed through the SCORE Tool Management process. + +--- + +## SCORE workflow at a glance + +```{mermaid} +flowchart LR + A["Create Tool Verification Report"] --> B["status = draft"] + B --> C["Evaluate Tool"] + C --> D["status = evaluated"] + D --> E{"Qualification required?"} + + E -->|"No"| H["Review / approve TVR"] + E -->|"Yes"| F["Qualify Tool"] + F --> G["status = qualified"] + G --> H + + H --> I["status = released"] + + H -. unsuccessful .-> J["status = rejected"] +``` + +The remainder of this guide follows these steps. + +--- + +## Step 1 — Create the Tool Verification Report + +### What to model + +The SCORE Tool Verification Report is represented by `doc_tool`. + +The current metamodel contains the following relevant attributes: + +- `status`: `draft | evaluated | qualified | released | rejected` +- `safety_affected`: `YES | NO` +- `security_affected`: `YES | NO` +- `tcl`: `LOW | HIGH` +- `tool_version`: optional in the current metamodel + +Example structure: + +````markdown +```{doc_tool} S-CORE Docs-as-Code Tool Verification Report +:id: doc_tool__s_core_docs_as_code +:status: draft +:safety_affected: YES +:security_affected: NO +:tcl: LOW +:tool_version: + +Describe the tool, its intended use, relevant configuration, environment, +constraints, inputs, outputs, and available documentation. +``` +```` + +```{note} +`tcl` is mandatory in the current metamodel even while the report is still +`draft`. Update it to the actual evaluation result once Step 5 is complete. +``` + +### External tool + +Identify the exact version and configuration that is actually used. + +Do not evaluate the S-CORE Docs-as-Code tool, Clang, or GitHub generically when +the project relies on a specific version, plugin set, configuration, or usage +restriction. + +### Self-developed tool + +Reference the project's normal version/release identification. The TVR must make +clear which released tool version the evaluation applies to. + +--- + +## Step 2 — Define how the project uses the tool + +SCORE evaluates the tool in the context of its **use cases**. + +A `tool_usecase` captures: + +> **What are we relying on this tool to do in our development process?** + +The current metamodel defines: + +```yaml +tool_usecase: + mandatory_links: + belongs_to: doc_tool + realized_by: tool_req + optional_links: + realizes: stkh_req +``` + +The normal relationship is therefore: + +```{mermaid} +flowchart LR + STKH["stkh_req"] + DT["doc_tool"] + UC["tool_usecase"] + TR["tool_req"] + + UC -->|"belongs_to
(mandatory)"| DT + UC -->|"realizes
(optional)"| STKH + UC -->|"realized_by
(mandatory)"| TR + + style STKH fill:#DAE8FC + style UC fill:#E1D5E7 + style TR fill:#F5F5F5 +``` + +### When is `tool_usecase` useful? + +Use `tool_usecase` when the stakeholder requirements are too high-level and the +tool requirements are too detailed to clearly express the usage context. + +For example: + +- stakeholder requirement: the system-development process must provide valid + requirements traceability, +- tool use case: validate requirement traceability during documentation builds, +- tool requirement: unresolved requirement links must be reported. + +If an existing requirement already expresses the usage context exactly, avoid +inventing additional behaviour in the use case. The use case can remain a thin +grouping/context element. + +```{important} +With the current metamodel, every `potential_tool_malfunction` has a mandatory +`parent_needs` link to a `tool_usecase`. Therefore, once you model +malfunctions, a `tool_usecase` is structurally required even if it merely +references an already well-scoped requirement. +``` + +### Tool model example + +````markdown +```{tool_usecase} Validate requirement traceability during documentation builds +:id: tool_usecase__docs_as_code__traceability +:belongs_to: doc_tool__s_core_docs_as_code +:realized_by: tool_req__docs_as_code__unresolved_links +:realizes: stkh_req__docs_as_code__traceability + +The project relies on the documentation tooling to identify invalid or +unresolved requirement links before documentation is accepted. +``` +```` + +A good use case describes project usage, not an implementation detail. + +**Avoid:** + +```text +Call Python function resolve_links(). +``` + +**Prefer:** + +```text +Validate requirement links during the documentation build. +``` + +### External tool + +Define the `tool_req` needs from **our expectations of the tool**, not by +importing the vendor's complete specification. + +Example: + +```text +Tool use case: +Validate requirement traceability during documentation builds. + +Tool requirement: +The tool shall report unresolved requirement links. +``` + +### Self-developed tool + +Prefer linking the use case to existing `tool_req` needs. + +Do not create separate "qualification requirements" when the normal tool +requirements already describe the behaviour relied upon by the project. + +--- + +## Step 3 — Identify potential tool malfunctions + +For every tool use case, identify relevant ways in which the expected tool +behaviour could fail. + +The current metamodel defines: + +```yaml +potential_tool_malfunction: + mandatory_options: + safety_affected: "^(YES|NO)$" + optional_options: + detection_sufficient: "^(YES|NO)$" + safety_measures: ^.+$ + mandatory_links: + parent_needs: tool_usecase + violates: stkh_req, tool_req +``` + +The relationship is: + +```{mermaid} +flowchart LR + UC["tool_usecase"] + PM["potential_tool_malfunction"] + STKH["stkh_req"] + TR["tool_req"] + + PM -->|"parent_needs"| UC + PM -->|"violates"| STKH + PM -->|"violates"| TR + + style UC fill:#E1D5E7 + style PM fill:#F8CECC + style STKH fill:#DAE8FC + style TR fill:#F5F5F5 +``` + +### What is a useful malfunction? + +Describe **incorrect behaviour relevant to the use case**. + +**Too generic:** + +```text +The tool crashes. +The tool has a bug. +The tool produces a wrong result. +``` + +**Better:** + +```text +An unresolved requirement link is accepted as valid. +A valid requirement is omitted from generated documentation. +Generated source code does not represent the configured input model correctly. +``` + +A useful question is: + +> **What could the tool do incorrectly, or fail to do, such that one of the +> requirements we rely on is violated?** + +### `violates` + +Use `violates` for requirements whose expected behaviour is directly broken by +the malfunction. + +Typical targets are: + +- `stkh_req` +- `tool_req` + +Do not link arbitrary downstream requirements merely because the malfunction +could eventually contribute to an incorrect engineering decision. + +### Tool model example + +````markdown +```{potential_tool_malfunction} Unresolved requirement link is accepted as valid +:id: potential_tool_malfunction__docs_as_code__unresolved_link_accepted +:parent_needs: tool_usecase__docs_as_code__traceability +:violates: + tool_req__docs_as_code__unresolved_links, + stkh_req__docs_as_code__traceability +:safety_affected: YES +:detection_sufficient: NO + +An unresolved requirement link is accepted as valid and the documentation build +does not report the problem. +``` +```` + +There is no conceptual difference between external and self-developed tools in +this step. Malfunctions are derived from the intended usage and the requirements +the project relies on. + +`detection_sufficient` is conditional. It is required for +`safety_affected: YES`; it is not required for `safety_affected: NO` and is +omitted for non-safety malfunctions. A safety-relevant malfunction with +`detection_sufficient: YES` must also document a non-empty `safety_measures` +value. The malfunction body remains the place for the human-readable reasoning +that explains why the measure is sufficient or insufficient. + +--- + +## Step 4 — Evaluate safety impact and detection + +For each `potential_tool_malfunction`, answer two questions. + +### Does the malfunction affect safety? + +Use: + +```text +:safety_affected: YES +``` + +if the malfunction can: + +- introduce an error into safety-related work, or +- prevent an existing error from being detected. + +Use: + +```text +:safety_affected: NO +``` + +if the malfunction has no relevant safety impact in this usage context. + +This is a property of the **usage context and malfunction**, not a generic label +for the tool. + +The same tool can therefore have safety-relevant and non-safety-relevant use +cases. + +--- + +### Is detection sufficient? + +`detection_sufficient` asks: + +> **If this malfunction occurs during the intended use of the tool, do the +> measures that are part of that intended usage prevent or detect it with +> sufficient confidence?** + +These measures can be: + +- internal to the tool, +- provided by another tool, +- part of the surrounding workflow or process. + +Examples include: + +- generated source is compiled before integration, +- an independent consistency checker validates generated output, +- a review compares the generated artifact against its source, +- the tool performs an internal integrity check and rejects inconsistent input, +- a downstream tool independently validates critical information. + +Document the relevant measure using `safety_measures`. + +````markdown +```{potential_tool_malfunction} Generated source contains invalid syntax +:id: potential_tool_malfunction__generator__invalid_syntax +:parent_needs: tool_usecase__generator__generate_source +:violates: tool_req__generator__valid_source +:safety_affected: YES +:detection_sufficient: YES +:safety_measures: Generated source is compiled before it can be integrated. + +The generator may emit syntactically invalid source code. The mandatory +downstream compilation detects this malfunction before the output can be used. +``` +```` + +The important point is that the measure is part of the **defined intended +usage**. + +It does not matter whether the measure existed before the TVR was written or +was introduced as a result of the evaluation. If it becomes a mandatory part of +the intended usage, evaluate the malfunction with that usage concept. + +--- + +## Step 5 — Determine confidence and qualification need + +For the SCORE safety evaluation, the simplified decision is: + +```{mermaid} +flowchart TD + A{"safety_affected?"} + A -->|"NO"| B["HIGH confidence
No qualification required"] + A -->|"YES"| C{"detection_sufficient?"} + C -->|"YES"| B + C -->|"NO"| D["LOW confidence
Qualification required"] +``` + +| `safety_affected` | `detection_sufficient` | TVR `tcl` | Qualification | +|---|---|---|---| +| `NO` | not relevant | `HIGH` | not required | +| `YES` | `YES` | `HIGH` | not required | +| `YES` | `NO` | `LOW` | required | + +The tooling derives the TVR values from the owned graph rather than treating +them as independent labels: + +1. Follow `doc_tool` -> `tool_usecase` through the mandatory `belongs_to` link. +2. Follow each use case -> `potential_tool_malfunction` through + `parent_needs`. +3. Set `doc_tool.safety_affected` to `YES` if any owned malfunction is safety + affected; otherwise set it to `NO`. +4. Set `doc_tool.tcl` to `LOW` if any owned malfunction is safety affected and + has `detection_sufficient: NO`; otherwise set it to `HIGH`. + +The stored `safety_affected` and `tcl` values remain mandatory SCORE TVR data, +but validation reports a mismatch when they do not match this derivation. +Qualification evidence does not change either value and does not change a +malfunction's `detection_sufficient` result. + +After the evaluation is complete, update the `doc_tool`: + +```text +:status: evaluated +:tcl: HIGH +``` + +or: + +```text +:status: evaluated +:tcl: LOW +``` + +The report must clearly state whether qualification is required. + +### Multiple use cases and malfunctions + +The overall TVR result must reflect the relevant worst case. + +If any safety-relevant malfunction has insufficient detection, the tool cannot +be treated as `HIGH` merely because the other use cases are well protected. + +--- + +## Step 5a — Improve the usage concept instead of qualifying + +A `LOW` evaluation does not mean qualification is the only possible response. + +You may change the intended usage by adding an appropriate detection or +prevention measure. + +```{mermaid} +flowchart LR + subgraph Before + A1["Generator"] --> A2["Generated output"] + A2 --> A3["Used directly"] + end + + subgraph After + B1["Generator"] --> B2["Generated output"] + B2 --> B3["Independent check"] + B3 --> B4["Accepted output"] + end +``` + +Before: + +```text +:safety_affected: YES +:detection_sufficient: NO +``` + +After the independent check becomes a mandatory part of intended usage: + +```text +:safety_affected: YES +:safety_measures: Independent checker validates every generated artifact. +:detection_sufficient: YES +``` + +This requires **re-evaluating the affected malfunction because the usage concept +changed**. + +It is not a reclassification caused by qualification. + +--- + +## Step 6 — Qualify the tool if required + +Qualification is required when the evaluation results in insufficient +confidence and the project does not resolve that through a changed usage +concept. + +SCORE uses validation of the software tool as the qualification approach. + +The qualification view starts from `tool_req`: + +```{mermaid} +flowchart LR + TR["tool_req"] + TC["testcase"] + RESULT["Test result / evidence"] + + TC -->|"fully_verifies or
partially_verifies"| TR + TC --> RESULT + + style TR fill:#F5F5F5 +``` + +In the current S-CORE Docs-as-Code model, test execution results are represented +as `testcase` needs. Their normal links to requirements are: + +- `fully_verifies` +- `partially_verifies` + +For qualification, the relevant testcases should verify the `tool_req` needs +that define the behaviour relied upon by the tool use cases. + +### Qualification is not a detection measure + +Compare the two roles: + +```{mermaid} +flowchart TB + subgraph Usage["Detection during intended tool usage"] + U1["Tool execution"] --> U2["Output"] + U2 --> U3["Independent / internal check"] + U3 --> U4["Output accepted"] + end + + subgraph Qualification["Qualification evidence"] + Q1["Specific tool version"] --> Q2["Requirements-based testcase"] + Q2 --> Q3["Evidence that tool_req is satisfied"] + end +``` + +A qualification testcase answers: + +> Does this tool version satisfy the requirement? + +A safety measure used for `detection_sufficient` answers: + +> If the malfunction happens during real intended usage, will it be prevented or +> detected before its result is relied upon? + +In other words, a qualification test checks whether the tool version satisfies +a `tool_req`. `detection_sufficient` checks whether the measures used during +normal tool usage will detect or prevent a malfunction before its output is +relied upon. Passing a qualification test therefore does **not** by itself +justify setting `detection_sufficient: YES`. + +The same technical check can only count for both if it actually performs both +roles. + +--- + +### External tool qualification + +External tools are commonly validated largely as black boxes. + +Example requirement: + +````markdown +```{tool_req} Report unresolved requirement links +:id: tool_req__docs_as_code__unresolved_links + +The tool shall report an unresolved requirement link as an error. +``` +```` + +A requirements-based qualification test can then: + +1. create input containing a known unresolved link, +2. execute the exact configured tool version, +3. verify that the expected error is reported. + +The resulting SCORE `testcase` should link to the tool requirement via +`fully_verifies` or `partially_verifies`, as appropriate. + +Vendor documentation, release notes, and upstream tests can support the +argument, but the qualification evidence needs to address the behaviour that +**our project relies on**. + +--- + +### Self-developed tool qualification + +For a self-developed tool, do not automatically create a second qualification +test suite. + +If the normal development process already produces testcases that: + +- verify the relevant `tool_req` needs, +- execute against the tool version being qualified, +- provide suitable recorded results, + +reuse those testcases as qualification evidence. + +Prefer: + +```{mermaid} +flowchart LR + TR["Existing tool_req"] --> IMPL["Tool implementation"] + TC["Existing testcase"] -->|"fully_verifies / partially_verifies"| TR + TC --> E["Existing test evidence"] +``` + +over duplicating the same requirement and test solely for qualification. + +Additional qualification tests are needed only where the existing evidence is +insufficient. + +--- + +## Step 7 — Update the Tool Verification Report + +If no qualification was required, the evaluated report can proceed to review. + +If qualification was required and completed successfully: + +```text +:status: qualified +``` + +The evaluation result itself does **not** automatically change. + +This is a valid final state before release: + +```text +:tcl: LOW +:status: qualified +``` + +`LOW` records why qualification was necessary. + +`qualified` records that the additional qualification evidence has been +provided. + +Qualification does not turn: + +```text +:detection_sufficient: NO +``` + +into: + +```text +:detection_sufficient: YES +``` + +unless the intended usage was also changed by introducing a new mandatory +detection mechanism. + +--- + +## Step 8 — Review and release the Tool Verification Report + +Review the completed TVR against the SCORE Tool Verification Report Review +Checklist. + +The review should cover, where applicable: + +- unique tool identification, +- exact tool version, +- purpose and tool use cases, +- inputs and outputs, +- configuration, +- environment and limitations, +- documentation, +- usage constraints, +- potential malfunctions, +- safety impact, +- safety measures, +- error detection, +- overall confidence, +- qualification evidence where required. + +After successful approval: + +```text +:status: released +``` + +If verification is unsuccessful: + +```text +:status: rejected +``` + +See: + +- [SCORE Tool Management Workflow](https://eclipse-score.github.io/process_description/main/process_areas/tool_management/tool_management_workflow.html) +- [SCORE Tool Verification Report Template](https://eclipse-score.github.io/process_description/main/folder_templates/tools/tool_verification_report_template.html) +- [SCORE Tool Management Process Requirements](https://eclipse-score.github.io/process_description/main/process_areas/tool_management/guidance/tool_management_reqs.html) +- [SCORE Tool Verification Report Review Checklist](https://eclipse-score.github.io/process_description/main/process_areas/tool_management/guidance/tool_management_checklist.html) + +--- + +## Complete example — external tool + +Assume an external documentation tool is used to validate requirement +traceability. + +### Stakeholder requirement + +A real `stkh_req` must provide the mandatory metadata defined by the SCORE +metamodel. For example: + +````markdown +```{stkh_req} Provide valid verification traceability +:id: stkh_req__docs_as_code__traceability +:reqtype: Process +:safety: ASIL_B +:security: NO +:status: valid +:rationale: Invalid traceability can hide missing verification. +:valid_from: v1.0 + +Safety-related requirements shall have valid verification traceability. +``` +```` + +### Tool requirement + +````markdown +```{tool_req} Report unresolved requirement links +:id: tool_req__docs_as_code__unresolved_links +:satisfies: stkh_req__docs_as_code__traceability + +The tool shall report unresolved requirement links as errors. +``` +```` + +### Tool use case + +````markdown +```{tool_usecase} Validate requirement traceability during documentation builds +:id: tool_usecase__docs_as_code__traceability +:belongs_to: doc_tool__s_core_docs_as_code +:realized_by: tool_req__docs_as_code__unresolved_links +:realizes: stkh_req__docs_as_code__traceability + +The project relies on the documentation tool to detect invalid traceability +before generated documentation is accepted. +``` +```` + +### Potential malfunction + +````markdown +```{potential_tool_malfunction} Unresolved requirement link is accepted as valid +:id: potential_tool_malfunction__docs_as_code__unresolved_link_accepted +:parent_needs: tool_usecase__docs_as_code__traceability +:violates: + tool_req__docs_as_code__unresolved_links, + stkh_req__docs_as_code__traceability +:safety_affected: YES +:detection_sufficient: NO + +An unresolved requirement link is accepted as valid and no error is reported. +``` +```` + +The resulting model is: + +```{mermaid} +flowchart LR + STKH["stkh_req__docs_as_code__traceability"] + UC["tool_usecase__docs_as_code__traceability"] + TR["tool_req__docs_as_code__unresolved_links"] + PM["potential_tool_malfunction__docs_as_code__
unresolved_link_accepted"] + + UC -->|"realizes"| STKH + UC -->|"realized_by"| TR + PM -->|"parent_needs"| UC + PM -->|"violates"| STKH + PM -->|"violates"| TR + + style STKH fill:#DAE8FC + style UC fill:#E1D5E7 + style TR fill:#F5F5F5 + style PM fill:#F8CECC +``` + +Because the malfunction is safety relevant and detection is insufficient: + +```text +tcl = LOW +qualification required = YES +``` + +Qualification then validates +`tool_req__docs_as_code__unresolved_links`. + +The resulting test execution must produce a `testcase` that links to the tool +requirement using the normal SCORE verification links, for example +`fully_verifies`. + +The TVR remains `LOW`, but after successful qualification its lifecycle can +progress: + +```{mermaid} +flowchart LR + A["evaluated
tcl = LOW"] --> B["qualification tests pass"] + B --> C["qualified
tcl = LOW"] + C --> D["released
tcl = LOW"] +``` + +--- + +## Complete example — self-developed tool + +Assume the project develops its own model-to-code generator. + +### Existing requirements + +````markdown +```{stkh_req} Generate software from the approved model +:id: stkh_req__generator__approved_model +:reqtype: Functional +:safety: ASIL_B +:security: NO +:status: valid +:rationale: Production code must represent the approved behavioural model. +:valid_from: v1.0 + +Software source shall be generated from the approved model. +``` +```` + +````markdown +```{tool_req} Preserve configured state transitions +:id: tool_req__generator__state_transitions +:satisfies: stkh_req__generator__approved_model + +The generator shall preserve configured state transitions in generated source +code. +``` +```` + +### Tool use case + +````markdown +```{tool_usecase} Generate production source from the approved model +:id: tool_usecase__generator__generate_source +:belongs_to: doc_tool__s_core_docs_as_code +:realized_by: tool_req__generator__state_transitions +:realizes: stkh_req__generator__approved_model + +The project relies on the generator to transform the approved model into +production source code. +``` +```` + +### Potential malfunction + +````markdown +```{potential_tool_malfunction} State transition is omitted from generated source +:id: potential_tool_malfunction__generator__missing_transition +:parent_needs: tool_usecase__generator__generate_source +:violates: + tool_req__generator__state_transitions, + stkh_req__generator__approved_model +:safety_affected: YES +:detection_sufficient: NO + +A state transition present in the approved input model is omitted from the +generated source code. +``` +```` + +Result: + +```text +tcl = LOW +qualification required = YES +``` + +If an existing requirements-based test already verifies +`tool_req__generator__state_transitions` for the released generator version, +reuse its generated `testcase` and test evidence for qualification. + +Do not create another requirement and another test solely to label them +"qualification". + +--- + +## When to evaluate the tool again + +Re-evaluate affected malfunctions when information used by the evaluation +changes. + +Typical triggers include: + +- a new or changed tool use case, +- new or changed stakeholder requirements, +- new or changed tool requirements, +- a changed tool version, +- a changed relevant configuration, +- a changed integration/environment, +- a newly identified potential malfunction, +- a new, changed, or removed safety measure, +- a changed surrounding workflow/toolchain, +- evidence that an assumed detection measure is not sufficient. + +Do **not** re-evaluate merely because qualification tests passed. + +Qualification adds evidence about the tool. It does not by itself change the +intended usage or the detection measures. + +```{mermaid} +flowchart TD + A{"What changed?"} + + A -->|"Tool version / use case / requirement /
workflow / safety measure"| B["Re-evaluate affected malfunctions"] + A -->|"Only qualification tests executed"| C["No re-evaluation required"] + + B --> D["Update TVR evaluation if result changed"] +``` + +--- + +## Common mistakes + +### Treating `tool_usecase` as a new requirement level + +Do not invent new normative behaviour in `tool_usecase`. + +It is a **usage-context / grouping element** between stakeholder intent and +concrete tool behaviour. + +```{mermaid} +flowchart LR + STKH["Stakeholder intent"] + UC["Usage context"] + TR["Concrete tool behaviour"] + + UC -->|"realizes"| STKH + UC -->|"realized_by"| TR +``` + +--- + +### Using generic malfunctions + +Avoid: + +```text +The tool crashes. +The tool has a bug. +The result is wrong. +``` + +Prefer: + +```text +A missing link is accepted as valid. +A required artifact is omitted from generated output. +The generated value differs from the configured source value. +``` + +--- + +### Treating every existing test as a safety measure + +A test executed during development or release validation is not automatically a +measure for `detection_sufficient`. + +Ask: + +> If the malfunction occurs during actual intended tool usage, does this +> mechanism prevent or detect it before the result is relied upon? + +If not, it is not sufficient justification for: + +```text +:detection_sufficient: YES +``` + +--- + +### Treating qualification as reclassification + +Qualification may leave the evaluation unchanged: + +```text +:safety_affected: YES +:detection_sufficient: NO +:tcl: LOW +:status: qualified +``` + +This is not contradictory. + +`LOW` describes the evaluation. + +`qualified` describes the additional evidence. + +--- + +### Duplicating external specifications + +Do not copy an external tool's complete specification into `tool_req`. + +Model the behaviour **our project relies on**. + +--- + +### Duplicating self-developed tests + +Do not build a parallel qualification-test world when existing SCORE testcases +already verify the relevant `tool_req` needs with suitable evidence. + +--- + +## Quick reference + +```{mermaid} +flowchart TD + A["1. Identify exact tool version / configuration"] + B["2. Create doc_tool
status = draft"] + C["3. Define tool_usecase"] + D["4. Link tool_req
and optionally stkh_req"] + E["5. Identify potential_tool_malfunction"] + F["6. Set safety_affected"] + G["7. Document safety_measures
and detection_sufficient"] + H["8. Determine tcl"] + I["status = evaluated"] + J{"tcl = LOW?"} + K["Verify relevant tool_req
with testcase evidence"] + L["status = qualified"] + M["Review / approve TVR"] + N["status = released"] + + A --> B --> C --> D --> E --> F --> G --> H --> I --> J + J -->|"No"| M + J -->|"Yes"| K --> L --> M + M --> N +``` + +In short: + +```text +tool_usecase + -> defines the context in which we rely on the tool + +potential_tool_malfunction + -> describes how that reliance can fail + +safety_affected + -> says whether the failure matters for safety + +safety_measures + detection_sufficient + -> describe whether intended usage prevents/detects the failure + +LOW + -> qualification is required + +testcase -> fully_verifies/partially_verifies -> tool_req + -> provides qualification evidence + +qualified + -> qualification evidence exists; it does not mean tcl became HIGH +``` + +## SCORE references + +- [Tool Management Workflow](https://eclipse-score.github.io/process_description/main/process_areas/tool_management/tool_management_workflow.html) +- [Tool Verification Report Template](https://eclipse-score.github.io/process_description/main/folder_templates/tools/tool_verification_report_template.html) +- [Tool Management Process Requirements](https://eclipse-score.github.io/process_description/main/process_areas/tool_management/guidance/tool_management_reqs.html) +- [Tool Verification Report Review Checklist](https://eclipse-score.github.io/process_description/main/process_areas/tool_management/guidance/tool_management_checklist.html) diff --git a/docs/internals/requirements/classification.rst b/docs/internals/requirements/classification.rst new file mode 100644 index 000000000..4a110e8e6 --- /dev/null +++ b/docs/internals/requirements/classification.rst @@ -0,0 +1,527 @@ +.. + # ******************************************************************************* + # Copyright (c) 2025 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Doc-as-Code Tool Evaluation and Classification +============================================== + +This page records the evaluation of the docs-as-code tool in the usage +contexts on which the project relies. It is the classification part of the +Tool Verification Report. The resulting qualification evidence and lifecycle +state are described in :doc:`qualification`. + +Introduction +------------ + +Scope and purpose +~~~~~~~~~~~~~~~~~ + +The S-CORE Docs-as-Code tool (Bazel module ``score_docs_as_code``) builds HTML +documentation from RST/Markdown sources — process description, requirements, and +traceability — and validates content against the S-CORE metamodel. + +Inputs and outputs +~~~~~~~~~~~~~~~~~~ + +* **Inputs:** RST/Markdown sources, Sphinx configuration (``conf.py``), the S-CORE + metamodel (``metamodel.yaml``), Bazel build files, source-code links + (``sourcelinks_json``) and test results (``testlinks``). +* **Outputs:** HTML documentation (``_build/``), needs/traceability data + (``needs.json``), coverage/linkage statistics (``metrics.json``). + +.. mermaid:: + + graph LR + src@{ shape: docs, label: "RST/Markdown sources (+ assets)" } + code@{ shape: docs, label: "C++/Rust/Python sources" } + srclinks@{ shape: doc, label: "sourcelinks" } + cfg@{ shape: docs, label: "Config (conf.py, metamodel.yaml, Bazel)" } + tests@{ shape: docs, label: "Test results" } + dac@{ shape: subproc, label: "Doc-as-Code" } + html@{ shape: docs, label: "HTML docs" } + needs@{ shape: doc, label: "needs.json" } + metrics@{ shape: docs, label: "metrics.json" } + + src --> dac + code --> srclinks --> dac + cfg --> dac + tests --> dac + dac --> html + dac --> needs + dac --> metrics + +Available information +~~~~~~~~~~~~~~~~~~~~~ +* Repository: https://github.com/eclipse-score/docs-as-code +* Documentation: https://eclipse-score.github.io/docs-as-code/v8.1.2/ +* Bazel module name: ``score_docs_as_code`` + +Installation and integration +---------------------------- + +Installation +~~~~~~~~~~~~ + +The tool is consumed as a Bazel module. Declare the dependency in +``MODULE.bazel``:: + + bazel_dep(name = "score_docs_as_code", version = "8.1.2") + +and the S-CORE registry in ``.bazelrc``:: + + common --registry=https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/ + common --registry=https://bcr.bazel.build + +Invoke the ``docs()`` macro from the root ``BUILD`` file:: + + load("@score_docs_as_code//:docs.bzl", "docs") + docs( + project = "My Project", + project_url = "https://github.com/eclipse-score/my-project", + source_dir = "docs", + ) + +For local development, ``bazel run //:ide_support`` creates a Python virtual +environment (``.venv_docs``) with all Sphinx extensions pre-installed for IDE +support (Esbonio). The macro's build targets (``//:docs``, ``//:docs_check``, +``//:docs_link_check``, ``//:traceability_gate``, ``//:live_preview``, ``//:ide_support``) +are documented in the `Build commands reference +`_. + +Tool sources live in the ``docs-as-code`` repository under ``src/extensions/`` +(Sphinx extensions) and ``docs.bzl`` (Bazel macros). The default metamodel is +bundled at ``@score_docs_as_code//src/extensions/score_metamodel:metamodel_yaml`` +and may be overridden via the ``metamodel`` parameter. See the +`Bazel macros reference +`_ +for macro parameters and the +`score_metamodel design +`_ +for metamodel definition and validation checks. + +Integration +~~~~~~~~~~~ +The tool is the central documentation hub of the S-CORE Bazel toolchain, used +by all modules to build, check, and publish documentation. + +Cross-module linking supports two modes: + +- **External needs import:** reference another module's ``:needs_json_file`` target + via the ``external_needs`` parameter of ``docs()`` to cross-reference need + IDs across modules (e.g., ``:need:`gd_req__example_id```). +- **Bundle mounting:** mount another module's ``:docs_bundle`` target via the + ``bundles`` parameter; the mounted sources join the consuming build, with + placement controlled by ``mount_at`` (docname prefix) and ``attach_to`` + (toctree anchor). See `How to mount external sources + `_. + +Within a repository, Sphinx combines documentation sources (RST/Markdown), +needs JSON, source-code links (``sourcelinks_json``) and test metadata through +the S-CORE extensions (``score_metamodel``, ``score_metrics``, ``score_mounts``) +to produce HTML, ``needs.json`` and ``metrics.json``. + + +Environment +~~~~~~~~~~~ +- **Operating system:** Linux — the S-CORE DevContainer (canonical, + recommended), WSL2, or native. +- **Build system:** Bazel (``rules_python``, ``sphinxdocs``) fetches all + toolchains and dependencies, including a remote JDK 17 for PlantUML diagrams + when no local Java is present. + +Safety evaluation +----------------- + +Evaluation model +~~~~~~~~~~~~~~~~ + +The evaluation follows the structured SCORE Tool Management model: + +.. mermaid:: + + graph LR + DT["doc_tool\nTool Verification Report"] + UC["tool_usecase\nUsage context"] + PM["potential_tool_malfunction\nIncorrect behaviour in that context"] + STKH["stkh_req"] + TR["tool_req"] + + UC -->|"belongs_to"| DT + PM -->|"parent_needs"| UC + UC -->|"realizes (optional)"| STKH + UC -->|"realized_by"| TR + PM -->|"violates"| STKH + PM -->|"violates"| TR + + style DT fill:#F5F5F5 + style UC fill:#E1D5E7 + style PM fill:#F8CECC + +``tool_usecase`` is a usage-context and grouping element. It does not add a +new requirements abstraction level. ``potential_tool_malfunction`` describes +incorrect tool behaviour in that context. Its body contains the human-readable +argumentation for the classification; the structured options record the +classification inputs: + +* ``safety_affected`` is mandatory. +* ``detection_sufficient`` is required only for safety-relevant malfunctions. +* ``safety_measures`` describes mechanisms active during intended tool usage. + It is required when ``detection_sufficient`` is ``YES``. +* A safety-relevant malfunction with ``detection_sufficient: NO`` violates at + least one ``tool_req`` so it can be qualified. + +Non-safety malfunctions omit ``detection_sufficient``. Qualification tests are +not safety measures and do not change this classification. + +Use cases were derived from the process requirements +and the docs-as-code +`Tool Requirements `_. + +The facts below are shared by use cases and only referenced in each +Malfunctions cell. + +.. _basis-ci: + +Build/CI behavior + Builds run with ``-W``; any warning trips CI. The + safety-relevant danger is the *silent* failure — a missing warning + or a wrong output published undetected. + A loud CI abort is safe: no wrong output enters the baseline. + +.. _pr_review: + +PR Review + Repository contents are the source of truth + and every change is reviewed by a committer + (:need:`rl__committer`, :need:`doc_concept__wp_inspections`). + Still, for silent wrong outputs the gated CI stays green. + +.. _basis-ti1: + +Derived-view + The rendered HTML output is a derived view; + the authoritative safety artifacts are mostly the source-controlled work products. + There are two exceptions, the architecture views (see M4) and backlinks (see M8). + Rendering/preview defects affect reviewer convenience, not safety evidence. + + +.. tool_usecase:: Build/CI behavior + :id: tool_usecase__docs_as_code__build_ci + :belongs_to: doc_tool__doc_as_code + :realized_by: + tool_req__docs_doc_types, + tool_req__docs_common_attr_safety_link_check, + tool_req__docs_test_linkage_metrics, + tool_req__docs_test_link_testcase, + tool_req__docs_req_link_covers_aou + :version: 1 + + Builds run with ``-W``; any warning trips CI. The + safety-relevant danger is the *silent* failure — a missing warning + or a wrong output published undetected. + A loud CI abort is safe: no wrong output enters the baseline. + + .. potential_tool_malfunction:: Document metamodel enforcement + :id: potential_tool_malfunction__docs_as_code__m1 + :safety_affected: YES + :detection_sufficient: NO + :safety_measures: PR review + :violates: + tool_req__docs_doc_types, + tool_req__docs_doc_generic_mandatory, + tool_req__docs_common_attr_id, + tool_req__docs_common_attr_id_scheme, + tool_req__docs_common_attr_status, + tool_req__docs_common_attr_description, + tool_req__docs_common_attr_title, + tool_req__docs_common_attr_desc_wording, + tool_req__docs_common_attr_security, + tool_req__docs_common_attr_safety, + tool_req__docs_common_attr_version, + tool_req__docs_common_attr_suspicious, + tool_req__docs_req_attr_rationale, + tool_req__docs_req_attr_reqtype, + tool_req__docs_req_attr_testcov, + tool_req__docs_req_attr_validity_correctness, + tool_req__docs_req_attr_validity_consistency, + tool_req__arch_linkage_safety, + tool_req__arch_consistency_interf, + tool_req__docs_tvr_safety, + tool_req__docs_tvr_security, + tool_req__docs_tvr_status, + tool_req__docs_tvr_version, + tool_req__docs_tvr_confidence_level, + tool_req__docs_saf_attrs_mitigated_by, + tool_req__docs_saf_attrs_mitigation_issue, + tool_req__docs_saf_attrs_sufficient, + tool_req__docs_saf_attrs_sufficient_check, + tool_req__docs_saf_attrs_content, + tool_req__docs_saf_attrs_violates, + tool_req__docs_saf_attr_fmea_fault_id, + tool_req__docs_saf_attr_fmea_failure_effect + :version: 1 + + **Document metamodel enforcement** — enforce document types, mandatory attributes (id, status, security, safety, realizes), etc. + See, for example, :need:`gd_req__doc_attr_status`, :need:`gd_req__req_attr_uid`, :need:`gd_req__req_attr_safety`, :need:`gd_req__arch_attr_safety`, :need:`gd_req__req_check_mandatory`. + + `Silent false-negative `_, too-permissive + ``metamodel.yaml`` regex accepted with no guard, or a check bug skips a case. + + Impact on safety: yes. + Impact safety measures available: yes: `PR review `_. + Impact safety detection sufficient: no: Qualify metamodel enforcement. + Further additional safety measure required: yes (qualification). + Confidence (automatic calculation): low. + + .. potential_tool_malfunction:: Safety-critical linking enforcement + :id: potential_tool_malfunction__docs_as_code__m2 + :safety_affected: YES + :detection_sufficient: NO + :safety_measures: PR review + :violates: + tool_req__docs_common_attr_safety_link_check, + tool_req__docs_req_arch_link_safety_to_arch, + tool_req__docs_req_link_satisfies_allowed, + tool_req__docs_req_link_covers_aou, + tool_req__docs_arch_link_fulfils, + tool_req__docs_arch_link_fulfils_aou, + tool_req__docs_arch_link_aou_check, + tool_req__docs_arch_link_safety_to_req, + tool_req__docs_arch_link_security + :version: 1 + + **Safety-critical linking enforcement**. + See :need:`gd_req__req_linkage_safety`. + + `Silent false-negative `_: Allow links which cannot be safe derivations. + + Impact on safety: yes. + Impact safety measures available: yes: `PR review `_. + Impact safety detection sufficient: no: Qualify graph checks. + Further additional safety measure required: yes (qualification). + + The clearest gap is ``satisfied_by`` (and arguably ``covers``), which carry the same "target at least as safe" obligation as the checked ``fulfils``/``implements`` yet are unconstrained. + + Confidence (automatic calculation): low. + + .. potential_tool_malfunction:: Requirements coverage statistics + :id: potential_tool_malfunction__docs_as_code__m3 + :safety_affected: YES + :detection_sufficient: NO + :violates: + tool_req__docs_test_linkage_metrics, + tool_req__docs_verification_report_need + :version: 1 + + **Requirements coverage statistics** — count, per requirement type, the requirements carrying a ``testlink``, compute link-coverage percentages. + See :need:`gd_req__verification_reporting`. + + `Silent wrong-output `_: a coverage statistic computed wrong. + + Impact on safety: yes. + Impact safety measures available: no. + Impact safety detection sufficient: no: Qualify coverage statistics. + Further additional safety measure required: yes (qualification). + Confidence (automatic calculation): low. + + .. potential_tool_malfunction:: Test linkage + :id: potential_tool_malfunction__docs_as_code__m5 + :safety_affected: YES + :detection_sufficient: NO + :violates: + tool_req__docs_test_link_testcase, + tool_req__docs_test_linkage_metrics, + tool_req__docs_verification_report_need, + tool_req__docs_test_metadata_mandatory_1, + tool_req__docs_test_metadata_mandatory_2, + tool_req__docs_test_metadata_link_levels + :version: 1 + + **Test linkage** — for each ``testcase`` need, resolve its ``partially_verifies``/``fully_verifies`` references against the needs set. + See :need:`gd_req__req_attr_testlink`, :need:`gd_req__verification_reporting`. + + `Silent wrong-output `_: + Safety case believes the requirement is tested where it is not. + + Impact on safety: yes. + Impact safety measures available: no. + Impact safety detection sufficient: no: Qualify linkage statistics. + Further additional safety measure required: yes (qualification). + Confidence (automatic calculation): low. + + .. potential_tool_malfunction:: Test reference check + :id: potential_tool_malfunction__docs_as_code__m6 + :safety_affected: YES + :detection_sufficient: NO + :safety_measures: PR review + :violates: tool_req__docs_test_link_testcase + :version: 1 + + **Test reference check**. + See :need:`gd_req__req_attr_testlink`. + + `Silent wrong-output `_: + Test references an outdated/missing requirement. + + Impact on safety: yes. + Impact safety measures available: yes: `PR review `_. + Impact safety detection sufficient: no: Qualify test reference check. + Further additional safety measure required: yes (qualification). + Confidence (automatic calculation): low. + + .. potential_tool_malfunction:: Listing assumptions of use + :id: potential_tool_malfunction__docs_as_code__m7 + :safety_affected: YES + :detection_sufficient: NO + :safety_measures: PR review + :violates: + tool_req__docs_req_types, + tool_req__docs_req_link_covers_aou, + tool_req__docs_arch_link_fulfils_aou + :version: 1 + + **Listing assumptions of use** — safety manuals use ``needtable`` to communicate safety-critical assumptions of use to users. + See :need:`gd_guidl__saf_man`, :need:`wp__platform_safety_manual`. + + `Silent wrong-output `_: ``aou_req`` items might be missing or wrong. + + Impact on safety: yes. + Impact safety measures available: yes: `PR review `_. + Impact safety detection sufficient: no: Qualify ``needtable``. + Further additional safety measure required: yes: qualification. + Confidence (automatic calculation): low. + +.. tool_usecase:: PR Review + :id: tool_usecase__docs_as_code__pr_review + :belongs_to: doc_tool__doc_as_code + :realized_by: tool_req__docs_doc_types + :version: 1 + + Repository contents are the source of truth + and every change is reviewed by a committer + (:need:`rl__committer`, :need:`doc_concept__wp_inspections`). + Still, for silent wrong outputs the gated CI stays green. + +.. tool_usecase:: Derived-view + :id: tool_usecase__docs_as_code__derived_view + :belongs_to: doc_tool__doc_as_code + :realized_by: + tool_req__docs_arch_views, + tool_req__docs_verification_report_need, + tool_req__docs_req_link_satisfies_allowed + :version: 1 + + The rendered HTML output is a derived view; + the authoritative safety artifacts are mostly the source-controlled work products. + There are two exceptions, the architecture views (see M4) and backlinks (see M8). + Rendering/preview defects affect reviewer convenience, not safety evidence. + + .. potential_tool_malfunction:: Architecture visualization + :id: potential_tool_malfunction__docs_as_code__m4 + :safety_affected: YES + :detection_sufficient: YES + :safety_measures: PR review includes architecture inspection + :violates: tool_req__docs_arch_views + :version: 1 + + **Architecture visualization** — generate architecture diagrams. + See :need:`gd_req__arch_viewpoints`. + + `Silent wrong-output `_: a diagram misrepresents the architecture. + + Impact on safety: yes. + Impact safety measures available: yes: `PR review `_ includes architecture inspection. + Impact safety detection sufficient: yes. + Further additional safety measure required: no. + Confidence (automatic calculation): high. + + .. potential_tool_malfunction:: Backlinks + :id: potential_tool_malfunction__docs_as_code__m8 + :safety_affected: YES + :detection_sufficient: NO + :violates: tool_req__docs_req_link_satisfies_allowed + :version: 1 + + **Backlinks** — for bi-directional traceability, generate correct backlinks for links between Needs items. + See :need:`doc_concept__general_traceability`. + + `Silent wrong-output `_: Generated backlinks are wrong or missing. + + Impact on safety: yes. + Impact safety measures available: no. + Impact safety detection sufficient: no: Qualify backlinks in HTML. + Further additional safety measure required: yes (qualification). + Confidence (automatic calculation): low. + + .. potential_tool_malfunction:: Documentation generation + :id: potential_tool_malfunction__docs_as_code__m9 + :safety_affected: NO + :violates: tool_req__docs_doc_types + :version: 1 + + **Documentation generation** — apart from the aspects **not covered by previous malfunctions**. + See :need:`gd_req__doc_attributes_manual`, :need:`gd_req__doc_attr_status`. + + Incomplete, outdated, or mis-rendered HTML. + + Impact on safety: no: `Derived-view `_. + Impact safety measures available: no. + Impact safety detection sufficient: not applicable for a non-safety malfunction. + Further additional safety measure required: no. + Confidence (automatic calculation): high. + +Security evaluation +------------------- +The threat model reduces to a single class: **source tampering**. The tool has +no runtime attack surface — it is a build-time Sphinx extension reading +source-controlled inputs and writing generated output. + +.. list-table:: S-CORE Docs-as-Code security evaluation + :header-rows: 1 + :widths: 1 2 8 2 6 4 2 + + * - Threat identification + - Use case description + - Threats + - Impact on security? + - Impact security measures available? + - Impact security detection sufficient? + - Further additional security measure required? + * - T1 + - | **Source tampering** — applies to all tool use cases. + | See :need:`gd_req__req_attr_security`, :need:`gd_req__arch_attr_security`, :need:`gd_req__req_linkage`, + | :need:`gd_req__req_traceability`, :need:`gd_req__arch_linkage_security_trace`. + - | An attacker with write access tampers with sources, configuration, or + | extension code to weaken/disable security checks or inject misleading + | content into published output. + - yes + - yes: `PR review `_. + - yes + - no + +Result +------ +The stored TVR values are derived from the owned graph: + +* ``doc_tool.safety_affected`` is ``YES`` because at least one owned + malfunction is safety affected. +* ``doc_tool.tcl`` is ``LOW`` because at least one owned safety-relevant + malfunction has ``detection_sufficient: NO``. + +The final Tool Confidence Level is therefore **LOW**, the worst case across all +use cases. This result is not changed by qualification. Qualification evidence +is handled in :doc:`qualification`, and a qualified or released report remains +``tcl: LOW``. + +S-CORE Docs-as-Code requires qualification +for use in safety-related software development according to ISO 26262. diff --git a/docs/internals/requirements/index.rst b/docs/internals/requirements/index.rst index 01ccf2655..efbcf06e4 100644 --- a/docs/internals/requirements/index.rst +++ b/docs/internals/requirements/index.rst @@ -32,6 +32,10 @@ Pages itself, including test results and testcase metadata. - ``requirement_coverage`` shows per-requirement test and code linkage, using the same metrics as CI quality gates. +- ``classification`` describes the owned tool use cases, potential + malfunctions and the derived safety/TCL evaluation. +- ``qualification`` describes the qualification evidence and lifecycle rules + that follow from a LOW classification. .. toctree:: :maxdepth: 1 @@ -41,3 +45,5 @@ Pages requirements requirement_coverage tooling_verification + classification + qualification diff --git a/docs/internals/requirements/qualification.rst b/docs/internals/requirements/qualification.rst new file mode 100644 index 000000000..59e557e0e --- /dev/null +++ b/docs/internals/requirements/qualification.rst @@ -0,0 +1,176 @@ +.. + # ******************************************************************************* + # Copyright (c) 2025 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Doc-as-Code Tool Qualification +============================== + +This page is the qualification part of the Tool Verification Report. The +classification of intended usage, potential malfunctions, safety impact and +detection is maintained in :doc:`classification`. The two pages remain +separate because they answer different questions in the SCORE workflow: + +* **Classification:** can intended tool usage fail, and is that failure + sufficiently detected or prevented during intended usage? +* **Qualification:** does the concrete tool version satisfy the + ``tool_req`` needs on which the project relies? + +The complete workflow and authoring guidance are documented in +:doc:`../../how-to/perform_tool_verification`. + +Tool Verification Report +------------------------ + +The report owns its evaluation through ``belongs_to`` links from +``tool_usecase`` needs. The post-template follows that graph and renders the +evaluation and qualification views below automatically. + +.. doc_tool:: Doc-as-Code + :id: doc_tool__doc_as_code + :status: evaluated + :version: 3 + :tool_version: v8.1.2 + :tcl: LOW + :safety_affected: YES + :security_affected: YES + :realizes: wp__tool_verification_report[version==1] + :tags: tool_management, tools_documentation + :post_template: tool_qualification_report + +Qualification basis +------------------- + +The classification page contains the owned use cases and malfunctions. The +qualification scope is derived from this graph: + +.. mermaid:: + + graph LR + DT["doc_tool"] + UC["owned tool_usecase"] + PM["LOW potential_tool_malfunction"] + TR["tool_req requiring qualification"] + TC["testcase"] + + DT -->|"belongs_to backlink"| UC + UC -->|"parent_needs backlink"| PM + PM -->|"violates"| TR + TC -->|"fully_verifies / partially_verifies"| TR + +Qualification is required when a malfunction is safety affected and has +``detection_sufficient: NO``. Such a malfunction must violate at least one +``tool_req``. Stakeholder requirements may be linked as additional context, +but a stakeholder requirement alone cannot be qualified by tool evidence. + +Evidence interpretation +----------------------- + +Qualification reuses the existing SCORE testcase and requirement coverage +model. A relevant ``tool_req`` is completely qualified when at least one +``testcase``: + +* has ``result: passed``; and +* links to the requirement through ``fully_verifies``. + +``partially_verifies`` remains useful traceability evidence, but it does not +complete qualification by itself. The report template renders every relevant +tool requirement, its testcase links and the recorded result. It also keeps +requirements without verification links visible so missing evidence cannot be +mistaken for completed qualification. + +Qualification is not a safety measure. A passed qualification testcase does +not change ``detection_sufficient`` and does not change the derived TCL. + +Requirements and test evidence +------------------------------ + +Tool requirements are defined in the docs-as-code internal requirements +documentation. Each ``tool_req`` represents behaviour on which the project +relies, such as mandatory attribute enforcement, linkage rules, graph checks, +test linkage or report generation. + +Testcase results and metadata are published in +`Tooling Verification `_. +The existing requirement coverage view and ``metrics.json`` remain the source +for verification links and testcase results; qualification does not introduce +a second test type or a parallel qualification-test framework. + +External and self-developed tools use the same qualification flow. External +tools are typically validated as black boxes against the requirements our +project relies on. Self-developed tools should reuse suitable development +requirements and tests whenever those tests provide evidence for the concrete +tool version. + +Lifecycle state +--------------- + +The report status records workflow progress rather than an arbitrary label: + +.. list-table:: Tool Verification Report status + :header-rows: 1 + :widths: 18 62 + + * - Status + - Meaning in this model + * - ``draft`` + - Work in progress; evaluation data may be incomplete. + * - ``evaluated`` + - Owned use cases and structurally valid malfunctions are present, and the + stored safety relevance and TCL match the derived graph values. + * - ``qualified`` + - The report has ``tcl: LOW`` and every qualification-relevant + ``tool_req`` has successful full-verification evidence. + * - ``released`` + - Evaluation is consistent; LOW reports also meet the qualification + evidence rule. HIGH reports do not acquire a qualification requirement + merely because they are released. + * - ``rejected`` + - Review outcome; it does not claim that qualification was completed. + +Qualification does not reclassify the tool. These are valid final states: + +.. code-block:: text + + tcl: LOW + status: qualified + + tcl: LOW + status: released + +The TCL remains LOW because the underlying safety-relevant malfunction still +has insufficient detection. Only a change to the intended usage or its safety +and detection measures can change that classification. + +Generated report views +---------------------- + +The ``tool_qualification_report`` post-template automatically generates: + +* a complete evaluation table with use case, malfunction, violated + requirements, safety impact, safety measures and detection sufficiency; +* the derived safety relevance, tool confidence and qualification-required + summary; and +* for LOW reports, the qualification matrix and testcase evidence for each + relevant ``tool_req``. + +This avoids duplicating the graph manually in a separate qualification table. +The structured Needs remain authoritative; the rendered tables are derived +views. + +Current report state +-------------------- + +The report above is intentionally ``status: evaluated``. Its classification +derives ``tcl: LOW`` and therefore requires qualification evidence before it +can progress to ``qualified`` or ``released``. The generated qualification +view shows the current evidence state for each relevant requirement. diff --git a/src/extensions/score_metamodel/checks/tool_qualification.py b/src/extensions/score_metamodel/checks/tool_qualification.py new file mode 100644 index 000000000..f03307431 --- /dev/null +++ b/src/extensions/score_metamodel/checks/tool_qualification.py @@ -0,0 +1,351 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Validation for the structured SCORE tool-management workflow. + +The metamodel declares the data shape. This module validates the conditional +meaning of that data and derives the TVR safety/TCL values from the owned +use-case and malfunction graph. +""" + +from collections.abc import Iterable +from typing import Any, cast + +from score_metamodel import CheckLogger, graph_check, local_check +from sphinx.application import Sphinx +from sphinx_needs.data import NeedsView +from sphinx_needs.need_item import NeedItem + + +def _link_values(need: NeedItem, link_name: str) -> list[str]: + """Return an outgoing link field as plain IDs.""" + try: + link_values = cast(list[str], need.get_links(link_name, as_str=True)) + return [str(item) for item in link_values] + except KeyError: + # The unit-test NeedItem helper exposes link fields through get(), while + # collected Sphinx-Needs items expose them through get_links. + pass + value = need.get(link_name, []) + if value is None: + return [] + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [str(item) for item in cast(list[Any], value)] + return [str(value)] + + +def _base_id(need_id: str) -> str: + """Remove an optional Sphinx-Needs version filter from a link ID.""" + return need_id.split("[", 1)[0] + + +def _need_index(needs: Iterable[NeedItem]) -> dict[str, NeedItem]: + """Index Needs by both their exact and unqualified IDs.""" + index: dict[str, NeedItem] = {} + for need in needs: + index[need["id"]] = need + index.setdefault(_base_id(need["id"]), need) + return index + + +def _linked_targets( + source: NeedItem, link_name: str, index: dict[str, NeedItem] +) -> list[NeedItem]: + """Resolve outgoing links from a Need, ignoring unknown targets.""" + targets: list[NeedItem] = [] + for target_id in _link_values(source, link_name): + target = index.get(target_id) or index.get(_base_id(target_id)) + if target is not None: + targets.append(target) + return targets + + +def _owned_usecases( + doc_tool: NeedItem, all_needs: Iterable[NeedItem] +) -> list[NeedItem]: + """Return use cases explicitly owned by a Tool Verification Report.""" + return [ + need + for need in all_needs + if need["type"] == "tool_usecase" + and doc_tool["id"] + in {_base_id(link) for link in _link_values(need, "belongs_to")} + ] + + +def _malfunctions_for_usecase( + usecase: NeedItem, all_needs: Iterable[NeedItem] +) -> list[NeedItem]: + """Return malfunctions whose explicit parent is the given use case.""" + usecase_id = _base_id(usecase["id"]) + return [ + need + for need in all_needs + if need["type"] == "potential_tool_malfunction" + and usecase_id + in {_base_id(link) for link in _link_values(need, "parent_needs")} + ] + + +def _malfunctions_for_report( + doc_tool: NeedItem, all_needs: Iterable[NeedItem] +) -> list[NeedItem]: + """Return all malfunctions belonging to a Tool Verification Report.""" + usecases = _owned_usecases(doc_tool, all_needs) + found: list[NeedItem] = [] + found_ids: set[str] = set() + for usecase in usecases: + for malfunction in _malfunctions_for_usecase(usecase, all_needs): + if malfunction["id"] not in found_ids: + found.append(malfunction) + found_ids.add(malfunction["id"]) + return found + + +def _is_present(value: Any) -> bool: + """Treat non-empty strings and non-empty collections as present.""" + if isinstance(value, str): + return bool(value.strip()) + return bool(value) + + +def derive_tvr_values(malfunctions: Iterable[NeedItem]) -> tuple[str, str]: + """Derive TVR safety relevance and TCL from malfunction evaluations.""" + malfunction_list = list(malfunctions) + safety_affected = ( + "YES" + if any(need.get("safety_affected") == "YES" for need in malfunction_list) + else "NO" + ) + tcl = ( + "LOW" + if any( + need.get("safety_affected") == "YES" + and need.get("detection_sufficient") == "NO" + for need in malfunction_list + ) + else "HIGH" + ) + return safety_affected, tcl + + +def _qualification_requirements( + malfunctions: Iterable[NeedItem], index: dict[str, NeedItem] +) -> list[NeedItem]: + """Return unique tool requirements violated by LOW-confidence malfunctions.""" + requirements: list[NeedItem] = [] + requirement_ids: set[str] = set() + for malfunction in malfunctions: + if not ( + malfunction.get("safety_affected") == "YES" + and malfunction.get("detection_sufficient") == "NO" + ): + continue + for requirement in _linked_targets(malfunction, "violates", index): + if ( + requirement["type"] == "tool_req" + and requirement["id"] not in requirement_ids + ): + requirements.append(requirement) + requirement_ids.add(requirement["id"]) + return requirements + + +def _successful_full_testcase( + requirement: NeedItem, all_needs: Iterable[NeedItem] +) -> bool: + """Return whether a passed testcase fully verifies the requirement. + + Existing SCORE coverage distinguishes full and partial verification. A + qualification claim needs complete evidence for the tool requirement, so + partial links alone do not satisfy this workflow check. + """ + requirement_id = _base_id(requirement["id"]) + for testcase in all_needs: + if testcase["type"] != "testcase" or testcase.get("result") != "passed": + continue + fully_verified = { + _base_id(link) for link in _link_values(testcase, "fully_verifies") + } + if requirement_id in fully_verified: + return True + return False + + +def _check_low_malfunction_links( + needs: Iterable[NeedItem], index: dict[str, NeedItem], log: CheckLogger +) -> None: + """Require qualification targets for every LOW-confidence malfunction.""" + for malfunction in ( + need for need in needs if need["type"] == "potential_tool_malfunction" + ): + is_low = ( + malfunction.get("safety_affected") == "YES" + and malfunction.get("detection_sufficient") == "NO" + ) + if not is_low: + continue + + has_tool_requirement = any( + target["type"] == "tool_req" + for target in _linked_targets(malfunction, "violates", index) + ) + if not has_tool_requirement: + log.warning_for_need( + malfunction, + "LOW-confidence malfunctions must violate at least one `tool_req` " + "so that qualification evidence can be produced.", + category="tool-qualification", + ) + + +def _validate_report_values( + doc_tool: NeedItem, + malfunctions: list[NeedItem], + log: CheckLogger, +) -> str: + """Validate the stored TVR summary against its owned malfunctions.""" + expected_safety, expected_tcl = derive_tvr_values(malfunctions) + if doc_tool.get("safety_affected") != expected_safety: + log.warning_for_need( + doc_tool, + f"`safety_affected` is {doc_tool.get('safety_affected')!r}, but " + f"the owned malfunction graph derives {expected_safety!r}.", + category="tool-qualification", + ) + if doc_tool.get("tcl") != expected_tcl: + log.warning_for_need( + doc_tool, + f"`tcl` is {doc_tool.get('tcl')!r}, but the owned malfunction " + f"graph derives {expected_tcl!r}.", + category="tool-qualification", + ) + return expected_tcl + + +def _validate_report_status( + doc_tool: NeedItem, + status: Any, + expected_tcl: str, + malfunctions: list[NeedItem], + index: dict[str, NeedItem], + needs: list[NeedItem], + log: CheckLogger, +) -> None: + """Validate qualification and release requirements for a TVR status.""" + if status not in ("qualified", "released"): + return + if status == "qualified" and expected_tcl != "LOW": + log.warning_for_need( + doc_tool, + f"`status: {status}` is only valid for a TVR with `tcl: LOW`; " + "HIGH-confidence reports do not require qualification.", + category="tool-qualification", + ) + return + if expected_tcl != "LOW": + # A HIGH-confidence report can progress directly from evaluated to + # released. Qualification is only a workflow state for LOW TCL. + return + + requirements = _qualification_requirements(malfunctions, index) + for requirement in requirements: + if _successful_full_testcase(requirement, needs): + continue + log.warning_for_need( + doc_tool, + f"qualification evidence is incomplete: `{requirement['id']}` " + "needs a passed testcase with a `fully_verifies` link.", + category="tool-qualification", + ) + + +@local_check +def check_tool_malfunction_evaluation( + _: Sphinx, need: NeedItem, log: CheckLogger +) -> None: + """Validate conditional malfunction options. + + Detection is only meaningful for a safety-relevant malfunction. For a + safety-relevant malfunction, a positive detection claim also needs the + free-text safety measure that explains what detects or prevents it. + """ + if need["type"] != "potential_tool_malfunction": + return + + safety_affected = need.get("safety_affected") + detection_sufficient = need.get("detection_sufficient") + safety_measures = need.get("safety_measures") + + if safety_affected == "YES" and detection_sufficient not in ("YES", "NO"): + log.warning_for_need( + need, + "safety-relevant malfunctions must define `detection_sufficient` " + "as YES or NO.", + category="tool-qualification", + ) + elif ( + safety_affected == "YES" + and detection_sufficient == "YES" + and not _is_present(safety_measures) + ): + log.warning_for_need( + need, + "`detection_sufficient: YES` requires a non-empty `safety_measures` value.", + category="tool-qualification", + ) + elif safety_affected == "NO" and _is_present(detection_sufficient): + log.warning_for_need( + need, + "non-safety malfunctions must not define `detection_sufficient`.", + category="tool-qualification", + ) + + +@graph_check +def check_tool_qualification_workflow( + _: Sphinx, all_needs: NeedsView, log: CheckLogger +) -> None: + """Validate ownership, qualification prerequisites, and TVR workflow states.""" + needs = list(all_needs.values()) + index = _need_index(needs) + + _check_low_malfunction_links(needs, index, log) + + for doc_tool in (need for need in needs if need["type"] == "doc_tool"): + status = doc_tool.get("status") + if status in ("draft", "rejected"): + continue + + usecases = _owned_usecases(doc_tool, needs) + if not usecases: + log.warning_for_need( + doc_tool, + "non-draft Tool Verification Reports must own at least one " + "`tool_usecase` through `belongs_to`.", + category="tool-qualification", + ) + continue + + malfunctions = _malfunctions_for_report(doc_tool, needs) + expected_tcl = _validate_report_values(doc_tool, malfunctions, log) + _validate_report_status( + doc_tool, + status, + expected_tcl, + malfunctions, + index, + needs, + log, + ) diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index acbced5cc..e0d9d4f24 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -262,6 +262,42 @@ needs_types: realizes: workproduct parts: 2 + # Tool qualification reports keep use cases top-level for readable rendering. + # Their malfunctions are nested below them and use parent_needs as their + # semantic relationship. + # If either stakeholder requirements or tool requirements reflect this level, + # then tool_usecase is NOT relevant. + tool_usecase: + title: Tool Use Case + description: | + Defines the usage context in which the tool is evaluated. + A tool use case does not introduce additional requirements. + It groups the stakeholder and tool requirements relevant to a + particular way the project relies on the tool. + color: "#E1D5E7" + mandatory_links: + belongs_to: doc_tool + realized_by: tool_req + optional_links: + realizes: stkh_req + parts: 3 + + # A malfunction describes a violation of one or more requirements. Safety + # measures are deliberately kept as free text on the malfunction because + # they are report evidence, not independently traceable model elements. + potential_tool_malfunction: + title: Potential Tool Malfunction + color: "#F8CECC" + mandatory_options: + safety_affected: "^(YES|NO)$" + optional_options: + detection_sufficient: "^(YES|NO)$" + safety_measures: ^.+$ + mandatory_links: + parent_needs: tool_usecase + violates: stkh_req, tool_req + parts: 3 + # Requirements # req-Id: tool_req__docs_req_types stkh_req: @@ -1065,6 +1101,17 @@ needs_extra_links: incoming: realized by outgoing: realizes + realized_by: + incoming: realized by + outgoing: realizes + + # Sphinx-Needs stores nested Need relationships under parent_needs. Use + # report-oriented labels so generated links describe the containment rather + # than exposing the implementation-oriented word "parent". + parent_needs: + incoming: contains + outgoing: contained by + # Decision Record affects: incoming: affected by diff --git a/src/extensions/score_metamodel/tests/rst/architecture/architecture_tests.rst b/src/extensions/score_metamodel/tests/rst/architecture/architecture_tests.rst index 47b5e97f3..f7d79c308 100644 --- a/src/extensions/score_metamodel/tests/rst/architecture/architecture_tests.rst +++ b/src/extensions/score_metamodel/tests/rst/architecture/architecture_tests.rst @@ -15,6 +15,7 @@ .. test_metadata:: Test Architecture Needs :id: test_metadata__architecture + :fully_verifies_list: potential_tool_malfunction__docs_as_code__m4 :partially_verifies_list: tool_req__docs_arch_types :test_type: requirements_based :derivation_technique: requirements_based diff --git a/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst b/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst index 112bb702c..218f31fb6 100644 --- a/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst +++ b/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst @@ -14,6 +14,7 @@ .. test_metadata:: :id: test_metadata__metamodel_graph_checks + :fully_verifies_list: potential_tool_malfunction__docs_as_code__m2 :partially_verifies_list: tool_req__docs_common_attr_safety_link_check :test_type: requirements_based :derivation_technique: requirements_based @@ -78,3 +79,36 @@ :status: valid :derived_from: feat_req__parent0__abcd :expect: unknown outgoing link + + +.. doc_tool:: Nested tool qualification report + :id: doc_tool__nested_qualification + :status: evaluated + :safety_affected: YES + :security_affected: NO + :tcl: LOW + :version: 1 + +.. tool_usecase:: Nested tool use case + :id: tool_usecase__blabla__nested + :belongs_to: doc_tool__nested_qualification + :realized_by: tool_req__test__qualification + :version: 1 + + A use case contains its tool malfunctions as nested needs. + + .. potential_tool_malfunction:: Nested tool malfunction + :id: potential_tool_malfunction__blabla__nested + :safety_affected: YES + :detection_sufficient: NO + :safety_measures: Review the generated result before release. + :violates: tool_req__test__qualification + :version: 1 + + The generated result may contain a silent error. + +.. tool_req:: Test tool requirement + :id: tool_req__test__qualification + :version: 1 + + The test fixture uses this requirement as the target of tool qualification links. diff --git a/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst b/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst new file mode 100644 index 000000000..5f411f8e3 --- /dev/null +++ b/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst @@ -0,0 +1,81 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +.. test_metadata:: + :id: test_metadata__tool_qualification + :test_type: requirements_based + :derivation_technique: requirements_based + + Exercises the conditional local validation of tool malfunctions. + +.. doc_tool:: Draft tool verification report + :id: doc_tool__tool_qualification_checks + :status: draft + :safety_affected: NO + :security_affected: NO + :tcl: HIGH + :version: 1 + +.. tool_req:: Tool qualification check requirement + :id: tool_req__tool_qualification_checks + :version: 1 + +.. tool_usecase:: Tool qualification check use case + :id: tool_usecase__tool_qualification_checks + :belongs_to: doc_tool__tool_qualification_checks + :realized_by: tool_req__tool_qualification_checks + :version: 1 + +.. potential_tool_malfunction:: Missing detection value + :id: potential_tool_malfunction__tool_qualification_missing_detection + :parent_needs: tool_usecase__tool_qualification_checks + :violates: tool_req__tool_qualification_checks + :safety_affected: YES + :version: 1 + :expect: safety-relevant malfunctions must define + +.. potential_tool_malfunction:: Positive detection without measure + :id: potential_tool_malfunction__tool_qualification_missing_measure + :parent_needs: tool_usecase__tool_qualification_checks + :violates: tool_req__tool_qualification_checks + :safety_affected: YES + :detection_sufficient: YES + :version: 1 + :expect: non-empty `safety_measures` + +.. potential_tool_malfunction:: Insufficient detection + :id: potential_tool_malfunction__tool_qualification_insufficient_detection + :parent_needs: tool_usecase__tool_qualification_checks + :violates: tool_req__tool_qualification_checks + :safety_affected: YES + :detection_sufficient: NO + :version: 1 + :expect_not: safety-relevant malfunctions must define + +.. potential_tool_malfunction:: Non-safety malfunction + :id: potential_tool_malfunction__tool_qualification_non_safety + :parent_needs: tool_usecase__tool_qualification_checks + :violates: tool_req__tool_qualification_checks + :safety_affected: NO + :version: 1 + :expect_not: non-safety malfunctions must not define + +.. potential_tool_malfunction:: Meaningless detection value + :id: potential_tool_malfunction__tool_qualification_meaningless_detection + :parent_needs: tool_usecase__tool_qualification_checks + :violates: tool_req__tool_qualification_checks + :safety_affected: NO + :detection_sufficient: YES + :version: 1 + :expect: non-safety malfunctions must not define diff --git a/src/extensions/score_metamodel/tests/test_metamodel_load.py b/src/extensions/score_metamodel/tests/test_metamodel_load.py index 9b621e3ea..a1044e8fc 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel_load.py +++ b/src/extensions/score_metamodel/tests/test_metamodel_load.py @@ -14,6 +14,7 @@ from unittest.mock import mock_open, patch import pytest +from attribute_plugin import add_test_properties # type: ignore[import-untyped] from score_metamodel import ProhibitedWordCheck, load_metamodel_data MODEL_DIR = Path(__file__).absolute().parent / "model" @@ -188,6 +189,11 @@ def test_sphinx_needs_builtin_links_are_accepted(tmp_path: Path): } +@add_test_properties( + fully_verifies=["potential_tool_malfunction__docs_as_code__m1"], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) def test_all_undeclared_links_are_reported_at_once(tmp_path: Path): """Every offending link is listed, so one run shows all the work to do.""" model = _write_model( @@ -205,6 +211,49 @@ def test_all_undeclared_links_are_reported_at_once(tmp_path: Path): assert "ghost_two" in message +@add_test_properties( + fully_verifies=["potential_tool_malfunction__docs_as_code__m1"], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) def test_shipped_metamodel_declares_every_link_it_uses(): """The metamodel shipped with this extension must satisfy the check.""" load_metamodel_data() + + +@add_test_properties( + fully_verifies=["potential_tool_malfunction__docs_as_code__m1"], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) +def test_tool_qualification_types_model_nested_traceability(): + """Tool malfunctions are nested under use cases and carry safety text.""" + result = load_metamodel_data() + types = {need_type["directive"]: need_type for need_type in result.needs_types} + + assert types["tool_usecase"]["mandatory_links_str"] == { + "belongs_to": "doc_tool", + "realized_by": "tool_req", + } + assert types["tool_usecase"]["optional_links_str"] == { + "realizes": "stkh_req", + } + assert types["potential_tool_malfunction"]["mandatory_links_str"] == { + "parent_needs": "tool_usecase", + "violates": "stkh_req, tool_req", + } + malfunction_options = types["potential_tool_malfunction"] + assert malfunction_options["mandatory_options"]["safety_affected"] == "^(YES|NO)$" + assert "detection_sufficient" not in malfunction_options["mandatory_options"] + assert ( + malfunction_options["optional_options"]["detection_sufficient"] == "^(YES|NO)$" + ) + assert malfunction_options["optional_options"]["safety_measures"] == "^.+$" + assert ( + types["potential_tool_malfunction"]["optional_options"]["safety_measures"] + == "^.+$" + ) + assert result.needs_links["parent_needs"] == { + "incoming": "contains", + "outgoing": "contained by", + } diff --git a/src/extensions/score_metamodel/tests/test_tool_qualification.py b/src/extensions/score_metamodel/tests/test_tool_qualification.py new file mode 100644 index 000000000..e336a58ff --- /dev/null +++ b/src/extensions/score_metamodel/tests/test_tool_qualification.py @@ -0,0 +1,333 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Tests for the structured SCORE tool-management workflow checks.""" + +from typing import Any +from unittest.mock import MagicMock + +import pytest +from sphinx_needs.need_item import NeedItem + +from src.extensions.score_metamodel.checks.tool_qualification import ( + check_tool_malfunction_evaluation, + check_tool_qualification_workflow, + derive_tvr_values, +) +from src.extensions.score_metamodel.tests import fake_check_logger, need + + +def _run_local(malfunction: NeedItem, logger: Any = None) -> Any: + logger = logger or fake_check_logger() + check_tool_malfunction_evaluation(MagicMock(), malfunction, logger) + return logger + + +def _graph_needs(*needs: NeedItem) -> MagicMock: + all_needs = MagicMock() + all_needs.values.return_value = list(needs) + return all_needs + + +def _low_report( + *, + status: str = "evaluated", + testcase: NeedItem | None = None, + violated: list[str] | None = None, +) -> MagicMock: + report = need( + id="doc_tool__test_report", + type="doc_tool", + status=status, + safety_affected="YES", + tcl="LOW", + ) + usecase = need( + id="tool_usecase__test_report__context", + type="tool_usecase", + belongs_to=["doc_tool__test_report"], + realized_by=["tool_req__test_report__requirement"], + ) + requirement = need( + id="tool_req__test_report__requirement", + type="tool_req", + ) + malfunction = need( + id="potential_tool_malfunction__test_report__failure", + type="potential_tool_malfunction", + safety_affected="YES", + detection_sufficient="NO", + parent_needs=["tool_usecase__test_report__context"], + violates=violated or ["tool_req__test_report__requirement"], + ) + needs = [report, usecase, requirement, malfunction] + if testcase is not None: + needs.append(testcase) + return _graph_needs(*needs) + + +@pytest.mark.parametrize( + ("safety_affected", "detection_sufficient", "safety_measures", "message"), + [ + ("YES", None, None, "safety-relevant malfunctions"), + ("YES", "YES", "", "non-empty `safety_measures`"), + ], +) +def test_safety_malfunction_requires_conditional_evaluation_data( + safety_affected: str, + detection_sufficient: str | None, + safety_measures: str | None, + message: str, +): + """Safety-relevant malfunctions require detection and a positive measure.""" + logger = _run_local( + need( + id="potential_tool_malfunction__local_invalid", + type="potential_tool_malfunction", + safety_affected=safety_affected, + detection_sufficient=detection_sufficient, + safety_measures=safety_measures, + ) + ) + + logger.assert_warning(message) + + +def test_positive_detection_requires_non_empty_safety_measure(): + """A positive detection claim must explain its detecting/preventing measure.""" + logger = _run_local( + need( + id="potential_tool_malfunction__local_no_measure", + type="potential_tool_malfunction", + safety_affected="YES", + detection_sufficient="YES", + safety_measures=" ", + ) + ) + + logger.assert_warning("non-empty `safety_measures`") + + +def test_positive_detection_with_measure_is_valid(): + """A safety-relevant malfunction may be HIGH when its measure is documented.""" + logger = _run_local( + need( + id="potential_tool_malfunction__local_valid", + type="potential_tool_malfunction", + safety_affected="YES", + detection_sufficient="YES", + safety_measures="Independent review of the generated result.", + ) + ) + + logger.assert_no_warnings() + + +def test_safety_malfunction_with_insufficient_detection_needs_no_measure(): + """LOW evaluation is valid without a measure because it requires qualification.""" + logger = _run_local( + need( + id="potential_tool_malfunction__local_low", + type="potential_tool_malfunction", + safety_affected="YES", + detection_sufficient="NO", + ) + ) + + logger.assert_no_warnings() + + +def test_non_safety_malfunction_does_not_need_detection(): + """Non-safety malfunctions omit detection because it is not meaningful there.""" + logger = _run_local( + need( + id="potential_tool_malfunction__local_non_safety", + type="potential_tool_malfunction", + safety_affected="NO", + ) + ) + + logger.assert_no_warnings() + + +def test_non_safety_malfunction_rejects_meaningless_detection(): + """The local check rejects detection data that has no safety meaning.""" + logger = _run_local( + need( + id="potential_tool_malfunction__local_meaningless_detection", + type="potential_tool_malfunction", + safety_affected="NO", + detection_sufficient="YES", + ) + ) + + logger.assert_warning("must not define `detection_sufficient`") + + +def test_low_malfunction_with_tool_requirement_is_valid(): + """A LOW malfunction is qualification-ready when it violates a tool requirement.""" + logger = fake_check_logger() + check_tool_qualification_workflow(MagicMock(), _low_report(), logger) + + logger.assert_no_warnings() + + +def test_low_malfunction_with_only_stakeholder_requirement_is_invalid(): + """A stakeholder requirement alone cannot receive tool qualification evidence.""" + stakeholder = need(id="stkh_req__test_report__requirement", type="stkh_req") + all_needs = _low_report(violated=["stkh_req__test_report__requirement"]) + all_needs.values.return_value.append(stakeholder) + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_warning("must violate at least one `tool_req`") + + +def test_mismatching_derived_safety_value_is_invalid(): + """The stored TVR safety flag must match its owned malfunction graph.""" + all_needs = _low_report() + all_needs.values.return_value[0]["safety_affected"] = "NO" + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_warning("graph derives 'YES'") + + +def test_mismatching_derived_tcl_is_invalid(): + """The stored TVR TCL must match the LOW malfunction classification.""" + all_needs = _low_report() + all_needs.values.return_value[0]["tcl"] = "HIGH" + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_warning("graph derives 'LOW'") + + +def test_mixed_use_cases_derive_low_from_one_low_malfunction(): + """One LOW malfunction keeps a report LOW even when another use case is HIGH.""" + all_needs = _low_report() + high_usecase = need( + id="tool_usecase__test_report__high_context", + type="tool_usecase", + belongs_to=["doc_tool__test_report"], + ) + high_malfunction = need( + id="potential_tool_malfunction__test_report__high_failure", + type="potential_tool_malfunction", + safety_affected="YES", + detection_sufficient="YES", + safety_measures="Independent review.", + parent_needs=[high_usecase["id"]], + ) + all_needs.values.return_value.extend([high_usecase, high_malfunction]) + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_no_warnings() + + +def test_qualified_low_report_requires_successful_full_evidence(): + """A qualified LOW report needs passed full-verification evidence.""" + all_needs = _low_report(status="qualified") + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_warning("qualification evidence is incomplete") + + +def test_qualified_low_report_with_passed_full_evidence_is_valid(): + """A passed testcase with full verification completes LOW qualification.""" + testcase = need( + id="testcase__test_report__qualification", + type="testcase", + result="passed", + fully_verifies=["tool_req__test_report__requirement"], + ) + all_needs = _low_report(status="qualified", testcase=testcase) + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_no_warnings() + + +def test_partial_evidence_alone_does_not_complete_low_qualification(): + """Partial verification is traceability evidence, not complete qualification.""" + testcase = need( + id="testcase__test_report__partial_qualification", + type="testcase", + result="passed", + partially_verifies=["tool_req__test_report__requirement"], + ) + all_needs = _low_report(status="qualified", testcase=testcase) + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_warning("qualification evidence is incomplete") + + +def test_released_low_report_also_requires_successful_full_evidence(): + """Release cannot bypass qualification for a LOW report.""" + all_needs = _low_report(status="released") + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_warning("qualification evidence is incomplete") + + +def test_high_report_does_not_require_qualification(): + """HIGH reports may be evaluated or released without qualification tests.""" + report = need( + id="doc_tool__high_report", + type="doc_tool", + status="released", + safety_affected="NO", + tcl="HIGH", + ) + usecase = need( + id="tool_usecase__high_report__context", + type="tool_usecase", + belongs_to=[report["id"]], + ) + malfunction = need( + id="potential_tool_malfunction__high_report__failure", + type="potential_tool_malfunction", + safety_affected="NO", + parent_needs=[usecase["id"]], + ) + logger = fake_check_logger() + + check_tool_qualification_workflow( + MagicMock(), _graph_needs(report, usecase, malfunction), logger + ) + + logger.assert_no_warnings() + + +def test_qualification_does_not_reclassify_detection_or_tcl(): + """Successful qualification leaves the malfunction and derived TCL unchanged.""" + malfunction = need( + id="potential_tool_malfunction__qualification_is_not_classification", + type="potential_tool_malfunction", + safety_affected="YES", + detection_sufficient="NO", + ) + + assert derive_tvr_values([malfunction]) == ("YES", "LOW") + assert malfunction["detection_sufficient"] == "NO" diff --git a/src/extensions/score_metrics/tests/test_traceability_metrics.py b/src/extensions/score_metrics/tests/test_traceability_metrics.py index ac34eb628..a283f0dbd 100644 --- a/src/extensions/score_metrics/tests/test_traceability_metrics.py +++ b/src/extensions/score_metrics/tests/test_traceability_metrics.py @@ -153,7 +153,10 @@ def test_safe_percent_zero(value1: int, value2: int, expected: float) -> None: @add_test_properties( - partially_verifies=["tool_req__docs_test_linkage_metrics"], + partially_verifies=[ + "tool_req__docs_test_linkage_metrics", + "potential_tool_malfunction__docs_as_code__m3", + ], test_type="requirements-based", derivation_technique="requirements-analysis", ) @@ -231,7 +234,10 @@ def test_calculate_requirement_metrics_non_fully_linked_2() -> None: @add_test_properties( - partially_verifies=["tool_req__docs_test_linkage_metrics"], + partially_verifies=[ + "tool_req__docs_test_linkage_metrics", + "potential_tool_malfunction__docs_as_code__m3", + ], test_type="interface-test", derivation_technique="design-analysis", ) diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index de7383f62..209834dc6 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -23,6 +23,7 @@ from sphinx.testing.util import SphinxTestApp from sphinx_needs.data import SphinxNeedsData +from score_pytest.attribute_plugin import add_test_properties from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.needlinks import NeedLink from src.extensions.score_source_code_linker.repo_source_links import RepoInfo @@ -514,6 +515,14 @@ def compare_grouped_json_files(file1: Path, golden_file: Path): ) +@add_test_properties( + fully_verifies=[ + "potential_tool_malfunction__docs_as_code__m8", + "potential_tool_malfunction__docs_as_code__m9", + ], + test_type="requirements-based", + derivation_technique="requirements-analysis", +) def test_source_link_integration_ok( sphinx_app_setup: Callable[[], SphinxTestApp], example_source_link_text_all_ok: dict[str, list[NeedLink]], diff --git a/src/extensions/score_source_code_linker/tests/test_testlink.py b/src/extensions/score_source_code_linker/tests/test_testlink.py index 9d3e8ecab..5114d0dde 100644 --- a/src/extensions/score_source_code_linker/tests/test_testlink.py +++ b/src/extensions/score_source_code_linker/tests/test_testlink.py @@ -99,6 +99,10 @@ def test_clean_text_removes_ansi_and_html_unescapes(): @add_test_properties( partially_verifies=["tool_req__docs_test_link_testcase"], + fully_verifies=[ + "potential_tool_malfunction__docs_as_code__m5", + "potential_tool_malfunction__docs_as_code__m6", + ], test_type="requirements-based", derivation_technique="requirements-analysis", ) diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index 86db75245..7aa1a362c 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -277,6 +277,10 @@ def test_parse_properties(): @add_test_properties( partially_verifies=["tool_req__docs_test_link_testcase"], + fully_verifies=[ + "potential_tool_malfunction__docs_as_code__m5", + "potential_tool_malfunction__docs_as_code__m6", + ], test_type="requirements-based", derivation_technique="requirements-analysis", ) diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need new file mode 100644 index 000000000..32be18c41 --- /dev/null +++ b/src/needs_templates/tool_qualification_report.need @@ -0,0 +1,213 @@ +{# + score: render-after-needs-collection + + Reusable post-template for a ``doc_tool`` qualification report. The report + owns its use cases through ``belongs_to``; malfunctions are then resolved + through ``parent_needs``. This keeps the rendered view driven by the same + graph that the workflow checks validate. +#} +{% set usecases = linked_needs(id, "belongs_to_back")|selectattr("type", "equalto", "tool_usecase")|list %} + +{% macro text(value, fallback="—") -%} +{{ (value|default("", true)|replace("\n", " ")|trim) or fallback }} +{%- endmacro %} + +{% macro need_reference(need) -%} +{% if need.get("external_url") %}`{{ need["id"] }} <{{ need["external_url"] }}>`_{% else %}:need:`{{ need["id"] }}`{% endif %} +{%- endmacro %} + +{% macro need_list(needs) -%} +{%- if needs -%} +{%- for need in needs -%} +{{ "| " if loop.first else "\n | " }}{{ need_reference(need) }}{{ "," if not loop.last else "" }} +{%- endfor -%} +{%- else -%}—{%- endif -%} +{%- endmacro %} + +{% macro requirements_for(malfunction, with_testlinks) -%} +{%- set matching = namespace(items=[]) -%} +{%- for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list -%} +{%- set fully_verifying_tests = linked_needs(requirement["id"], "fully_verifies_back")|list -%} +{%- set partially_verifying_tests = linked_needs(requirement["id"], "partially_verifies_back")|list -%} +{%- set has_testlinks = fully_verifying_tests or partially_verifying_tests -%} +{%- if (with_testlinks and has_testlinks) or (not with_testlinks and not has_testlinks) -%} +{%- set matching.items = matching.items + [requirement] -%} +{%- endif -%} +{%- endfor -%} +{{- need_list(matching.items) -}} +{%- endmacro %} + +{% macro tests_for(malfunction, link_name) -%} +{%- set matching = namespace(items=[], ids=[]) -%} +{%- for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list -%} +{%- for testcase in linked_needs(requirement["id"], link_name) -%} +{%- if testcase["id"] not in matching.ids -%} +{%- set matching.items = matching.items + [testcase] -%} +{%- set matching.ids = matching.ids + [testcase["id"]] -%} +{%- endif -%} +{%- endfor -%} +{%- endfor -%} +{{- need_list(matching.items) -}} +{%- endmacro %} + +{% macro qualification_tests(requirement) -%} +{%- set matching = namespace(items=[], ids=[]) -%} +{%- for link_name in ["fully_verifies_back", "partially_verifies_back"] -%} +{%- for testcase in linked_needs(requirement["id"], link_name) -%} +{%- if testcase["id"] not in matching.ids -%} +{%- set matching.items = matching.items + [testcase] -%} +{%- set matching.ids = matching.ids + [testcase["id"]] -%} +{%- endif -%} +{%- endfor -%} +{%- endfor -%} +{%- if matching.items -%} +{%- for testcase in matching.items -%} +{{ "| " if loop.first else "\n | " }}{{ need_reference(testcase) }} ({{ text(testcase.get("result")) }}){{ "," if not loop.last else "" }} +{%- endfor -%} +{%- else -%}—{%- endif -%} +{%- endmacro %} + +{% set summary = namespace(safety=false, low=false) %} +{% for usecase in usecases %} +{% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} +{% if malfunction.get("safety_affected") == "YES" %}{% set summary.safety = true %}{% endif %} +{% if malfunction.get("safety_affected") == "YES" and malfunction.get("detection_sufficient") == "NO" %}{% set summary.low = true %}{% endif %} +{% endfor %} +{% endfor %} + +**Automatically generated overview starts here.** + +Tool evaluation +^^^^^^^^^^^^^^^ + +The following view is derived from the owned ``tool_usecase`` and +``potential_tool_malfunction`` Needs. + +.. list-table:: Tool evaluation + :header-rows: 1 + :widths: 20 24 25 12 25 16 + + * - Use case + - Potential malfunction + - Violates + - Safety affected + - Safety measures + - Detection sufficient +{% for usecase in usecases %} +{% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} + * - {{ usecase["title"] }} ({{ need_reference(usecase) }}) + - {{ malfunction["title"] }} ({{ need_reference(malfunction) }}) + - {{ need_list(linked_needs(malfunction["id"], "violates")) }} + - {{ text(malfunction.get("safety_affected")) }} + - {{ text(malfunction.get("safety_measures")) }} + - {{ text(malfunction.get("detection_sufficient")) }} +{% endfor %} +{% endfor %} +{% if not usecases %} + * - Report data is collected after the document Needs are merged. + - — + - — + - — + - — + - — +{% endif %} + +Derived summary +~~~~~~~~~~~~~~~ + +* Safety affected: **{{ "YES" if summary.safety else "NO" }}** +* Tool confidence: **{{ "LOW" if summary.low else "HIGH" }}** +* Qualification required: **{{ "YES" if summary.low else "NO" }}** + +Tool qualification matrix +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The qualification matrix contains only safety-relevant malfunctions whose +detection is insufficient. Every such malfunction must violate at least one +``tool_req``. + +.. list-table:: Tool qualification matrix + :header-rows: 1 + :widths: 22 18 30 30 + + * - Malfunction + - Use case description + - Tool requirements (with testlinks) + - Tool requirements (without testlinks) +{% for usecase in usecases %} +{% set malfunctions = linked_needs(usecase["id"], "parent_needs_back")|selectattr("safety_affected", "equalto", "YES")|selectattr("detection_sufficient", "equalto", "NO")|list %} +{% for malfunction in malfunctions %} + * - {{ malfunction["title"] }} ({{ need_reference(malfunction) }}) + - {{ text(usecase.get("title")) }} + - {{ requirements_for(malfunction, true) }} + - {{ requirements_for(malfunction, false) }} +{% endfor %} +{% endfor %} +{% if not summary.low %} + * - No qualification-relevant malfunction. + - — + - — + - — +{% endif %} + +Qualification evidence +^^^^^^^^^^^^^^^^^^^^^^ + +Qualification is complete only when every relevant ``tool_req`` has a passed +testcase linked with ``fully_verifies``. A partial verification remains useful +traceability evidence but does not by itself complete qualification. + +.. list-table:: Qualification evidence + :header-rows: 1 + :widths: 38 35 27 + + * - Tool requirement requiring qualification + - Testcase(s) + - Verification status/result +{% for usecase in usecases %} +{% for malfunction in linked_needs(usecase["id"], "parent_needs_back")|selectattr("safety_affected", "equalto", "YES")|selectattr("detection_sufficient", "equalto", "NO")|list %} +{% for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list %} + * - {{ need_reference(requirement) }} + - {{ qualification_tests(requirement) }} + - {{ "passed" if linked_needs(requirement["id"], "fully_verifies_back")|selectattr("result", "equalto", "passed")|list else "not fully qualified" }} +{% endfor %} +{% endfor %} +{% endfor %} +{% if not summary.low %} + * - No qualification evidence required. + - — + - — +{% endif %} + +Traceability evidence +^^^^^^^^^^^^^^^^^^^^^ + +The evidence below keeps safety measures and generated testcase links visible +for the qualification-relevant malfunctions. + +.. list-table:: Malfunction traceability + :header-rows: 1 + :widths: 20 25 25 15 15 + + * - Malfunction + - Violates requirements + - Safety measure + - Fully verified by + - Partially verified by +{% for usecase in usecases %} +{% set malfunctions = linked_needs(usecase["id"], "parent_needs_back")|selectattr("safety_affected", "equalto", "YES")|selectattr("detection_sufficient", "equalto", "NO")|list %} +{% for malfunction in malfunctions %} + * - {{ malfunction["title"] }} ({{ need_reference(malfunction) }}) + - {{ need_list(linked_needs(malfunction["id"], "violates")) }} + - {{ text(malfunction.get("safety_measures")) }} + - {{ tests_for(malfunction, "fully_verifies_back") }} + - {{ tests_for(malfunction, "partially_verifies_back") }} +{% endfor %} +{% endfor %} +{% if not summary.low %} + * - No qualification-relevant malfunction. + - — + - — + - — + - — +{% endif %} diff --git a/src/tests/docs_bzl/scenarios/basic_docs/_expected/needs_json/needs.json b/src/tests/docs_bzl/scenarios/basic_docs/_expected/needs_json/needs.json index f9f99829c..89c5f2937 100644 --- a/src/tests/docs_bzl/scenarios/basic_docs/_expected/needs_json/needs.json +++ b/src/tests/docs_bzl/scenarios/basic_docs/_expected/needs_json/needs.json @@ -329,6 +329,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -910,6 +919,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1036,6 +1063,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/data_bundle_needs/needs.json b/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/data_bundle_needs/needs.json index abd52ca91..5ac00b709 100644 --- a/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/data_bundle_needs/needs.json +++ b/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/data_bundle_needs/needs.json @@ -329,6 +329,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -910,6 +919,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1036,6 +1063,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/isolated_source_bundle_needs/needs.json b/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/isolated_source_bundle_needs/needs.json index 3fa5be430..0b76a0d2b 100644 --- a/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/isolated_source_bundle_needs/needs.json +++ b/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/isolated_source_bundle_needs/needs.json @@ -329,6 +329,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -910,6 +919,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1036,6 +1063,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/needs_json/needs.json b/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/needs_json/needs.json index 847698fe4..499a9e115 100644 --- a/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/needs_json/needs.json +++ b/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/needs_json/needs.json @@ -329,6 +329,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -910,6 +919,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1036,6 +1063,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/_expected/needs_json/needs.json b/src/tests/docs_bzl/scenarios/nested_bundles/_expected/needs_json/needs.json index 745bfab3a..68819f6ef 100644 --- a/src/tests/docs_bzl/scenarios/nested_bundles/_expected/needs_json/needs.json +++ b/src/tests/docs_bzl/scenarios/nested_bundles/_expected/needs_json/needs.json @@ -346,6 +346,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -927,6 +936,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1053,6 +1080,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/reference_integration/_expected/needs_local.json b/src/tests/docs_bzl/scenarios/reference_integration/_expected/needs_local.json index a7e834174..c7262602f 100644 --- a/src/tests/docs_bzl/scenarios/reference_integration/_expected/needs_local.json +++ b/src/tests/docs_bzl/scenarios/reference_integration/_expected/needs_local.json @@ -329,6 +329,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -910,6 +919,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1036,6 +1063,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/reference_integration/legacy_module/_expected/needs_json/needs.json b/src/tests/docs_bzl/scenarios/reference_integration/legacy_module/_expected/needs_json/needs.json index 949930eeb..6b98f239d 100644 --- a/src/tests/docs_bzl/scenarios/reference_integration/legacy_module/_expected/needs_json/needs.json +++ b/src/tests/docs_bzl/scenarios/reference_integration/legacy_module/_expected/needs_json/needs.json @@ -346,6 +346,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -927,6 +936,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1053,6 +1080,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/reference_integration/legacy_module/docs/components/component/_expected/needs_local.json b/src/tests/docs_bzl/scenarios/reference_integration/legacy_module/docs/components/component/_expected/needs_local.json index 4cab24c9b..963678b0f 100644 --- a/src/tests/docs_bzl/scenarios/reference_integration/legacy_module/docs/components/component/_expected/needs_local.json +++ b/src/tests/docs_bzl/scenarios/reference_integration/legacy_module/docs/components/component/_expected/needs_local.json @@ -346,6 +346,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -927,6 +936,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1053,6 +1080,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/reference_integration/modern_module/_expected/needs_json/needs.json b/src/tests/docs_bzl/scenarios/reference_integration/modern_module/_expected/needs_json/needs.json index df6189811..74b870f4b 100644 --- a/src/tests/docs_bzl/scenarios/reference_integration/modern_module/_expected/needs_json/needs.json +++ b/src/tests/docs_bzl/scenarios/reference_integration/modern_module/_expected/needs_json/needs.json @@ -429,6 +429,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -1010,6 +1019,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1136,6 +1163,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/reference_integration/modern_module/docs/components/unlinked_component/_expected/needs_local.json b/src/tests/docs_bzl/scenarios/reference_integration/modern_module/docs/components/unlinked_component/_expected/needs_local.json index 477fa06de..83f3b07e9 100644 --- a/src/tests/docs_bzl/scenarios/reference_integration/modern_module/docs/components/unlinked_component/_expected/needs_local.json +++ b/src/tests/docs_bzl/scenarios/reference_integration/modern_module/docs/components/unlinked_component/_expected/needs_local.json @@ -345,6 +345,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -926,6 +935,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1052,6 +1079,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/reference_integration/score_platform/_expected/needs_json/needs.json b/src/tests/docs_bzl/scenarios/reference_integration/score_platform/_expected/needs_json/needs.json index e62aa5ce3..5ad1b36d0 100644 --- a/src/tests/docs_bzl/scenarios/reference_integration/score_platform/_expected/needs_json/needs.json +++ b/src/tests/docs_bzl/scenarios/reference_integration/score_platform/_expected/needs_json/needs.json @@ -373,6 +373,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -954,6 +963,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1080,6 +1107,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/subdirectory_bundle/consumer/_expected/needs_json/needs.json b/src/tests/docs_bzl/scenarios/subdirectory_bundle/consumer/_expected/needs_json/needs.json index 14b5ad062..3274e6fb1 100644 --- a/src/tests/docs_bzl/scenarios/subdirectory_bundle/consumer/_expected/needs_json/needs.json +++ b/src/tests/docs_bzl/scenarios/subdirectory_bundle/consumer/_expected/needs_json/needs.json @@ -375,6 +375,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -956,6 +965,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1082,6 +1109,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", diff --git a/src/tests/docs_bzl/scenarios/subdirectory_bundle/producer/_expected/needs_json/needs.json b/src/tests/docs_bzl/scenarios/subdirectory_bundle/producer/_expected/needs_json/needs.json index 5c7780452..76ceeb3fe 100644 --- a/src/tests/docs_bzl/scenarios/subdirectory_bundle/producer/_expected/needs_json/needs.json +++ b/src/tests/docs_bzl/scenarios/subdirectory_bundle/producer/_expected/needs_json/needs.json @@ -360,6 +360,15 @@ }, "type": "array" }, + "detection_sufficient": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "docname": { "default": null, "description": "Name of the document where the need is defined (None if external).", @@ -941,6 +950,24 @@ "null" ] }, + "realized_by": { + "default": [], + "description": "Link field", + "field_type": "links", + "items": { + "type": "string" + }, + "type": "array" + }, + "realized_by_back": { + "default": [], + "description": "Backlink field", + "field_type": "backlinks", + "items": { + "type": "string" + }, + "type": "array" + }, "realizes": { "default": [], "description": "Link field", @@ -1067,6 +1094,15 @@ "null" ] }, + "safety_measures": { + "default": "", + "description": "Added by needs_fields config", + "field_type": "extra", + "type": [ + "string", + "null" + ] + }, "safety_relevant": { "default": "", "description": "Added by needs_fields config", From da1cf1185bbbd05a1dbb328ea6c8b3bef2a24397 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 23 Sep 2026 10:53:36 +0200 Subject: [PATCH 02/17] fix: restore docs preview checks --- .../internals/requirements/classification.rst | 6 ++-- docs/internals/requirements/qualification.rst | 2 +- .../checks/tool_qualification.py | 7 ++++ .../score_metamodel/tests/__init__.py | 2 ++ .../tests/test_tool_qualification.py | 33 +++++++++++++++++++ .../tool_qualification_report.need | 4 ++- 6 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/internals/requirements/classification.rst b/docs/internals/requirements/classification.rst index 4a110e8e6..4cd17fa94 100644 --- a/docs/internals/requirements/classification.rst +++ b/docs/internals/requirements/classification.rst @@ -217,7 +217,7 @@ Derived-view .. tool_usecase:: Build/CI behavior :id: tool_usecase__docs_as_code__build_ci - :belongs_to: doc_tool__doc_as_code + :belongs_to: doc_tool__score_docs_as_code :realized_by: tool_req__docs_doc_types, tool_req__docs_common_attr_safety_link_check, @@ -403,7 +403,7 @@ Derived-view .. tool_usecase:: PR Review :id: tool_usecase__docs_as_code__pr_review - :belongs_to: doc_tool__doc_as_code + :belongs_to: doc_tool__score_docs_as_code :realized_by: tool_req__docs_doc_types :version: 1 @@ -414,7 +414,7 @@ Derived-view .. tool_usecase:: Derived-view :id: tool_usecase__docs_as_code__derived_view - :belongs_to: doc_tool__doc_as_code + :belongs_to: doc_tool__score_docs_as_code :realized_by: tool_req__docs_arch_views, tool_req__docs_verification_report_need, diff --git a/docs/internals/requirements/qualification.rst b/docs/internals/requirements/qualification.rst index 59e557e0e..c7d0fa3a6 100644 --- a/docs/internals/requirements/qualification.rst +++ b/docs/internals/requirements/qualification.rst @@ -36,7 +36,7 @@ The report owns its evaluation through ``belongs_to`` links from evaluation and qualification views below automatically. .. doc_tool:: Doc-as-Code - :id: doc_tool__doc_as_code + :id: doc_tool__score_docs_as_code :status: evaluated :version: 3 :tool_version: v8.1.2 diff --git a/src/extensions/score_metamodel/checks/tool_qualification.py b/src/extensions/score_metamodel/checks/tool_qualification.py index f03307431..50f891af9 100644 --- a/src/extensions/score_metamodel/checks/tool_qualification.py +++ b/src/extensions/score_metamodel/checks/tool_qualification.py @@ -330,6 +330,13 @@ def check_tool_qualification_workflow( usecases = _owned_usecases(doc_tool, needs) if not usecases: + # Existing SCORE repositories may still contain legacy doc_tool + # reports whose requirements are modeled directly on the report. + # The structured workflow is opt-in through its post-template; do + # not make those reports fail merely because this extension now + # knows about tool_usecase ownership. + if not doc_tool.get("post_template"): + continue log.warning_for_need( doc_tool, "non-draft Tool Verification Reports must own at least one " diff --git a/src/extensions/score_metamodel/tests/__init__.py b/src/extensions/score_metamodel/tests/__init__.py index 21307ca43..ca5e1ef53 100644 --- a/src/extensions/score_metamodel/tests/__init__.py +++ b/src/extensions/score_metamodel/tests/__init__.py @@ -154,6 +154,7 @@ def need(**kwargs: Any) -> NeedItem: "content", "pre_content", "post_content", + "post_template", } # Extract extras (any remaining kwargs not in core or source/content) @@ -176,6 +177,7 @@ def need(**kwargs: Any) -> NeedItem: content=kwargs.get("content", ""), pre_content=kwargs.get("pre_content"), post_content=kwargs.get("post_content"), + post_template=kwargs.get("post_template"), ) return NeedItem( diff --git a/src/extensions/score_metamodel/tests/test_tool_qualification.py b/src/extensions/score_metamodel/tests/test_tool_qualification.py index e336a58ff..915599327 100644 --- a/src/extensions/score_metamodel/tests/test_tool_qualification.py +++ b/src/extensions/score_metamodel/tests/test_tool_qualification.py @@ -181,6 +181,39 @@ def test_low_malfunction_with_tool_requirement_is_valid(): logger.assert_no_warnings() +def test_legacy_report_without_structured_workflow_is_ignored(): + """Legacy reports without the structured post-template remain compatible.""" + report = need( + id="doc_tool__legacy_report", + type="doc_tool", + status="evaluated", + safety_affected="NO", + tcl="HIGH", + ) + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), _graph_needs(report), logger) + + logger.assert_no_warnings() + + +def test_structured_report_requires_owned_usecase(): + """Structured reports still require explicit tool-use-case ownership.""" + report = need( + id="doc_tool__structured_report", + type="doc_tool", + status="evaluated", + safety_affected="NO", + tcl="HIGH", + post_template="tool_qualification_report", + ) + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), _graph_needs(report), logger) + + logger.assert_warning("must own at least one `tool_usecase`") + + def test_low_malfunction_with_only_stakeholder_requirement_is_invalid(): """A stakeholder requirement alone cannot receive tool qualification evidence.""" stakeholder = need(id="stkh_req__test_report__requirement", type="stkh_req") diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index 32be18c41..8e4f5a549 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -164,16 +164,18 @@ traceability evidence but does not by itself complete qualification. * - Tool requirement requiring qualification - Testcase(s) - Verification status/result +{% set qualification_rows = namespace(items=[]) %} {% for usecase in usecases %} {% for malfunction in linked_needs(usecase["id"], "parent_needs_back")|selectattr("safety_affected", "equalto", "YES")|selectattr("detection_sufficient", "equalto", "NO")|list %} {% for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list %} +{% set qualification_rows.items = qualification_rows.items + [requirement["id"]] %} * - {{ need_reference(requirement) }} - {{ qualification_tests(requirement) }} - {{ "passed" if linked_needs(requirement["id"], "fully_verifies_back")|selectattr("result", "equalto", "passed")|list else "not fully qualified" }} {% endfor %} {% endfor %} {% endfor %} -{% if not summary.low %} +{% if not qualification_rows.items %} * - No qualification evidence required. - — - — From 993c5d01159a1f88e67f310d1ab17780bdff1d8a Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 23 Sep 2026 22:27:15 +0200 Subject: [PATCH 03/17] address review comments --- docs/how-to/perform_tool_verification.md | 74 ++++++++++++++++++------ 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/docs/how-to/perform_tool_verification.md b/docs/how-to/perform_tool_verification.md index 29435aa78..2cde38656 100644 --- a/docs/how-to/perform_tool_verification.md +++ b/docs/how-to/perform_tool_verification.md @@ -23,7 +23,11 @@ and execute that process with the available S-CORE Docs-as-Code model. ```{note} This guide focuses on the safety-related evaluation and qualification aspects -of SCORE Tool Management. Security-related evaluation remains unsolved for now. +of SCORE Tool Management. Security evaluation is outside this guide. Use the +[SCORE Security Analysis Guideline](https://eclipse-score.github.io/process_description/main/process_areas/security_analysis/guidance/security_analysis_guideline.html) +and applicable work products to identify relevant assets, threats, and impacts. +For example, data integrity may matter for a tool use case while data +availability does not; reflect that result in the TVR. ``` --- @@ -32,8 +36,8 @@ of SCORE Tool Management. Security-related evaluation remains unsolved for now. The important distinction is between: -- **Tool Verification Report (`doc_tool`)** — records the SCORE tool-management - state and the overall evaluation result. +- **Tool Verification Report (TVR, `doc_tool`)** — records the SCORE + tool-management state and the overall evaluation result. - **Tool requirements (`tool_req`)** — what the project relies on the tool to do. - **Tool use cases (`tool_usecase`)** — the usage context in which the project relies on those requirements. @@ -133,14 +137,15 @@ The remainder of this guide follows these steps. ### What to model -The SCORE Tool Verification Report is represented by `doc_tool`. +The SCORE Tool Verification Report (TVR) is represented by `doc_tool`. The current metamodel contains the following relevant attributes: - `status`: `draft | evaluated | qualified | released | rejected` - `safety_affected`: `YES | NO` - `security_affected`: `YES | NO` -- `tcl`: `LOW | HIGH` +- `tcl`: `LOW | HIGH` — the SCORE Tool Confidence Level (SCORE TCL) used by + this model - `tool_version`: optional in the current metamodel Example structure: @@ -490,7 +495,7 @@ flowchart TD C -->|"NO"| D["LOW confidence
Qualification required"] ``` -| `safety_affected` | `detection_sufficient` | TVR `tcl` | Qualification | +| `safety_affected` | `detection_sufficient` | TVR SCORE TCL (`tcl`) | Qualification | |---|---|---|---| | `NO` | not relevant | `HIGH` | not required | | `YES` | `YES` | `HIGH` | not required | @@ -512,6 +517,13 @@ but validation reports a mismatch when they do not match this derivation. Qualification evidence does not change either value and does not change a malfunction's `detection_sufficient` result. +The report declares the confidence result through `:tcl: HIGH` or +`:tcl: LOW`. In this model, `HIGH` means no qualification is required and `LOW` +means qualification is required. Qualification can change the report status from +`evaluated` to `qualified` and add linked test evidence, but it does not change +the evaluation result: `tcl` remains `HIGH` or `LOW`, and +`detection_sufficient` remains `YES` or `NO`. + After the evaluation is complete, update the `doc_tool`: ```text @@ -526,7 +538,8 @@ or: :tcl: LOW ``` -The report must clearly state whether qualification is required. +The report must state the SCORE TCL through `:tcl:` and the resulting +qualification need. A `LOW` report shall explain the required evidence. ### Multiple use cases and malfunctions @@ -565,6 +578,10 @@ Before: :detection_sufficient: NO ``` +An **independent check** is a separate opportunity to prevent or detect the +failure; it may be a downstream tool, compiler, consistency check, or review and +does not require a different organisation. + After the independent check becomes a mandatory part of intended usage: ```text @@ -576,6 +593,11 @@ After the independent check becomes a mandatory part of intended usage: This requires **re-evaluating the affected malfunction because the usage concept changed**. +The check must be mandatory for every applicable artifact or execution and gate +acceptance. Verify this through workflow or test-campaign evidence, such as a +CI gate, configured checker, or recorded review, and describe it in +`safety_measures`. An occasional or merely proposed check is insufficient. + It is not a reclassification caused by qualification. --- @@ -663,24 +685,30 @@ The tool shall report an unresolved requirement link as an error. ``` ```` -A requirements-based qualification test can then: +This is an example of a requirements-based qualification test; other campaign +tests need not be requirements-based: 1. create input containing a known unresolved link, -2. execute the exact configured tool version, +2. execute the configured tool version with the relevant configured invocation, 3. verify that the expected error is reported. +The campaign shall record the exact tool version, relevant configuration, +environment and dependencies, invocation, input, expected and actual results, +and outcome. + The resulting SCORE `testcase` should link to the tool requirement via `fully_verifies` or `partially_verifies`, as appropriate. -Vendor documentation, release notes, and upstream tests can support the -argument, but the qualification evidence needs to address the behaviour that -**our project relies on**. +Vendor documentation, release notes, and **upstream tests**—tests maintained +by the vendor or originating project—can support the argument. They remain +supporting information unless accepted under the recorded conditions and traced +to the relevant `tool_req`. --- ### Self-developed tool qualification -For a self-developed tool, do not automatically create a second qualification +For a self-developed tool, do not automatically create a separate qualification test suite. If the normal development process already produces testcases that: @@ -753,10 +781,10 @@ detection mechanism. Review the completed TVR against the SCORE Tool Verification Report Review Checklist. -The review should cover, where applicable: +The review shall describe, where applicable: - unique tool identification, -- exact tool version, +- exact tool version and the relevant test-campaign execution conditions, - purpose and tool use cases, - inputs and outputs, - configuration, @@ -996,7 +1024,8 @@ Typical triggers include: - a newly identified potential malfunction, - a new, changed, or removed safety measure, - a changed surrounding workflow/toolchain, -- evidence that an assumed detection measure is not sufficient. +- evidence that a previously sufficient detection measure is no longer + sufficient. Do **not** re-evaluate merely because qualification tests passed. @@ -1022,7 +1051,9 @@ flowchart TD Do not invent new normative behaviour in `tool_usecase`. It is a **usage-context / grouping element** between stakeholder intent and -concrete tool behaviour. +concrete tool behaviour. For example, "validate requirement traceability during +documentation builds" is a use case; "the tool shall report unresolved links" +is the requirement. The use case adds context, not another requirement level. ```{mermaid} flowchart LR @@ -1061,6 +1092,11 @@ The generated value differs from the configured source value. A test executed during development or release validation is not automatically a measure for `detection_sufficient`. +Development and qualification tests provide evidence about a tool version; +detection sufficiency requires a mechanism that runs during intended use and +prevents or detects the malfunction before the result is relied upon. Therefore, +adding a campaign test does not by itself make detection sufficient. + Ask: > If the malfunction occurs during actual intended tool usage, does this @@ -1097,6 +1133,10 @@ This is not contradictory. Do not copy an external tool's complete specification into `tool_req`. +Do not copy almost the entire external C&Q or qualification specification from +the Internet either; capture only the behaviour and assumptions our project +relies on. + Model the behaviour **our project relies on**. --- From 047925bab1ebde551729e281addc81e2e64264e1 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Thu, 24 Sep 2026 00:46:10 +0200 Subject: [PATCH 04/17] Improve tool verification report --- docs/how-to/perform_tool_verification.md | 258 +++++++++-------- docs/internals/requirements/index.rst | 10 +- docs/internals/requirements/qualification.rst | 176 ------------ docs/internals/requirements/requirements.rst | 16 +- ...assification.rst => tool_verification.rst} | 263 +++--------------- .../checks/tool_qualification.py | 46 +-- src/extensions/score_metamodel/metamodel.yaml | 21 +- .../tests/rst/graph/test_metamodel_graph.rst | 2 - .../rst/graph/test_tool_qualification.rst | 7 +- .../tests/test_metamodel_load.py | 8 +- .../tests/test_tool_qualification.py | 58 ++-- .../tool_qualification_report.need | 129 +++++++-- 12 files changed, 349 insertions(+), 645 deletions(-) delete mode 100644 docs/internals/requirements/qualification.rst rename docs/internals/requirements/{classification.rst => tool_verification.rst} (56%) diff --git a/docs/how-to/perform_tool_verification.md b/docs/how-to/perform_tool_verification.md index 2cde38656..8ea0bf8fa 100644 --- a/docs/how-to/perform_tool_verification.md +++ b/docs/how-to/perform_tool_verification.md @@ -40,7 +40,7 @@ The important distinction is between: tool-management state and the overall evaluation result. - **Tool requirements (`tool_req`)** — what the project relies on the tool to do. - **Tool use cases (`tool_usecase`)** — the usage context in which the project - relies on those requirements. + relies on the tool. - **Potential tool malfunctions (`potential_tool_malfunction`)** — ways in which the expected tool behaviour can fail. - **Safety measures** — measures active during intended tool usage that prevent @@ -58,9 +58,9 @@ flowchart LR TC["testcase
Qualification Evidence"] UC -->|"realizes
(optional)"| STKH - UC -->|"realized_by"| TR + UC -->|"realized_by
(optional)"| TR - PM -->|"violates"| UC + PM -->|"nested under
(parent_needs)"| UC PM -->|"violates"| STKH PM -->|"violates"| TR @@ -139,14 +139,18 @@ The remainder of this guide follows these steps. The SCORE Tool Verification Report (TVR) is represented by `doc_tool`. -The current metamodel contains the following relevant attributes: +The TVR uses the following relevant attributes: - `status`: `draft | evaluated | qualified | released | rejected` -- `safety_affected`: `YES | NO` - `security_affected`: `YES | NO` -- `tcl`: `LOW | HIGH` — the SCORE Tool Confidence Level (SCORE TCL) used by - this model -- `tool_version`: optional in the current metamodel +- `tool_version`: optional + +The generated TVR report additionally shows: + +- `safety_affected`, derived from the owned malfunctions +- SCORE Tool Confidence Level (SCORE TCL), derived from the owned malfunctions + +The report author does not provide these summary values. Example structure: @@ -154,9 +158,7 @@ Example structure: ```{doc_tool} S-CORE Docs-as-Code Tool Verification Report :id: doc_tool__s_core_docs_as_code :status: draft -:safety_affected: YES :security_affected: NO -:tcl: LOW :tool_version: Describe the tool, its intended use, relevant configuration, environment, @@ -165,22 +167,23 @@ constraints, inputs, outputs, and available documentation. ```` ```{note} -`tcl` is mandatory in the current metamodel even while the report is still -`draft`. Update it to the actual evaluation result once Step 5 is complete. +The report author provides the tool identity, status, and security relevance. +The safety summary and SCORE TCL are derived from the owned malfunction graph. ``` ### External tool Identify the exact version and configuration that is actually used. -Do not evaluate the S-CORE Docs-as-Code tool, Clang, or GitHub generically when -the project relies on a specific version, plugin set, configuration, or usage -restriction. +Evaluate the exact tool instance used by the project, including its version, +plugins, configuration, and usage restrictions. ### Self-developed tool -Reference the project's normal version/release identification. The TVR must make -clear which released tool version the evaluation applies to. +Reference the project's normal version, release, or source-revision +identification. For example, a Git commit can identify the evaluated tool when +the relevant source and build context are clear. The TVR must make clear which +specific tool version or revision the evaluation applies to. --- @@ -192,14 +195,14 @@ A `tool_usecase` captures: > **What are we relying on this tool to do in our development process?** -The current metamodel defines: +Represent a `tool_usecase` with the following relationships: ```yaml tool_usecase: mandatory_links: belongs_to: doc_tool - realized_by: tool_req optional_links: + realized_by: tool_req realizes: stkh_req ``` @@ -214,17 +217,20 @@ flowchart LR UC -->|"belongs_to
(mandatory)"| DT UC -->|"realizes
(optional)"| STKH - UC -->|"realized_by
(mandatory)"| TR + UC -->|"realized_by
(optional)"| TR style STKH fill:#DAE8FC style UC fill:#E1D5E7 style TR fill:#F5F5F5 ``` -### When is `tool_usecase` useful? +### How to model `tool_usecase` -Use `tool_usecase` when the stakeholder requirements are too high-level and the -tool requirements are too detailed to clearly express the usage context. +Every `evaluated`, `qualified`, or `released` TVR must own at least one +`tool_usecase`. The use case records the context in which the project relies on +the tool. Link it to a `tool_req` when a matching requirement exists; the +`realized_by` link is optional so the context can be documented before that +requirement has been defined. For example: @@ -234,14 +240,15 @@ For example: - tool requirement: unresolved requirement links must be reported. If an existing requirement already expresses the usage context exactly, avoid -inventing additional behaviour in the use case. The use case can remain a thin -grouping/context element. +inventing additional behaviour in the use case. Keep it as a thin +grouping/context element, whether or not it links to a `tool_req`. ```{important} -With the current metamodel, every `potential_tool_malfunction` has a mandatory -`parent_needs` link to a `tool_usecase`. Therefore, once you model -malfunctions, a `tool_usecase` is structurally required even if it merely -references an already well-scoped requirement. +Write each `potential_tool_malfunction` nested inside its `tool_usecase`. The +nested structure creates the `parent_needs` relationship automatically; do not +add a separate `parent_needs` option. Therefore, once you model malfunctions, +place them under a `tool_usecase` even if that use case merely provides context +for an already well-scoped requirement. ``` ### Tool model example @@ -301,7 +308,7 @@ requirements already describe the behaviour relied upon by the project. For every tool use case, identify relevant ways in which the expected tool behaviour could fail. -The current metamodel defines: +Represent a `potential_tool_malfunction` with the following options and links: ```yaml potential_tool_malfunction: @@ -311,6 +318,7 @@ potential_tool_malfunction: detection_sufficient: "^(YES|NO)$" safety_measures: ^.+$ mandatory_links: + # Established by nesting the malfunction inside its tool use case. parent_needs: tool_usecase violates: stkh_req, tool_req ``` @@ -324,7 +332,7 @@ flowchart LR STKH["stkh_req"] TR["tool_req"] - PM -->|"parent_needs"| UC + PM -->|"nested under
(parent_needs)"| UC PM -->|"violates"| STKH PM -->|"violates"| TR @@ -374,10 +382,17 @@ could eventually contribute to an incorrect engineering decision. ### Tool model example -````markdown -```{potential_tool_malfunction} Unresolved requirement link is accepted as valid +`````markdown +::::{tool_usecase} Validate requirement traceability during documentation builds +:id: tool_usecase__docs_as_code__traceability +:belongs_to: doc_tool__s_core_docs_as_code +:realized_by: tool_req__docs_as_code__unresolved_links +:realizes: stkh_req__docs_as_code__traceability + +The project relies on the documentation tool to detect invalid traceability. + +:::{potential_tool_malfunction} Unresolved requirement link is accepted as valid :id: potential_tool_malfunction__docs_as_code__unresolved_link_accepted -:parent_needs: tool_usecase__docs_as_code__traceability :violates: tool_req__docs_as_code__unresolved_links, stkh_req__docs_as_code__traceability @@ -386,8 +401,9 @@ could eventually contribute to an incorrect engineering decision. An unresolved requirement link is accepted as valid and the documentation build does not report the problem. -``` -```` +::: +:::: +````` There is no conceptual difference between external and self-developed tools in this step. Malfunctions are derived from the intended usage and the requirements @@ -459,10 +475,13 @@ Examples include: Document the relevant measure using `safety_measures`. -````markdown -```{potential_tool_malfunction} Generated source contains invalid syntax +`````markdown +::::{tool_usecase} Generate production source from the approved model +:id: tool_usecase__generator__generate_source +:belongs_to: doc_tool__s_core_docs_as_code + +:::{potential_tool_malfunction} Generated source contains invalid syntax :id: potential_tool_malfunction__generator__invalid_syntax -:parent_needs: tool_usecase__generator__generate_source :violates: tool_req__generator__valid_source :safety_affected: YES :detection_sufficient: YES @@ -470,8 +489,9 @@ Document the relevant measure using `safety_measures`. The generator may emit syntactically invalid source code. The mandatory downstream compilation detects this malfunction before the output can be used. -``` -```` +::: +:::: +````` The important point is that the measure is part of the **defined intended usage**. @@ -482,7 +502,7 @@ the intended usage, evaluate the malfunction with that usage concept. --- -## Step 5 — Determine confidence and qualification need +## Step 5 — Determine classification and qualification need For the SCORE safety evaluation, the simplified decision is: @@ -495,51 +515,50 @@ flowchart TD C -->|"NO"| D["LOW confidence
Qualification required"] ``` -| `safety_affected` | `detection_sufficient` | TVR SCORE TCL (`tcl`) | Qualification | +| `safety_affected` | `detection_sufficient` | TVR SCORE TCL | Qualification | |---|---|---|---| | `NO` | not relevant | `HIGH` | not required | | `YES` | `YES` | `HIGH` | not required | | `YES` | `NO` | `LOW` | required | -The tooling derives the TVR values from the owned graph rather than treating -them as independent labels: +The workflow automatically calculates the TVR summary from the owned +malfunction graph: -1. Follow `doc_tool` -> `tool_usecase` through the mandatory `belongs_to` link. -2. Follow each use case -> `potential_tool_malfunction` through - `parent_needs`. -3. Set `doc_tool.safety_affected` to `YES` if any owned malfunction is safety - affected; otherwise set it to `NO`. -4. Set `doc_tool.tcl` to `LOW` if any owned malfunction is safety affected and - has `detection_sufficient: NO`; otherwise set it to `HIGH`. +1. It follows `doc_tool` -> `tool_usecase` through the mandatory `belongs_to` + link. +2. It follows each use case -> nested `potential_tool_malfunction` needs + through the generated `parent_needs` relationship. +3. It reports safety affected as `YES` if any owned malfunction is safety + affected; otherwise it reports `NO`. +4. It reports SCORE TCL as `LOW` if any owned malfunction is safety affected + and has `detection_sufficient: NO`; otherwise it reports `HIGH`. -The stored `safety_affected` and `tcl` values remain mandatory SCORE TVR data, -but validation reports a mismatch when they do not match this derivation. -Qualification evidence does not change either value and does not change a -malfunction's `detection_sufficient` result. +The generated report exposes the classification result as `HIGH` or `LOW`. In +this model, `HIGH` means no qualification is required and `LOW` means +qualification is required. A `LOW` result identifies the evidence that must be +provided; it does not mean that qualification has already happened. The +evaluation result and `detection_sufficient` remain unchanged by qualification. -The report declares the confidence result through `:tcl: HIGH` or -`:tcl: LOW`. In this model, `HIGH` means no qualification is required and `LOW` -means qualification is required. Qualification can change the report status from -`evaluated` to `qualified` and add linked test evidence, but it does not change -the evaluation result: `tcl` remains `HIGH` or `LOW`, and -`detection_sufficient` remains `YES` or `NO`. - -After the evaluation is complete, update the `doc_tool`: +After the evaluation is complete, the generated report exposes the derived +summary. No manual `safety_affected` or `tcl` values are required on the +`doc_tool`. ```text :status: evaluated -:tcl: HIGH +SCORE TCL: HIGH +Qualification required: NO ``` or: ```text :status: evaluated -:tcl: LOW +SCORE TCL: LOW +Qualification required: YES ``` -The report must state the SCORE TCL through `:tcl:` and the resulting -qualification need. A `LOW` report shall explain the required evidence. +For a `LOW` result, the generated qualification view identifies the required +tool requirements and their current evidence status. ### Multiple use cases and malfunctions @@ -604,9 +623,10 @@ It is not a reclassification caused by qualification. ## Step 6 — Qualify the tool if required -Qualification is required when the evaluation results in insufficient -confidence and the project does not resolve that through a changed usage -concept. +Qualification is an evidence activity required when the evaluation results in +insufficient confidence and the project does not resolve that through a changed +usage concept. It does not happen merely because the qualification view is +generated. SCORE uses validation of the software tool as the qualification approach. @@ -630,8 +650,10 @@ as `testcase` needs. Their normal links to requirements are: - `fully_verifies` - `partially_verifies` -For qualification, the relevant testcases should verify the `tool_req` needs -that define the behaviour relied upon by the tool use cases. +For each qualification-relevant `tool_req`, qualification requires at least one +recorded testcase with `result: passed` and a `fully_verifies` link to that +requirement. A `partially_verifies` link is useful supplemental traceability, +but does not complete qualification by itself. ### Qualification is not a detection measure @@ -696,8 +718,10 @@ The campaign shall record the exact tool version, relevant configuration, environment and dependencies, invocation, input, expected and actual results, and outcome. -The resulting SCORE `testcase` should link to the tool requirement via -`fully_verifies` or `partially_verifies`, as appropriate. +The resulting SCORE `testcase` must link to the tool requirement via +`fully_verifies` for the requirement to count as qualified. A +`partially_verifies` link may be recorded as supplemental evidence, but does not +complete qualification by itself. Vendor documentation, release notes, and **upstream tests**—tests maintained by the vendor or originating project—can support the argument. They remain @@ -715,7 +739,8 @@ If the normal development process already produces testcases that: - verify the relevant `tool_req` needs, - execute against the tool version being qualified, -- provide suitable recorded results, +- provide a recorded `result: passed`, and +- link to each relevant requirement with `fully_verifies`, reuse those testcases as qualification evidence. @@ -739,18 +764,17 @@ insufficient. If no qualification was required, the evaluated report can proceed to review. -If qualification was required and completed successfully: - -```text -:status: qualified -``` +If qualification was required and every relevant `tool_req` has the required +passed `fully_verifies` evidence, set the report status to `qualified`. The evaluation result itself does **not** automatically change. -This is a valid final state before release: +Changing the status does not create qualification evidence; it records that the +required evidence has already been provided. + +This is a valid final state before release; the generated summary remains LOW: ```text -:tcl: LOW :status: qualified ``` @@ -826,8 +850,7 @@ traceability. ### Stakeholder requirement -A real `stkh_req` must provide the mandatory metadata defined by the SCORE -metamodel. For example: +A real `stkh_req` must provide the required SCORE metadata. For example: ````markdown ```{stkh_req} Provide valid verification traceability @@ -854,10 +877,10 @@ The tool shall report unresolved requirement links as errors. ``` ```` -### Tool use case +### Tool use case and potential malfunction -````markdown -```{tool_usecase} Validate requirement traceability during documentation builds +`````markdown +::::{tool_usecase} Validate requirement traceability during documentation builds :id: tool_usecase__docs_as_code__traceability :belongs_to: doc_tool__s_core_docs_as_code :realized_by: tool_req__docs_as_code__unresolved_links @@ -865,15 +888,9 @@ The tool shall report unresolved requirement links as errors. The project relies on the documentation tool to detect invalid traceability before generated documentation is accepted. -``` -```` - -### Potential malfunction -````markdown -```{potential_tool_malfunction} Unresolved requirement link is accepted as valid +:::{potential_tool_malfunction} Unresolved requirement link is accepted as valid :id: potential_tool_malfunction__docs_as_code__unresolved_link_accepted -:parent_needs: tool_usecase__docs_as_code__traceability :violates: tool_req__docs_as_code__unresolved_links, stkh_req__docs_as_code__traceability @@ -881,8 +898,9 @@ before generated documentation is accepted. :detection_sufficient: NO An unresolved requirement link is accepted as valid and no error is reported. -``` -```` +::: +:::: +````` The resulting model is: @@ -894,8 +912,8 @@ flowchart LR PM["potential_tool_malfunction__docs_as_code__
unresolved_link_accepted"] UC -->|"realizes"| STKH - UC -->|"realized_by"| TR - PM -->|"parent_needs"| UC + UC -->|"realized_by (optional)"| TR + PM -->|"nested under
(parent_needs)"| UC PM -->|"violates"| STKH PM -->|"violates"| TR @@ -908,7 +926,7 @@ flowchart LR Because the malfunction is safety relevant and detection is insufficient: ```text -tcl = LOW +SCORE TCL = LOW qualification required = YES ``` @@ -924,9 +942,9 @@ progress: ```{mermaid} flowchart LR - A["evaluated
tcl = LOW"] --> B["qualification tests pass"] - B --> C["qualified
tcl = LOW"] - C --> D["released
tcl = LOW"] + A["evaluated
SCORE TCL = LOW"] --> B["qualification tests pass"] + B --> C["qualified
SCORE TCL = LOW"] + C --> D["released
SCORE TCL = LOW"] ``` --- @@ -961,10 +979,10 @@ code. ``` ```` -### Tool use case +### Tool use case and potential malfunction -````markdown -```{tool_usecase} Generate production source from the approved model +`````markdown +::::{tool_usecase} Generate production source from the approved model :id: tool_usecase__generator__generate_source :belongs_to: doc_tool__s_core_docs_as_code :realized_by: tool_req__generator__state_transitions @@ -972,15 +990,9 @@ code. The project relies on the generator to transform the approved model into production source code. -``` -```` - -### Potential malfunction -````markdown -```{potential_tool_malfunction} State transition is omitted from generated source +:::{potential_tool_malfunction} State transition is omitted from generated source :id: potential_tool_malfunction__generator__missing_transition -:parent_needs: tool_usecase__generator__generate_source :violates: tool_req__generator__state_transitions, stkh_req__generator__approved_model @@ -989,13 +1001,14 @@ production source code. A state transition present in the approved input model is omitted from the generated source code. -``` -```` +::: +:::: +````` Result: ```text -tcl = LOW +SCORE TCL = LOW qualification required = YES ``` @@ -1062,7 +1075,7 @@ flowchart LR TR["Concrete tool behaviour"] UC -->|"realizes"| STKH - UC -->|"realized_by"| TR + UC -->|"realized_by (optional)"| TR ``` --- @@ -1117,7 +1130,6 @@ Qualification may leave the evaluation unchanged: ```text :safety_affected: YES :detection_sufficient: NO -:tcl: LOW :status: qualified ``` @@ -1155,13 +1167,13 @@ flowchart TD A["1. Identify exact tool version / configuration"] B["2. Create doc_tool
status = draft"] C["3. Define tool_usecase"] - D["4. Link tool_req
and optionally stkh_req"] + D["4. Link available tool_req
and optionally stkh_req"] E["5. Identify potential_tool_malfunction"] - F["6. Set safety_affected"] + F["6. Read generated TVR summary"] G["7. Document safety_measures
and detection_sufficient"] - H["8. Determine tcl"] + H["8. Read generated SCORE TCL"] I["status = evaluated"] - J{"tcl = LOW?"} + J{"SCORE TCL = LOW?"} K["Verify relevant tool_req
with testcase evidence"] L["status = qualified"] M["Review / approve TVR"] @@ -1195,7 +1207,7 @@ testcase -> fully_verifies/partially_verifies -> tool_req -> provides qualification evidence qualified - -> qualification evidence exists; it does not mean tcl became HIGH + -> qualification evidence exists; it does not change the generated SCORE TCL ``` ## SCORE references diff --git a/docs/internals/requirements/index.rst b/docs/internals/requirements/index.rst index efbcf06e4..7db237e7e 100644 --- a/docs/internals/requirements/index.rst +++ b/docs/internals/requirements/index.rst @@ -32,10 +32,9 @@ Pages itself, including test results and testcase metadata. - ``requirement_coverage`` shows per-requirement test and code linkage, using the same metrics as CI quality gates. -- ``classification`` describes the owned tool use cases, potential - malfunctions and the derived safety/TCL evaluation. -- ``qualification`` describes the qualification evidence and lifecycle rules - that follow from a LOW classification. +- ``tool_verification`` is the authoritative Tool Verification Report, including + the owned use cases, potential malfunctions, qualification evidence and + lifecycle state. .. toctree:: :maxdepth: 1 @@ -45,5 +44,4 @@ Pages requirements requirement_coverage tooling_verification - classification - qualification + tool_verification diff --git a/docs/internals/requirements/qualification.rst b/docs/internals/requirements/qualification.rst deleted file mode 100644 index c7d0fa3a6..000000000 --- a/docs/internals/requirements/qualification.rst +++ /dev/null @@ -1,176 +0,0 @@ -.. - # ******************************************************************************* - # Copyright (c) 2025 Contributors to the Eclipse Foundation - # - # See the NOTICE file(s) distributed with this work for additional - # information regarding copyright ownership. - # - # This program and the accompanying materials are made available under the - # terms of the Apache License Version 2.0 which is available at - # https://www.apache.org/licenses/LICENSE-2.0 - # - # SPDX-License-Identifier: Apache-2.0 - # ******************************************************************************* - -Doc-as-Code Tool Qualification -============================== - -This page is the qualification part of the Tool Verification Report. The -classification of intended usage, potential malfunctions, safety impact and -detection is maintained in :doc:`classification`. The two pages remain -separate because they answer different questions in the SCORE workflow: - -* **Classification:** can intended tool usage fail, and is that failure - sufficiently detected or prevented during intended usage? -* **Qualification:** does the concrete tool version satisfy the - ``tool_req`` needs on which the project relies? - -The complete workflow and authoring guidance are documented in -:doc:`../../how-to/perform_tool_verification`. - -Tool Verification Report ------------------------- - -The report owns its evaluation through ``belongs_to`` links from -``tool_usecase`` needs. The post-template follows that graph and renders the -evaluation and qualification views below automatically. - -.. doc_tool:: Doc-as-Code - :id: doc_tool__score_docs_as_code - :status: evaluated - :version: 3 - :tool_version: v8.1.2 - :tcl: LOW - :safety_affected: YES - :security_affected: YES - :realizes: wp__tool_verification_report[version==1] - :tags: tool_management, tools_documentation - :post_template: tool_qualification_report - -Qualification basis -------------------- - -The classification page contains the owned use cases and malfunctions. The -qualification scope is derived from this graph: - -.. mermaid:: - - graph LR - DT["doc_tool"] - UC["owned tool_usecase"] - PM["LOW potential_tool_malfunction"] - TR["tool_req requiring qualification"] - TC["testcase"] - - DT -->|"belongs_to backlink"| UC - UC -->|"parent_needs backlink"| PM - PM -->|"violates"| TR - TC -->|"fully_verifies / partially_verifies"| TR - -Qualification is required when a malfunction is safety affected and has -``detection_sufficient: NO``. Such a malfunction must violate at least one -``tool_req``. Stakeholder requirements may be linked as additional context, -but a stakeholder requirement alone cannot be qualified by tool evidence. - -Evidence interpretation ------------------------ - -Qualification reuses the existing SCORE testcase and requirement coverage -model. A relevant ``tool_req`` is completely qualified when at least one -``testcase``: - -* has ``result: passed``; and -* links to the requirement through ``fully_verifies``. - -``partially_verifies`` remains useful traceability evidence, but it does not -complete qualification by itself. The report template renders every relevant -tool requirement, its testcase links and the recorded result. It also keeps -requirements without verification links visible so missing evidence cannot be -mistaken for completed qualification. - -Qualification is not a safety measure. A passed qualification testcase does -not change ``detection_sufficient`` and does not change the derived TCL. - -Requirements and test evidence ------------------------------- - -Tool requirements are defined in the docs-as-code internal requirements -documentation. Each ``tool_req`` represents behaviour on which the project -relies, such as mandatory attribute enforcement, linkage rules, graph checks, -test linkage or report generation. - -Testcase results and metadata are published in -`Tooling Verification `_. -The existing requirement coverage view and ``metrics.json`` remain the source -for verification links and testcase results; qualification does not introduce -a second test type or a parallel qualification-test framework. - -External and self-developed tools use the same qualification flow. External -tools are typically validated as black boxes against the requirements our -project relies on. Self-developed tools should reuse suitable development -requirements and tests whenever those tests provide evidence for the concrete -tool version. - -Lifecycle state ---------------- - -The report status records workflow progress rather than an arbitrary label: - -.. list-table:: Tool Verification Report status - :header-rows: 1 - :widths: 18 62 - - * - Status - - Meaning in this model - * - ``draft`` - - Work in progress; evaluation data may be incomplete. - * - ``evaluated`` - - Owned use cases and structurally valid malfunctions are present, and the - stored safety relevance and TCL match the derived graph values. - * - ``qualified`` - - The report has ``tcl: LOW`` and every qualification-relevant - ``tool_req`` has successful full-verification evidence. - * - ``released`` - - Evaluation is consistent; LOW reports also meet the qualification - evidence rule. HIGH reports do not acquire a qualification requirement - merely because they are released. - * - ``rejected`` - - Review outcome; it does not claim that qualification was completed. - -Qualification does not reclassify the tool. These are valid final states: - -.. code-block:: text - - tcl: LOW - status: qualified - - tcl: LOW - status: released - -The TCL remains LOW because the underlying safety-relevant malfunction still -has insufficient detection. Only a change to the intended usage or its safety -and detection measures can change that classification. - -Generated report views ----------------------- - -The ``tool_qualification_report`` post-template automatically generates: - -* a complete evaluation table with use case, malfunction, violated - requirements, safety impact, safety measures and detection sufficiency; -* the derived safety relevance, tool confidence and qualification-required - summary; and -* for LOW reports, the qualification matrix and testcase evidence for each - relevant ``tool_req``. - -This avoids duplicating the graph manually in a separate qualification table. -The structured Needs remain authoritative; the rendered tables are derived -views. - -Current report state --------------------- - -The report above is intentionally ``status: evaluated``. Its classification -derives ``tcl: LOW`` and therefore requires qualification evidence before it -can progress to ``qualified`` or ``released``. The generated qualification -view shows the current evidence state for each relevant requirement. diff --git a/docs/internals/requirements/requirements.rst b/docs/internals/requirements/requirements.rst index c6a158fcd..d670c0d60 100644 --- a/docs/internals/requirements/requirements.rst +++ b/docs/internals/requirements/requirements.rst @@ -879,11 +879,11 @@ Testing :parent_covered: YES :satisfies: gd_req__tool_attr_safety_affected[version==1], gd_req__tool_check_mandatory[version==1] - Docs-as-Code shall enforce that every Tool Verification Report (`doc_tool`) includes a - ``safety_affected`` attribute with one of the following values: + Docs-as-Code shall derive the Tool Verification Report safety classification + from its owned potential tool malfunctions: - * YES - * NO + * ``YES`` if at least one owned malfunction is safety affected + * ``NO`` otherwise .. tool_req:: Enforce security classification :id: tool_req__docs_tvr_security @@ -936,11 +936,11 @@ Testing :satisfies: gd_req__tool_attr_tcl[version==1] :parent_covered: YES - Docs-as-Code shall enforce that every Tool Verification Report (`doc_tool`) includes a - `tcl` attribute with one of the following values: + Docs-as-Code shall derive the Tool Confidence Level of a Tool Verification + Report from its owned potential tool malfunctions: - * LOW - * HIGH + * ``LOW`` if an owned safety-relevant malfunction has insufficient detection + * ``HIGH`` otherwise ⚙️ Process / Other ################### diff --git a/docs/internals/requirements/classification.rst b/docs/internals/requirements/tool_verification.rst similarity index 56% rename from docs/internals/requirements/classification.rst rename to docs/internals/requirements/tool_verification.rst index 4cd17fa94..91e8e56d4 100644 --- a/docs/internals/requirements/classification.rst +++ b/docs/internals/requirements/tool_verification.rst @@ -12,207 +12,34 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -Doc-as-Code Tool Evaluation and Classification -============================================== +Doc-as-Code Tool Verification Report +==================================== -This page records the evaluation of the docs-as-code tool in the usage -contexts on which the project relies. It is the classification part of the -Tool Verification Report. The resulting qualification evidence and lifecycle -state are described in :doc:`qualification`. +This page is the authoritative Tool Verification Report for the S-CORE Docs-as-Code tool. It records the classification of intended usage, potential malfunctions, safety impact and detection, together with qualification evidence and lifecycle state. -Introduction ------------- - -Scope and purpose -~~~~~~~~~~~~~~~~~ +Tool under evaluation +--------------------- The S-CORE Docs-as-Code tool (Bazel module ``score_docs_as_code``) builds HTML -documentation from RST/Markdown sources — process description, requirements, and -traceability — and validates content against the S-CORE metamodel. - -Inputs and outputs -~~~~~~~~~~~~~~~~~~ - -* **Inputs:** RST/Markdown sources, Sphinx configuration (``conf.py``), the S-CORE - metamodel (``metamodel.yaml``), Bazel build files, source-code links - (``sourcelinks_json``) and test results (``testlinks``). -* **Outputs:** HTML documentation (``_build/``), needs/traceability data - (``needs.json``), coverage/linkage statistics (``metrics.json``). - -.. mermaid:: - - graph LR - src@{ shape: docs, label: "RST/Markdown sources (+ assets)" } - code@{ shape: docs, label: "C++/Rust/Python sources" } - srclinks@{ shape: doc, label: "sourcelinks" } - cfg@{ shape: docs, label: "Config (conf.py, metamodel.yaml, Bazel)" } - tests@{ shape: docs, label: "Test results" } - dac@{ shape: subproc, label: "Doc-as-Code" } - html@{ shape: docs, label: "HTML docs" } - needs@{ shape: doc, label: "needs.json" } - metrics@{ shape: docs, label: "metrics.json" } - - src --> dac - code --> srclinks --> dac - cfg --> dac - tests --> dac - dac --> html - dac --> needs - dac --> metrics - -Available information -~~~~~~~~~~~~~~~~~~~~~ -* Repository: https://github.com/eclipse-score/docs-as-code -* Documentation: https://eclipse-score.github.io/docs-as-code/v8.1.2/ -* Bazel module name: ``score_docs_as_code`` - -Installation and integration ----------------------------- - -Installation -~~~~~~~~~~~~ - -The tool is consumed as a Bazel module. Declare the dependency in -``MODULE.bazel``:: - - bazel_dep(name = "score_docs_as_code", version = "8.1.2") - -and the S-CORE registry in ``.bazelrc``:: - - common --registry=https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/ - common --registry=https://bcr.bazel.build - -Invoke the ``docs()`` macro from the root ``BUILD`` file:: - - load("@score_docs_as_code//:docs.bzl", "docs") - docs( - project = "My Project", - project_url = "https://github.com/eclipse-score/my-project", - source_dir = "docs", - ) - -For local development, ``bazel run //:ide_support`` creates a Python virtual -environment (``.venv_docs``) with all Sphinx extensions pre-installed for IDE -support (Esbonio). The macro's build targets (``//:docs``, ``//:docs_check``, -``//:docs_link_check``, ``//:traceability_gate``, ``//:live_preview``, ``//:ide_support``) -are documented in the `Build commands reference -`_. - -Tool sources live in the ``docs-as-code`` repository under ``src/extensions/`` -(Sphinx extensions) and ``docs.bzl`` (Bazel macros). The default metamodel is -bundled at ``@score_docs_as_code//src/extensions/score_metamodel:metamodel_yaml`` -and may be overridden via the ``metamodel`` parameter. See the -`Bazel macros reference -`_ -for macro parameters and the -`score_metamodel design -`_ -for metamodel definition and validation checks. - -Integration -~~~~~~~~~~~ -The tool is the central documentation hub of the S-CORE Bazel toolchain, used -by all modules to build, check, and publish documentation. - -Cross-module linking supports two modes: - -- **External needs import:** reference another module's ``:needs_json_file`` target - via the ``external_needs`` parameter of ``docs()`` to cross-reference need - IDs across modules (e.g., ``:need:`gd_req__example_id```). -- **Bundle mounting:** mount another module's ``:docs_bundle`` target via the - ``bundles`` parameter; the mounted sources join the consuming build, with - placement controlled by ``mount_at`` (docname prefix) and ``attach_to`` - (toctree anchor). See `How to mount external sources - `_. - -Within a repository, Sphinx combines documentation sources (RST/Markdown), -needs JSON, source-code links (``sourcelinks_json``) and test metadata through -the S-CORE extensions (``score_metamodel``, ``score_metrics``, ``score_mounts``) -to produce HTML, ``needs.json`` and ``metrics.json``. - - -Environment -~~~~~~~~~~~ -- **Operating system:** Linux — the S-CORE DevContainer (canonical, - recommended), WSL2, or native. -- **Build system:** Bazel (``rules_python``, ``sphinxdocs``) fetches all - toolchains and dependencies, including a remote JDK 17 for PlantUML diagrams - when no local Java is present. - -Safety evaluation ------------------ - -Evaluation model -~~~~~~~~~~~~~~~~ - -The evaluation follows the structured SCORE Tool Management model: - -.. mermaid:: - - graph LR - DT["doc_tool\nTool Verification Report"] - UC["tool_usecase\nUsage context"] - PM["potential_tool_malfunction\nIncorrect behaviour in that context"] - STKH["stkh_req"] - TR["tool_req"] - - UC -->|"belongs_to"| DT - PM -->|"parent_needs"| UC - UC -->|"realizes (optional)"| STKH - UC -->|"realized_by"| TR - PM -->|"violates"| STKH - PM -->|"violates"| TR - - style DT fill:#F5F5F5 - style UC fill:#E1D5E7 - style PM fill:#F8CECC - -``tool_usecase`` is a usage-context and grouping element. It does not add a -new requirements abstraction level. ``potential_tool_malfunction`` describes -incorrect tool behaviour in that context. Its body contains the human-readable -argumentation for the classification; the structured options record the -classification inputs: - -* ``safety_affected`` is mandatory. -* ``detection_sufficient`` is required only for safety-relevant malfunctions. -* ``safety_measures`` describes mechanisms active during intended tool usage. - It is required when ``detection_sufficient`` is ``YES``. -* A safety-relevant malfunction with ``detection_sufficient: NO`` violates at - least one ``tool_req`` so it can be qualified. - -Non-safety malfunctions omit ``detection_sufficient``. Qualification tests are -not safety measures and do not change this classification. - -Use cases were derived from the process requirements -and the docs-as-code -`Tool Requirements `_. - -The facts below are shared by use cases and only referenced in each -Malfunctions cell. - -.. _basis-ci: - -Build/CI behavior - Builds run with ``-W``; any warning trips CI. The - safety-relevant danger is the *silent* failure — a missing warning - or a wrong output published undetected. - A loud CI abort is safe: no wrong output enters the baseline. +documentation and traceability data from RST/Markdown sources. It evaluates +those sources with the S-CORE extensions and metamodel, and produces the +generated documentation, ``needs.json`` and ``metrics.json`` used by the +downstream process. -.. _pr_review: - -PR Review - Repository contents are the source of truth - and every change is reviewed by a committer - (:need:`rl__committer`, :need:`doc_concept__wp_inspections`). - Still, for silent wrong outputs the gated CI stays green. +Report record +------------- -.. _basis-ti1: +.. doc_tool:: Doc-as-Code + :id: doc_tool__score_docs_as_code + :status: evaluated + :version: 3 + :tool_version: v8.1.2 + :security_affected: YES + :realizes: wp__tool_verification_report[version==1] + :post_template: tool_qualification_report -Derived-view - The rendered HTML output is a derived view; - the authoritative safety artifacts are mostly the source-controlled work products. - There are two exceptions, the architecture views (see M4) and backlinks (see M8). - Rendering/preview defects affect reviewer convenience, not safety evidence. +Details +------- .. tool_usecase:: Build/CI behavior @@ -274,11 +101,11 @@ Derived-view **Document metamodel enforcement** — enforce document types, mandatory attributes (id, status, security, safety, realizes), etc. See, for example, :need:`gd_req__doc_attr_status`, :need:`gd_req__req_attr_uid`, :need:`gd_req__req_attr_safety`, :need:`gd_req__arch_attr_safety`, :need:`gd_req__req_check_mandatory`. - `Silent false-negative `_, too-permissive + **Silent false-negative**: too-permissive ``metamodel.yaml`` regex accepted with no guard, or a check bug skips a case. Impact on safety: yes. - Impact safety measures available: yes: `PR review `_. + Impact safety measures available: yes: PR review. Impact safety detection sufficient: no: Qualify metamodel enforcement. Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. @@ -303,10 +130,10 @@ Derived-view **Safety-critical linking enforcement**. See :need:`gd_req__req_linkage_safety`. - `Silent false-negative `_: Allow links which cannot be safe derivations. + **Silent false-negative**: allow links which cannot be safe derivations. Impact on safety: yes. - Impact safety measures available: yes: `PR review `_. + Impact safety measures available: yes: PR review. Impact safety detection sufficient: no: Qualify graph checks. Further additional safety measure required: yes (qualification). @@ -326,7 +153,7 @@ Derived-view **Requirements coverage statistics** — count, per requirement type, the requirements carrying a ``testlink``, compute link-coverage percentages. See :need:`gd_req__verification_reporting`. - `Silent wrong-output `_: a coverage statistic computed wrong. + **Silent wrong-output**: a coverage statistic computed wrong. Impact on safety: yes. Impact safety measures available: no. @@ -350,7 +177,7 @@ Derived-view **Test linkage** — for each ``testcase`` need, resolve its ``partially_verifies``/``fully_verifies`` references against the needs set. See :need:`gd_req__req_attr_testlink`, :need:`gd_req__verification_reporting`. - `Silent wrong-output `_: + **Silent wrong-output**: Safety case believes the requirement is tested where it is not. Impact on safety: yes. @@ -370,11 +197,11 @@ Derived-view **Test reference check**. See :need:`gd_req__req_attr_testlink`. - `Silent wrong-output `_: + **Silent wrong-output**: Test references an outdated/missing requirement. Impact on safety: yes. - Impact safety measures available: yes: `PR review `_. + Impact safety measures available: yes: PR review. Impact safety detection sufficient: no: Qualify test reference check. Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. @@ -393,10 +220,10 @@ Derived-view **Listing assumptions of use** — safety manuals use ``needtable`` to communicate safety-critical assumptions of use to users. See :need:`gd_guidl__saf_man`, :need:`wp__platform_safety_manual`. - `Silent wrong-output `_: ``aou_req`` items might be missing or wrong. + **Silent wrong-output**: ``aou_req`` items might be missing or wrong. Impact on safety: yes. - Impact safety measures available: yes: `PR review `_. + Impact safety measures available: yes: PR review. Impact safety detection sufficient: no: Qualify ``needtable``. Further additional safety measure required: yes: qualification. Confidence (automatic calculation): low. @@ -437,10 +264,10 @@ Derived-view **Architecture visualization** — generate architecture diagrams. See :need:`gd_req__arch_viewpoints`. - `Silent wrong-output `_: a diagram misrepresents the architecture. + **Silent wrong-output**: a diagram misrepresents the architecture. Impact on safety: yes. - Impact safety measures available: yes: `PR review `_ includes architecture inspection. + Impact safety measures available: yes: PR review includes architecture inspection. Impact safety detection sufficient: yes. Further additional safety measure required: no. Confidence (automatic calculation): high. @@ -455,7 +282,7 @@ Derived-view **Backlinks** — for bi-directional traceability, generate correct backlinks for links between Needs items. See :need:`doc_concept__general_traceability`. - `Silent wrong-output `_: Generated backlinks are wrong or missing. + **Silent wrong-output**: generated backlinks are wrong or missing. Impact on safety: yes. Impact safety measures available: no. @@ -474,7 +301,8 @@ Derived-view Incomplete, outdated, or mis-rendered HTML. - Impact on safety: no: `Derived-view `_. + Impact on safety: no: rendered-view defects affect reviewer convenience, + not safety evidence. Impact safety measures available: no. Impact safety detection sufficient: not applicable for a non-safety malfunction. Further additional safety measure required: no. @@ -505,23 +333,6 @@ source-controlled inputs and writing generated output. | extension code to weaken/disable security checks or inject misleading | content into published output. - yes - - yes: `PR review `_. + - yes: PR review. - yes - no - -Result ------- -The stored TVR values are derived from the owned graph: - -* ``doc_tool.safety_affected`` is ``YES`` because at least one owned - malfunction is safety affected. -* ``doc_tool.tcl`` is ``LOW`` because at least one owned safety-relevant - malfunction has ``detection_sufficient: NO``. - -The final Tool Confidence Level is therefore **LOW**, the worst case across all -use cases. This result is not changed by qualification. Qualification evidence -is handled in :doc:`qualification`, and a qualified or released report remains -``tcl: LOW``. - -S-CORE Docs-as-Code requires qualification -for use in safety-related software development according to ISO 26262. diff --git a/src/extensions/score_metamodel/checks/tool_qualification.py b/src/extensions/score_metamodel/checks/tool_qualification.py index 50f891af9..06da18837 100644 --- a/src/extensions/score_metamodel/checks/tool_qualification.py +++ b/src/extensions/score_metamodel/checks/tool_qualification.py @@ -210,30 +210,6 @@ def _check_low_malfunction_links( ) -def _validate_report_values( - doc_tool: NeedItem, - malfunctions: list[NeedItem], - log: CheckLogger, -) -> str: - """Validate the stored TVR summary against its owned malfunctions.""" - expected_safety, expected_tcl = derive_tvr_values(malfunctions) - if doc_tool.get("safety_affected") != expected_safety: - log.warning_for_need( - doc_tool, - f"`safety_affected` is {doc_tool.get('safety_affected')!r}, but " - f"the owned malfunction graph derives {expected_safety!r}.", - category="tool-qualification", - ) - if doc_tool.get("tcl") != expected_tcl: - log.warning_for_need( - doc_tool, - f"`tcl` is {doc_tool.get('tcl')!r}, but the owned malfunction " - f"graph derives {expected_tcl!r}.", - category="tool-qualification", - ) - return expected_tcl - - def _validate_report_status( doc_tool: NeedItem, status: Any, @@ -249,7 +225,8 @@ def _validate_report_status( if status == "qualified" and expected_tcl != "LOW": log.warning_for_need( doc_tool, - f"`status: {status}` is only valid for a TVR with `tcl: LOW`; " + f"`status: {status}` is only valid for a TVR with generated " + "SCORE TCL LOW; " "HIGH-confidence reports do not require qualification.", category="tool-qualification", ) @@ -330,23 +307,22 @@ def check_tool_qualification_workflow( usecases = _owned_usecases(doc_tool, needs) if not usecases: - # Existing SCORE repositories may still contain legacy doc_tool - # reports whose requirements are modeled directly on the report. - # The structured workflow is opt-in through its post-template; do - # not make those reports fail merely because this extension now - # knows about tool_usecase ownership. - if not doc_tool.get("post_template"): - continue log.warning_for_need( doc_tool, - "non-draft Tool Verification Reports must own at least one " - "`tool_usecase` through `belongs_to`.", + "evaluated, qualified, and released Tool Verification Reports " + "must own at least one `tool_usecase` through `belongs_to`.", category="tool-qualification", ) continue malfunctions = _malfunctions_for_report(doc_tool, needs) - expected_tcl = _validate_report_values(doc_tool, malfunctions, log) + safety_affected, expected_tcl = derive_tvr_values(malfunctions) + # These summary values are properties of the report's owned + # malfunction graph. Store them on the report so exported Needs and + # report consumers see the same classification that the workflow + # validation uses; authors only provide the malfunction-level inputs. + doc_tool["safety_affected"] = safety_affected + doc_tool["tcl"] = expected_tcl _validate_report_status( doc_tool, status, diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index e0d9d4f24..0d4b1b147 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -245,13 +245,16 @@ needs_types: mandatory_options: # req-Id: tool_req__docs_tvr_status status: ^(draft|evaluated|qualified|released|rejected)$ - # req-Id: tool_req__docs_tvr_safety - safety_affected: "^(YES|NO)$" # req-Id: tool_req__docs_tvr_security security_affected: "^(YES|NO)$" + optional_options: + # These summary fields are rendered from the owned malfunction graph by + # the structured TVR workflow. They are optional so process templates + # can declare their output shape; report authors must not duplicate them. + # req-Id: tool_req__docs_tvr_safety + safety_affected: "^(YES|NO)$" # req-Id: tool_req__docs_tvr_confidence_level tcl: "^(LOW|HIGH)$" - optional_options: author: ^.*$ approver: ^.*$ reviewer: ^.*$ @@ -265,20 +268,22 @@ needs_types: # Tool qualification reports keep use cases top-level for readable rendering. # Their malfunctions are nested below them and use parent_needs as their # semantic relationship. - # If either stakeholder requirements or tool requirements reflect this level, - # then tool_usecase is NOT relevant. + # Evaluated, qualified, and released tool qualification reports require at + # least one tool_usecase as the context anchor for their evaluation. + # Requirement links remain optional so the context can be recorded before all + # requirements exist. tool_usecase: title: Tool Use Case description: | Defines the usage context in which the tool is evaluated. A tool use case does not introduce additional requirements. - It groups the stakeholder and tool requirements relevant to a - particular way the project relies on the tool. + Where applicable, it groups the stakeholder and tool requirements + relevant to a particular way the project relies on the tool. color: "#E1D5E7" mandatory_links: belongs_to: doc_tool - realized_by: tool_req optional_links: + realized_by: tool_req realizes: stkh_req parts: 3 diff --git a/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst b/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst index 218f31fb6..6be988d58 100644 --- a/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst +++ b/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst @@ -84,9 +84,7 @@ .. doc_tool:: Nested tool qualification report :id: doc_tool__nested_qualification :status: evaluated - :safety_affected: YES :security_affected: NO - :tcl: LOW :version: 1 .. tool_usecase:: Nested tool use case diff --git a/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst b/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst index 5f411f8e3..272f5f3b8 100644 --- a/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst +++ b/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst @@ -22,9 +22,7 @@ .. doc_tool:: Draft tool verification report :id: doc_tool__tool_qualification_checks :status: draft - :safety_affected: NO :security_affected: NO - :tcl: HIGH :version: 1 .. tool_req:: Tool qualification check requirement @@ -37,6 +35,11 @@ :realized_by: tool_req__tool_qualification_checks :version: 1 +.. tool_usecase:: Tool usage context without a pre-existing requirement + :id: tool_usecase__tool_qualification_unmapped + :belongs_to: doc_tool__tool_qualification_checks + :version: 1 + .. potential_tool_malfunction:: Missing detection value :id: potential_tool_malfunction__tool_qualification_missing_detection :parent_needs: tool_usecase__tool_qualification_checks diff --git a/src/extensions/score_metamodel/tests/test_metamodel_load.py b/src/extensions/score_metamodel/tests/test_metamodel_load.py index a1044e8fc..c0a85e2ed 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel_load.py +++ b/src/extensions/score_metamodel/tests/test_metamodel_load.py @@ -231,11 +231,17 @@ def test_tool_qualification_types_model_nested_traceability(): result = load_metamodel_data() types = {need_type["directive"]: need_type for need_type in result.needs_types} + report_options = types["doc_tool"] + assert "safety_affected" not in report_options["mandatory_options"] + assert report_options["optional_options"]["safety_affected"] == "^(YES|NO)$" + assert "tcl" not in report_options["mandatory_options"] + assert report_options["optional_options"]["tcl"] == "^(LOW|HIGH)$" + assert types["tool_usecase"]["mandatory_links_str"] == { "belongs_to": "doc_tool", - "realized_by": "tool_req", } assert types["tool_usecase"]["optional_links_str"] == { + "realized_by": "tool_req", "realizes": "stkh_req", } assert types["potential_tool_malfunction"]["mandatory_links_str"] == { diff --git a/src/extensions/score_metamodel/tests/test_tool_qualification.py b/src/extensions/score_metamodel/tests/test_tool_qualification.py index 915599327..e5374d136 100644 --- a/src/extensions/score_metamodel/tests/test_tool_qualification.py +++ b/src/extensions/score_metamodel/tests/test_tool_qualification.py @@ -48,8 +48,8 @@ def _low_report( id="doc_tool__test_report", type="doc_tool", status=status, - safety_affected="YES", - tcl="LOW", + safety_affected="", + tcl="", ) usecase = need( id="tool_usecase__test_report__context", @@ -181,31 +181,27 @@ def test_low_malfunction_with_tool_requirement_is_valid(): logger.assert_no_warnings() -def test_legacy_report_without_structured_workflow_is_ignored(): - """Legacy reports without the structured post-template remain compatible.""" - report = need( - id="doc_tool__legacy_report", - type="doc_tool", - status="evaluated", - safety_affected="NO", - tcl="HIGH", - ) +def test_report_summary_is_derived_when_summary_fields_are_omitted(): + """A TVR stores summary values derived from its malfunction graph.""" + all_needs = _low_report() logger = fake_check_logger() - check_tool_qualification_workflow(MagicMock(), _graph_needs(report), logger) + check_tool_qualification_workflow(MagicMock(), all_needs, logger) logger.assert_no_warnings() + report = all_needs.values.return_value[0] + assert report["safety_affected"] == "YES" + assert report["tcl"] == "LOW" -def test_structured_report_requires_owned_usecase(): - """Structured reports still require explicit tool-use-case ownership.""" +def test_report_requires_owned_usecase(): + """A non-draft report requires explicit tool-use-case ownership.""" report = need( - id="doc_tool__structured_report", + id="doc_tool__report_without_usecase", type="doc_tool", status="evaluated", - safety_affected="NO", - tcl="HIGH", - post_template="tool_qualification_report", + safety_affected="", + tcl="", ) logger = fake_check_logger() @@ -226,28 +222,6 @@ def test_low_malfunction_with_only_stakeholder_requirement_is_invalid(): logger.assert_warning("must violate at least one `tool_req`") -def test_mismatching_derived_safety_value_is_invalid(): - """The stored TVR safety flag must match its owned malfunction graph.""" - all_needs = _low_report() - all_needs.values.return_value[0]["safety_affected"] = "NO" - logger = fake_check_logger() - - check_tool_qualification_workflow(MagicMock(), all_needs, logger) - - logger.assert_warning("graph derives 'YES'") - - -def test_mismatching_derived_tcl_is_invalid(): - """The stored TVR TCL must match the LOW malfunction classification.""" - all_needs = _low_report() - all_needs.values.return_value[0]["tcl"] = "HIGH" - logger = fake_check_logger() - - check_tool_qualification_workflow(MagicMock(), all_needs, logger) - - logger.assert_warning("graph derives 'LOW'") - - def test_mixed_use_cases_derive_low_from_one_low_malfunction(): """One LOW malfunction keeps a report LOW even when another use case is HIGH.""" all_needs = _low_report() @@ -330,8 +304,8 @@ def test_high_report_does_not_require_qualification(): id="doc_tool__high_report", type="doc_tool", status="released", - safety_affected="NO", - tcl="HIGH", + safety_affected="", + tcl="", ) usecase = need( id="tool_usecase__high_report__context", diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index 8e4f5a549..3ae04fb58 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -1,17 +1,38 @@ {# score: render-after-needs-collection - Reusable post-template for a ``doc_tool`` qualification report. The report - owns its use cases through ``belongs_to``; malfunctions are then resolved - through ``parent_needs``. This keeps the rendered view driven by the same - graph that the workflow checks validate. + Reusable post-template for a ``doc_tool`` qualification report. #} {% set usecases = linked_needs(id, "belongs_to_back")|selectattr("type", "equalto", "tool_usecase")|list %} +{% set malfunction_scope = namespace(ids=[]) %} +{% set qualification_requirement_scope = namespace(ids=[]) %} +{% for usecase in usecases %} +{% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} +{% if malfunction["id"] not in malfunction_scope.ids %} +{% set malfunction_scope.ids = malfunction_scope.ids + [malfunction["id"]] %} +{% endif %} +{% if malfunction.get("safety_affected") == "YES" and malfunction.get("detection_sufficient") == "NO" %} +{% for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list %} +{% if requirement["id"] not in qualification_requirement_scope.ids %} +{% set qualification_requirement_scope.ids = qualification_requirement_scope.ids + [requirement["id"]] %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} +{% endfor %} {% macro text(value, fallback="—") -%} {{ (value|default("", true)|replace("\n", " ")|trim) or fallback }} {%- endmacro %} +{% macro plantuml_text(value, fallback="—") -%} +{{ ((value|default("", true)|replace("\n", " ")|trim) or fallback)|replace('"', "'") }} +{%- endmacro %} + +{% macro id_filter_list(ids) -%} +[{% for need_id in ids %}"{{ need_id }}"{% if not loop.last %}, {% endif %}{% endfor %}] +{%- endmacro %} + {% macro need_reference(need) -%} {% if need.get("external_url") %}`{{ need["id"] }} <{{ need["external_url"] }}>`_{% else %}:need:`{{ need["id"] }}`{% endif %} {%- endmacro %} @@ -75,15 +96,62 @@ {% endfor %} {% endfor %} -**Automatically generated overview starts here.** +Conclusion +^^^^^^^^^^ + +*Generated from linked report records.* + +The following values are generated on the ``doc_tool`` from its owned +``potential_tool_malfunction`` records: + +* ``safety_affected: {{ "YES" if summary.safety else "NO" }}`` means that + {{ "at least one owned malfunction is safety affected" if summary.safety else "no owned malfunction is safety affected" }}. +* ``tcl: {{ "LOW" if summary.low else "HIGH" }}`` means that + {{ "at least one safety-relevant malfunction has insufficient detection" if summary.low else "no safety-relevant malfunction has insufficient detection" }}. +* Qualification is **{{ "required" if summary.low else "not required" }}** + because the generated ``tcl`` is ``{{ "LOW" if summary.low else "HIGH" }}``. + +Tool verification graph +^^^^^^^^^^^^^^^^^^^^^^^ + +*Generated from linked report records.* + +.. plantuml:: + + @startuml + left to right direction + skinparam componentStyle rectangle + + rectangle "Tool Verification Report" as report [[#{{ id }}]] + rectangle "Tool requirements" as requirements [[#tool-qualification-matrix]] + rectangle "Passed fully-verifying testcases" as testcases [[#qualification-evidence]] + testcases --> requirements : fully verifies +{% for usecase in usecases %} +{% set usecase_alias = "usecase_" ~ loop.index %} + rectangle "{{ plantuml_text(usecase.get("title")) }}" as {{ usecase_alias }} [[#{{ usecase["id"] }}]] + report --> {{ usecase_alias }} +{% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} +{% set malfunction_alias = usecase_alias ~ "_malfunction_" ~ loop.index %} + rectangle "{{ plantuml_text(malfunction.get("title")) }}" as {{ malfunction_alias }} [[#{{ malfunction["id"] }}]] + {{ usecase_alias }} --> {{ malfunction_alias }} +{% if malfunction.get("safety_affected") == "YES" and malfunction.get("detection_sufficient") == "NO" %} + {{ malfunction_alias }} --> requirements : violates +{% endif %} +{% endfor %} +{% endfor %} +{% if not usecases %} + rectangle "No tool use cases" as no_usecases + report --> no_usecases +{% endif %} + + @enduml -Tool evaluation -^^^^^^^^^^^^^^^ +Evaluation overview +^^^^^^^^^^^^^^^^^^^ -The following view is derived from the owned ``tool_usecase`` and -``potential_tool_malfunction`` Needs. +*Generated from linked report records.* -.. list-table:: Tool evaluation +.. list-table:: Evaluation overview :header-rows: 1 :widths: 20 24 25 12 25 16 @@ -104,7 +172,7 @@ The following view is derived from the owned ``tool_usecase`` and {% endfor %} {% endfor %} {% if not usecases %} - * - Report data is collected after the document Needs are merged. + * - No tool use cases. - — - — - — @@ -112,16 +180,41 @@ The following view is derived from the owned ``tool_usecase`` and - — {% endif %} -Derived summary -~~~~~~~~~~~~~~~ +Evaluation statistics +^^^^^^^^^^^^^^^^^^^^^ + +*Generated from linked report records.* + +.. grid:: 1 2 2 2 + :gutter: 3 + + .. grid-item:: + + .. needpie:: Detection sufficiency + :labels: Sufficient, Insufficient, Not assessed + :colors: #37a12d, #f0a500, #ca2828 + :legend: + + type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and detection_sufficient == "YES" + type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and detection_sufficient == "NO" + type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and not detection_sufficient -* Safety affected: **{{ "YES" if summary.safety else "NO" }}** -* Tool confidence: **{{ "LOW" if summary.low else "HIGH" }}** -* Qualification required: **{{ "YES" if summary.low else "NO" }}** + .. grid-item:: + + .. needpie:: Qualification requirement verification + :labels: Fully verified, Partially verified, Not verified + :colors: #37a12d, #f0a500, #ca2828 + :legend: + + type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and fully_verifies_back + type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and partially_verifies_back and not fully_verifies_back + type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and not fully_verifies_back and not partially_verifies_back Tool qualification matrix ^^^^^^^^^^^^^^^^^^^^^^^^^ +*Generated from linked report records.* + The qualification matrix contains only safety-relevant malfunctions whose detection is insufficient. Every such malfunction must violate at least one ``tool_req``. @@ -153,6 +246,8 @@ detection is insufficient. Every such malfunction must violate at least one Qualification evidence ^^^^^^^^^^^^^^^^^^^^^^ +*Generated from linked report records.* + Qualification is complete only when every relevant ``tool_req`` has a passed testcase linked with ``fully_verifies``. A partial verification remains useful traceability evidence but does not by itself complete qualification. @@ -184,6 +279,8 @@ traceability evidence but does not by itself complete qualification. Traceability evidence ^^^^^^^^^^^^^^^^^^^^^ +*Generated from linked report records.* + The evidence below keeps safety measures and generated testcase links visible for the qualification-relevant malfunctions. From 2c40f1e7af7cf14052b228674dec1aa6d39f9994 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Thu, 24 Sep 2026 09:25:45 +0200 Subject: [PATCH 05/17] fix: keep tool verification graph links in report --- .../tool_qualification_report.need | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index 3ae04fb58..3f6090466 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -25,8 +25,8 @@ {{ (value|default("", true)|replace("\n", " ")|trim) or fallback }} {%- endmacro %} -{% macro plantuml_text(value, fallback="—") -%} -{{ ((value|default("", true)|replace("\n", " ")|trim) or fallback)|replace('"', "'") }} +{% macro mermaid_text(value, fallback="—") -%} +{{ ((value|default("", true)|replace("\n", " ")|trim) or fallback)|replace("\\", "\\\\")|replace('"', '\\"') }} {%- endmacro %} {% macro id_filter_list(ids) -%} @@ -116,36 +116,41 @@ Tool verification graph *Generated from linked report records.* -.. plantuml:: - - @startuml - left to right direction - skinparam componentStyle rectangle - - rectangle "Tool Verification Report" as report [[#{{ id }}]] - rectangle "Tool requirements" as requirements [[#tool-qualification-matrix]] - rectangle "Passed fully-verifying testcases" as testcases [[#qualification-evidence]] - testcases --> requirements : fully verifies +.. mermaid:: + + --- + config: + layout: elk + securityLevel: loose + --- + flowchart LR + report["Tool Verification Report"] + requirements["Tool requirements"] + testcases["Passed fully-verifying testcases"] + testcases -->|fully verifies| requirements + click report href "#{{ id }}" + click requirements href "#tool-qualification-matrix" + click testcases href "#qualification-evidence" {% for usecase in usecases %} {% set usecase_alias = "usecase_" ~ loop.index %} - rectangle "{{ plantuml_text(usecase.get("title")) }}" as {{ usecase_alias }} [[#{{ usecase["id"] }}]] + {{ usecase_alias }}["{{ mermaid_text(usecase.get("title")) }}"] report --> {{ usecase_alias }} + click {{ usecase_alias }} href "#{{ usecase["id"] }}" {% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} {% set malfunction_alias = usecase_alias ~ "_malfunction_" ~ loop.index %} - rectangle "{{ plantuml_text(malfunction.get("title")) }}" as {{ malfunction_alias }} [[#{{ malfunction["id"] }}]] + {{ malfunction_alias }}["{{ mermaid_text(malfunction.get("title")) }}"] {{ usecase_alias }} --> {{ malfunction_alias }} + click {{ malfunction_alias }} href "#{{ malfunction["id"] }}" {% if malfunction.get("safety_affected") == "YES" and malfunction.get("detection_sufficient") == "NO" %} - {{ malfunction_alias }} --> requirements : violates + {{ malfunction_alias }} -->|violates| requirements {% endif %} {% endfor %} {% endfor %} {% if not usecases %} - rectangle "No tool use cases" as no_usecases + no_usecases["No tool use cases"] report --> no_usecases {% endif %} - @enduml - Evaluation overview ^^^^^^^^^^^^^^^^^^^ From 96124666a4030d88554222a939f818efa756c9fb Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Thu, 24 Sep 2026 09:29:53 +0200 Subject: [PATCH 06/17] docs: summarize tool qualification results in conclusion --- .../tool_qualification_report.need | 95 ++++++++++++------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index 3f6090466..0716250c0 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -104,12 +104,67 @@ Conclusion The following values are generated on the ``doc_tool`` from its owned ``potential_tool_malfunction`` records: -* ``safety_affected: {{ "YES" if summary.safety else "NO" }}`` means that - {{ "at least one owned malfunction is safety affected" if summary.safety else "no owned malfunction is safety affected" }}. -* ``tcl: {{ "LOW" if summary.low else "HIGH" }}`` means that - {{ "at least one safety-relevant malfunction has insufficient detection" if summary.low else "no safety-relevant malfunction has insufficient detection" }}. +* ``safety_affected: {{ "YES" if summary.safety else "NO" }}`` was determined + because {{ "at least one owned malfunction is safety affected" if summary.safety else "no owned malfunction is safety affected" }}; + it means that {{ "the intended usage contains a safety-relevant malfunction" if summary.safety else "no modeled malfunction affects safety" }}. +* Detection sufficiency is determined per safety-relevant malfunction. + ``detection_sufficient: YES`` was determined because the intended usage + contains a measure that prevents or detects the malfunction before its output + is relied upon; it means that qualification is not needed for that + malfunction. ``NO`` was determined because no such sufficient measure is + established; it means that qualification evidence is needed unless the usage + concept is changed. +* ``tcl: {{ "LOW" if summary.low else "HIGH" }}`` was determined because + {{ "at least one safety-relevant malfunction has insufficient detection" if summary.low else "no safety-relevant malfunction has insufficient detection" }}; + it means that qualification is {{ "required" if summary.low else "not required" }}. * Qualification is **{{ "required" if summary.low else "not required" }}** - because the generated ``tcl`` is ``{{ "LOW" if summary.low else "HIGH" }}``. + because the generated ``tcl`` is ``{{ "LOW" if summary.low else "HIGH" }}``; + this means that {{ "each relevant tool requirement needs passed, fully-verifying testcase evidence before the report can become qualified or released" if summary.low else "no qualification evidence is required by this workflow" }}. + +.. grid:: 1 2 2 2 + :gutter: 3 + + .. grid-item:: + + .. needpie:: Safety relevance of malfunctions + :labels: Safety affected, Not safety affected + :colors: #ca2828, #37a12d + :legend: + + type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" + type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "NO" + + .. grid-item:: + + .. needpie:: Detection sufficiency + :labels: Sufficient, Insufficient, Not assessed + :colors: #37a12d, #f0a500, #ca2828 + :legend: + + type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and detection_sufficient == "YES" + type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and detection_sufficient == "NO" + type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and not detection_sufficient + + .. grid-item:: + + .. needpie:: SCORE TCL + :labels: HIGH, LOW + :colors: #37a12d, #f0a500 + :legend: + + type == "doc_tool" and id == "{{ id }}" and tcl == "HIGH" + type == "doc_tool" and id == "{{ id }}" and tcl == "LOW" + + .. grid-item:: + + .. needpie:: Qualification requirement verification + :labels: Fully verified, Partially verified, Not verified + :colors: #37a12d, #f0a500, #ca2828 + :legend: + + type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and fully_verifies_back + type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and partially_verifies_back and not fully_verifies_back + type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and not fully_verifies_back and not partially_verifies_back Tool verification graph ^^^^^^^^^^^^^^^^^^^^^^^ @@ -185,36 +240,6 @@ Evaluation overview - — {% endif %} -Evaluation statistics -^^^^^^^^^^^^^^^^^^^^^ - -*Generated from linked report records.* - -.. grid:: 1 2 2 2 - :gutter: 3 - - .. grid-item:: - - .. needpie:: Detection sufficiency - :labels: Sufficient, Insufficient, Not assessed - :colors: #37a12d, #f0a500, #ca2828 - :legend: - - type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and detection_sufficient == "YES" - type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and detection_sufficient == "NO" - type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and not detection_sufficient - - .. grid-item:: - - .. needpie:: Qualification requirement verification - :labels: Fully verified, Partially verified, Not verified - :colors: #37a12d, #f0a500, #ca2828 - :legend: - - type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and fully_verifies_back - type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and partially_verifies_back and not fully_verifies_back - type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and not fully_verifies_back and not partially_verifies_back - Tool qualification matrix ^^^^^^^^^^^^^^^^^^^^^^^^^ From bd5cb6a4daaedf68598f2ba016fee87b3b0744db Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Thu, 24 Sep 2026 09:54:53 +0200 Subject: [PATCH 07/17] docs: distinguish tool verification graph node types --- .../tool_qualification_report.need | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index 0716250c0..ab61f4a56 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -171,6 +171,9 @@ Tool verification graph *Generated from linked report records.* +The box labels show the Need type. Colors distinguish the report, tool +requirements, testcases, use cases, and potential malfunctions. + .. mermaid:: --- @@ -179,21 +182,32 @@ Tool verification graph securityLevel: loose --- flowchart LR - report["Tool Verification Report"] - requirements["Tool requirements"] - testcases["Passed fully-verifying testcases"] + classDef report fill:#D5E8D4,stroke:#82B366,color:#000 + classDef requirement fill:#DAE8FC,stroke:#6C8EBF,color:#000 + classDef testcase fill:#FFF2CC,stroke:#D6B656,color:#000 + classDef usecase fill:#E1D5E7,stroke:#9673A6,color:#000 + classDef malfunction fill:#F8CECC,stroke:#B85450,color:#000 + + report["Tool Verification Report (doc_tool)"] + requirements["Tool requirements (tool_req)"] + testcases["Passed fully-verifying testcases (testcase)"] + class report report + class requirements requirement + class testcases testcase testcases -->|fully verifies| requirements click report href "#{{ id }}" click requirements href "#tool-qualification-matrix" click testcases href "#qualification-evidence" {% for usecase in usecases %} {% set usecase_alias = "usecase_" ~ loop.index %} - {{ usecase_alias }}["{{ mermaid_text(usecase.get("title")) }}"] + {{ usecase_alias }}["{{ mermaid_text(usecase.get("title")) }} (tool_usecase)"] + class {{ usecase_alias }} usecase report --> {{ usecase_alias }} click {{ usecase_alias }} href "#{{ usecase["id"] }}" {% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} {% set malfunction_alias = usecase_alias ~ "_malfunction_" ~ loop.index %} - {{ malfunction_alias }}["{{ mermaid_text(malfunction.get("title")) }}"] + {{ malfunction_alias }}["{{ mermaid_text(malfunction.get("title")) }} (potential_tool_malfunction)"] + class {{ malfunction_alias }} malfunction {{ usecase_alias }} --> {{ malfunction_alias }} click {{ malfunction_alias }} href "#{{ malfunction["id"] }}" {% if malfunction.get("safety_affected") == "YES" and malfunction.get("detection_sufficient") == "NO" %} @@ -202,7 +216,8 @@ Tool verification graph {% endfor %} {% endfor %} {% if not usecases %} - no_usecases["No tool use cases"] + no_usecases["No tool use cases (tool_usecase)"] + class no_usecases usecase report --> no_usecases {% endif %} From f8ad937ae848b390c6e13c802414919a278d85e1 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Thu, 24 Sep 2026 09:58:11 +0200 Subject: [PATCH 08/17] docs: make tool qualification conclusion scannable --- .../tool_qualification_report.need | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index ab61f4a56..89b9f8875 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -88,11 +88,21 @@ {%- else -%}—{%- endif -%} {%- endmacro %} -{% set summary = namespace(safety=false, low=false) %} +{% set summary = namespace(total=0, safety=0, detection_yes=0, detection_no=0, detection_unassessed=0, low=false) %} {% for usecase in usecases %} {% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} -{% if malfunction.get("safety_affected") == "YES" %}{% set summary.safety = true %}{% endif %} -{% if malfunction.get("safety_affected") == "YES" and malfunction.get("detection_sufficient") == "NO" %}{% set summary.low = true %}{% endif %} +{% set summary.total = summary.total + 1 %} +{% if malfunction.get("safety_affected") == "YES" %} +{% set summary.safety = summary.safety + 1 %} +{% if malfunction.get("detection_sufficient") == "YES" %} +{% set summary.detection_yes = summary.detection_yes + 1 %} +{% elif malfunction.get("detection_sufficient") == "NO" %} +{% set summary.detection_no = summary.detection_no + 1 %} +{% set summary.low = true %} +{% else %} +{% set summary.detection_unassessed = summary.detection_unassessed + 1 %} +{% endif %} +{% endif %} {% endfor %} {% endfor %} @@ -101,25 +111,23 @@ Conclusion *Generated from linked report records.* -The following values are generated on the ``doc_tool`` from its owned +The generated conclusion is summarized from the owned ``potential_tool_malfunction`` records: -* ``safety_affected: {{ "YES" if summary.safety else "NO" }}`` was determined - because {{ "at least one owned malfunction is safety affected" if summary.safety else "no owned malfunction is safety affected" }}; - it means that {{ "the intended usage contains a safety-relevant malfunction" if summary.safety else "no modeled malfunction affects safety" }}. -* Detection sufficiency is determined per safety-relevant malfunction. - ``detection_sufficient: YES`` was determined because the intended usage - contains a measure that prevents or detects the malfunction before its output - is relied upon; it means that qualification is not needed for that - malfunction. ``NO`` was determined because no such sufficient measure is - established; it means that qualification evidence is needed unless the usage - concept is changed. -* ``tcl: {{ "LOW" if summary.low else "HIGH" }}`` was determined because - {{ "at least one safety-relevant malfunction has insufficient detection" if summary.low else "no safety-relevant malfunction has insufficient detection" }}; - it means that qualification is {{ "required" if summary.low else "not required" }}. -* Qualification is **{{ "required" if summary.low else "not required" }}** - because the generated ``tcl`` is ``{{ "LOW" if summary.low else "HIGH" }}``; - this means that {{ "each relevant tool requirement needs passed, fully-verifying testcase evidence before the report can become qualified or released" if summary.low else "no qualification evidence is required by this workflow" }}. +* **Safety:** ``{{ summary.safety }}/{{ summary.total }}`` potential tool + malfunctions affect safety → ``safety_affected: {{ "YES" if summary.safety else "NO" }}``. +* **Detection:** ``{{ summary.detection_yes }}/{{ summary.safety }}`` safety- + relevant malfunctions have sufficient detection, ``{{ summary.detection_no }}`` + do not, and ``{{ summary.detection_unassessed }}`` are not assessed. +* **TCL:** ``{{ summary.detection_no }}`` safety-relevant malfunctions have + insufficient detection → ``tcl: {{ "LOW" if summary.detection_no else "HIGH" }}``. +* **Qualification:** ``{{ qualification_requirement_scope.ids|length }}`` tool + requirements are in qualification scope → **qualification is + {{ "required" if summary.detection_no else "not required" }}**. + +Here, ``safety_affected: YES`` means that at least one modeled malfunction +affects safety. ``tcl: LOW`` means that the relevant tool requirements need +passed, fully-verifying testcase evidence. .. grid:: 1 2 2 2 :gutter: 3 From 60a8c1f93af4bb32dd3f00ba67d581d1e271e46c Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Thu, 24 Sep 2026 10:20:24 +0200 Subject: [PATCH 09/17] docs: refine tool qualification report summary --- .../tool_qualification_report.need | 80 ++++++++++++------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index 89b9f8875..d2562a499 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -33,6 +33,15 @@ [{% for need_id in ids %}"{{ need_id }}"{% if not loop.last %}, {% endif %}{% endfor %}] {%- endmacro %} +{% set usecase_malfunction_scope = namespace(with_malfunctions=[], without_malfunctions=[]) %} +{% for usecase in usecases %} +{% if linked_needs(usecase["id"], "parent_needs_back")|list %} +{% set usecase_malfunction_scope.with_malfunctions = usecase_malfunction_scope.with_malfunctions + [usecase["id"]] %} +{% else %} +{% set usecase_malfunction_scope.without_malfunctions = usecase_malfunction_scope.without_malfunctions + [usecase["id"]] %} +{% endif %} +{% endfor %} + {% macro need_reference(need) -%} {% if need.get("external_url") %}`{{ need["id"] }} <{{ need["external_url"] }}>`_{% else %}:need:`{{ need["id"] }}`{% endif %} {%- endmacro %} @@ -88,7 +97,7 @@ {%- else -%}—{%- endif -%} {%- endmacro %} -{% set summary = namespace(total=0, safety=0, detection_yes=0, detection_no=0, detection_unassessed=0, low=false) %} +{% set summary = namespace(total=0, safety=0, detection_yes=0, detection_no=0, detection_unassessed=0, low=false, fully_covered=0) %} {% for usecase in usecases %} {% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} {% set summary.total = summary.total + 1 %} @@ -99,6 +108,16 @@ {% elif malfunction.get("detection_sufficient") == "NO" %} {% set summary.detection_no = summary.detection_no + 1 %} {% set summary.low = true %} +{% set coverage = namespace(total=0, passed=0) %} +{% for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list %} +{% set coverage.total = coverage.total + 1 %} +{% if linked_needs(requirement["id"], "fully_verifies_back")|selectattr("result", "equalto", "passed")|list %} +{% set coverage.passed = coverage.passed + 1 %} +{% endif %} +{% endfor %} +{% if coverage.total > 0 and coverage.passed == coverage.total %} +{% set summary.fully_covered = summary.fully_covered + 1 %} +{% endif %} {% else %} {% set summary.detection_unassessed = summary.detection_unassessed + 1 %} {% endif %} @@ -116,27 +135,33 @@ The generated conclusion is summarized from the owned * **Safety:** ``{{ summary.safety }}/{{ summary.total }}`` potential tool malfunctions affect safety → ``safety_affected: {{ "YES" if summary.safety else "NO" }}``. -* **Detection:** ``{{ summary.detection_yes }}/{{ summary.safety }}`` safety- - relevant malfunctions have sufficient detection, ``{{ summary.detection_no }}`` - do not, and ``{{ summary.detection_unassessed }}`` are not assessed. -* **TCL:** ``{{ summary.detection_no }}`` safety-relevant malfunctions have - insufficient detection → ``tcl: {{ "LOW" if summary.detection_no else "HIGH" }}``. -* **Qualification:** ``{{ qualification_requirement_scope.ids|length }}`` tool - requirements are in qualification scope → **qualification is - {{ "required" if summary.detection_no else "not required" }}**. - -Here, ``safety_affected: YES`` means that at least one modeled malfunction -affects safety. ``tcl: LOW`` means that the relevant tool requirements need -passed, fully-verifying testcase evidence. +* **Detection:** ``{{ summary.detection_no }}/{{ summary.safety }}`` safety- + relevant malfunctions have insufficient detection → ``tcl: {{ "LOW" if summary.detection_no else "HIGH" }}``. +* **Qualification:** → qualification of ``{{ qualification_requirement_scope.ids|length }}`` tool + requirements is **{{ "required" if summary.detection_no else "not required" }}**. +* **Testcase coverage:** ``{{ summary.fully_covered }}/{{ summary.detection_no }}`` + qualification-relevant malfunctions are fully covered by passed, + fully-verifying testcases. + .. grid:: 1 2 2 2 :gutter: 3 .. grid-item:: - .. needpie:: Safety relevance of malfunctions + .. needpie:: Use case malfunction coverage + :labels: Has malfunctions, No malfunctions + :colors: #4E79A7, #BDBDBD + :legend: + + type == "tool_usecase" and id in {{ id_filter_list(usecase_malfunction_scope.with_malfunctions) }} + type == "tool_usecase" and id in {{ id_filter_list(usecase_malfunction_scope.without_malfunctions) }} + + .. grid-item:: + + .. needpie:: Safety-affected malfunctions :labels: Safety affected, Not safety affected - :colors: #ca2828, #37a12d + :colors: #4E79A7, #BDBDBD :legend: type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" @@ -144,9 +169,9 @@ passed, fully-verifying testcase evidence. .. grid-item:: - .. needpie:: Detection sufficiency + .. needpie:: Detection status :labels: Sufficient, Insufficient, Not assessed - :colors: #37a12d, #f0a500, #ca2828 + :colors: #4E79A7, #E15759, #BDBDBD :legend: type == "potential_tool_malfunction" and id in {{ id_filter_list(malfunction_scope.ids) }} and safety_affected == "YES" and detection_sufficient == "YES" @@ -155,19 +180,19 @@ passed, fully-verifying testcase evidence. .. grid-item:: - .. needpie:: SCORE TCL - :labels: HIGH, LOW - :colors: #37a12d, #f0a500 + .. needpie:: Qualification scope + :labels: tool requirements in scope, tool requirements out of scope + :colors: #4E79A7, #BDBDBD :legend: - type == "doc_tool" and id == "{{ id }}" and tcl == "HIGH" - type == "doc_tool" and id == "{{ id }}" and tcl == "LOW" + type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} + type == "tool_req" and not (id in {{ id_filter_list(qualification_requirement_scope.ids) }}) .. grid-item:: - .. needpie:: Qualification requirement verification + .. needpie:: Qualification evidence :labels: Fully verified, Partially verified, Not verified - :colors: #37a12d, #f0a500, #ca2828 + :colors: #4E79A7, #F28E2B, #E15759 :legend: type == "tool_req" and id in {{ id_filter_list(qualification_requirement_scope.ids) }} and fully_verifies_back @@ -180,7 +205,7 @@ Tool verification graph *Generated from linked report records.* The box labels show the Need type. Colors distinguish the report, tool -requirements, testcases, use cases, and potential malfunctions. +requirements, use cases, and potential malfunctions. .. mermaid:: @@ -192,20 +217,15 @@ requirements, testcases, use cases, and potential malfunctions. flowchart LR classDef report fill:#D5E8D4,stroke:#82B366,color:#000 classDef requirement fill:#DAE8FC,stroke:#6C8EBF,color:#000 - classDef testcase fill:#FFF2CC,stroke:#D6B656,color:#000 classDef usecase fill:#E1D5E7,stroke:#9673A6,color:#000 classDef malfunction fill:#F8CECC,stroke:#B85450,color:#000 report["Tool Verification Report (doc_tool)"] requirements["Tool requirements (tool_req)"] - testcases["Passed fully-verifying testcases (testcase)"] class report report class requirements requirement - class testcases testcase - testcases -->|fully verifies| requirements click report href "#{{ id }}" click requirements href "#tool-qualification-matrix" - click testcases href "#qualification-evidence" {% for usecase in usecases %} {% set usecase_alias = "usecase_" ~ loop.index %} {{ usecase_alias }}["{{ mermaid_text(usecase.get("title")) }} (tool_usecase)"] From d6d273e5c0704eeec8ff0ef6ec043aea72538353 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 25 Sep 2026 15:52:03 +0200 Subject: [PATCH 10/17] Address tool verification review feedback --- docs/how-to/perform_tool_verification.md | 164 ++++++++++-------- .../requirements/tool_verification.rst | 69 ++++++-- src/extensions/score_metamodel/metamodel.yaml | 14 +- .../tests/rst/graph/test_metamodel_graph.rst | 1 - .../rst/graph/test_tool_qualification.rst | 1 - .../tests/test_metamodel_load.py | 7 +- .../tests/test_tool_qualification.py | 9 +- .../tool_qualification_report.need | 24 +++ 8 files changed, 175 insertions(+), 114 deletions(-) diff --git a/docs/how-to/perform_tool_verification.md b/docs/how-to/perform_tool_verification.md index 8ea0bf8fa..fb97f0a86 100644 --- a/docs/how-to/perform_tool_verification.md +++ b/docs/how-to/perform_tool_verification.md @@ -38,7 +38,8 @@ The important distinction is between: - **Tool Verification Report (TVR, `doc_tool`)** — records the SCORE tool-management state and the overall evaluation result. -- **Tool requirements (`tool_req`)** — what the project relies on the tool to do. +- **Tool requirements (`tool_req`)** — capabilities or behaviours provided by + the tool, independent of whether a particular project use case uses them. - **Tool use cases (`tool_usecase`)** — the usage context in which the project relies on the tool. - **Potential tool malfunctions (`potential_tool_malfunction`)** — ways in which @@ -53,29 +54,25 @@ flowchart LR UC["tool_usecase
Tool Use Case"] PM["potential_tool_malfunction
Potential Tool Malfunction"] - STKH["stkh_req
Stakeholder Requirement"] + UP["upstream requirements
(stkh_req, gd_req, feat_req, comp_req)"] TR["tool_req
Tool Requirement"] TC["testcase
Qualification Evidence"] - UC -->|"realizes
(optional)"| STKH - UC -->|"realized_by
(optional)"| TR - PM -->|"nested under
(parent_needs)"| UC - PM -->|"violates"| STKH PM -->|"violates"| TR + TR -->|"satisfies
(optional)"| UP TC -->|"fully_verifies /
partially_verifies"| TR - TR -->|"realizes
(optional)"| STKH style UC fill:#E1D5E7 style PM fill:#F8CECC - style STKH fill:#DAE8FC + style UP fill:#DAE8FC style TR fill:#F5F5F5 ``` -The two analysis views answer different questions: +Two analysis perspectives answer different questions: -| View | Question | +| Perspective | Question | |---|---| | **Evaluation / classification** | How can our intended use of the tool fail, and would the intended usage detect or prevent that failure? | | **Qualification** | Do we have sufficient evidence that the specific tool version satisfies the tool requirements we rely on? | @@ -99,7 +96,7 @@ evidence. | Topic | External tool | Self-developed tool | |---|---|---| | Version | Pin the exact external version and relevant configuration | Identify the exact internal release/version | -| `tool_req` | Define the behaviour **our project relies on** | Reuse existing tool requirements where suitable | +| `tool_req` | Select the tool capabilities relevant to the evaluated use cases | Reuse existing tool requirements where suitable | | Implementation | Usually treated largely as a black box | Architecture and implementation may be available | | Existing tests | Vendor/upstream tests can be supporting information | Existing development tests may already provide qualification evidence | | Qualification | Usually validate our `tool_req` against the configured external tool | Reuse suitable requirements-based tests where possible | @@ -152,6 +149,21 @@ The generated TVR report additionally shows: The report author does not provide these summary values. +Record the tool's scope and purpose, relevant configuration, environment, constraints, +inputs, outputs, and available documentation when creating the TVR. Inputs and outputs +define the usage boundary and are needed to identify potential malfunctions. + +The introduction should make the purpose and intended use visible before the +detailed evaluation. Model each intended use as a `tool_usecase`; the generated +report places an overview of those use cases before the evaluation result. + +```{mermaid} +flowchart LR + P["Tool purpose and scope"] --> U["Intended use cases
(tool_usecase)"] + U --> M["Potential malfunctions"] + M --> Q["Qualification of relevant
(tool_req) capabilities"] +``` + Example structure: ````markdown @@ -161,8 +173,8 @@ Example structure: :security_affected: NO :tool_version: -Describe the tool, its intended use, relevant configuration, environment, -constraints, inputs, outputs, and available documentation. +Describe the tool, its purpose and intended use, relevant configuration, +environment, constraints, inputs, outputs, and available documentation. ``` ```` @@ -201,47 +213,49 @@ Represent a `tool_usecase` with the following relationships: tool_usecase: mandatory_links: belongs_to: doc_tool - optional_links: - realized_by: tool_req - realizes: stkh_req ``` The normal relationship is therefore: ```{mermaid} flowchart LR - STKH["stkh_req"] + UP["upstream requirements
(stkh_req, gd_req, comp_req, feat_req)"] DT["doc_tool"] UC["tool_usecase"] TR["tool_req"] UC -->|"belongs_to
(mandatory)"| DT - UC -->|"realizes
(optional)"| STKH - UC -->|"realized_by
(optional)"| TR + TR -->|"satisfies
(optional)"| UP - style STKH fill:#DAE8FC + style UP fill:#DAE8FC style UC fill:#E1D5E7 style TR fill:#F5F5F5 ``` +Tool requirements may satisfy upstream requirements, including `stkh_req`, +`gd_req`, `feat_req`, and `comp_req`. The relevant upstream requirements should +be linked through `tool_req.satisfies` when relating a tool capability to an +upstream need. + ### How to model `tool_usecase` Every `evaluated`, `qualified`, or `released` TVR must own at least one `tool_usecase`. The use case records the context in which the project relies on -the tool. Link it to a `tool_req` when a matching requirement exists; the -`realized_by` link is optional so the context can be documented before that -requirement has been defined. +the tool. The use case itself has no direct requirement links; the relevant +`tool_req` needs are associated through the nested malfunctions, and upstream +requirements are linked from those `tool_req` needs. For example: -- stakeholder requirement: the system-development process must provide valid - requirements traceability, +- upstream requirement (e.g. `stkh_req`, `gd_req`, `feat_req`, `comp_req`): the + system-development process must provide valid requirements traceability, - tool use case: validate requirement traceability during documentation builds, - tool requirement: unresolved requirement links must be reported. If an existing requirement already expresses the usage context exactly, avoid inventing additional behaviour in the use case. Keep it as a thin -grouping/context element, whether or not it links to a `tool_req`. +grouping/context element. The relevant `tool_req` needs remain linked through +the nested malfunctions. ```{important} Write each `potential_tool_malfunction` nested inside its `tool_usecase`. The @@ -257,8 +271,6 @@ for an already well-scoped requirement. ```{tool_usecase} Validate requirement traceability during documentation builds :id: tool_usecase__docs_as_code__traceability :belongs_to: doc_tool__s_core_docs_as_code -:realized_by: tool_req__docs_as_code__unresolved_links -:realizes: stkh_req__docs_as_code__traceability The project relies on the documentation tooling to identify invalid or unresolved requirement links before documentation is accepted. @@ -281,7 +293,7 @@ Validate requirement links during the documentation build. ### External tool -Define the `tool_req` needs from **our expectations of the tool**, not by +Define the `tool_req` needs from the tool capability being evaluated, not by importing the vendor's complete specification. Example: @@ -296,10 +308,10 @@ The tool shall report unresolved requirement links. ### Self-developed tool -Prefer linking the use case to existing `tool_req` needs. +Prefer reusing existing `tool_req` needs in the nested malfunctions. Do not create separate "qualification requirements" when the normal tool -requirements already describe the behaviour relied upon by the project. +requirements already describe the capability relevant to the project use case. --- @@ -320,7 +332,7 @@ potential_tool_malfunction: mandatory_links: # Established by nesting the malfunction inside its tool use case. parent_needs: tool_usecase - violates: stkh_req, tool_req + violates: tool_req ``` The relationship is: @@ -329,16 +341,16 @@ The relationship is: flowchart LR UC["tool_usecase"] PM["potential_tool_malfunction"] - STKH["stkh_req"] + UP["upstream requirement"] TR["tool_req"] PM -->|"nested under
(parent_needs)"| UC - PM -->|"violates"| STKH PM -->|"violates"| TR + TR -->|"satisfies"| UP style UC fill:#E1D5E7 style PM fill:#F8CECC - style STKH fill:#DAE8FC + style UP fill:#DAE8FC style TR fill:#F5F5F5 ``` @@ -374,28 +386,21 @@ the malfunction. Typical targets are: -- `stkh_req` - `tool_req` -Do not link arbitrary downstream requirements merely because the malfunction -could eventually contribute to an incorrect engineering decision. - ### Tool model example `````markdown ::::{tool_usecase} Validate requirement traceability during documentation builds :id: tool_usecase__docs_as_code__traceability :belongs_to: doc_tool__s_core_docs_as_code -:realized_by: tool_req__docs_as_code__unresolved_links -:realizes: stkh_req__docs_as_code__traceability The project relies on the documentation tool to detect invalid traceability. :::{potential_tool_malfunction} Unresolved requirement link is accepted as valid :id: potential_tool_malfunction__docs_as_code__unresolved_link_accepted :violates: - tool_req__docs_as_code__unresolved_links, - stkh_req__docs_as_code__traceability + tool_req__docs_as_code__unresolved_links :safety_affected: YES :detection_sufficient: NO @@ -406,15 +411,16 @@ does not report the problem. ````` There is no conceptual difference between external and self-developed tools in -this step. Malfunctions are derived from the intended usage and the requirements -the project relies on. +this step. Malfunctions are derived from the intended usage and the tool +capabilities relevant to that usage. + +`detection_sufficient` is only required in case of `safety_affected: YES`. A +safety-relevant malfunction with `detection_sufficient: YES` must also document a +non-empty `safety_measures` value. The malfunction body remains the place for the +human-readable reasoning that explains why the measure is sufficient or insufficient. -`detection_sufficient` is conditional. It is required for -`safety_affected: YES`; it is not required for `safety_affected: NO` and is -omitted for non-safety malfunctions. A safety-relevant malfunction with -`detection_sufficient: YES` must also document a non-empty `safety_measures` -value. The malfunction body remains the place for the human-readable reasoning -that explains why the measure is sufficient or insufficient. +The same `tool_req` may be relevant to multiple use cases; +evaluate its potential malfunctions separately for each usage context. --- @@ -848,9 +854,11 @@ See: Assume an external documentation tool is used to validate requirement traceability. -### Stakeholder requirement +### Upstream requirement -A real `stkh_req` must provide the required SCORE metadata. For example: +An upstream requirement is represented by a concrete SCORE requirement type, such +as `stkh_req`, `gd_req`, `feat_req`, or `comp_req`. This example uses the +existing `stkh_req` type: ````markdown ```{stkh_req} Provide valid verification traceability @@ -883,8 +891,6 @@ The tool shall report unresolved requirement links as errors. ::::{tool_usecase} Validate requirement traceability during documentation builds :id: tool_usecase__docs_as_code__traceability :belongs_to: doc_tool__s_core_docs_as_code -:realized_by: tool_req__docs_as_code__unresolved_links -:realizes: stkh_req__docs_as_code__traceability The project relies on the documentation tool to detect invalid traceability before generated documentation is accepted. @@ -892,8 +898,7 @@ before generated documentation is accepted. :::{potential_tool_malfunction} Unresolved requirement link is accepted as valid :id: potential_tool_malfunction__docs_as_code__unresolved_link_accepted :violates: - tool_req__docs_as_code__unresolved_links, - stkh_req__docs_as_code__traceability + tool_req__docs_as_code__unresolved_links :safety_affected: YES :detection_sufficient: NO @@ -906,18 +911,16 @@ The resulting model is: ```{mermaid} flowchart LR - STKH["stkh_req__docs_as_code__traceability"] + UP["stkh_req__docs_as_code__traceability"] UC["tool_usecase__docs_as_code__traceability"] TR["tool_req__docs_as_code__unresolved_links"] PM["potential_tool_malfunction__docs_as_code__
unresolved_link_accepted"] - UC -->|"realizes"| STKH - UC -->|"realized_by (optional)"| TR PM -->|"nested under
(parent_needs)"| UC - PM -->|"violates"| STKH PM -->|"violates"| TR + TR -->|"satisfies"| UP - style STKH fill:#DAE8FC + style UP fill:#DAE8FC style UC fill:#E1D5E7 style TR fill:#F5F5F5 style PM fill:#F8CECC @@ -985,8 +988,6 @@ code. ::::{tool_usecase} Generate production source from the approved model :id: tool_usecase__generator__generate_source :belongs_to: doc_tool__s_core_docs_as_code -:realized_by: tool_req__generator__state_transitions -:realizes: stkh_req__generator__approved_model The project relies on the generator to transform the approved model into production source code. @@ -994,8 +995,7 @@ production source code. :::{potential_tool_malfunction} State transition is omitted from generated source :id: potential_tool_malfunction__generator__missing_transition :violates: - tool_req__generator__state_transitions, - stkh_req__generator__approved_model + tool_req__generator__state_transitions :safety_affected: YES :detection_sufficient: NO @@ -1029,7 +1029,7 @@ changes. Typical triggers include: - a new or changed tool use case, -- new or changed stakeholder requirements, +- new or changed upstream requirements (`stkh_req`, `gd_req`, `feat_req`, or `comp_req`), - new or changed tool requirements, - a changed tool version, - a changed relevant configuration, @@ -1063,19 +1063,28 @@ flowchart TD Do not invent new normative behaviour in `tool_usecase`. -It is a **usage-context / grouping element** between stakeholder intent and -concrete tool behaviour. For example, "validate requirement traceability during +A `tool_usecase` describes how the project uses the tool. A `tool_req` describes +a capability or behaviour provided by the tool, regardless of whether the project +uses it in a particular use case. The use case provides context and grouping +without introducing an additional normative requirement level. + +For example, "validate requirement traceability during documentation builds" is a use case; "the tool shall report unresolved links" -is the requirement. The use case adds context, not another requirement level. +is the tool requirement. + +Another example is the pair of use cases "Produce the platform components binaries" +and "Create unit test binaries", which both map to the tool requirement +"the compiler shall create an executable from C++ source code". + +The use case adds context, not another requirement level. ```{mermaid} flowchart LR - STKH["Stakeholder intent"] + UP["Upstream requirement"] UC["Usage context"] TR["Concrete tool behaviour"] - UC -->|"realizes"| STKH - UC -->|"realized_by (optional)"| TR + TR -->|"satisfies"| UP ``` --- @@ -1146,10 +1155,11 @@ This is not contradictory. Do not copy an external tool's complete specification into `tool_req`. Do not copy almost the entire external C&Q or qualification specification from -the Internet either; capture only the behaviour and assumptions our project -relies on. +the Internet either; capture only the capabilities and assumptions relevant to +the evaluated use cases. -Model the behaviour **our project relies on**. +Model the tool capability being evaluated; describe the project context in the +corresponding `tool_usecase`. --- @@ -1167,7 +1177,7 @@ flowchart TD A["1. Identify exact tool version / configuration"] B["2. Create doc_tool
status = draft"] C["3. Define tool_usecase"] - D["4. Link available tool_req
and optionally stkh_req"] + D["4. Link available tool_req
to upstream requirements"] E["5. Identify potential_tool_malfunction"] F["6. Read generated TVR summary"] G["7. Document safety_measures
and detection_sufficient"] diff --git a/docs/internals/requirements/tool_verification.rst b/docs/internals/requirements/tool_verification.rst index 91e8e56d4..a64db60d8 100644 --- a/docs/internals/requirements/tool_verification.rst +++ b/docs/internals/requirements/tool_verification.rst @@ -17,14 +17,57 @@ Doc-as-Code Tool Verification Report This page is the authoritative Tool Verification Report for the S-CORE Docs-as-Code tool. It records the classification of intended usage, potential malfunctions, safety impact and detection, together with qualification evidence and lifecycle state. -Tool under evaluation ---------------------- +Introduction +------------ + +Scope and purpose +~~~~~~~~~~~~~~~~~ The S-CORE Docs-as-Code tool (Bazel module ``score_docs_as_code``) builds HTML -documentation and traceability data from RST/Markdown sources. It evaluates -those sources with the S-CORE extensions and metamodel, and produces the -generated documentation, ``needs.json`` and ``metrics.json`` used by the -downstream process. +documentation from RST/Markdown sources, including process descriptions, +requirements, and traceability data. It validates the sources with the S-CORE +extensions and metamodel. + +Inputs and outputs +~~~~~~~~~~~~~~~~~~ + +* **Inputs:** RST/Markdown sources, Sphinx configuration, the S-CORE metamodel, + Bazel build files, source-code links, and test results. +* **Outputs:** HTML documentation, traceability data (``needs.json``), and + coverage and linkage statistics (``metrics.json``). + +.. mermaid:: + + flowchart LR + sources["RST/Markdown sources"] --> tool["S-CORE Docs-as-Code"] + config["Configuration and metamodel"] --> tool + links["Source-code links"] --> tool + tests["Test results"] --> tool + tool --> html["HTML documentation"] + tool --> needs["needs.json"] + tool --> metrics["metrics.json"] + +Available information +~~~~~~~~~~~~~~~~~~~~~ + +* Repository: https://github.com/eclipse-score/docs-as-code +* Documentation: https://eclipse-score.github.io/docs-as-code/v8.1.2/ +* Bazel module: ``score_docs_as_code`` + +Installation and integration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The tool is consumed as a Bazel module and integrated through the repository's +documentation build targets. The evaluated configuration is the one defined by +the repository's ``MODULE.bazel``, ``BUILD`` files, Sphinx configuration, and +S-CORE metamodel. The relevant checks are run through ``//:docs_check`` and +the associated documentation and traceability targets. + +Environment +~~~~~~~~~~~ + +The evaluation runs in the repository's supported Bazel environment on Linux, +using the configured Sphinx, Python, and diagram-generation toolchains. Report record ------------- @@ -38,6 +81,9 @@ Report record :realizes: wp__tool_verification_report[version==1] :post_template: tool_qualification_report + Evaluates the S-CORE Docs-as-Code tool for building and checking + documentation and traceability data from RST/Markdown sources. + Details ------- @@ -45,12 +91,6 @@ Details .. tool_usecase:: Build/CI behavior :id: tool_usecase__docs_as_code__build_ci :belongs_to: doc_tool__score_docs_as_code - :realized_by: - tool_req__docs_doc_types, - tool_req__docs_common_attr_safety_link_check, - tool_req__docs_test_linkage_metrics, - tool_req__docs_test_link_testcase, - tool_req__docs_req_link_covers_aou :version: 1 Builds run with ``-W``; any warning trips CI. The @@ -231,7 +271,6 @@ Details .. tool_usecase:: PR Review :id: tool_usecase__docs_as_code__pr_review :belongs_to: doc_tool__score_docs_as_code - :realized_by: tool_req__docs_doc_types :version: 1 Repository contents are the source of truth @@ -242,10 +281,6 @@ Details .. tool_usecase:: Derived-view :id: tool_usecase__docs_as_code__derived_view :belongs_to: doc_tool__score_docs_as_code - :realized_by: - tool_req__docs_arch_views, - tool_req__docs_verification_report_need, - tool_req__docs_req_link_satisfies_allowed :version: 1 The rendered HTML output is a derived view; diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index 0d4b1b147..fa6558c0b 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -270,21 +270,19 @@ needs_types: # semantic relationship. # Evaluated, qualified, and released tool qualification reports require at # least one tool_usecase as the context anchor for their evaluation. - # Requirement links remain optional so the context can be recorded before all - # requirements exist. + # Tool requirements are linked through malfunctions; upstream requirements + # are linked from those tool requirements. tool_usecase: title: Tool Use Case description: | Defines the usage context in which the tool is evaluated. A tool use case does not introduce additional requirements. - Where applicable, it groups the stakeholder and tool requirements - relevant to a particular way the project relies on the tool. + Where applicable, its nested malfunctions provide context for the tool + requirements relevant to a particular way the project relies on the tool. + Those tool requirements may satisfy upstream requirements. color: "#E1D5E7" mandatory_links: belongs_to: doc_tool - optional_links: - realized_by: tool_req - realizes: stkh_req parts: 3 # A malfunction describes a violation of one or more requirements. Safety @@ -300,7 +298,7 @@ needs_types: safety_measures: ^.+$ mandatory_links: parent_needs: tool_usecase - violates: stkh_req, tool_req + violates: tool_req parts: 3 # Requirements diff --git a/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst b/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst index 6be988d58..5e66f5b16 100644 --- a/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst +++ b/src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst @@ -90,7 +90,6 @@ .. tool_usecase:: Nested tool use case :id: tool_usecase__blabla__nested :belongs_to: doc_tool__nested_qualification - :realized_by: tool_req__test__qualification :version: 1 A use case contains its tool malfunctions as nested needs. diff --git a/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst b/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst index 272f5f3b8..664093b5a 100644 --- a/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst +++ b/src/extensions/score_metamodel/tests/rst/graph/test_tool_qualification.rst @@ -32,7 +32,6 @@ .. tool_usecase:: Tool qualification check use case :id: tool_usecase__tool_qualification_checks :belongs_to: doc_tool__tool_qualification_checks - :realized_by: tool_req__tool_qualification_checks :version: 1 .. tool_usecase:: Tool usage context without a pre-existing requirement diff --git a/src/extensions/score_metamodel/tests/test_metamodel_load.py b/src/extensions/score_metamodel/tests/test_metamodel_load.py index c0a85e2ed..651fd28a7 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel_load.py +++ b/src/extensions/score_metamodel/tests/test_metamodel_load.py @@ -240,13 +240,10 @@ def test_tool_qualification_types_model_nested_traceability(): assert types["tool_usecase"]["mandatory_links_str"] == { "belongs_to": "doc_tool", } - assert types["tool_usecase"]["optional_links_str"] == { - "realized_by": "tool_req", - "realizes": "stkh_req", - } + assert types["tool_usecase"]["optional_links_str"] == {} assert types["potential_tool_malfunction"]["mandatory_links_str"] == { "parent_needs": "tool_usecase", - "violates": "stkh_req, tool_req", + "violates": "tool_req", } malfunction_options = types["potential_tool_malfunction"] assert malfunction_options["mandatory_options"]["safety_affected"] == "^(YES|NO)$" diff --git a/src/extensions/score_metamodel/tests/test_tool_qualification.py b/src/extensions/score_metamodel/tests/test_tool_qualification.py index e5374d136..a58a03089 100644 --- a/src/extensions/score_metamodel/tests/test_tool_qualification.py +++ b/src/extensions/score_metamodel/tests/test_tool_qualification.py @@ -55,7 +55,6 @@ def _low_report( id="tool_usecase__test_report__context", type="tool_usecase", belongs_to=["doc_tool__test_report"], - realized_by=["tool_req__test_report__requirement"], ) requirement = need( id="tool_req__test_report__requirement", @@ -210,11 +209,11 @@ def test_report_requires_owned_usecase(): logger.assert_warning("must own at least one `tool_usecase`") -def test_low_malfunction_with_only_stakeholder_requirement_is_invalid(): - """A stakeholder requirement alone cannot receive tool qualification evidence.""" - stakeholder = need(id="stkh_req__test_report__requirement", type="stkh_req") +def test_low_malfunction_with_only_upstream_requirement_is_invalid(): + """An upstream requirement alone cannot receive tool qualification evidence.""" + upstream = need(id="stkh_req__test_report__requirement", type="stkh_req") all_needs = _low_report(violated=["stkh_req__test_report__requirement"]) - all_needs.values.return_value.append(stakeholder) + all_needs.values.return_value.append(upstream) logger = fake_check_logger() check_tool_qualification_workflow(MagicMock(), all_needs, logger) diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index d2562a499..d8a70949e 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -125,6 +125,30 @@ {% endfor %} {% endfor %} +Purpose and intended use +^^^^^^^^^^^^^^^^^^^^^^^^ + +*Generated from linked report records.* + +The tool is evaluated for the following intended use cases. The use case title +is the concise statement of the project context in which the tool is used; +the detailed evaluation is shown in the sections below. + +.. list-table:: Intended use cases + :header-rows: 1 + :widths: 72 28 + + * - Intended use + - Record +{% for usecase in usecases %} + * - {{ text(usecase.get("title")) }} + - {{ need_reference(usecase) }} +{% endfor %} +{% if not usecases %} + * - No tool use case is documented. + - — +{% endif %} + Conclusion ^^^^^^^^^^ From c44ce173d53bae14401492c89781863d86cd0699 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 25 Sep 2026 16:39:09 +0200 Subject: [PATCH 11/17] Address remaining tool qualification review feedback --- .../checks/tool_qualification.py | 45 ++++++++----------- .../score_metamodel/tests/__init__.py | 19 +++++--- .../tests/test_metamodel__init__.py | 3 +- .../tests/test_tool_qualification.py | 16 +------ 4 files changed, 34 insertions(+), 49 deletions(-) diff --git a/src/extensions/score_metamodel/checks/tool_qualification.py b/src/extensions/score_metamodel/checks/tool_qualification.py index 06da18837..30833c598 100644 --- a/src/extensions/score_metamodel/checks/tool_qualification.py +++ b/src/extensions/score_metamodel/checks/tool_qualification.py @@ -18,7 +18,7 @@ """ from collections.abc import Iterable -from typing import Any, cast +from typing import Any from score_metamodel import CheckLogger, graph_check, local_check from sphinx.application import Sphinx @@ -28,21 +28,9 @@ def _link_values(need: NeedItem, link_name: str) -> list[str]: """Return an outgoing link field as plain IDs.""" - try: - link_values = cast(list[str], need.get_links(link_name, as_str=True)) - return [str(item) for item in link_values] - except KeyError: - # The unit-test NeedItem helper exposes link fields through get(), while - # collected Sphinx-Needs items expose them through get_links. - pass - value = need.get(link_name, []) - if value is None: + if link_name not in need.iter_links_keys(): return [] - if isinstance(value, str): - return [value] - if isinstance(value, list): - return [str(item) for item in cast(list[Any], value)] - return [str(value)] + return [str(item) for item in need.get_links(link_name, as_str=True)] def _base_id(need_id: str) -> str: @@ -51,7 +39,7 @@ def _base_id(need_id: str) -> str: def _need_index(needs: Iterable[NeedItem]) -> dict[str, NeedItem]: - """Index Needs by both their exact and unqualified IDs.""" + """Index needs by stored IDs and IDs without a version selector.""" index: dict[str, NeedItem] = {} for need in needs: index[need["id"]] = need @@ -113,13 +101,6 @@ def _malfunctions_for_report( return found -def _is_present(value: Any) -> bool: - """Treat non-empty strings and non-empty collections as present.""" - if isinstance(value, str): - return bool(value.strip()) - return bool(value) - - def derive_tvr_values(malfunctions: Iterable[NeedItem]) -> tuple[str, str]: """Derive TVR safety relevance and TCL from malfunction evaluations.""" malfunction_list = list(malfunctions) @@ -231,7 +212,7 @@ def _validate_report_status( category="tool-qualification", ) return - if expected_tcl != "LOW": + if expected_tcl == "HIGH": # A HIGH-confidence report can progress directly from evaluated to # released. Qualification is only a workflow state for LOW TCL. return @@ -275,14 +256,14 @@ def check_tool_malfunction_evaluation( elif ( safety_affected == "YES" and detection_sufficient == "YES" - and not _is_present(safety_measures) + and not safety_measures ): log.warning_for_need( need, "`detection_sufficient: YES` requires a non-empty `safety_measures` value.", category="tool-qualification", ) - elif safety_affected == "NO" and _is_present(detection_sufficient): + elif safety_affected == "NO" and detection_sufficient is not None: log.warning_for_need( need, "non-safety malfunctions must not define `detection_sufficient`.", @@ -295,7 +276,17 @@ def check_tool_qualification_workflow( _: Sphinx, all_needs: NeedsView, log: CheckLogger ) -> None: """Validate ownership, qualification prerequisites, and TVR workflow states.""" - needs = list(all_needs.values()) + needs = list( + all_needs.filter_types( + [ + "doc_tool", + "tool_usecase", + "potential_tool_malfunction", + "tool_req", + "testcase", + ] + ).values() + ) index = _need_index(needs) _check_low_malfunction_links(needs, index, log) diff --git a/src/extensions/score_metamodel/tests/__init__.py b/src/extensions/score_metamodel/tests/__init__.py index ca5e1ef53..d436ad88b 100644 --- a/src/extensions/score_metamodel/tests/__init__.py +++ b/src/extensions/score_metamodel/tests/__init__.py @@ -110,12 +110,6 @@ def assert_info(self, expected_substring: str, expect_location: bool = True): def need(**kwargs: Any) -> NeedItem: """Convenience function to create a NeedItem object with some defaults.""" - # Extract links (any list field that's not a core field) - link_keys = { - "links", - } - links = {k: kwargs.pop(k, []) for k in list(link_keys) if k in kwargs} - # Set defaults for core fields kwargs.setdefault("id", "test_need") kwargs.setdefault("type", "requirement") @@ -157,6 +151,19 @@ def need(**kwargs: Any) -> NeedItem: "post_template", } + # Test fixtures represent link values as lists. Store those values in the + # same NeedItem.links field used by Sphinx-Needs; scalar non-core values + # remain regular extras. Link names are configured dynamically by + # Sphinx-Needs, so keeping a second global list here would be brittle. + link_keys = { + key + for key, value in kwargs.items() + if key not in core_keys + and key not in source_content_keys + and isinstance(value, list) + } + links = {key: kwargs.pop(key) for key in link_keys} + # Extract extras (any remaining kwargs not in core or source/content) extras = { k: v diff --git a/src/extensions/score_metamodel/tests/test_metamodel__init__.py b/src/extensions/score_metamodel/tests/test_metamodel__init__.py index f7d36eae0..ba4de2659 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel__init__.py +++ b/src/extensions/score_metamodel/tests/test_metamodel__init__.py @@ -122,7 +122,7 @@ def test_custom_values_override_defaults(self): assert n["tags"] == ["tag1", "tag2"] def test_link_fields_extracted(self): - """Verify link fields are extracted and accessible via .get().""" + """Verify link fields use NeedItem's link storage and accessors.""" n = need( complies=["std_req_1", "std_req_2"], input=["wp_input_1"], @@ -136,6 +136,7 @@ def test_link_fields_extracted(self): assert n.get("output", []) == ["wp_output_1", "wp_output_2"] assert n.get("contains", []) == ["item_1"] assert n.get("satisfies", []) == ["req_1"] + assert n.get_links("satisfies", as_str=True) == ["req_1"] def test_extra_fields_in_extras(self): """Verify extra fields (not core, not links) go into extras.""" diff --git a/src/extensions/score_metamodel/tests/test_tool_qualification.py b/src/extensions/score_metamodel/tests/test_tool_qualification.py index a58a03089..ad882363f 100644 --- a/src/extensions/score_metamodel/tests/test_tool_qualification.py +++ b/src/extensions/score_metamodel/tests/test_tool_qualification.py @@ -35,6 +35,7 @@ def _run_local(malfunction: NeedItem, logger: Any = None) -> Any: def _graph_needs(*needs: NeedItem) -> MagicMock: all_needs = MagicMock() all_needs.values.return_value = list(needs) + all_needs.filter_types.return_value = all_needs return all_needs @@ -101,21 +102,6 @@ def test_safety_malfunction_requires_conditional_evaluation_data( logger.assert_warning(message) -def test_positive_detection_requires_non_empty_safety_measure(): - """A positive detection claim must explain its detecting/preventing measure.""" - logger = _run_local( - need( - id="potential_tool_malfunction__local_no_measure", - type="potential_tool_malfunction", - safety_affected="YES", - detection_sufficient="YES", - safety_measures=" ", - ) - ) - - logger.assert_warning("non-empty `safety_measures`") - - def test_positive_detection_with_measure_is_valid(): """A safety-relevant malfunction may be HIGH when its measure is documented.""" logger = _run_local( From 93b8393bdee27070b46ee6a9ee166bb493d73e87 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 25 Sep 2026 16:57:06 +0200 Subject: [PATCH 12/17] Align tool verification with SCORE source --- docs/internals/requirements/tool_verification.rst | 6 ++++-- src/extensions/score_metamodel/checks/tool_qualification.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/internals/requirements/tool_verification.rst b/docs/internals/requirements/tool_verification.rst index a64db60d8..00ffb3e7c 100644 --- a/docs/internals/requirements/tool_verification.rst +++ b/docs/internals/requirements/tool_verification.rst @@ -104,7 +104,6 @@ Details :detection_sufficient: NO :safety_measures: PR review :violates: - tool_req__docs_doc_types, tool_req__docs_doc_generic_mandatory, tool_req__docs_common_attr_id, tool_req__docs_common_attr_id_scheme, @@ -134,8 +133,11 @@ Details tool_req__docs_saf_attrs_sufficient_check, tool_req__docs_saf_attrs_content, tool_req__docs_saf_attrs_violates, + tool_req__docs_saf_attrs_mandatory, tool_req__docs_saf_attr_fmea_fault_id, - tool_req__docs_saf_attr_fmea_failure_effect + tool_req__docs_saf_attr_fmea_failure_effect, + tool_req__docs_sec_attr_stride_threat_id, + tool_req__docs_sec_attrs_mandatory :version: 1 **Document metamodel enforcement** — enforce document types, mandatory attributes (id, status, security, safety, realizes), etc. diff --git a/src/extensions/score_metamodel/checks/tool_qualification.py b/src/extensions/score_metamodel/checks/tool_qualification.py index 30833c598..e19a95d0b 100644 --- a/src/extensions/score_metamodel/checks/tool_qualification.py +++ b/src/extensions/score_metamodel/checks/tool_qualification.py @@ -263,7 +263,7 @@ def check_tool_malfunction_evaluation( "`detection_sufficient: YES` requires a non-empty `safety_measures` value.", category="tool-qualification", ) - elif safety_affected == "NO" and detection_sufficient is not None: + elif safety_affected == "NO" and detection_sufficient: log.warning_for_need( need, "non-safety malfunctions must not define `detection_sufficient`.", From 5b8c9112a37ed263ee4ffa61ec6d6c8cfcce98d9 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 25 Sep 2026 19:55:47 +0200 Subject: [PATCH 13/17] Allow TVRs without structured use cases --- docs/how-to/perform_tool_verification.md | 13 ++++++++----- .../score_metamodel/checks/tool_qualification.py | 9 +++------ src/extensions/score_metamodel/metamodel.yaml | 5 +++-- .../tests/rst/options/test_doc_tool_attrs.rst | 12 ++++++------ .../tests/test_tool_qualification.py | 8 +++++--- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/how-to/perform_tool_verification.md b/docs/how-to/perform_tool_verification.md index fb97f0a86..759d79e9c 100644 --- a/docs/how-to/perform_tool_verification.md +++ b/docs/how-to/perform_tool_verification.md @@ -239,11 +239,14 @@ upstream need. ### How to model `tool_usecase` -Every `evaluated`, `qualified`, or `released` TVR must own at least one -`tool_usecase`. The use case records the context in which the project relies on -the tool. The use case itself has no direct requirement links; the relevant -`tool_req` needs are associated through the nested malfunctions, and upstream -requirements are linked from those `tool_req` needs. +In the structured verification workflow, an `evaluated`, `qualified`, or +`released` TVR owns one or more `tool_usecase` records. A use case records the +context in which the project relies on the tool. The use case itself has no +direct requirement links; the relevant `tool_req` needs are associated through +the nested malfunctions, and upstream requirements are linked from those +`tool_req` needs. Existing TVRs without structured use cases remain accepted +for compatibility, but the workflow checker does not derive summary values for +them. For example: diff --git a/src/extensions/score_metamodel/checks/tool_qualification.py b/src/extensions/score_metamodel/checks/tool_qualification.py index e19a95d0b..f13ae4414 100644 --- a/src/extensions/score_metamodel/checks/tool_qualification.py +++ b/src/extensions/score_metamodel/checks/tool_qualification.py @@ -298,12 +298,9 @@ def check_tool_qualification_workflow( usecases = _owned_usecases(doc_tool, needs) if not usecases: - log.warning_for_need( - doc_tool, - "evaluated, qualified, and released Tool Verification Reports " - "must own at least one `tool_usecase` through `belongs_to`.", - category="tool-qualification", - ) + # Reports without structured use cases are legacy records. Leave + # their manually supplied summary fields unchanged until they are + # migrated to the structured workflow. continue malfunctions = _malfunctions_for_report(doc_tool, needs) diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index b8f12352e..fa318b3d1 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -270,8 +270,9 @@ needs_types: # Tool qualification reports keep use cases top-level for readable rendering. # Their malfunctions are nested below them and use parent_needs as their # semantic relationship. - # Evaluated, qualified, and released tool qualification reports require at - # least one tool_usecase as the context anchor for their evaluation. + # Structured tool qualification reports use tool_usecase as the context + # anchor for their evaluation. Legacy reports without use cases remain + # accepted for compatibility and are left unchanged by the workflow check. # Tool requirements are linked through malfunctions; upstream requirements # are linked from those tool requirements. tool_usecase: diff --git a/src/extensions/score_metamodel/tests/rst/options/test_doc_tool_attrs.rst b/src/extensions/score_metamodel/tests/rst/options/test_doc_tool_attrs.rst index 1cd74cd29..b22b67982 100644 --- a/src/extensions/score_metamodel/tests/rst/options/test_doc_tool_attrs.rst +++ b/src/extensions/score_metamodel/tests/rst/options/test_doc_tool_attrs.rst @@ -13,17 +13,17 @@ # ******************************************************************************* -.. test_metadata:: Mandatory Document and TVR Attributes +.. test_metadata:: Document and TVR Attribute Validation :id: test_metadata__doc_tool_attrs :fully_verifies_list: tool_req__docs_doc_generic_mandatory,tool_req__docs_tvr_safety,tool_req__docs_tvr_security,tool_req__docs_tvr_status,tool_req__docs_tvr_confidence_level :partially_verifies_list: tool_req__docs_tvr_version :test_type: requirements_based :derivation_technique: requirements_based - Tests the mandatory attributes of Generic Documents (document) - and Tool Verification Reports (doc_tool) as defined in the metamodel. - Every mandatory attribute is exercised with an accepted value, an - unaccepted value, and an omitted value. + Tests the mandatory attributes of Generic Documents (document) and Tool + Verification Reports (doc_tool), plus validation of TVR summary attribute + values, as defined in the metamodel. Every mandatory attribute is + exercised with an accepted value, an unaccepted value, and an omitted value. The tool_version attribute is not enforced yet for migration purposes. @@ -162,4 +162,4 @@ .. doc_tool:: TVR With Missing Mandatory Attributes :id: doc_tool__attr_missing :version: 1 - :expect: doc_tool__attr_missing: is missing required attribute: `status`., doc_tool__attr_missing: is missing required attribute: `safety_affected`., doc_tool__attr_missing: is missing required attribute: `security_affected`., doc_tool__attr_missing: is missing required attribute: `tcl`. + :expect: doc_tool__attr_missing: is missing required attribute: `status`., doc_tool__attr_missing: is missing required attribute: `security_affected`. diff --git a/src/extensions/score_metamodel/tests/test_tool_qualification.py b/src/extensions/score_metamodel/tests/test_tool_qualification.py index ad882363f..76615a7c1 100644 --- a/src/extensions/score_metamodel/tests/test_tool_qualification.py +++ b/src/extensions/score_metamodel/tests/test_tool_qualification.py @@ -179,8 +179,8 @@ def test_report_summary_is_derived_when_summary_fields_are_omitted(): assert report["tcl"] == "LOW" -def test_report_requires_owned_usecase(): - """A non-draft report requires explicit tool-use-case ownership.""" +def test_report_without_usecase_is_left_unchanged(): + """Legacy reports without use cases remain compatible with the workflow check.""" report = need( id="doc_tool__report_without_usecase", type="doc_tool", @@ -192,7 +192,9 @@ def test_report_requires_owned_usecase(): check_tool_qualification_workflow(MagicMock(), _graph_needs(report), logger) - logger.assert_warning("must own at least one `tool_usecase`") + logger.assert_no_warnings() + assert report["safety_affected"] == "" + assert report["tcl"] == "" def test_low_malfunction_with_only_upstream_requirement_is_invalid(): From a0030a36eea3f14a5b613ca8b75ed14a85f53b57 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 25 Sep 2026 22:07:18 +0200 Subject: [PATCH 14/17] Address Copilot tool verification feedback --- docs/how-to/perform_tool_verification.md | 8 ++ .../checks/tool_qualification.py | 4 +- .../tests/rst/options/test_doc_tool_attrs.rst | 7 ++ .../tests/test_tool_qualification.py | 29 ++++++ .../score_sphinx_needs_templates/BUILD | 8 +- .../tests/test_needs_templates.py | 95 +++++++++++++++++++ .../tool_qualification_report.need | 2 + 7 files changed, 151 insertions(+), 2 deletions(-) diff --git a/docs/how-to/perform_tool_verification.md b/docs/how-to/perform_tool_verification.md index 759d79e9c..dd476f553 100644 --- a/docs/how-to/perform_tool_verification.md +++ b/docs/how-to/perform_tool_verification.md @@ -197,6 +197,14 @@ identification. For example, a Git commit can identify the evaluated tool when the relevant source and build context are clear. The TVR must make clear which specific tool version or revision the evaluation applies to. +```{note} +The current SCORE tool-management model assumes one active TVR per tool. It +does not currently support maintaining multiple parallel TVRs for different +versions of the same tool, nor does it scope qualification testcases to a +particular TVR/version. This limitation must be addressed before that usage +pattern is introduced. +``` + --- ## Step 2 — Define how the project uses the tool diff --git a/src/extensions/score_metamodel/checks/tool_qualification.py b/src/extensions/score_metamodel/checks/tool_qualification.py index f13ae4414..618b5268d 100644 --- a/src/extensions/score_metamodel/checks/tool_qualification.py +++ b/src/extensions/score_metamodel/checks/tool_qualification.py @@ -150,7 +150,9 @@ def _successful_full_testcase( Existing SCORE coverage distinguishes full and partial verification. A qualification claim needs complete evidence for the tool requirement, so - partial links alone do not satisfy this workflow check. + partial links alone do not satisfy this workflow check. The current model + assumes one active TVR per tool; testcases are not yet linked to a specific + TVR or tool version, so parallel version-specific TVRs are not supported. """ requirement_id = _base_id(requirement["id"]) for testcase in all_needs: diff --git a/src/extensions/score_metamodel/tests/rst/options/test_doc_tool_attrs.rst b/src/extensions/score_metamodel/tests/rst/options/test_doc_tool_attrs.rst index b22b67982..a6f51ba8c 100644 --- a/src/extensions/score_metamodel/tests/rst/options/test_doc_tool_attrs.rst +++ b/src/extensions/score_metamodel/tests/rst/options/test_doc_tool_attrs.rst @@ -159,6 +159,13 @@ :tcl: MEDIUM :expect: doc_tool__attr_bad_tcl.tcl (MEDIUM): does not follow pattern +.. doc_tool:: TVR Without Tool Version During Migration + :id: doc_tool__attr_missing_tool_version + :version: 1 + :status: evaluated + :security_affected: NO + :expect_not: is missing required attribute: `tool_version` + .. doc_tool:: TVR With Missing Mandatory Attributes :id: doc_tool__attr_missing :version: 1 diff --git a/src/extensions/score_metamodel/tests/test_tool_qualification.py b/src/extensions/score_metamodel/tests/test_tool_qualification.py index 76615a7c1..974712b04 100644 --- a/src/extensions/score_metamodel/tests/test_tool_qualification.py +++ b/src/extensions/score_metamodel/tests/test_tool_qualification.py @@ -259,6 +259,35 @@ def test_qualified_low_report_with_passed_full_evidence_is_valid(): logger.assert_no_warnings() +def test_qualified_low_report_requires_evidence_for_each_requirement(): + """Qualification evidence is required for every LOW tool requirement.""" + testcase = need( + id="testcase__test_report__first_requirement", + type="testcase", + result="passed", + fully_verifies=["tool_req__test_report__requirement"], + ) + second_requirement = need( + id="tool_req__test_report__second_requirement", + type="tool_req", + ) + second_malfunction = need( + id="potential_tool_malfunction__test_report__second_failure", + type="potential_tool_malfunction", + safety_affected="YES", + detection_sufficient="NO", + parent_needs=["tool_usecase__test_report__context"], + violates=[second_requirement["id"]], + ) + all_needs = _low_report(status="qualified", testcase=testcase) + all_needs.values.return_value.extend([second_requirement, second_malfunction]) + logger = fake_check_logger() + + check_tool_qualification_workflow(MagicMock(), all_needs, logger) + + logger.assert_warning("tool_req__test_report__second_requirement") + + def test_partial_evidence_alone_does_not_complete_low_qualification(): """Partial verification is traceability evidence, not complete qualification.""" testcase = need( diff --git a/src/extensions/score_sphinx_needs_templates/BUILD b/src/extensions/score_sphinx_needs_templates/BUILD index 21ebcfa31..2bc323480 100644 --- a/src/extensions/score_sphinx_needs_templates/BUILD +++ b/src/extensions/score_sphinx_needs_templates/BUILD @@ -32,7 +32,13 @@ py_library( score_pytest( name = "unit_tests", + # The template regression test builds a temporary Sphinx HTML project, + # so it needs more time than a pure Python unit test. + size = "medium", srcs = ["tests/test_needs_templates.py"], - deps = [":score_sphinx_needs_templates"], + deps = [ + ":score_sphinx_needs_templates", + "//src/extensions/score_metamodel:score_metamodel", + ], pytest_config = "//:pyproject.toml", ) diff --git a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py index 85224bf13..3c132393f 100644 --- a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py +++ b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py @@ -13,9 +13,11 @@ """Tests for graph traversal helpers used by Sphinx-Needs templates.""" from collections.abc import Iterable +from pathlib import Path import pytest import score_sphinx_needs_templates as templates +from sphinx.testing.util import SphinxTestApp class FakeLink: @@ -169,3 +171,96 @@ def test_any_req_in_report_version_false_if_no_requirement_matches() -> None: b["valid_from"] = "v3.0" assert any_req_in_report_version([a, b], "v1.0") is False + + +def test_qualification_evidence_deduplicates_shared_requirements( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Render one qualification-evidence row for a shared tool requirement. + + A tool requirement is reusable across project use cases. This fixture models + two use cases, each with a different LOW-confidence malfunction, while both + malfunctions violate the same tool requirement. The qualification evidence + table is requirement-scoped, so it must show that requirement once and + combine its evidence rather than rendering one row for each malfunction. + + This is an integration test because the duplicate would be introduced by + the Jinja post-template's nested use-case/malfunction loops; checking the + Need graph alone would not verify the rendered report. + """ + monkeypatch.setenv("BUILD_WORKSPACE_DIRECTORY", str(tmp_path)) + (tmp_path / "conf.py").write_text( + """ +extensions = ["sphinx_needs", "score_sphinx_needs_templates", "score_metamodel"] +master_doc = "index" +needs_id_regex = r"^[a-zA-Z0-9_]+$" +""", + encoding="utf-8", + ) + (tmp_path / "index.rst").write_text( + """ +.. doc_tool:: Shared requirement report + :id: doc_tool__shared_requirement_report + :status: evaluated + :security_affected: NO + :tool_version: v1 + :post_template: tool_qualification_report + +.. gd_req:: Shared process requirement + :id: gd_req__shared_requirement + + The process shall preserve shared qualification evidence. + +.. tool_req:: Shared tool requirement + :id: tool_req__shared_requirement + :satisfies: gd_req__shared_requirement + + The tool shall satisfy the shared capability. + +.. tool_usecase:: First use case + :id: tool_usecase__shared_requirement_first + :belongs_to: doc_tool__shared_requirement_report + + .. potential_tool_malfunction:: First malfunction + :id: potential_tool_malfunction__shared_requirement_first + :violates: tool_req__shared_requirement + :safety_affected: YES + :detection_sufficient: NO + :safety_measures: Independent review + +.. tool_usecase:: Second use case + :id: tool_usecase__shared_requirement_second + :belongs_to: doc_tool__shared_requirement_report + + .. potential_tool_malfunction:: Second malfunction + :id: potential_tool_malfunction__shared_requirement_second + :violates: tool_req__shared_requirement + :safety_affected: YES + :detection_sufficient: NO + :safety_measures: Independent review +""", + encoding="utf-8", + ) + + app = SphinxTestApp( + srcdir=tmp_path, + outdir=tmp_path / "_build", + buildername="html", + freshenv=True, + ) + try: + app.build(force_all=True) + html = (app.outdir / "index.html").read_text(encoding="utf-8") + finally: + app.cleanup() + + qualification_section = html.split( + '
', maxsplit=1 + )[1].split('
', maxsplit=1)[0] + assert ( + qualification_section.count( + "Shared tool requirement (tool_req__shared_requirement)" + ) + == 1 + ) diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index d8a70949e..a71b22d38 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -360,10 +360,12 @@ traceability evidence but does not by itself complete qualification. {% for usecase in usecases %} {% for malfunction in linked_needs(usecase["id"], "parent_needs_back")|selectattr("safety_affected", "equalto", "YES")|selectattr("detection_sufficient", "equalto", "NO")|list %} {% for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list %} +{% if requirement["id"] not in qualification_rows.items %} {% set qualification_rows.items = qualification_rows.items + [requirement["id"]] %} * - {{ need_reference(requirement) }} - {{ qualification_tests(requirement) }} - {{ "passed" if linked_needs(requirement["id"], "fully_verifies_back")|selectattr("result", "equalto", "passed")|list else "not fully qualified" }} +{% endif %} {% endfor %} {% endfor %} {% endfor %} From a0d7ce2305799da086b861207d38e0893228f34a Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 25 Sep 2026 22:26:50 +0200 Subject: [PATCH 15/17] Render tool use cases as linked summaries --- .../tests/test_needs_templates.py | 14 +++++++++++ .../tool_qualification_report.need | 24 +++++++++---------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py index 3c132393f..29adeee45 100644 --- a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py +++ b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py @@ -222,6 +222,8 @@ def test_qualification_evidence_deduplicates_shared_requirements( :id: tool_usecase__shared_requirement_first :belongs_to: doc_tool__shared_requirement_report + The tool is used to prepare the first project artifact. + .. potential_tool_malfunction:: First malfunction :id: potential_tool_malfunction__shared_requirement_first :violates: tool_req__shared_requirement @@ -233,6 +235,8 @@ def test_qualification_evidence_deduplicates_shared_requirements( :id: tool_usecase__shared_requirement_second :belongs_to: doc_tool__shared_requirement_report + The tool is used to prepare the second project artifact. + .. potential_tool_malfunction:: Second malfunction :id: potential_tool_malfunction__shared_requirement_second :violates: tool_req__shared_requirement @@ -255,6 +259,16 @@ def test_qualification_evidence_deduplicates_shared_requirements( finally: app.cleanup() + purpose_section = html.split('
', maxsplit=1)[ + 1 + ].split('
', maxsplit=1)[0] + assert "', maxsplit=1 )[1].split('
', maxsplit=1)[0] diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index a71b22d38..2b06271fb 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -46,6 +46,14 @@ {% if need.get("external_url") %}`{{ need["id"] }} <{{ need["external_url"] }}>`_{% else %}:need:`{{ need["id"] }}`{% endif %} {%- endmacro %} +{% macro need_title_reference(need) -%} +{% if need.get("external_url") %}`{{ text(need.get("title")) }} <{{ need["external_url"] }}>`_{% else %}:need:`{{ text(need.get("title")) }} <{{ need["id"] }}>`{% endif %} +{%- endmacro %} + +{% macro need_content(need) -%} +{{ text(need.get("content", "").split("\n\n.. potential_tool_malfunction::", 1)[0]) }} +{%- endmacro %} + {% macro need_list(needs) -%} {%- if needs -%} {%- for need in needs -%} @@ -131,22 +139,14 @@ Purpose and intended use *Generated from linked report records.* The tool is evaluated for the following intended use cases. The use case title -is the concise statement of the project context in which the tool is used; -the detailed evaluation is shown in the sections below. +is the concise statement of the project context in which the tool is used; the +detailed evaluation is shown in the sections below. -.. list-table:: Intended use cases - :header-rows: 1 - :widths: 72 28 - - * - Intended use - - Record {% for usecase in usecases %} - * - {{ text(usecase.get("title")) }} - - {{ need_reference(usecase) }} +* {{ need_title_reference(usecase) }}: {{ need_content(usecase) }} {% endfor %} {% if not usecases %} - * - No tool use case is documented. - - — +* No tool use case is documented. {% endif %} Conclusion From 354a6a4973df7564131acd2ff326d9e2fe922052 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 25 Sep 2026 22:45:23 +0200 Subject: [PATCH 16/17] Model tool use cases per malfunction --- .../requirements/tool_verification.rst | 220 +++++++++++------- 1 file changed, 132 insertions(+), 88 deletions(-) diff --git a/docs/internals/requirements/tool_verification.rst b/docs/internals/requirements/tool_verification.rst index 00ffb3e7c..1c7d49ba6 100644 --- a/docs/internals/requirements/tool_verification.rst +++ b/docs/internals/requirements/tool_verification.rst @@ -87,18 +87,36 @@ Report record Details ------- - -.. tool_usecase:: Build/CI behavior - :id: tool_usecase__docs_as_code__build_ci +The safety evaluation uses the following shared facts: + +* **Build/CI behavior:** Builds run with ``-W``; any warning trips CI. The + safety-relevant danger is the *silent* failure — a missing warning or a wrong + output published undetected. A loud CI abort is safe: no wrong output enters + the baseline. +* **PR Review:** Repository contents are the source of truth and every change + is reviewed by a committer (:need:`rl__committer`, + :need:`doc_concept__wp_inspections`). Still, for silent wrong outputs the + gated CI stays green. +* **Derived-view:** The rendered HTML output is a derived view; the + authoritative safety artifacts are mostly the source-controlled work + products. There are two exceptions, the architecture views and backlinks. + Rendering/preview defects affect reviewer convenience, not safety evidence. + +Each of the following tool capabilities is evaluated as an intended use case +with its corresponding potential malfunction. + +.. tool_usecase:: Document metamodel enforcement + :id: tool_usecase__docs_as_code__metamodel :belongs_to: doc_tool__score_docs_as_code :version: 1 - Builds run with ``-W``; any warning trips CI. The - safety-relevant danger is the *silent* failure — a missing warning - or a wrong output published undetected. - A loud CI abort is safe: no wrong output enters the baseline. + Enforce document types and mandatory attributes such as id, status, + security, safety, and realizes. See, for example, + :need:`gd_req__doc_attr_status`, :need:`gd_req__req_attr_uid`, + :need:`gd_req__req_attr_safety`, :need:`gd_req__arch_attr_safety`, + and :need:`gd_req__req_check_mandatory`. - .. potential_tool_malfunction:: Document metamodel enforcement + .. potential_tool_malfunction:: Document metamodel enforcement failure :id: potential_tool_malfunction__docs_as_code__m1 :safety_affected: YES :detection_sufficient: NO @@ -140,11 +158,8 @@ Details tool_req__docs_sec_attrs_mandatory :version: 1 - **Document metamodel enforcement** — enforce document types, mandatory attributes (id, status, security, safety, realizes), etc. - See, for example, :need:`gd_req__doc_attr_status`, :need:`gd_req__req_attr_uid`, :need:`gd_req__req_attr_safety`, :need:`gd_req__arch_attr_safety`, :need:`gd_req__req_check_mandatory`. - - **Silent false-negative**: too-permissive - ``metamodel.yaml`` regex accepted with no guard, or a check bug skips a case. + **Silent false-negative:** a too-permissive ``metamodel.yaml`` regex is + accepted without a guard, or a check bug skips a case. Impact on safety: yes. Impact safety measures available: yes: PR review. @@ -152,7 +167,15 @@ Details Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. - .. potential_tool_malfunction:: Safety-critical linking enforcement +.. tool_usecase:: Safety-critical linking enforcement + :id: tool_usecase__docs_as_code__safety_links + :belongs_to: doc_tool__score_docs_as_code + :version: 1 + + Enforce that links between safety-relevant needs preserve the required + safety relationships. See :need:`gd_req__req_linkage_safety`. + + .. potential_tool_malfunction:: Safety-critical linking enforcement failure :id: potential_tool_malfunction__docs_as_code__m2 :safety_affected: YES :detection_sufficient: NO @@ -169,21 +192,30 @@ Details tool_req__docs_arch_link_security :version: 1 - **Safety-critical linking enforcement**. - See :need:`gd_req__req_linkage_safety`. - - **Silent false-negative**: allow links which cannot be safe derivations. + **Silent false-negative:** links are allowed which cannot be safe + derivations. Impact on safety: yes. Impact safety measures available: yes: PR review. Impact safety detection sufficient: no: Qualify graph checks. Further additional safety measure required: yes (qualification). - The clearest gap is ``satisfied_by`` (and arguably ``covers``), which carry the same "target at least as safe" obligation as the checked ``fulfils``/``implements`` yet are unconstrained. + The clearest gap is ``satisfied_by`` (and arguably ``covers``), which + carry the same "target at least as safe" obligation as the checked + ``fulfils``/``implements`` yet are unconstrained. Confidence (automatic calculation): low. - .. potential_tool_malfunction:: Requirements coverage statistics +.. tool_usecase:: Requirements coverage statistics + :id: tool_usecase__docs_as_code__coverage + :belongs_to: doc_tool__score_docs_as_code + :version: 1 + + Count, per requirement type, the requirements carrying a ``testlink`` and + compute link-coverage percentages. See + :need:`gd_req__verification_reporting`. + + .. potential_tool_malfunction:: Requirements coverage statistics failure :id: potential_tool_malfunction__docs_as_code__m3 :safety_affected: YES :detection_sufficient: NO @@ -192,10 +224,7 @@ Details tool_req__docs_verification_report_need :version: 1 - **Requirements coverage statistics** — count, per requirement type, the requirements carrying a ``testlink``, compute link-coverage percentages. - See :need:`gd_req__verification_reporting`. - - **Silent wrong-output**: a coverage statistic computed wrong. + **Silent wrong-output:** a coverage statistic is computed incorrectly. Impact on safety: yes. Impact safety measures available: no. @@ -203,7 +232,41 @@ Details Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. - .. potential_tool_malfunction:: Test linkage +.. tool_usecase:: Architecture visualization + :id: tool_usecase__docs_as_code__architecture + :belongs_to: doc_tool__score_docs_as_code + :version: 1 + + Generate architecture diagrams. See :need:`gd_req__arch_viewpoints`. + + .. potential_tool_malfunction:: Architecture visualization failure + :id: potential_tool_malfunction__docs_as_code__m4 + :safety_affected: YES + :detection_sufficient: YES + :safety_measures: PR review includes architecture inspection + :violates: tool_req__docs_arch_views + :version: 1 + + **Silent wrong-output:** a diagram misrepresents the architecture. + + Impact on safety: yes. + Impact safety measures available: yes: PR review includes architecture + inspection. + Impact safety detection sufficient: yes. + Further additional safety measure required: no. + Confidence (automatic calculation): high. + +.. tool_usecase:: Test linkage + :id: tool_usecase__docs_as_code__test_linkage + :belongs_to: doc_tool__score_docs_as_code + :version: 1 + + For each ``testcase`` need, resolve its + ``partially_verifies``/``fully_verifies`` references against the needs set. + See :need:`gd_req__req_attr_testlink` and + :need:`gd_req__verification_reporting`. + + .. potential_tool_malfunction:: Test linkage failure :id: potential_tool_malfunction__docs_as_code__m5 :safety_affected: YES :detection_sufficient: NO @@ -216,11 +279,8 @@ Details tool_req__docs_test_metadata_link_levels :version: 1 - **Test linkage** — for each ``testcase`` need, resolve its ``partially_verifies``/``fully_verifies`` references against the needs set. - See :need:`gd_req__req_attr_testlink`, :need:`gd_req__verification_reporting`. - - **Silent wrong-output**: - Safety case believes the requirement is tested where it is not. + **Silent wrong-output:** the safety case believes the requirement is + tested where it is not. Impact on safety: yes. Impact safety measures available: no. @@ -228,7 +288,15 @@ Details Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. - .. potential_tool_malfunction:: Test reference check +.. tool_usecase:: Test reference check + :id: tool_usecase__docs_as_code__test_refs + :belongs_to: doc_tool__score_docs_as_code + :version: 1 + + Check that test references are present and point to the intended + requirements. See :need:`gd_req__req_attr_testlink`. + + .. potential_tool_malfunction:: Test reference check failure :id: potential_tool_malfunction__docs_as_code__m6 :safety_affected: YES :detection_sufficient: NO @@ -236,11 +304,8 @@ Details :violates: tool_req__docs_test_link_testcase :version: 1 - **Test reference check**. - See :need:`gd_req__req_attr_testlink`. - - **Silent wrong-output**: - Test references an outdated/missing requirement. + **Silent wrong-output:** a test references an outdated or missing + requirement. Impact on safety: yes. Impact safety measures available: yes: PR review. @@ -248,7 +313,16 @@ Details Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. - .. potential_tool_malfunction:: Listing assumptions of use +.. tool_usecase:: Listing assumptions of use + :id: tool_usecase__docs_as_code__assumptions + :belongs_to: doc_tool__score_docs_as_code + :version: 1 + + Use ``needtable`` to communicate safety-critical assumptions of use in + safety manuals. See :need:`gd_guidl__saf_man` and + :need:`wp__platform_safety_manual`. + + .. potential_tool_malfunction:: Listing assumptions of use failure :id: potential_tool_malfunction__docs_as_code__m7 :safety_affected: YES :detection_sufficient: NO @@ -259,67 +333,30 @@ Details tool_req__docs_arch_link_fulfils_aou :version: 1 - **Listing assumptions of use** — safety manuals use ``needtable`` to communicate safety-critical assumptions of use to users. - See :need:`gd_guidl__saf_man`, :need:`wp__platform_safety_manual`. - - **Silent wrong-output**: ``aou_req`` items might be missing or wrong. + **Silent wrong-output:** ``aou_req`` items might be missing or wrong. Impact on safety: yes. Impact safety measures available: yes: PR review. Impact safety detection sufficient: no: Qualify ``needtable``. - Further additional safety measure required: yes: qualification. + Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. -.. tool_usecase:: PR Review - :id: tool_usecase__docs_as_code__pr_review - :belongs_to: doc_tool__score_docs_as_code - :version: 1 - - Repository contents are the source of truth - and every change is reviewed by a committer - (:need:`rl__committer`, :need:`doc_concept__wp_inspections`). - Still, for silent wrong outputs the gated CI stays green. - -.. tool_usecase:: Derived-view - :id: tool_usecase__docs_as_code__derived_view +.. tool_usecase:: Backlinks + :id: tool_usecase__docs_as_code__backlinks :belongs_to: doc_tool__score_docs_as_code :version: 1 - The rendered HTML output is a derived view; - the authoritative safety artifacts are mostly the source-controlled work products. - There are two exceptions, the architecture views (see M4) and backlinks (see M8). - Rendering/preview defects affect reviewer convenience, not safety evidence. - - .. potential_tool_malfunction:: Architecture visualization - :id: potential_tool_malfunction__docs_as_code__m4 - :safety_affected: YES - :detection_sufficient: YES - :safety_measures: PR review includes architecture inspection - :violates: tool_req__docs_arch_views - :version: 1 - - **Architecture visualization** — generate architecture diagrams. - See :need:`gd_req__arch_viewpoints`. + Generate correct backlinks for links between Needs items to provide + bi-directional traceability. See :need:`doc_concept__general_traceability`. - **Silent wrong-output**: a diagram misrepresents the architecture. - - Impact on safety: yes. - Impact safety measures available: yes: PR review includes architecture inspection. - Impact safety detection sufficient: yes. - Further additional safety measure required: no. - Confidence (automatic calculation): high. - - .. potential_tool_malfunction:: Backlinks + .. potential_tool_malfunction:: Backlinks failure :id: potential_tool_malfunction__docs_as_code__m8 :safety_affected: YES :detection_sufficient: NO :violates: tool_req__docs_req_link_satisfies_allowed :version: 1 - **Backlinks** — for bi-directional traceability, generate correct backlinks for links between Needs items. - See :need:`doc_concept__general_traceability`. - - **Silent wrong-output**: generated backlinks are wrong or missing. + **Silent wrong-output:** generated backlinks are wrong or missing. Impact on safety: yes. Impact safety measures available: no. @@ -327,21 +364,28 @@ Details Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. - .. potential_tool_malfunction:: Documentation generation +.. tool_usecase:: Documentation generation + :id: tool_usecase__docs_as_code__generation + :belongs_to: doc_tool__score_docs_as_code + :version: 1 + + Generate complete and correct HTML apart from the aspects covered by the + other use cases. See :need:`gd_req__doc_attributes_manual` and + :need:`gd_req__doc_attr_status`. + + .. potential_tool_malfunction:: Documentation generation failure :id: potential_tool_malfunction__docs_as_code__m9 :safety_affected: NO :violates: tool_req__docs_doc_types :version: 1 - **Documentation generation** — apart from the aspects **not covered by previous malfunctions**. - See :need:`gd_req__doc_attributes_manual`, :need:`gd_req__doc_attr_status`. - - Incomplete, outdated, or mis-rendered HTML. + **Wrong output:** incomplete, outdated, or mis-rendered HTML. Impact on safety: no: rendered-view defects affect reviewer convenience, not safety evidence. Impact safety measures available: no. - Impact safety detection sufficient: not applicable for a non-safety malfunction. + Impact safety detection sufficient: not applicable for a non-safety + malfunction. Further additional safety measure required: no. Confidence (automatic calculation): high. From e32e63e0bcb5e2aa2d36dd853168d966971fedf0 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 25 Sep 2026 23:50:35 +0200 Subject: [PATCH 17/17] Refine tool qualification report matrix --- .../requirements/tool_verification.rst | 36 ++-- .../tests/test_needs_templates.py | 61 ++++-- .../tool_qualification_report.need | 193 +++++++----------- 3 files changed, 134 insertions(+), 156 deletions(-) diff --git a/docs/internals/requirements/tool_verification.rst b/docs/internals/requirements/tool_verification.rst index 1c7d49ba6..cc588cd5e 100644 --- a/docs/internals/requirements/tool_verification.rst +++ b/docs/internals/requirements/tool_verification.rst @@ -105,7 +105,7 @@ The safety evaluation uses the following shared facts: Each of the following tool capabilities is evaluated as an intended use case with its corresponding potential malfunction. -.. tool_usecase:: Document metamodel enforcement +.. tool_usecase:: Enforce document types and attributes :id: tool_usecase__docs_as_code__metamodel :belongs_to: doc_tool__score_docs_as_code :version: 1 @@ -116,7 +116,7 @@ with its corresponding potential malfunction. :need:`gd_req__req_attr_safety`, :need:`gd_req__arch_attr_safety`, and :need:`gd_req__req_check_mandatory`. - .. potential_tool_malfunction:: Document metamodel enforcement failure + .. potential_tool_malfunction:: Invalid document passes metamodel checks :id: potential_tool_malfunction__docs_as_code__m1 :safety_affected: YES :detection_sufficient: NO @@ -167,7 +167,7 @@ with its corresponding potential malfunction. Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. -.. tool_usecase:: Safety-critical linking enforcement +.. tool_usecase:: Enforce safety-critical links :id: tool_usecase__docs_as_code__safety_links :belongs_to: doc_tool__score_docs_as_code :version: 1 @@ -175,7 +175,7 @@ with its corresponding potential malfunction. Enforce that links between safety-relevant needs preserve the required safety relationships. See :need:`gd_req__req_linkage_safety`. - .. potential_tool_malfunction:: Safety-critical linking enforcement failure + .. potential_tool_malfunction:: Unsafe link passes validation :id: potential_tool_malfunction__docs_as_code__m2 :safety_affected: YES :detection_sufficient: NO @@ -206,7 +206,7 @@ with its corresponding potential malfunction. Confidence (automatic calculation): low. -.. tool_usecase:: Requirements coverage statistics +.. tool_usecase:: Calculate requirement coverage :id: tool_usecase__docs_as_code__coverage :belongs_to: doc_tool__score_docs_as_code :version: 1 @@ -215,7 +215,7 @@ with its corresponding potential malfunction. compute link-coverage percentages. See :need:`gd_req__verification_reporting`. - .. potential_tool_malfunction:: Requirements coverage statistics failure + .. potential_tool_malfunction:: Requirement coverage is wrong :id: potential_tool_malfunction__docs_as_code__m3 :safety_affected: YES :detection_sufficient: NO @@ -232,14 +232,14 @@ with its corresponding potential malfunction. Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. -.. tool_usecase:: Architecture visualization +.. tool_usecase:: Generate architecture diagrams :id: tool_usecase__docs_as_code__architecture :belongs_to: doc_tool__score_docs_as_code :version: 1 Generate architecture diagrams. See :need:`gd_req__arch_viewpoints`. - .. potential_tool_malfunction:: Architecture visualization failure + .. potential_tool_malfunction:: Architecture diagram is wrong :id: potential_tool_malfunction__docs_as_code__m4 :safety_affected: YES :detection_sufficient: YES @@ -256,7 +256,7 @@ with its corresponding potential malfunction. Further additional safety measure required: no. Confidence (automatic calculation): high. -.. tool_usecase:: Test linkage +.. tool_usecase:: Resolve testcase verification links :id: tool_usecase__docs_as_code__test_linkage :belongs_to: doc_tool__score_docs_as_code :version: 1 @@ -266,7 +266,7 @@ with its corresponding potential malfunction. See :need:`gd_req__req_attr_testlink` and :need:`gd_req__verification_reporting`. - .. potential_tool_malfunction:: Test linkage failure + .. potential_tool_malfunction:: Requirement appears tested when it is not :id: potential_tool_malfunction__docs_as_code__m5 :safety_affected: YES :detection_sufficient: NO @@ -288,7 +288,7 @@ with its corresponding potential malfunction. Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. -.. tool_usecase:: Test reference check +.. tool_usecase:: Validate testcase references :id: tool_usecase__docs_as_code__test_refs :belongs_to: doc_tool__score_docs_as_code :version: 1 @@ -296,7 +296,7 @@ with its corresponding potential malfunction. Check that test references are present and point to the intended requirements. See :need:`gd_req__req_attr_testlink`. - .. potential_tool_malfunction:: Test reference check failure + .. potential_tool_malfunction:: Test reference is missing or outdated :id: potential_tool_malfunction__docs_as_code__m6 :safety_affected: YES :detection_sufficient: NO @@ -313,7 +313,7 @@ with its corresponding potential malfunction. Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. -.. tool_usecase:: Listing assumptions of use +.. tool_usecase:: List assumptions of use :id: tool_usecase__docs_as_code__assumptions :belongs_to: doc_tool__score_docs_as_code :version: 1 @@ -322,7 +322,7 @@ with its corresponding potential malfunction. safety manuals. See :need:`gd_guidl__saf_man` and :need:`wp__platform_safety_manual`. - .. potential_tool_malfunction:: Listing assumptions of use failure + .. potential_tool_malfunction:: Assumption of use is missing or wrong :id: potential_tool_malfunction__docs_as_code__m7 :safety_affected: YES :detection_sufficient: NO @@ -341,7 +341,7 @@ with its corresponding potential malfunction. Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. -.. tool_usecase:: Backlinks +.. tool_usecase:: Generate traceability backlinks :id: tool_usecase__docs_as_code__backlinks :belongs_to: doc_tool__score_docs_as_code :version: 1 @@ -349,7 +349,7 @@ with its corresponding potential malfunction. Generate correct backlinks for links between Needs items to provide bi-directional traceability. See :need:`doc_concept__general_traceability`. - .. potential_tool_malfunction:: Backlinks failure + .. potential_tool_malfunction:: Backlink is wrong or missing :id: potential_tool_malfunction__docs_as_code__m8 :safety_affected: YES :detection_sufficient: NO @@ -364,7 +364,7 @@ with its corresponding potential malfunction. Further additional safety measure required: yes (qualification). Confidence (automatic calculation): low. -.. tool_usecase:: Documentation generation +.. tool_usecase:: Generate HTML documentation :id: tool_usecase__docs_as_code__generation :belongs_to: doc_tool__score_docs_as_code :version: 1 @@ -373,7 +373,7 @@ with its corresponding potential malfunction. other use cases. See :need:`gd_req__doc_attributes_manual` and :need:`gd_req__doc_attr_status`. - .. potential_tool_malfunction:: Documentation generation failure + .. potential_tool_malfunction:: HTML output is incomplete or wrong :id: potential_tool_malfunction__docs_as_code__m9 :safety_affected: NO :violates: tool_req__docs_doc_types diff --git a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py index 29adeee45..3c477507b 100644 --- a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py +++ b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py @@ -173,21 +173,20 @@ def test_any_req_in_report_version_false_if_no_requirement_matches() -> None: assert any_req_in_report_version([a, b], "v1.0") is False -def test_qualification_evidence_deduplicates_shared_requirements( +def test_tool_qualification_matrix_groups_shared_requirements( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """Render one qualification-evidence row for a shared tool requirement. + """Render shared tool requirements in the qualification matrix. A tool requirement is reusable across project use cases. This fixture models two use cases, each with a different LOW-confidence malfunction, while both - malfunctions violate the same tool requirement. The qualification evidence - table is requirement-scoped, so it must show that requirement once and - combine its evidence rather than rendering one row for each malfunction. + malfunctions violate the same tool requirement. The matrix must expose the + three verification categories and place the unlinked requirement in the + corresponding column for each malfunction. - This is an integration test because the duplicate would be introduced by - the Jinja post-template's nested use-case/malfunction loops; checking the - Need graph alone would not verify the rendered report. + This is an integration test because the category assignment and table + structure are produced by the Jinja post-template. """ monkeypatch.setenv("BUILD_WORKSPACE_DIRECTORY", str(tmp_path)) (tmp_path / "conf.py").write_text( @@ -269,12 +268,46 @@ def test_qualification_evidence_deduplicates_shared_requirements( assert "First malfunction" not in purpose_section assert "safety_affected" not in purpose_section - qualification_section = html.split( - '
', maxsplit=1 - )[1].split('
', maxsplit=1)[0] + evaluation_section = html.split('
', maxsplit=1)[ + 1 + ].split('
', maxsplit=1)[0] + assert "Violates" not in evaluation_section + assert "Safety affected" in evaluation_section + assert "SCORE TCL" in evaluation_section + assert "Capability being evaluated." in evaluation_section + assert "table.tool-qualification-report tbody a" in evaluation_section + assert "LOW" in evaluation_section + assert "First use case (" not in evaluation_section + assert "First malfunction (" not in evaluation_section + assert '
First use case
' in evaluation_section + assert ( + '
' + "tool_usecase__shared_requirement_first
" in evaluation_section + ) + assert 'href="#tool_usecase__shared_requirement_first"' in evaluation_section + assert ( + 'href="#potential_tool_malfunction__shared_requirement_first"' + in evaluation_section + ) + + matrix_section = html.split('
', maxsplit=1)[ + 1 + ] + assert "Fully verified tool requirements" in matrix_section + assert "Partially verified tool requirements" in matrix_section + assert "Unverified tool requirements" in matrix_section + assert "Tool requirements (with testlinks)" not in matrix_section + assert "Tool requirements (without testlinks)" not in matrix_section + assert matrix_section.index("First use case") < matrix_section.index( + "First malfunction" + ) assert ( - qualification_section.count( - "Shared tool requirement (tool_req__shared_requirement)" + matrix_section.count( + 'tool_req__shared_requirement' ) - == 1 + == 2 ) + assert '
' not in html + assert '
' not in html diff --git a/src/needs_templates/tool_qualification_report.need b/src/needs_templates/tool_qualification_report.need index 2b06271fb..4d6b275d6 100644 --- a/src/needs_templates/tool_qualification_report.need +++ b/src/needs_templates/tool_qualification_report.need @@ -43,7 +43,7 @@ {% endfor %} {% macro need_reference(need) -%} -{% if need.get("external_url") %}`{{ need["id"] }} <{{ need["external_url"] }}>`_{% else %}:need:`{{ need["id"] }}`{% endif %} +{% if need.get("external_url") %}`{{ need["id"] }} <{{ need["external_url"] }}>`_{% else %}`{{ need["id"] }} <#{{ need["id"] }}>`_{% endif %} {%- endmacro %} {% macro need_title_reference(need) -%} @@ -62,49 +62,20 @@ {%- else -%}—{%- endif -%} {%- endmacro %} -{% macro requirements_for(malfunction, with_testlinks) -%} +{% macro requirements_for(malfunction, verification) -%} {%- set matching = namespace(items=[]) -%} {%- for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list -%} {%- set fully_verifying_tests = linked_needs(requirement["id"], "fully_verifies_back")|list -%} {%- set partially_verifying_tests = linked_needs(requirement["id"], "partially_verifies_back")|list -%} -{%- set has_testlinks = fully_verifying_tests or partially_verifying_tests -%} -{%- if (with_testlinks and has_testlinks) or (not with_testlinks and not has_testlinks) -%} +{%- if (verification == "fully" and fully_verifying_tests) + or (verification == "partially" and partially_verifying_tests and not fully_verifying_tests) + or (verification == "unlinked" and not fully_verifying_tests and not partially_verifying_tests) -%} {%- set matching.items = matching.items + [requirement] -%} {%- endif -%} {%- endfor -%} {{- need_list(matching.items) -}} {%- endmacro %} -{% macro tests_for(malfunction, link_name) -%} -{%- set matching = namespace(items=[], ids=[]) -%} -{%- for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list -%} -{%- for testcase in linked_needs(requirement["id"], link_name) -%} -{%- if testcase["id"] not in matching.ids -%} -{%- set matching.items = matching.items + [testcase] -%} -{%- set matching.ids = matching.ids + [testcase["id"]] -%} -{%- endif -%} -{%- endfor -%} -{%- endfor -%} -{{- need_list(matching.items) -}} -{%- endmacro %} - -{% macro qualification_tests(requirement) -%} -{%- set matching = namespace(items=[], ids=[]) -%} -{%- for link_name in ["fully_verifies_back", "partially_verifies_back"] -%} -{%- for testcase in linked_needs(requirement["id"], link_name) -%} -{%- if testcase["id"] not in matching.ids -%} -{%- set matching.items = matching.items + [testcase] -%} -{%- set matching.ids = matching.ids + [testcase["id"]] -%} -{%- endif -%} -{%- endfor -%} -{%- endfor -%} -{%- if matching.items -%} -{%- for testcase in matching.items -%} -{{ "| " if loop.first else "\n | " }}{{ need_reference(testcase) }} ({{ text(testcase.get("result")) }}){{ "," if not loop.last else "" }} -{%- endfor -%} -{%- else -%}—{%- endif -%} -{%- endmacro %} - {% set summary = namespace(total=0, safety=0, detection_yes=0, detection_no=0, detection_unassessed=0, low=false, fully_covered=0) %} {% for usecase in usecases %} {% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} @@ -278,24 +249,60 @@ Evaluation overview *Generated from linked report records.* +.. raw:: html + + + .. list-table:: Evaluation overview :header-rows: 1 - :widths: 20 24 25 12 25 16 - - * - Use case - - Potential malfunction - - Violates - - Safety affected - - Safety measures - - Detection sufficient + :class: evaluation-overview tool-qualification-report + :widths: 20 25 12 20 13 10 + + * - **Use case** + *Capability being evaluated.* + - **Potential malfunction** + *Failure mode if the capability is wrong.* + - **Safety affected** + *Can this affect safety?* + - **Safety measures** + *Measures used during normal operation.* + - **Detection sufficient** + *Detected before the output is relied on?* + - **SCORE TCL** + *Per-malfunction classification.* {% for usecase in usecases %} {% for malfunction in linked_needs(usecase["id"], "parent_needs_back") %} - * - {{ usecase["title"] }} ({{ need_reference(usecase) }}) - - {{ malfunction["title"] }} ({{ need_reference(malfunction) }}) - - {{ need_list(linked_needs(malfunction["id"], "violates")) }} + * - | {{ text(usecase.get("title")) }} + | {{ need_reference(usecase) }} + - | {{ text(malfunction.get("title")) }} + | {{ need_reference(malfunction) }} - {{ text(malfunction.get("safety_affected")) }} - {{ text(malfunction.get("safety_measures")) }} - {{ text(malfunction.get("detection_sufficient")) }} + - {{ "LOW" if malfunction.get("safety_affected") == "YES" and malfunction.get("detection_sufficient") == "NO" else "HIGH" }} {% endfor %} {% endfor %} {% if not usecases %} @@ -314,92 +321,30 @@ Tool qualification matrix The qualification matrix contains only safety-relevant malfunctions whose detection is insufficient. Every such malfunction must violate at least one -``tool_req``. +``tool_req``. Requirements are grouped by their strongest available testcase +link: full verification, partial verification without full verification, or no +verification link. .. list-table:: Tool qualification matrix :header-rows: 1 - :widths: 22 18 30 30 + :class: tool-qualification-report + :widths: 20 25 18 18 19 - * - Malfunction - - Use case description - - Tool requirements (with testlinks) - - Tool requirements (without testlinks) -{% for usecase in usecases %} -{% set malfunctions = linked_needs(usecase["id"], "parent_needs_back")|selectattr("safety_affected", "equalto", "YES")|selectattr("detection_sufficient", "equalto", "NO")|list %} -{% for malfunction in malfunctions %} - * - {{ malfunction["title"] }} ({{ need_reference(malfunction) }}) - - {{ text(usecase.get("title")) }} - - {{ requirements_for(malfunction, true) }} - - {{ requirements_for(malfunction, false) }} -{% endfor %} -{% endfor %} -{% if not summary.low %} - * - No qualification-relevant malfunction. - - — - - — - - — -{% endif %} - -Qualification evidence -^^^^^^^^^^^^^^^^^^^^^^ - -*Generated from linked report records.* - -Qualification is complete only when every relevant ``tool_req`` has a passed -testcase linked with ``fully_verifies``. A partial verification remains useful -traceability evidence but does not by itself complete qualification. - -.. list-table:: Qualification evidence - :header-rows: 1 - :widths: 38 35 27 - - * - Tool requirement requiring qualification - - Testcase(s) - - Verification status/result -{% set qualification_rows = namespace(items=[]) %} -{% for usecase in usecases %} -{% for malfunction in linked_needs(usecase["id"], "parent_needs_back")|selectattr("safety_affected", "equalto", "YES")|selectattr("detection_sufficient", "equalto", "NO")|list %} -{% for requirement in linked_needs(malfunction["id"], "violates")|selectattr("type", "equalto", "tool_req")|list %} -{% if requirement["id"] not in qualification_rows.items %} -{% set qualification_rows.items = qualification_rows.items + [requirement["id"]] %} - * - {{ need_reference(requirement) }} - - {{ qualification_tests(requirement) }} - - {{ "passed" if linked_needs(requirement["id"], "fully_verifies_back")|selectattr("result", "equalto", "passed")|list else "not fully qualified" }} -{% endif %} -{% endfor %} -{% endfor %} -{% endfor %} -{% if not qualification_rows.items %} - * - No qualification evidence required. - - — - - — -{% endif %} - -Traceability evidence -^^^^^^^^^^^^^^^^^^^^^ - -*Generated from linked report records.* - -The evidence below keeps safety measures and generated testcase links visible -for the qualification-relevant malfunctions. - -.. list-table:: Malfunction traceability - :header-rows: 1 - :widths: 20 25 25 15 15 - - * - Malfunction - - Violates requirements - - Safety measure - - Fully verified by - - Partially verified by + * - Use case + - Potential malfunction + - Fully verified tool requirements + - Partially verified tool requirements + - Unverified tool requirements {% for usecase in usecases %} {% set malfunctions = linked_needs(usecase["id"], "parent_needs_back")|selectattr("safety_affected", "equalto", "YES")|selectattr("detection_sufficient", "equalto", "NO")|list %} {% for malfunction in malfunctions %} - * - {{ malfunction["title"] }} ({{ need_reference(malfunction) }}) - - {{ need_list(linked_needs(malfunction["id"], "violates")) }} - - {{ text(malfunction.get("safety_measures")) }} - - {{ tests_for(malfunction, "fully_verifies_back") }} - - {{ tests_for(malfunction, "partially_verifies_back") }} + * - | {{ text(usecase.get("title")) }} + | {{ need_reference(usecase) }} + - | {{ text(malfunction.get("title")) }} + | {{ need_reference(malfunction) }} + - {{ requirements_for(malfunction, "fully") }} + - {{ requirements_for(malfunction, "partially") }} + - {{ requirements_for(malfunction, "unlinked") }} {% endfor %} {% endfor %} {% if not summary.low %}