From 89c6009d770915235595867564346b1a7aaa94f2 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 17:15:13 +0200 Subject: [PATCH 01/11] feat(provider-tck): emit a machine-readable conformance report Setting PROVIDER_TCK_REPORT_DIR makes each suite write its run to /.json against the report schema in the specification repository (open-feature/spec#425, part of open-feature/spec#424). Unset means no report, which is the default and is not an error. An environment variable rather than a TckConfig field, so that emitting a report is a property of the run and not of the code: CI sets it, a local run does not, and no adopter changes a line to publish one. Several suites in one pytest session each write their own file, so flagd's two resolvers would not collide. The load-bearing part is the per-scenario list. Appendix F requires that a scenario skipped for an undeclared capability is reported as skipped with the reason and never as passed, and nothing downstream can check that against a summary line. Recording every scenario's outcome individually makes the rule checkable by the consumer instead of dependent on the runner. It is also required to be complete, because a document that quietly dropped what it skipped would satisfy the letter of the rule and still mislead whoever read it. pytest, unlike godog, reports a skip honestly -- so the interesting divergence here is elsewhere. The one scenario the Python SDK cannot satisfy is marked xfail, so the run finishes green; the provider still did not satisfy it, and the document says failed with the reason. An expected failure is a recorded deviation, not an excused one. Scenarios are therefore enumerated at collection and resolved at the end of the session rather than as fixtures run, which is also what keeps a scenario skipped by a marker -- whose fixtures never run at all -- from vanishing from the document. Identity comes from spec_revision.json, generated by hatch_build_sync.py beside the copied assets and force-included into the wheel. It has to be captured at build time: the submodule that knows the answer is not in the distribution, so an installed copy has nothing left to ask. A build that cannot reach git -- an unpacked sdist -- warns and records "unknown" rather than inventing a commit. Both the commit and the tree hash are recorded, the tree because it identifies the assets alone: unchanged by unrelated edits elsewhere in the specification, so two runs of identical assets agree even when pinned to different commits, and checkable because `git rev-parse :specification/assets/provider-tck` reproduces it. Two smaller decisions. The provider is identified by the name it reports through its own metadata, with TckConfig.name recorded as the configuration, because TckConfig.name is chosen to read well in a failure message -- "flagd-rpc" -- and a provider with two materially different modes produces two reports that are not interchangeable. And how the backend was driven is read off an optional control_api property rather than added to the BackendControl protocol, so that adding it leaves every existing control complete and one that stays quiet simply omits the field. The tests assert the two properties a consumer is entitled to assume -- that no scenario the capability gate stopped is ever reported as passed, and that every collected scenario appears exactly once, counted against pytest's own collection rather than against a number written down beside it. Signed-off-by: Simon Schrottner --- tools/openfeature-provider-tck/.gitignore | 3 + tools/openfeature-provider-tck/README.md | 82 ++- tools/openfeature-provider-tck/hatch_build.py | 17 +- .../hatch_build_sync.py | 72 ++- tools/openfeature-provider-tck/pyproject.toml | 4 + .../contrib/tools/provider_tck/__init__.py | 4 + .../contrib/tools/provider_tck/capability.py | 12 + .../contrib/tools/provider_tck/control.py | 14 + .../contrib/tools/provider_tck/emitter.py | 297 ++++++++++ .../contrib/tools/provider_tck/inprocess.py | 10 + .../contrib/tools/provider_tck/plugin.py | 34 +- .../contrib/tools/provider_tck/report.py | 532 ++++++++++++++++++ .../contrib/tools/provider_tck/state.py | 7 + .../provider_tck/steps/provider_steps.py | 17 + .../tests/test_report.py | 438 ++++++++++++++ 15 files changed, 1534 insertions(+), 9 deletions(-) create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py create mode 100644 tools/openfeature-provider-tck/tests/test_report.py diff --git a/tools/openfeature-provider-tck/.gitignore b/tools/openfeature-provider-tck/.gitignore index 06664622..04ba5649 100644 --- a/tools/openfeature-provider-tck/.gitignore +++ b/tools/openfeature-provider-tck/.gitignore @@ -5,3 +5,6 @@ src/openfeature/contrib/tools/provider_tck/features/ src/openfeature/contrib/tools/provider_tck/flag_data/ src/openfeature/contrib/tools/provider_tck/control-api.yaml +# Generated alongside them, from the submodule pin, so a conformance report can +# name the spec revision it ran against. +src/openfeature/contrib/tools/provider_tck/spec_revision.json diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md index 040141f7..7017f0db 100644 --- a/tools/openfeature-provider-tck/README.md +++ b/tools/openfeature-provider-tck/README.md @@ -218,6 +218,72 @@ is recorded by the pin and nowhere else, so the two cannot drift apart unnoticed This mirrors what `openfeature-flagd-api-testkit` already does for the flagd test harness. +## Conformance reports + +Set `PROVIDER_TCK_REPORT_DIR` and each suite writes a machine-readable record of its run to +`/.json`, conforming to the [report schema][report-schema] in the specification. + +```console +$ PROVIDER_TCK_REPORT_DIR=./reports pytest +provider-tck [in-memory]: report written to reports/in-memory.json (1 failed, 5 not-declared, 23 passed) + +$ jq '.scenarios | group_by(.outcome) | map({(.[0].outcome): length}) | add' reports/in-memory.json +{ + "failed": 1, + "not-declared": 5, + "passed": 23 +} +``` + +It is an environment variable rather than a `TckConfig` field so that emitting a report is a property +of the *run* and not of the code: CI sets it, a developer running the suite locally does not, and no +adopter changes a line to publish one. Unset means no report, which is not an error. Several suites +in one pytest session each write their own file, so flagd's two resolvers would not collide. + +### Why every scenario is listed + +Appendix F requires that a scenario skipped for an undeclared capability is reported as skipped +**with the reason** and never as passed. A consumer cannot check that against a summary line, so the +report records the outcome of *every* scenario individually — and is required to be complete, because +a document that quietly dropped what it skipped would satisfy the letter of the rule and still +mislead whoever read it. + +Which also means the report is not a transcription of pytest's summary. The run above finishes green: +the one scenario the Python SDK cannot satisfy is marked `xfail` (finding 1), so pytest counts it as +expected and exits zero. The provider still did not satisfy it, and the document says `failed` with +the reason — an expected failure is a recorded deviation, not an excused one. + +Four outcomes rather than two, because "did not run" is not one thing: + +| Outcome | Means | +| --- | --- | +| `passed` | the scenario ran and passed | +| `failed` | the scenario ran and failed, including a known deviation marked `xfail` | +| `not-declared` | skipped because the provider did not declare a capability the scenario is tagged with | +| `not-applicable` | skipped for any other reason — a marker an adopter applied, a step calling `pytest.skip` | + +### What identifies a report + +`tck.specRevision` and `tck.assetsTree` come from `spec_revision.json`, which `hatch_build_sync.py` +generates from the submodule alongside the copied assets. It has to be captured at build time: the +submodule is not in the wheel, so an installed copy has nothing left to ask. A build that cannot +reach git — an unpacked sdist, say — warns and records `unknown` rather than inventing a commit. + +The tree hash is carried as well as the commit because it identifies the assets alone. It is +unchanged by unrelated edits elsewhere in the specification, so two runs that executed identical +assets report the same value even when pinned to different commits — and it is checkable, since +`git rev-parse :specification/assets/provider-tck` must reproduce it. + +`provider.name` is what the provider reports through its own metadata, not `TckConfig.name`. +`TckConfig.name` is chosen to read well in a failure message — `flagd-rpc` — which makes it the +*configuration*, and it is reported as such. One provider with two materially different modes +produces two reports that are not interchangeable. + +`backend.controlApi` is read off an optional `control_api` property on your `BackendControl`, +returning `"http"` or `"in-process"`. It is not a member of the protocol: adding one would make every +existing control incomplete for the sake of one string, and a control that stays quiet simply omits +the field. + ## The self-tests | Suite | Subject | Why | @@ -225,12 +291,15 @@ This mirrors what `openfeature-flagd-api-testkit` already does for the flagd tes | `test_in_memory_conformance` | the SDK's `InMemoryProvider` | reference adoption for a backend-less provider | | `test_controllable_conformance` | `ControllableInMemoryProvider` | the only suite that exercises the configuration-change path — see finding 2 | | `test_in_process_control` | `InProcessControl` | pins what the Gherkin cannot assert about itself | +| `test_report` | the conformance report | checks the two properties a consumer is entitled to assume | ``` -54 passed, 9 skipped, 2 xfailed +78 passed, 9 skipped, 2 xfailed ``` -No Docker, no network, under a second. +No Docker and no network. The conformance suites take under a second; `test_report` takes most of a +minute, because the properties it checks are properties of a whole pytest session and it runs four of +them in subprocesses to check them. Neither in-memory suite declares `@lifecycle`, so the three lifecycle scenarios are skipped in both. That is the point: with no backend to reach, they would pass without testing anything — which is @@ -242,7 +311,14 @@ what they did while the feature was gated on `@events`. cannot assert one *reached* the backend. That needs an echo operation on the control API. - **No HTTP control client yet.** It arrives with the first containerised adopter. - **Caching, hooks and flag metadata** are not covered. - +- **A report cannot name a Scenario Outline row portably.** Every row of an outline shares one + scenario name, and the report schema has nowhere to put the row, so several entries would be + indistinguishable — including, here, one that differs in outcome from its siblings. This + implementation qualifies the name with pytest's example id (`... [boolean-flag-Integer-1]`), which + is unambiguous but is not what another language would produce for the same row. Raised on + [open-feature/spec#424](https://github.com/open-feature/spec/issues/424). + +[report-schema]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/report/conformance-report.schema.json [appendix-a]: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md [appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md [spec]: https://github.com/open-feature/spec diff --git a/tools/openfeature-provider-tck/hatch_build.py b/tools/openfeature-provider-tck/hatch_build.py index 4b6f1d84..3e187984 100644 --- a/tools/openfeature-provider-tck/hatch_build.py +++ b/tools/openfeature-provider-tck/hatch_build.py @@ -18,7 +18,14 @@ # the single definition of what gets copied where -- would not be importable. sys.path.insert(0, str(Path(__file__).parent)) -from hatch_build_sync import FILES, PACKAGE_REL, SPEC_ASSETS, TREES, sync +from hatch_build_sync import ( + FILES, + PACKAGE_REL, + REVISION_FILE, + SPEC_ASSETS, + TREES, + sync, +) class SpecAssetsCopyHook(BuildHookInterface): @@ -26,7 +33,13 @@ class SpecAssetsCopyHook(BuildHookInterface): def initialize(self, version: str, build_data: dict) -> None: root = Path(self.root) - copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + # The generated revision file travels with the assets it describes. It + # has to be built here rather than read at run time, because the + # submodule that knows the answer is not in the wheel and a conformance + # report has to name the revision it ran against. + copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + [ + root / PACKAGE_REL / REVISION_FILE + ] # Building from a checkout: refresh from the submodule, so what ships is # always the revision the pin names. Building from an sdist: there is no diff --git a/tools/openfeature-provider-tck/hatch_build_sync.py b/tools/openfeature-provider-tck/hatch_build_sync.py index f31bc55b..acee9bf0 100644 --- a/tools/openfeature-provider-tck/hatch_build_sync.py +++ b/tools/openfeature-provider-tck/hatch_build_sync.py @@ -10,11 +10,16 @@ needs no submodule: the copies are inside the distribution. """ +import json import shutil +import subprocess +import warnings from pathlib import Path ROOT = Path(__file__).parent -SPEC_ASSETS = (ROOT / "spec/specification/assets/provider-tck").resolve() +SPEC_ROOT = (ROOT / "spec").resolve() +ASSETS_PATH_IN_SPEC = "specification/assets/provider-tck" +SPEC_ASSETS = (SPEC_ROOT / ASSETS_PATH_IN_SPEC).resolve() PACKAGE_REL = Path("src/openfeature/contrib/tools/provider_tck") DEST_BASE = ROOT / PACKAGE_REL @@ -28,6 +33,27 @@ TREES = [("gherkin", "features"), ("flags", "flag_data")] FILES = [("openapi/control-api.yaml", "control-api.yaml")] +REVISION_FILE = "spec_revision.json" +"""Which revision of the specification the copied assets came from. + +Recorded at build time because the answer is only available at build time: the +submodule that holds it is not in the wheel, and a conformance report that cannot +name the revision it ran against cannot be compared with another. It is generated +by the same command that copies the assets, which is what keeps the two from +disagreeing. + +Not committed, for the same reason the assets are not: the submodule pin is the +single record of which revision this package targets. +""" + +UNKNOWN_REVISION = "unknown" +"""Seven characters, the minimum the report schema accepts. + +A build that cannot reach git says it does not know rather than inventing a +commit, and still produces a document that validates. Which happens for real: +building from a source tarball has no ``.git`` to ask. +""" + def sync() -> None: if not SPEC_ASSETS.exists(): @@ -51,6 +77,50 @@ def sync() -> None: dest.unlink() shutil.copy2(SPEC_ASSETS / src_name, dest) + write_revision() + + +def write_revision() -> None: + """Record the spec commit and the asset tree these copies came from. + + The tree hash is carried as well as the commit because it identifies the + assets alone: it does not change when an unrelated part of the specification + does, so two runs that executed identical assets report the same value even + when pinned to different commits. It is also checkable rather than merely + asserted, since ``git rev-parse :specification/assets/provider-tck`` + must reproduce it. + """ + commit = _git("rev-parse", "HEAD") or UNKNOWN_REVISION + tree = _git("rev-parse", f"HEAD:{ASSETS_PATH_IN_SPEC}") or "" + (DEST_BASE / REVISION_FILE).write_text( + json.dumps({"specRevision": commit, "assetsTree": tree}, indent=2) + "\n", + encoding="utf-8", + ) + + +def _git(*args: str) -> str: + """Run git inside the submodule, returning its output or an empty string. + + A build must not hard-fail because git is absent or the checkout is not a + repository -- both are ordinary when building from an unpacked sdist. The + failure is reported as a warning and the identity degrades to ``unknown``, + which is legible in the resulting report rather than silently wrong. + """ + command = ["git", "-C", str(SPEC_ROOT), *args] + try: + completed = subprocess.run( # noqa: S603 + command, capture_output=True, check=True, text=True + ) + except (OSError, subprocess.CalledProcessError) as error: + warnings.warn( + f"could not determine the spec revision ({' '.join(command)}: {error}); " + f"conformance reports from this build will not name the revision they " + f"ran against", + stacklevel=2, + ) + return "" + return completed.stdout.strip() + if __name__ == "__main__": sync() diff --git a/tools/openfeature-provider-tck/pyproject.toml b/tools/openfeature-provider-tck/pyproject.toml index ff0cbe43..5e23849a 100644 --- a/tools/openfeature-provider-tck/pyproject.toml +++ b/tools/openfeature-provider-tck/pyproject.toml @@ -58,6 +58,10 @@ artifacts = [ "src/openfeature/contrib/tools/provider_tck/features/", "src/openfeature/contrib/tools/provider_tck/flag_data/", "src/openfeature/contrib/tools/provider_tck/control-api.yaml", + # Which spec revision those assets came from, generated beside them. The + # submodule is not in the wheel, so a conformance report emitted by an + # installed copy has no other way to name the revision it ran against. + "src/openfeature/contrib/tools/provider_tck/spec_revision.json", ] [tool.hatch.build.hooks.custom] diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py index 8b615296..8f4e651b 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -62,15 +62,19 @@ def tck_config(): ControllableInMemoryProvider, canonical_flag_set, ) +from .report import REPORT_DIR_ENV, SCHEMA_VERSION, Outcome __all__ = [ "ALL_CAPABILITIES", "CHANGING_FLAG_KEY", + "REPORT_DIR_ENV", + "SCHEMA_VERSION", "BackendControl", "Capability", "ConnectionControl", "ControllableInMemoryProvider", "InProcessControl", + "Outcome", "TckConfig", "UnsupportedControlError", "canonical_flag_set", diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py index 6021b461..c285b276 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py @@ -120,6 +120,7 @@ def __str__(self) -> str: """ _BY_MARKER: dict[str, Capability] = {c.value: c for c in Capability} +_BY_TAG: dict[str, Capability] = {c.tag: c for c in Capability} def capability_for_marker(name: str) -> Capability | None: @@ -129,3 +130,14 @@ def capability_for_marker(name: str) -> Capability | None: the canonical feature files carry organisational tags freely. """ return _BY_MARKER.get(name) + + +def capability_for_tag(tag: str) -> Capability | None: + """Map a Gherkin tag, leading at-sign included, onto the capability it gates. + + The tag form rather than the marker form because that is what the + conformance report carries: the report records a scenario's tags as the + feature files spell them, and deciding whether a failure counts against a + capability means reading them back. + """ + return _BY_TAG.get(tag) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py index 0e83e5bd..e92dd18d 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py @@ -73,6 +73,20 @@ def change_flag(self) -> None: def description(self) -> str: """A short description of what is being controlled, for messages a human reads.""" + # OPTIONAL: ``control_api`` + # + # A control may also offer a ``control_api`` property returning ``"http"`` + # for the normative HTTP control API, or ``"in-process"`` for the narrow + # allowance made for providers with no backend. The conformance report + # records it, so that a claim of in-process control by a provider that does + # have a backend can be treated with the suspicion it deserves. + # + # It is deliberately not a member of this protocol. Adding one would make + # every existing control incomplete for the sake of one string, and there is + # nothing useful the TCK can do with a control that has not said: it cannot + # tell from the outside whether a control spoke HTTP or reached into the + # process, so the field is simply omitted. See ``report.control_api_of``. + @typing.runtime_checkable class ConnectionControl(typing.Protocol): diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py new file mode 100644 index 00000000..abfe3c64 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py @@ -0,0 +1,297 @@ +"""The pytest half of the conformance report: turning a run into the document. + +Kept apart from :mod:`report`, which knows what a report *is* and nothing about +pytest. Everything here is translation -- a pytest node into a scenario, a +:class:`pytest.TestReport` into an :class:`~.report.Outcome`, the end of a +session into a file on disk. + +The translation that matters is the one for skips. pytest reports a skip +honestly, unlike some runners, but "skipped" alone does not distinguish a +capability the provider never declared from a scenario the run had some other +reason not to execute, and the report format does. So the decision is made +against the scenario's own tags and the suite's declared capabilities rather than +against the wording of a skip message. +""" + +from __future__ import annotations + +import os +import typing +from pathlib import Path + +import pytest + +from .config import TckConfig +from .report import ( + REPORT_DIR_ENV, + Outcome, + PhaseOutcome, + ReportCollector, + ScenarioIdentity, + normalise_tags, + report_file_name, + write_report, +) + +__all__ = ["COLLECTOR_KEY", "ReportEmitter", "classify_phase", "scenario_identity"] + +COLLECTOR_KEY = pytest.StashKey[ReportCollector]() +"""Where the session's collector lives, so a fixture can reach it from a request.""" + +_MAX_REASON = 500 +"""How much of a failure message the report carries. + +A reason is for a person reading a comparison page, not for debugging: whoever +ran the suite has the traceback. Whole tracebacks in a published document also +leak local paths. +""" + + +def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: + """Describe a pytest node as a Gherkin scenario, or return ``None``. + + ``__scenario__`` is what pytest-bdd hangs on the function it generates, so + its presence is also the test for "is this a TCK scenario at all" -- and it + is readable at collection, without running a single fixture, which is what + lets a scenario skipped before its first step still be accounted for. + """ + scenario = getattr(getattr(node, "function", None), "__scenario__", None) + if scenario is None: + return None + + feature = getattr(scenario, "feature", None) + tags: set[str] = set(getattr(scenario, "tags", None) or ()) + tags |= set(getattr(feature, "tags", None) or ()) + rule = getattr(scenario, "rule", None) + if rule is not None: + tags |= set(getattr(rule, "tags", None) or ()) + + return ScenarioIdentity( + feature=Path(str(getattr(feature, "filename", ""))).stem, + name=_scenario_name(node, str(getattr(scenario, "name", ""))), + tags=normalise_tags(tags), + ) + + +def _scenario_name(node: pytest.Item, name: str) -> str: + """Qualify a Scenario Outline's name with the example row that ran. + + Every row of an outline shares one scenario name, so a report using the name + alone would carry several entries a consumer cannot tell apart -- and in this + suite one row of an outline genuinely differs in outcome from its siblings. + The schema has nowhere to put the row, so it goes in the name, in the form + pytest already uses to select one: ``... [boolean-flag-Integer-1]``. + """ + example_id = getattr(getattr(node, "callspec", None), "id", "") + return f"{name} [{example_id}]" if example_id else name + + +def _group_of(node: pytest.Item) -> str: + """Which module a scenario was generated into. + + pytest-bdd's ``scenarios()`` injects its tests into the module that called + it, and a module resolves one ``tck_config``, so the module is what says + which suite a scenario belongs to. Two modules sharing a ``tck_config`` from + a conftest are two groups pointing at one suite, which is exactly right. + """ + return node.nodeid.partition("::")[0] + + +class ReportEmitter: + """Collects outcomes for the session and writes one report per suite. + + A plugin object rather than module-level hook functions because + ``pytest_runtest_logreport`` is handed a report and nothing else: the state + it has to reach has to come from somewhere, and an instance is a less + surprising somewhere than a module global. + """ + + def __init__(self, config: pytest.Config) -> None: + self.collector = ReportCollector() + config.stash[COLLECTOR_KEY] = self.collector + + def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: + """Enumerate every TCK scenario the session collected. + + At collection rather than as each runs, so that the document accounts for + scenarios that never got as far as running a fixture. + """ + for item in items: + identity = scenario_identity(item) + if identity is not None: + self.collector.collect(item.nodeid, _group_of(item), identity) + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + self.collector.observe(report.nodeid, _phase_outcome(report)) + + def pytest_sessionfinish(self, session: pytest.Session) -> None: + directory = os.environ.get(REPORT_DIR_ENV, "").strip() + if not directory: + return + self.write(session, Path(directory)) + + def write(self, session: pytest.Session, directory: Path) -> None: + """Write every suite's report, failing the session if one cannot be written. + + A run that asked for a report and silently did not get one is how a + publishing pipeline ends up serving a stale result forever, so both a + write failure and an incomplete document are loud and change the exit + status rather than being logged and forgotten. + """ + for problem in self.collector.resolve(classify_phase): + self._fail(session, f"provider-tck: {problem}") + + written: dict[str, str] = {} + for suite in self.collector.suites: + name = suite.config.name + file_name = report_file_name(name) + if written.get(file_name, name) != name: + self._fail( + session, + f"provider-tck: suites {written[file_name]!r} and {name!r} both " + f"write {file_name}; give them names that do not collide", + ) + continue + written[file_name] = name + + try: + path = write_report(directory, name, suite.build()) + except OSError as error: + self._fail( + session, + f"provider-tck [{name}]: could not write the conformance report " + f"to {directory}: {error}", + ) + continue + counts = ", ".join( + f"{count} {outcome}" + for outcome, count in sorted(suite.counts().items()) + ) + self._say( + session, f"provider-tck [{name}]: report written to {path} ({counts})" + ) + + def _say(self, session: pytest.Session, message: str) -> None: + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line(message) + + def _fail(self, session: pytest.Session, message: str) -> None: + self._say(session, message) + session.exitstatus = pytest.ExitCode.INTERNAL_ERROR + + +def _phase_outcome(report: pytest.TestReport) -> PhaseOutcome: + """Reduce a pytest phase report to what the conformance report needs.""" + xfail_reason: str | None = getattr(report, "wasxfail", None) + message = _skip_reason(report) if report.skipped else _failure_reason(report) + return PhaseOutcome( + when=report.when or "", + outcome=report.outcome, + xfail_reason=xfail_reason, + message=message, + duration=report.duration, + ) + + +def classify_phase( + phase: PhaseOutcome, identity: ScenarioIdentity, config: TckConfig +) -> tuple[Outcome, str] | None: + """Map one phase onto an outcome, or onto nothing. + + Nothing is the answer for a setup or teardown that simply worked: it says + nothing about the scenario, and letting it speak would overwrite what the + call phase already established. + """ + if phase.outcome == "skipped" and phase.xfail_reason is not None: + # An expected failure is still a failure. The provider did not satisfy + # the scenario, and a report calling it anything else would hide exactly + # the deviation the marker was added to keep visible. + return Outcome.FAILED, _reason(f"expected failure: {phase.xfail_reason}") + if phase.outcome == "failed": + return Outcome.FAILED, phase.message or "failed" + if phase.outcome == "skipped": + return _skipped(phase, identity, config) + if phase.when == "call": + return Outcome.PASSED, "" + return None + + +def _skipped( + phase: PhaseOutcome, identity: ScenarioIdentity, config: TckConfig +) -> tuple[Outcome, str]: + """Tell a capability skip apart from every other kind. + + Decided from the scenario's tags and the suite's declared capabilities rather + than from the skip message, because the message is prose and the distinction + is not. Anything else that skipped a scenario -- a marker an adopter applied, + a step calling ``pytest.skip`` -- is reported as not applicable: it did not + run, and not because a capability was left undeclared. + """ + undeclared = [ + capability.tag + for capability in identity.capabilities() + if not config.declares(capability) + ] + if undeclared: + return Outcome.NOT_DECLARED, phase.message or ( + f"provider does not declare {' '.join(undeclared)}" + ) + return Outcome.NOT_APPLICABLE, phase.message or "skipped" + + +def _skip_reason(report: pytest.TestReport) -> str: + longrepr = report.longrepr + if isinstance(longrepr, tuple) and len(longrepr) == 3: + return _reason(str(longrepr[2]).removeprefix("Skipped: ")) + return _reason(str(longrepr)) if longrepr else "" + + +def _failure_reason(report: pytest.TestReport) -> str: + message = getattr(getattr(report.longrepr, "reprcrash", None), "message", "") + if not message: + message = str(report.longrepr) if report.longrepr else "" + return _reason(message) + + +def _reason(message: str) -> str: + collapsed = " ".join(message.split()) + if len(collapsed) <= _MAX_REASON: + return collapsed + return collapsed[: _MAX_REASON - 1].rstrip() + "…" + + +def observe_provider_name( + config: pytest.Config, tck_config: TckConfig, provider_name: str | None +) -> None: + """Record what the provider called itself, for the suite the run is in. + + The provider's own metadata name rather than the suite name, because the two + answer different questions: the suite name is chosen to read well in a + failure message, which makes it the configuration and it is reported as one. + """ + collector: ReportCollector | None = config.stash.get(COLLECTOR_KEY, None) + if collector is not None and provider_name: + collector.suite_for(tck_config).observe_provider_name(provider_name) + + +def bind_scenario(request: pytest.FixtureRequest) -> None: + """Tell the collector which suite this scenario's module is testing. + + Called from an autouse fixture that the capability gate depends on, so that a + scenario the gate stops has still contributed its suite. Only one scenario of + a module has to get this far, but the gate skips whole capabilities at a + time, and a module all of whose scenarios were skipped would otherwise have + no report to be written to. + """ + collector: ReportCollector | None = request.config.stash.get(COLLECTOR_KEY, None) + if collector is None or scenario_identity(request.node) is None: + # Checked before asking for the config so that a test which is not a TCK + # scenario instantiates nothing, which is the same bargain the capability + # gate makes. + return + try: + tck_config = typing.cast(TckConfig, request.getfixturevalue("tck_config")) + except pytest.FixtureLookupError: + return + collector.bind(request.node.nodeid, tck_config) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py index 1d69254c..11586e0f 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py @@ -63,6 +63,16 @@ def __init__(self) -> None: def description(self) -> str: return "in-process control of an in-memory provider" + @property + def control_api(self) -> str: + """Report how this backend was driven, for the conformance report. + + ``in-process`` is the narrow allowance for providers with no backend, + which is exactly what this control exists for. A provider that does have + a backend and reports this is claiming something it should not. + """ + return "in-process" + def new_provider(self) -> FeatureProvider: """Create the provider for the scenario about to run, at the baseline. diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py index b8b1a73b..29aeb0d5 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py @@ -17,6 +17,7 @@ from .capability import Capability, capability_for_marker from .config import TckConfig +from .emitter import ReportEmitter, bind_scenario, observe_provider_name from .state import TckState # The step modules are registered as plugins in their own right, not merely @@ -32,22 +33,33 @@ def pytest_configure(config: pytest.Config) -> None: - """Register the capability tags as markers. + """Register the capability tags as markers, and the report emitter. pytest-bdd turns every Gherkin tag into a marker with ``getattr(pytest.mark, tag)`` without registering it, which raises ``PytestUnknownMarkWarning`` for each one -- noise at best, and a hard failure in a project configured with ``-W error``. + + The emitter is registered unconditionally even though it writes nothing + unless :data:`~.report.REPORT_DIR_ENV` is set. Accumulating the outcomes + costs a dictionary entry per scenario, and deciding at the end of the session + rather than at the start is one fewer way for a run to discover too late that + it was not recording. """ for capability in Capability: config.addinivalue_line( "markers", f"{capability.value}: OpenFeature provider TCK capability {capability.tag}", ) + config.pluginmanager.register( + ReportEmitter(config), "openfeature-provider-tck-report" + ) @pytest.fixture -def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: +def tck_state( + request: pytest.FixtureRequest, tck_config: TckConfig +) -> typing.Iterator[TckState]: """Per-scenario state, carried between step definitions.""" # Resetting here rather than in an autouse fixture ties the reset to the # scenarios that actually use the TCK, and guarantees it happens after the @@ -56,11 +68,22 @@ def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: tck_config.control.prepare_scenario() state = TckState(config=tck_config) yield state + # The provider is identified in the report by what it called itself, and the + # only thing that ever holds an instance is the scenario that made one. + observe_provider_name(request.config, tck_config, state.provider_name) state.teardown() @pytest.fixture(autouse=True) -def _tck_capability_gate(request: pytest.FixtureRequest) -> None: +def _tck_report_binding(request: pytest.FixtureRequest) -> None: + """Attribute this scenario to its suite before anything can skip it.""" + bind_scenario(request) + + +@pytest.fixture(autouse=True) +def _tck_capability_gate( + request: pytest.FixtureRequest, _tck_report_binding: None +) -> None: """Skip a scenario whose capability the provider did not declare. ``pytest.skip`` here reports the scenario as skipped **with the reason**, @@ -75,6 +98,11 @@ def _tck_capability_gate(request: pytest.FixtureRequest) -> None: Checking markers first also means the gate costs nothing, and instantiates nothing, for tests that are not TCK scenarios. + + ``_tck_report_binding`` is requested rather than left to autouse ordering so + that the scenario has reached its suite before this fixture can skip it. A + scenario skipped here is exactly the one the conformance report must account + for, and one that never reached a suite could not be reported at all. """ gated = [ capability diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py new file mode 100644 index 00000000..8979f580 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py @@ -0,0 +1,532 @@ +"""The machine-readable conformance report: what a run of the suite claims. + +A run of the suite produces a pass or a fail on a terminal, which is enough for +the person who started it and useless to anyone else. The report is the same run +written down in a form something other than a human can read -- a comparison +page, an aggregator, a release gate -- against a schema owned by the +specification rather than by this package, so that four languages emit the same +document. + +The load-bearing part is the per-scenario list. Appendix F requires that a +scenario skipped for an undeclared capability is reported as skipped *with the +reason* and never as passed, and a summary line cannot be checked against that +rule by anything downstream. Recording every scenario's outcome individually +makes the rule checkable by the consumer instead of dependent on each runner's +summary being trustworthy -- and the outcomes are required to be complete, +because a report that silently omitted what it skipped would satisfy the letter +of the rule while still misleading its reader. + +See https://github.com/open-feature/spec/issues/424 for the format and +``specification/assets/provider-tck/report/`` for the schema. +""" + +from __future__ import annotations + +import importlib.metadata +import importlib.resources +import json +import re +import typing +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path + +from .capability import Capability, capability_for_tag +from .config import TckConfig + +__all__ = [ + "REPORT_DIR_ENV", + "SCHEMA_VERSION", + "Outcome", + "PhaseOutcome", + "ReportCollector", + "ScenarioIdentity", + "ScenarioRecord", + "SuiteReport", + "report_file_name", +] + +REPORT_DIR_ENV = "PROVIDER_TCK_REPORT_DIR" +"""Names the directory a conformance report is written to. + +An environment variable rather than a :class:`~.config.TckConfig` field, so that +emitting a report is a property of the *run* and not of the code: CI sets it, a +developer running the suite locally does not, and no adopter changes a line to +publish one. Each suite writes ``/.json``, so several suites in one +pytest session -- flagd's RPC and in-process resolvers, say -- each produce their +own file without colliding. + +Unset means no report, which is the default and is not an error. +""" + +SCHEMA_VERSION = "1" +"""The major version of the report schema this emitter produces.""" + +TCK_IMPLEMENTATION = "python-sdk-contrib/tools/openfeature-provider-tck" +"""Which TCK implementation produced the report, as the schema spells it.""" + +PROVIDER_LANGUAGE = "python" + +SDK_DISTRIBUTION = "openfeature-sdk" +TCK_DISTRIBUTION = "openfeature-provider-tck" + +UNKNOWN = "unknown" +"""Stands in for an identity that could not be read. + +Seven characters, which is the schema's minimum for ``tck.specRevision``, so a +build that could not reach git still emits a document that validates and says +plainly that it does not know rather than inventing a commit. +""" + +_PACKAGE = "openfeature.contrib.tools.provider_tck" + +_REVISION_FILE = "spec_revision.json" +"""Written at build time from the spec submodule; see ``hatch_build_sync.py``. + +Read from a data file rather than the submodule because the submodule is not in +the published wheel: an adopter installing this package has no ``spec/`` +directory to interrogate, and the revision the assets came from is exactly what +the report has to name. +""" + +_TAG_PATTERN = re.compile(r"^[a-z0-9-]+$") +"""What the schema accepts as a tag, minus the leading at-sign. + +Tags that do not match are dropped rather than emitted, because an invalid +document helps nobody; the canonical feature files carry none, so this only bites +a feature file that has been forked, which is itself worth noticing. +""" + +_UNSAFE_IN_FILENAME = re.compile(r"[^A-Za-z0-9._-]") + + +class Outcome(str, Enum): + """The result of one scenario, or of one capability. + + Four rather than two, because "did not run" is not one thing. A capability + the provider chose not to declare is a different statement from one the + language makes impossible -- ``@strict-numeric-typing`` cannot hold in a + language with no integer type -- and reporting both as not declared would + show a whole language as missing something none of its providers can have. + """ + + PASSED = "passed" + FAILED = "failed" + NOT_DECLARED = "not-declared" + NOT_APPLICABLE = "not-applicable" + + +@dataclass(frozen=True) +class ScenarioIdentity: + """What a scenario is, independent of how it turned out. + + Established at collection, from the pytest-bdd node alone, so that a scenario + skipped before a single step ran is identified exactly as fully as one that + passed. That is what lets the report account for every scenario rather than + only for the ones that got far enough to be interesting. + """ + + feature: str + name: str + tags: tuple[str, ...] + + def capabilities(self) -> tuple[Capability, ...]: + """The capabilities this scenario's tags gate it behind.""" + gated = (capability_for_tag(tag) for tag in self.tags) + return tuple(capability for capability in gated if capability is not None) + + +@dataclass +class ScenarioRecord: + """One scenario's outcome, as the report will carry it.""" + + feature: str + """The feature file without its extension, e.g. ``errors``.""" + + name: str + tags: tuple[str, ...] + outcome: Outcome + reason: str = "" + duration_ms: float = 0.0 + + def as_json(self) -> dict[str, typing.Any]: + document: dict[str, typing.Any] = { + "feature": self.feature, + "name": self.name, + "outcome": self.outcome.value, + } + if self.tags: + document["tags"] = list(self.tags) + if self.reason: + document["reason"] = self.reason + if self.duration_ms: + document["durationMs"] = round(self.duration_ms, 3) + return document + + +@dataclass +class SuiteReport: + """What one suite -- one :class:`~.config.TckConfig` -- accumulates as it runs. + + Records are keyed by pytest node id rather than appended to a list, which is + what makes "every scenario appears exactly once" a property of the structure + instead of a promise made by the code that fills it. A scenario reports + through several phases (setup, call, teardown) and each of them finds the + same entry. + """ + + config: TckConfig + provider_name: str | None = None + records: dict[str, ScenarioRecord] = field(default_factory=dict) + durations: dict[str, float] = field(default_factory=dict) + + def observe_provider_name(self, name: str) -> None: + """Remember what the provider called itself through its own metadata. + + Last one wins, and they should all agree: a suite tests one provider. + """ + if name: + self.provider_name = name + + def add_duration(self, node_id: str, seconds: float) -> None: + """Add one phase's time to a scenario's total. + + Kept apart from the record rather than added to it, because a scenario's + first phase can take time before anything has decided its outcome, and + time spent on a scenario that ended up skipped is still time. + """ + self.durations[node_id] = self.durations.get(node_id, 0.0) + seconds * 1000.0 + + def set_outcome( + self, + node_id: str, + identity: ScenarioIdentity, + outcome: Outcome, + reason: str = "", + ) -> None: + """Record, or revise, one scenario's outcome. + + A failure is never revised away. A scenario whose steps passed and whose + teardown then blew up is a failed scenario, and the phase that reports + last must not be the one that decides. + """ + record = self.records.get(node_id) + if record is None: + self.records[node_id] = ScenarioRecord( + feature=identity.feature, + name=identity.name, + tags=identity.tags, + outcome=outcome, + reason=reason, + ) + return + if record.outcome is Outcome.FAILED: + return + record.outcome = outcome + record.reason = reason or record.reason + + @property + def sorted_records(self) -> list[ScenarioRecord]: + for node_id, record in self.records.items(): + record.duration_ms = self.durations.get(node_id, 0.0) + return sorted(self.records.values(), key=lambda r: (r.feature, r.name)) + + def counts(self) -> dict[str, int]: + """Outcome tallies, for a log line and for the tests that check them.""" + tally: dict[str, int] = {} + for record in self.records.values(): + tally[record.outcome.value] = tally.get(record.outcome.value, 0) + 1 + return tally + + def build(self) -> dict[str, typing.Any]: + """Assemble the report document.""" + records = self.sorted_records + spec_revision, assets_tree = spec_identity() + + tck: dict[str, typing.Any] = { + "implementation": TCK_IMPLEMENTATION, + "version": distribution_version(TCK_DISTRIBUTION), + "specRevision": spec_revision, + } + if assets_tree: + tck["assetsTree"] = assets_tree + + document: dict[str, typing.Any] = { + "schemaVersion": SCHEMA_VERSION, + "provider": { + # What the provider calls itself, not the suite name: the suite + # name is chosen to read well in a failure message -- "flagd-rpc" + # -- which makes it the configuration, and it is reported as one. + # A provider with two materially different modes therefore + # produces two reports that are not interchangeable. + "name": self.provider_name or self.config.name, + "language": PROVIDER_LANGUAGE, + "configuration": self.config.name, + }, + "sdk": { + "name": SDK_DISTRIBUTION, + "version": distribution_version(SDK_DISTRIBUTION), + }, + "tck": tck, + "capabilities": self._capabilities(records), + "scenarios": [record.as_json() for record in records], + } + + backend = self._backend() + if backend: + document["backend"] = backend + return document + + def _backend(self) -> dict[str, typing.Any]: + backend: dict[str, typing.Any] = {} + description = getattr(self.config.control, "description", "") + if isinstance(description, str) and description: + backend["description"] = description + control_api = control_api_of(self.config.control) + if control_api: + backend["controlApi"] = control_api + return backend + + def _capabilities( + self, records: list[ScenarioRecord] + ) -> dict[str, dict[str, typing.Any]]: + """Roll the per-scenario outcomes up to one verdict per capability. + + A capability is only reported as passed when everything gating on it + actually passed, and only reported as not declared when the provider did + not declare it -- in which case the reason says so, because "this + provider does not support configuration-change events" is exactly what + someone comparing providers came to find out. + """ + failed: set[Capability] = set() + for record in records: + if record.outcome is not Outcome.FAILED: + continue + for tag in record.tags: + capability = capability_for_tag(tag) + if capability is not None: + failed.add(capability) + + capabilities: dict[str, dict[str, typing.Any]] = {} + for capability in Capability: + if not self.config.declares(capability): + capabilities[capability.tag] = { + "state": Outcome.NOT_DECLARED.value, + "reason": ( + f"not declared by this provider's configuration; the " + f"{capability.tag} scenarios were skipped and did not " + f"contribute to this result" + ), + } + elif capability in failed: + capabilities[capability.tag] = { + "state": Outcome.FAILED.value, + "reason": f"at least one {capability.tag} scenario failed", + } + else: + capabilities[capability.tag] = {"state": Outcome.PASSED.value} + return capabilities + + +@dataclass(frozen=True) +class PhaseOutcome: + """One pytest phase report, reduced to what the conformance report needs. + + Reduced rather than kept, because a :class:`pytest.TestReport` holds a + formatted traceback and holding a session's worth of them to classify at the + end would be a memory leak with a nice name. + """ + + when: str + """``setup``, ``call`` or ``teardown``.""" + + outcome: str + """``passed``, ``failed`` or ``skipped``, as pytest decided.""" + + xfail_reason: str | None = None + """Set when pytest marked this an expected failure.""" + + message: str = "" + """The skip reason, or the failure's headline, already trimmed.""" + + duration: float = 0.0 + + +Classifier = typing.Callable[ + [PhaseOutcome, ScenarioIdentity, TckConfig], "tuple[Outcome, str] | None" +] + + +class ReportCollector: + """Session-wide accumulator: which scenario belongs to which suite, and how it went. + + One pytest session can run several suites -- the TCK's own tests run two, and + a provider with more than one resolver runs one per resolver -- so outcomes + are attributed to a suite rather than to the session, and each suite writes + its own file. + + Scenarios are enumerated at collection and resolved into records only at the + end of the session. The order matters. A scenario skipped by a marker never + runs a fixture, so a design that learned of a scenario when its fixtures ran + would leave it out of the document entirely -- and a report that silently + omits what it skipped satisfies "a skip is never reported as passed" while + still misleading the person reading it. + """ + + def __init__(self) -> None: + # Suites are keyed by the identity of their TckConfig, so two suites that + # happen to share a name stay distinct here; that collision is caught + # where it actually bites, when their file names turn out to be equal. + self._suites: dict[int, SuiteReport] = {} + self._suite_by_group: dict[str, SuiteReport] = {} + self._collected: dict[str, tuple[str, ScenarioIdentity]] = {} + self._phases: dict[str, list[PhaseOutcome]] = {} + + def collect(self, node_id: str, group: str, identity: ScenarioIdentity) -> None: + """Note that this scenario exists, and which group of tests it came from. + + The group is the module the scenario was generated into. pytest-bdd's + ``scenarios()`` injects its tests into the module that called it, and a + module resolves one ``tck_config``, so the module is what says which + suite a scenario belongs to -- and it says so without running anything. + """ + self._collected[node_id] = (group, identity) + + def observe(self, node_id: str, phase: PhaseOutcome) -> None: + """Record one phase's result for a scenario, if it is one of ours.""" + if node_id in self._collected: + self._phases.setdefault(node_id, []).append(phase) + + def bind(self, node_id: str, config: TckConfig) -> None: + """Learn which suite a group of scenarios is testing. + + Called from a fixture, because the ``TckConfig`` is a fixture value and + there is no way to know it without asking for it. Only one scenario of a + group has to get this far for the whole group to be attributed. + """ + entry = self._collected.get(node_id) + if entry is not None: + self._suite_by_group[entry[0]] = self.suite_for(config) + + def suite_for(self, config: TckConfig) -> SuiteReport: + return self._suites.setdefault(id(config), SuiteReport(config=config)) + + @property + def suites(self) -> list[SuiteReport]: + return list(self._suites.values()) + + def resolve(self, classify: Classifier) -> list[str]: + """Turn the collected phases into records, and report what could not be. + + Returns the problems, one string each, and they are meant to be shouted + about rather than logged: a scenario that ran but is missing from the + document is the one failure mode this format exists to rule out. + """ + problems: list[str] = [] + for node_id, (group, identity) in sorted(self._collected.items()): + suite = self._suite_by_group.get(group) + if suite is None: + problems.append( + f"{node_id}: no TckConfig was resolved for {group}, so its " + f"outcome belongs to no suite and is missing from every report" + ) + continue + phases = self._phases.get(node_id) + if not phases: + problems.append( + f"{node_id}: was collected but never ran, so the report for " + f"{suite.config.name!r} does not account for it" + ) + continue + for phase in phases: + classified = classify(phase, identity, suite.config) + if classified is not None: + outcome, reason = classified + suite.set_outcome(node_id, identity, outcome, reason) + suite.add_duration(node_id, phase.duration) + return problems + + +def control_api_of(control: object) -> str: + """Report how the backend was driven, if the control says. + + Read off an optional attribute rather than added to the + :class:`~.control.BackendControl` protocol, because a protocol member would + make every existing control incomplete for the sake of one string. A control + that does not offer it simply omits the field, which is the honest answer: + the TCK cannot infer from the outside whether a control spoke the normative + HTTP API or reached into the process. + """ + value = getattr(control, "control_api", None) + if isinstance(value, str) and value in {"http", "in-process"}: + return value + return "" + + +def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: + """Turn Gherkin tags as pytest-bdd holds them into the form the schema wants. + + pytest-bdd strips the leading at-sign; the schema requires it back. + """ + return tuple(sorted(f"@{tag}" for tag in tags if _TAG_PATTERN.match(tag))) + + +def report_file_name(suite_name: str) -> str: + """Turn a suite name into a file name. + + Suite names are chosen to read well in a failure message rather than to be + path-safe, so anything not obviously safe becomes a hyphen. Without this a + suite named ``flagd/rpc`` would quietly write outside the directory it was + given. + """ + cleaned = _UNSAFE_IN_FILENAME.sub("-", suite_name).strip("-.") + return f"{cleaned or 'report'}.json" + + +def write_report( + directory: Path, suite_name: str, document: dict[str, typing.Any] +) -> Path: + """Write one report, returning where it went.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / report_file_name(suite_name) + path.write_text( + json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + return path + + +def distribution_version(distribution: str) -> str: + """Read an installed distribution's version. + + Read rather than declared, because a declared version is a second place to + be wrong: the report would go on claiming 0.8.2 after a dependency bump moved + the actual code underneath it. + """ + try: + return importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + return UNKNOWN + + +def spec_identity() -> tuple[str, str]: + """Return the spec commit and asset tree these feature files came from. + + Captured at build time rather than read here, because the submodule that + holds the answer is not in the wheel. A build that could not reach git says + so with :data:`UNKNOWN` instead of inventing a commit, and an installation + old enough to predate the generated file degrades the same way rather than + failing to emit a report at all. + """ + reference = importlib.resources.files(_PACKAGE) / _REVISION_FILE + try: + data = json.loads(reference.read_text(encoding="utf-8")) + except (OSError, ValueError): + return UNKNOWN, "" + if not isinstance(data, dict): + return UNKNOWN, "" + revision = data.get("specRevision") + tree = data.get("assetsTree") + return ( + revision if isinstance(revision, str) and len(revision) >= 7 else UNKNOWN, + tree if isinstance(tree, str) and re.fullmatch(r"[0-9a-f]{40}", tree) else "", + ) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py index 71ea4150..1b1faa86 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py @@ -93,6 +93,13 @@ class TckState: config: TckConfig client: OpenFeatureClient | None = None + provider_name: str | None = None + """What the provider called itself through its own metadata. + + Observed rather than configured, because it is what the conformance report + identifies the provider by: ``TckConfig.name`` is chosen to read well in a + failure message, which makes it the *configuration* rather than the provider. + """ flag_key: str | None = None flag_type: FlagType | None = None default_value: typing.Any = None diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py index bca37fae..59520879 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py @@ -30,6 +30,7 @@ def a_stable_provider(tck_state: TckState) -> None: if provider is None: msg = "TckConfig.new_provider returned None" raise AssertionError(msg) + _observe_metadata_name(tck_state, provider) try: _set_provider_within(provider, config.domain, config.ready_timeout) @@ -79,6 +80,7 @@ def an_unavailable_provider(tck_state: TckState) -> None: if provider is None: msg = "TckConfig.new_unavailable_provider returned None" raise AssertionError(msg) + _observe_metadata_name(tck_state, provider) # A raising initialize is already converted to PROVIDER_ERROR by the SDK's # registry, so this is belt and braces: a provider that raises anyway must @@ -90,6 +92,21 @@ def an_unavailable_provider(tck_state: TckState) -> None: tck_state.client = api.get_client(config.domain) +def _observe_metadata_name(tck_state: TckState, provider: FeatureProvider) -> None: + """Note what the provider calls itself, for the conformance report. + + Before registration rather than after, so that a provider which fails to + initialise -- the ``@unavailable`` case, and any genuine failure -- is still + identified in the report by its own name. Metadata is a pure accessor by + contract, but a provider that raises from it must not take the scenario down + with it: the name is for a report, and no scenario asserts on it. + """ + with contextlib.suppress(Exception): + name = provider.get_metadata().name + if name: + tck_state.provider_name = name + + def _set_provider_within( provider: FeatureProvider, domain: str, timeout: float ) -> None: diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py new file mode 100644 index 00000000..60271f7c --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -0,0 +1,438 @@ +"""What the conformance report must never do. + +The report exists because a runner's summary cannot be checked by anything +downstream. So the tests that matter here are not about JSON shape; they are +about the two properties a consumer is entitled to assume, neither of which is +guaranteed by the code that happens to assemble the document: + +* a scenario skipped for an undeclared capability is never reported as passed, + and carries the reason it was skipped; +* every scenario the run collected appears exactly once, which is what makes the + first property checkable rather than merely asserted -- a document that quietly + dropped what it skipped would satisfy the letter of it and still mislead. + +Both are checked against a real pytest session in a subprocess, because both are +properties of how the suite runs rather than of how the document is assembled. +That session is also the only place all four outcomes occur together, and the +only place the document can be seen to disagree with the runner's summary -- +which it does, deliberately, for a known deviation. +""" + +from __future__ import annotations + +import collections +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.provider_tck import Capability, TckConfig +from openfeature.contrib.tools.provider_tck.emitter import classify_phase +from openfeature.contrib.tools.provider_tck.report import ( + REPORT_DIR_ENV, + Outcome, + PhaseOutcome, + ScenarioIdentity, + SuiteReport, + control_api_of, + normalise_tags, + report_file_name, + spec_identity, +) + +OUTCOMES = {outcome.value for outcome in Outcome} + +# The generated suite's name is deliberately not path-safe. +SUITE_NAME = "report/fixture" +SUITE_FILE = "report-fixture.json" + +UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so the report can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={{ + Capability.EVENTS, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING, + }}, + ) + + +scenarios(features_path()) +''' + +# One scenario skipped outright and one known deviation marked xfail, so the run +# produces all four outcomes and finishes green while the document does not. +_CONFTEST_MODULE = """\ +import pytest + +SKIPPED = "test_an_unknown_flag_key_returns_the_code_default" +DEVIATION = "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name == SKIPPED: + item.add_marker(pytest.mark.skip(reason="deliberately not run here")) + elif item.name == DEVIATION: + item.add_marker(pytest.mark.xfail(reason="python-sdk#619", strict=True)) +""" + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of the generated suite.""" + + directory: Path + result: subprocess.CompletedProcess[str] + document: dict[str, typing.Any] + + @property + def scenarios(self) -> list[dict[str, typing.Any]]: + scenarios: list[dict[str, typing.Any]] = self.document["scenarios"] + return scenarios + + +# -- helpers ----------------------------------------------------------------- + + +class _StubControl: + """A control that says nothing about how it drove the backend.""" + + @property + def description(self) -> str: + return "a stub" + + def prepare_scenario(self) -> None: + return None + + def change_flag(self) -> None: + return None + + +class _HttpControl(_StubControl): + @property + def control_api(self) -> str: + return "http" + + +def _config(**overrides: typing.Any) -> TckConfig: + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "capabilities": {Capability.EVENTS}, + } + settings.update(overrides) + return TckConfig(**settings) + + +def _identity(*tags: str) -> ScenarioIdentity: + return ScenarioIdentity(feature="events", name="a scenario", tags=tags) + + +def _phase(outcome: str, when: str = "call", **extra: typing.Any) -> PhaseOutcome: + return PhaseOutcome(when=when, outcome=outcome, **extra) + + +def _pytest( + *arguments: str, report_dir: Path | None = None +) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment.pop(REPORT_DIR_ENV, None) + if report_dir is not None: + environment[REPORT_DIR_ENV] = str(report_dir) + return subprocess.run( # noqa: S603 + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *arguments], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +def _write_suite(directory: Path) -> Path: + directory.mkdir(parents=True, exist_ok=True) + (directory / "test_suite.py").write_text( + _SUITE_MODULE.format(name=SUITE_NAME), encoding="utf-8" + ) + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + return directory + + +@pytest.fixture(scope="module") +def run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One real run of the generated suite, with a report asked for.""" + directory = _write_suite(tmp_path_factory.mktemp("suite")) + reports = tmp_path_factory.mktemp("reports") + result = _pytest(str(directory), report_dir=reports) + + path = reports / SUITE_FILE + assert path.exists(), ( + f"no report at {path}; pytest exited {result.returncode}\n" + f"{result.stdout}\n{result.stderr}" + ) + return Run( + directory=directory, + result=result, + document=json.loads(path.read_text(encoding="utf-8")), + ) + + +# -- the two properties that matter ------------------------------------------ + + +def test_a_capability_skip_is_never_reported_as_passed(run: Run) -> None: + """The rule Appendix F states, checked against the document, not the runner.""" + undeclared = { + tag + for tag, result in run.document["capabilities"].items() + if result["state"] == Outcome.NOT_DECLARED.value + } + assert undeclared, "the generated suite is meant to leave capabilities undeclared" + + gated = [s for s in run.scenarios if undeclared & set(s.get("tags", ()))] + assert gated, "the generated suite is meant to have scenarios behind those" + for scenario in gated: + assert scenario["outcome"] == Outcome.NOT_DECLARED.value, scenario + assert scenario.get("reason"), f"a skip must say why: {scenario}" + + +def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: + """The property that makes the rule above checkable rather than promised. + + Counted against pytest's own collection rather than against a number written + down here, so that adding a scenario to the specification cannot leave this + passing while the report loses one. + """ + names = [(s["feature"], s["name"]) for s in run.scenarios] + assert len(names) == len(set(names)), "a scenario is reported twice" + + collected = _pytest("--collect-only", str(run.directory)) + assert len(names) == sum( + 1 for line in collected.stdout.splitlines() if "::test_" in line + ) + + +def test_the_outcomes_account_for_every_scenario(run: Run) -> None: + counts = collections.Counter(s["outcome"] for s in run.scenarios) + assert set(counts) <= OUTCOMES, "an outcome outside the four the schema allows" + assert sum(counts.values()) == len(run.scenarios) + # All four occur, which is what makes the distinctions worth drawing. + assert set(counts) == OUTCOMES, counts + + +def test_the_document_does_not_repeat_the_runner_summary(run: Run) -> None: + """A known deviation is a failure in the report even when pytest finishes green. + + The suite marks the one scenario the Python SDK cannot satisfy as an expected + failure, so pytest exits zero. The provider still did not satisfy it, and a + document that agreed with the summary would hide exactly what the marker was + added to keep visible. + """ + assert run.result.returncode == 0, run.result.stdout + failed = [s for s in run.scenarios if s["outcome"] == Outcome.FAILED.value] + assert len(failed) == 1 + assert "python-sdk#619" in failed[0]["reason"] + + +def test_a_scenario_skipped_for_another_reason_is_not_a_missing_capability( + run: Run, +) -> None: + """A run that chose not to execute a scenario is a different fact from a gap. + + ``not-applicable`` rather than ``not-declared``, because nothing about the + provider's declared capabilities kept it from running -- and it appears at + all, even though a marker skip never runs a fixture. + """ + matching = [s for s in run.scenarios if s["name"] == UNKNOWN_KEY_SCENARIO] + assert len(matching) == 1 + assert matching[0]["outcome"] == Outcome.NOT_APPLICABLE.value + assert "deliberately not run here" in matching[0]["reason"] + + +# -- identity ---------------------------------------------------------------- + + +def test_the_provider_and_its_configuration_are_reported_separately(run: Run) -> None: + assert run.document["provider"]["name"] == "In-Memory Provider" + assert run.document["provider"]["configuration"] == SUITE_NAME + assert run.document["provider"]["language"] == "python" + + +def test_the_report_names_what_ran_it(run: Run) -> None: + assert run.document["schemaVersion"] == "1" + assert ( + run.document["tck"]["implementation"] + == "python-sdk-contrib/tools/openfeature-provider-tck" + ) + assert run.document["sdk"]["name"] == "openfeature-sdk" + assert run.document["sdk"]["version"] + assert len(run.document["tck"]["specRevision"]) >= 7 + assert run.document["backend"]["controlApi"] == "in-process" + + +def test_the_spec_revision_comes_from_the_build() -> None: + """Generated beside the assets, because the submodule is not in the wheel.""" + revision, tree = spec_identity() + assert len(revision) >= 7 + assert tree == "" or len(tree) == 40 + + +# -- opting in --------------------------------------------------------------- + + +def test_no_report_is_written_without_the_environment_variable( + tmp_path: Path, +) -> None: + """The default, and not an error: emitting is a property of the run.""" + directory = _write_suite(tmp_path / "suite") + result = _pytest(str(directory), report_dir=None) + assert result.returncode == 0, result.stdout + assert "report written" not in result.stdout + assert not list(tmp_path.rglob("*.json")) + + +def test_a_report_that_cannot_be_written_fails_the_run(tmp_path: Path) -> None: + """Loudly, because a pipeline that silently got no report serves a stale one. + + The destination is placed under a regular file, which no platform will let + ``mkdir`` turn into a directory. The run itself passes, so a non-zero exit + can only have come from the failure to write. + """ + blocker = tmp_path / "not-a-directory" + blocker.write_text("", encoding="utf-8") + directory = _write_suite(tmp_path / "suite") + result = _pytest(str(directory), report_dir=blocker / "reports") + assert "could not write the conformance report" in result.stdout + assert result.returncode != 0 + + +# -- assembling the document ------------------------------------------------- + + +def test_a_failure_is_not_revised_away_by_a_later_phase() -> None: + """A scenario whose steps passed and whose teardown blew up is a failure.""" + suite = SuiteReport(config=_config()) + identity = _identity() + suite.set_outcome("node", identity, Outcome.FAILED, "teardown exploded") + suite.set_outcome("node", identity, Outcome.PASSED) + assert suite.records["node"].outcome is Outcome.FAILED + assert suite.records["node"].reason == "teardown exploded" + + +def test_an_undeclared_capability_is_reported_with_a_reason() -> None: + document = SuiteReport(config=_config()).build() + assert document["capabilities"]["@events"] == {"state": Outcome.PASSED.value} + stale = document["capabilities"]["@stale"] + assert stale["state"] == Outcome.NOT_DECLARED.value + assert "@stale" in stale["reason"] + + +def test_a_capability_whose_scenario_failed_is_not_reported_as_passed() -> None: + suite = SuiteReport(config=_config()) + suite.set_outcome("node", _identity("@events"), Outcome.FAILED, "boom") + assert suite.build()["capabilities"]["@events"]["state"] == Outcome.FAILED.value + + +def test_the_provider_name_falls_back_to_the_suite_name() -> None: + """A suite whose every scenario was skipped never saw a provider. + + Reporting the suite name is more useful than the empty string the schema + would reject. + """ + assert SuiteReport(config=_config()).build()["provider"]["name"] == "stub" + + +def test_the_control_api_is_omitted_when_the_control_does_not_say() -> None: + assert "controlApi" not in SuiteReport(config=_config()).build()["backend"] + http = SuiteReport(config=_config(control=_HttpControl())).build() + assert http["backend"]["controlApi"] == "http" + + +def test_control_api_ignores_a_value_the_schema_would_reject() -> None: + class Odd(_StubControl): + control_api = "carrier pigeon" + + assert control_api_of(Odd()) == "" + + +@pytest.mark.parametrize( + ("suite_name", "expected"), + [ + ("in-memory", "in-memory.json"), + ("flagd/rpc", "flagd-rpc.json"), + ("../escape", "escape.json"), + ("...", "report.json"), + ], +) +def test_a_suite_name_cannot_write_outside_its_directory( + suite_name: str, expected: str +) -> None: + """Suite names are chosen to read well in a failure message, not to be paths.""" + assert report_file_name(suite_name) == expected + + +def test_only_tags_the_schema_accepts_are_carried() -> None: + assert normalise_tags({"events", "Not A Tag", "stale"}) == ("@events", "@stale") + + +# -- classifying one phase --------------------------------------------------- + + +def test_an_expected_failure_is_still_a_failure() -> None: + """An xfail marker records a known deviation; it does not excuse one.""" + classified = classify_phase( + _phase("skipped", xfail_reason="the SDK coerces a bool to an int"), + _identity(), + _config(), + ) + assert classified is not None + outcome, reason = classified + assert outcome is Outcome.FAILED + assert "the SDK coerces a bool to an int" in reason + + +def test_a_phase_that_merely_worked_says_nothing() -> None: + assert ( + classify_phase(_phase("passed", when="setup"), _identity(), _config()) is None + ) + assert classify_phase(_phase("passed", when="call"), _identity(), _config()) == ( + Outcome.PASSED, + "", + ) + + +def test_a_gated_skip_and_an_ungated_skip_are_different_outcomes() -> None: + config = _config(capabilities={Capability.EVENTS}) + gated = classify_phase(_phase("skipped", when="setup"), _identity("@stale"), config) + assert gated == (Outcome.NOT_DECLARED, "provider does not declare @stale") + + ungated = classify_phase( + _phase("skipped", when="setup"), _identity("@events"), config + ) + assert ungated == (Outcome.NOT_APPLICABLE, "skipped") From f30e1095c6fee50273f8d7d64850e3528143315d Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 21:26:14 +0200 Subject: [PATCH 02/11] feat(provider-tck): identify a Scenario Outline row by its parameters A report entry was identified by feature and name. Every row of a Scenario Outline shares one name, so the eleven rows of the type-mismatch matrix in errors.feature produced eleven entries nothing could tell apart -- and in the Python run one of the eleven fails while ten pass, which is exactly the case the report could not express. A consumer keying on feature and name kept whichever row it happened to see last. Each entry from an outline now carries the row it came from, as the Examples parameters keyed by column header, matching the "example" property added to the schema. Values are the cell contents verbatim as strings: Gherkin has no types, so "1" stays "1" rather than becoming a number the table never mentioned. pytest-bdd parametrizes the generated test over one dict per row, keyed by the header, so the row is read back off the node's callspec -- available at collection, which is what lets a row the capability gate skipped be identified as precisely as one that ran. This removes the workaround that appended pytest's own id for the row to the scenario name. It was the wrong shape twice over. The name is the feature file's name, and qualifying it made Python disagree with Go and JavaScript about a scenario all three ran, which defeats the cross-language comparison the report exists for. And a name format would be normative text -- a separator, an ordering, an escaping rule -- that four languages have to reproduce byte for byte, where drift is invisible until two reports silently fail to line up. The parameters are the identity, and they come from the feature file rather than from any runner. The uniqueness test now keys on feature, name and example together, which is the property this change exists to establish. The examples the report emits are checked against the Examples tables read out of the Gherkin by hand, rather than against pytest-bdd's parser, which is what produced them. Signed-off-by: Simon Schrottner --- .../contrib/tools/provider_tck/emitter.py | 50 ++++- .../contrib/tools/provider_tck/report.py | 21 +- .../tests/test_report.py | 198 ++++++++++++++++-- 3 files changed, 241 insertions(+), 28 deletions(-) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py index abfe3c64..35fc1028 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py @@ -38,6 +38,15 @@ COLLECTOR_KEY = pytest.StashKey[ReportCollector]() """Where the session's collector lives, so a fixture can reach it from a request.""" +_EXAMPLE_PARAM = "_pytest_bdd_example" +"""The parameter pytest-bdd renders a Scenario Outline over. + +An implementation detail of pytest-bdd, named here rather than spelled inline so +that a version bump that renames it fails in one place. The alternative -- asking +the scenario template for its examples -- would have to work out which row *this* +node is, which is the question the callspec already answers. +""" + _MAX_REASON = 500 """How much of a failure message the report carries. @@ -68,22 +77,43 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: return ScenarioIdentity( feature=Path(str(getattr(feature, "filename", ""))).stem, - name=_scenario_name(node, str(getattr(scenario, "name", ""))), + name=str(getattr(scenario, "name", "")), + example=_example_of(node), tags=normalise_tags(tags), ) -def _scenario_name(node: pytest.Item, name: str) -> str: - """Qualify a Scenario Outline's name with the example row that ran. +def _example_of(node: pytest.Item) -> tuple[tuple[str, str], ...]: + """The Examples row this node came from, keyed by column header. + + Every row of a Scenario Outline shares one scenario name, so the row is what + tells eleven otherwise identical entries apart -- and in this suite one row + of the type-mismatch matrix genuinely differs in outcome from its ten + siblings. The row goes in its own field rather than into a mangled name + because the parameters *are* the identity and they come from the feature + file, whereas a name format would be a rule about this runner: pytest-bdd's + own id for the row above is ``boolean-flag-Integer-1``, which no other + language's runner has any reason to reproduce. + + pytest-bdd renders an outline by parametrizing the generated test over one + dict per row, keyed by the Examples column header, and pytest hangs it on the + node's callspec. A scenario that is not an outline is not parametrized and + has no callspec at all, which is why the empty tuple -- and therefore an + omitted field -- is the answer for one. - Every row of an outline shares one scenario name, so a report using the name - alone would carry several entries a consumer cannot tell apart -- and in this - suite one row of an outline genuinely differs in outcome from its siblings. - The schema has nowhere to put the row, so it goes in the name, in the form - pytest already uses to select one: ``... [boolean-flag-Integer-1]``. + Values are passed through as the parser produced them: Gherkin cells are + strings, and the report says what the table said rather than guessing that + ``1`` was meant as a number. """ - example_id = getattr(getattr(node, "callspec", None), "id", "") - return f"{name} [{example_id}]" if example_id else name + params = getattr(getattr(node, "callspec", None), "params", None) + if not isinstance(params, dict): + return () + row = params.get(_EXAMPLE_PARAM) + if not isinstance(row, dict): + return () + # Column order, as the feature file wrote it, because dicts preserve + # insertion order and pytest-bdd builds this one from the header row. + return tuple((str(header), str(cell)) for header, cell in row.items()) def _group_of(node: pytest.Item) -> str: diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py index 8979f580..fbf03ba2 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py @@ -129,6 +129,13 @@ class ScenarioIdentity: feature: str name: str tags: tuple[str, ...] + example: tuple[tuple[str, str], ...] = () + """The Examples row, as header/cell pairs, for a scenario from an outline. + + Pairs rather than a mapping so that this stays hashable and ordered: the + order is the feature file's column order, and the report carries it through + rather than imposing one of its own. + """ def capabilities(self) -> tuple[Capability, ...]: """The capabilities this scenario's tags gate it behind.""" @@ -146,6 +153,10 @@ class ScenarioRecord: name: str tags: tuple[str, ...] outcome: Outcome + example: tuple[tuple[str, str], ...] = () + """The Examples row this entry came from; empty for a scenario that is not + an outline, in which case the field is omitted rather than emitted empty.""" + reason: str = "" duration_ms: float = 0.0 @@ -155,6 +166,8 @@ def as_json(self) -> dict[str, typing.Any]: "name": self.name, "outcome": self.outcome.value, } + if self.example: + document["example"] = dict(self.example) if self.tags: document["tags"] = list(self.tags) if self.reason: @@ -217,6 +230,7 @@ def set_outcome( name=identity.name, tags=identity.tags, outcome=outcome, + example=identity.example, reason=reason, ) return @@ -229,7 +243,12 @@ def set_outcome( def sorted_records(self) -> list[ScenarioRecord]: for node_id, record in self.records.items(): record.duration_ms = self.durations.get(node_id, 0.0) - return sorted(self.records.values(), key=lambda r: (r.feature, r.name)) + # Sorted by the whole identity, example included, so that two rows of one + # outline come out in a stable order rather than in whichever order the + # dictionary happened to be filled. + return sorted( + self.records.values(), key=lambda r: (r.feature, r.name, r.example) + ) def counts(self) -> dict[str, int]: """Outcome tallies, for a log line and for the tests that check them.""" diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py index 60271f7c..16cb7952 100644 --- a/tools/openfeature-provider-tck/tests/test_report.py +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -31,7 +31,11 @@ import pytest -from openfeature.contrib.tools.provider_tck import Capability, TckConfig +from openfeature.contrib.tools.provider_tck import ( + Capability, + TckConfig, + features_path, +) from openfeature.contrib.tools.provider_tck.emitter import classify_phase from openfeature.contrib.tools.provider_tck.report import ( REPORT_DIR_ENV, @@ -53,6 +57,14 @@ UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" +# The type-mismatch matrix: eleven Examples rows under one scenario name, one of +# which the Python SDK fails. It is the case the example field exists for. +MISMATCH_SCENARIO = "Requesting the wrong type returns the code default" + +# The row that fails, spelled as the feature file spells it -- strings, because +# Gherkin has no types and "1" is not 1. +DEVIATING_ROW = {"key": "boolean-flag", "requested": "Integer", "default": "1"} + _SUITE_MODULE = '''\ """A one-fixture adoption, generated so the report can be checked end to end.""" @@ -74,17 +86,18 @@ def tck_config(): name="{name}", control=control, new_provider=control.new_provider, - capabilities={{ - Capability.EVENTS, - Capability.OBJECT, - Capability.STRICT_NUMERIC_TYPING, - }}, + capabilities={capabilities}, ) scenarios(features_path()) ''' +CAPABILITIES = ( + "{Capability.EVENTS, Capability.OBJECT, Capability.STRICT_NUMERIC_TYPING}" +) +"""What the main generated suite declares: enough to produce all four outcomes.""" + # One scenario skipped outright and one known deviation marked xfail, so the run # produces all four outcomes and finishes green while the document does not. _CONFTEST_MODULE = """\ @@ -155,6 +168,47 @@ def _identity(*tags: str) -> ScenarioIdentity: return ScenarioIdentity(feature="events", name="a scenario", tags=tags) +def _identity_of(scenario: dict[str, typing.Any]) -> tuple[typing.Any, ...]: + """What identifies one entry: feature, name and the Examples row together.""" + example = scenario.get("example") or {} + return (scenario["feature"], scenario["name"], tuple(sorted(example.items()))) + + +def _examples_from_the_feature_file(feature: str, outline: str) -> list[dict[str, str]]: + """Read an outline's Examples tables straight out of the Gherkin. + + Hand-read rather than taken from pytest-bdd's parser, because the parser is + what produced the values under test: asking it what it should have said would + check nothing. It is a small reader for a small shape -- the tables in these + files are plain pipe-delimited rows -- and it exists so that "the report says + what the table said" is checked against the table. + """ + source = Path(features_path()) / f"{feature}.feature" + lines = source.read_text(encoding="utf-8").splitlines() + rows: list[dict[str, str]] = [] + headers: list[str] = [] + inside = False + + for line in lines: + stripped = line.strip() + if stripped.startswith(("Scenario:", "Scenario Outline:")): + inside = stripped.split(":", 1)[1].strip() == outline + headers = [] + elif not inside: + continue + elif stripped.startswith("Examples"): + headers = [] + elif stripped.startswith("|"): + cells = [cell.strip() for cell in stripped.strip("|").split("|")] + if headers: + rows.append(dict(zip(headers, cells, strict=True))) + else: + headers = cells + + assert rows, f"no Examples rows found for {outline!r} in {feature}.feature" + return rows + + def _phase(outcome: str, when: str = "call", **extra: typing.Any) -> PhaseOutcome: return PhaseOutcome(when=when, outcome=outcome, **extra) @@ -175,23 +229,32 @@ def _pytest( ) -def _write_suite(directory: Path) -> Path: +def _write_suite( + directory: Path, + name: str = SUITE_NAME, + capabilities: str = CAPABILITIES, + deviations: bool = True, +) -> Path: directory.mkdir(parents=True, exist_ok=True) (directory / "test_suite.py").write_text( - _SUITE_MODULE.format(name=SUITE_NAME), encoding="utf-8" + _SUITE_MODULE.format(name=name, capabilities=capabilities), encoding="utf-8" ) - (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + if deviations: + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") return directory -@pytest.fixture(scope="module") -def run(tmp_path_factory: pytest.TempPathFactory) -> Run: - """One real run of the generated suite, with a report asked for.""" - directory = _write_suite(tmp_path_factory.mktemp("suite")) +def _run_suite( + tmp_path_factory: pytest.TempPathFactory, + file_name: str = SUITE_FILE, + **suite: typing.Any, +) -> Run: + """Run one generated suite in a subprocess and read the report it wrote.""" + directory = _write_suite(tmp_path_factory.mktemp("suite"), **suite) reports = tmp_path_factory.mktemp("reports") result = _pytest(str(directory), report_dir=reports) - path = reports / SUITE_FILE + path = reports / file_name assert path.exists(), ( f"no report at {path}; pytest exited {result.returncode}\n" f"{result.stdout}\n{result.stderr}" @@ -203,6 +266,29 @@ def run(tmp_path_factory: pytest.TempPathFactory) -> Run: ) +@pytest.fixture(scope="module") +def run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One real run of the generated suite, with a report asked for.""" + return _run_suite(tmp_path_factory) + + +@pytest.fixture(scope="module") +def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """A run of a suite that leaves the capability gating an outline undeclared. + + ``@object`` is left undeclared so that a whole Scenario Outline is skipped + by the capability gate, which is the case that has to keep saying which row it + skipped. + """ + return _run_suite( + tmp_path_factory, + file_name="narrow.json", + name="narrow", + capabilities="{Capability.STRICT_NUMERIC_TYPING}", + deviations=False, + ) + + # -- the two properties that matter ------------------------------------------ @@ -225,15 +311,21 @@ def test_a_capability_skip_is_never_reported_as_passed(run: Run) -> None: def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: """The property that makes the rule above checkable rather than promised. + An entry is identified by feature, name **and example** together. Feature and + name alone are shared by every row of a Scenario Outline, so keying on them + would let eleven rows of the type-mismatch matrix collapse into one and this + test would not notice -- which is the ambiguity the example field exists to + remove. + Counted against pytest's own collection rather than against a number written down here, so that adding a scenario to the specification cannot leave this passing while the report loses one. """ - names = [(s["feature"], s["name"]) for s in run.scenarios] - assert len(names) == len(set(names)), "a scenario is reported twice" + identities = [_identity_of(s) for s in run.scenarios] + assert len(identities) == len(set(identities)), "a scenario is reported twice" collected = _pytest("--collect-only", str(run.directory)) - assert len(names) == sum( + assert len(identities) == sum( 1 for line in collected.stdout.splitlines() if "::test_" in line ) @@ -275,6 +367,78 @@ def test_a_scenario_skipped_for_another_reason_is_not_a_missing_capability( assert "deliberately not run here" in matching[0]["reason"] +# -- which row of an outline ------------------------------------------------- + + +def test_an_outline_row_is_named_by_its_example_not_by_its_name(run: Run) -> None: + """The eleven rows of the type-mismatch matrix are told apart, and only here. + + All eleven share one scenario name, which is the feature file's name and must + stay that way: it is what a report from Go or JavaScript carries for the same + row, and qualifying it with this runner's id for the row -- which an earlier + version of this emitter did -- makes the three disagree about a scenario they + all ran. + """ + rows = [s for s in run.scenarios if s["name"] == MISMATCH_SCENARIO] + expected = _examples_from_the_feature_file("errors", MISMATCH_SCENARIO) + assert len(rows) == len(expected) == 11 + + for row in rows: + assert row["name"] == MISMATCH_SCENARIO, "the name carries a runner's id" + + observed = [row["example"] for row in rows] + assert len(observed) == len({tuple(sorted(e.items())) for e in observed}) + assert sorted(map(sorted, (e.items() for e in observed))) == sorted( + map(sorted, (e.items() for e in expected)) + ) + + +def test_an_example_says_what_the_table_said(run: Run) -> None: + """Verbatim strings, because Gherkin has no types. + + A ``1`` in a table is the two-character cell the feature file contains, and a + report that emitted it as a number would be saying something the table did + not -- and would not validate, since the schema types the values as strings. + """ + rows = [s for s in run.scenarios if s["name"] == MISMATCH_SCENARIO] + for row in rows: + assert all(isinstance(value, str) for value in row["example"].values()), row + + failed = [row for row in rows if row["outcome"] == Outcome.FAILED.value] + assert len(failed) == 1 + assert failed[0]["example"] == DEVIATING_ROW + + +def test_a_scenario_that_is_not_an_outline_has_no_example(run: Run) -> None: + """Omitted rather than empty: there is no row, so there is nothing to say.""" + plain = [s for s in run.scenarios if s["name"] == UNKNOWN_KEY_SCENARIO] + assert len(plain) == 1 + assert "example" not in plain[0] + + +def test_a_capability_skipped_outline_row_still_carries_its_example( + narrow_run: Run, +) -> None: + """A skipped row is exactly as ambiguous as a failed one. + + Identity is established at collection, from the node alone, so it does not + depend on the scenario having run -- which is what lets a row the capability + gate stopped before its first step be told apart from its siblings just as + well as one that failed. + """ + outline = "Requesting a structured flag as a scalar returns the code default" + expected = _examples_from_the_feature_file("errors", outline) + rows = [s for s in narrow_run.scenarios if s["name"] == outline] + assert len(rows) == len(expected) + + for row in rows: + assert row["outcome"] == Outcome.NOT_DECLARED.value, row + assert row.get("example"), f"a skipped outline row must say which row: {row}" + assert sorted(map(sorted, (row["example"].items() for row in rows))) == sorted( + map(sorted, (e.items() for e in expected)) + ) + + # -- identity ---------------------------------------------------------------- From 191902227d4bc1d191cbceeebf739502d19d8665 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 21:33:04 +0200 Subject: [PATCH 03/11] fix(provider-tck): give a failed capability a reason, and stop claiming untested ones Two defects in the capability rollup, mirroring the fix already made in Go (go-sdk-contrib#944). A failed capability was emitted as {"state": "failed"} with no reason. The schema now requires a reason for any outcome other than passed, so that entry does not validate -- and it appears only when a provider is actually failing, which is precisely when the report matters. It now says how many of how many scenarios carrying the tag failed, and points at the per-scenario results for which and why. No test caught it because every self-test suite passes, so nothing that runs end to end ever reaches that branch. The test now drives the report builder directly with synthetic records, which is the only way to exercise a failure without breaking a provider on purpose. A declared capability that no scenario carries was reported as passed. @targeting is reserved -- it exists in the vocabulary but nothing tests it, because asserting that an evaluation context reached the backend needs an echo operation the control API does not have -- so a provider declaring it got a green result for a claim nothing had examined. That is the vacuous pass the capability vocabulary was introduced to eliminate, arriving through the report rather than through the suite. Such a capability is now omitted. The suite asked no question, so it has no answer to report, and a consumer sees the tag is absent rather than a pass it cannot rely on. Omitting is preferred to inventing a fifth outcome: the four in the schema are about what the provider did, and "the suite does not test this" is a fact about the suite. An undeclared capability is still reported with its reason whether or not any scenario carries it, because that is a fact about the provider rather than about the suite. Signed-off-by: Simon Schrottner --- .../contrib/tools/provider_tck/report.py | 36 ++++++++-- .../tests/test_report.py | 68 ++++++++++++++++--- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py index fbf03ba2..ab3625b7 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py @@ -316,15 +316,31 @@ def _capabilities( not declare it -- in which case the reason says so, because "this provider does not support configuration-change events" is exactly what someone comparing providers came to find out. + + A capability the provider declared and *no scenario carries* is omitted + rather than reported. ``@targeting`` is reserved: it exists in the + vocabulary but nothing tests it, because asserting that an evaluation + context reached the backend needs an echo operation the control API does + not have. Reporting it as passed would be a green result for a claim + nothing examined -- the vacuous pass the capability vocabulary exists to + eliminate, arriving through the report rather than through the suite. + Omitting beats inventing a fifth outcome: the four the schema allows are + about what the provider did, and "the suite does not test this" is a fact + about the suite. """ - failed: set[Capability] = set() + # Counted rather than flagged, so that a failure can say how much of what + # failed, and so that "no scenario exercises this at all" is a case the + # rollup can see rather than one it silently reads as success. + exercised: dict[Capability, int] = {} + failed: dict[Capability, int] = {} for record in records: - if record.outcome is not Outcome.FAILED: - continue for tag in record.tags: capability = capability_for_tag(tag) - if capability is not None: - failed.add(capability) + if capability is None: + continue + exercised[capability] = exercised.get(capability, 0) + 1 + if record.outcome is Outcome.FAILED: + failed[capability] = failed.get(capability, 0) + 1 capabilities: dict[str, dict[str, typing.Any]] = {} for capability in Capability: @@ -337,10 +353,16 @@ def _capabilities( f"contribute to this result" ), } - elif capability in failed: + elif not exercised.get(capability): + continue + elif failed.get(capability): capabilities[capability.tag] = { "state": Outcome.FAILED.value, - "reason": f"at least one {capability.tag} scenario failed", + "reason": ( + f"{failed[capability]} of {exercised[capability]} scenarios " + f"carrying {capability.tag} failed; the per-scenario results " + f"say which, and why" + ), } else: capabilities[capability.tag] = {"state": Outcome.PASSED.value} diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py index 16cb7952..d00dac57 100644 --- a/tools/openfeature-provider-tck/tests/test_report.py +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -274,17 +274,20 @@ def run(tmp_path_factory: pytest.TempPathFactory) -> Run: @pytest.fixture(scope="module") def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: - """A run of a suite that leaves the capability gating an outline undeclared. + """A run of a suite that declares one capability the suite never tests. - ``@object`` is left undeclared so that a whole Scenario Outline is skipped + ``@targeting`` is reserved -- it is in the vocabulary and no scenario carries + it. ``@object`` is left undeclared so that a whole Scenario Outline is skipped by the capability gate, which is the case that has to keep saying which row it - skipped. + skipped. ``@strict-numeric-typing`` is declared and does have a scenario, so + the omission of ``@targeting`` is specific rather than a general failure to + report capabilities. """ return _run_suite( tmp_path_factory, file_name="narrow.json", name="narrow", - capabilities="{Capability.STRICT_NUMERIC_TYPING}", + capabilities="{Capability.STRICT_NUMERIC_TYPING, Capability.TARGETING}", deviations=False, ) @@ -510,7 +513,9 @@ def test_a_failure_is_not_revised_away_by_a_later_phase() -> None: def test_an_undeclared_capability_is_reported_with_a_reason() -> None: - document = SuiteReport(config=_config()).build() + suite = SuiteReport(config=_config()) + suite.set_outcome("node", _identity("@events"), Outcome.PASSED) + document = suite.build() assert document["capabilities"]["@events"] == {"state": Outcome.PASSED.value} stale = document["capabilities"]["@stale"] assert stale["state"] == Outcome.NOT_DECLARED.value @@ -518,9 +523,56 @@ def test_an_undeclared_capability_is_reported_with_a_reason() -> None: def test_a_capability_whose_scenario_failed_is_not_reported_as_passed() -> None: - suite = SuiteReport(config=_config()) - suite.set_outcome("node", _identity("@events"), Outcome.FAILED, "boom") - assert suite.build()["capabilities"]["@events"]["state"] == Outcome.FAILED.value + """And says how much failed, because the schema requires a reason. + + Reached by driving the builder directly: every self-test suite that runs end + to end passes, so nothing else gets near this branch -- and an entry without a + reason would be rejected by the schema at exactly the moment the report + matters most, when a provider is failing. + """ + suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) + suite.set_outcome("failed", _identity("@events"), Outcome.FAILED, "boom") + suite.set_outcome("passed", _identity("@events"), Outcome.PASSED) + + events = suite.build()["capabilities"]["@events"] + assert events["state"] == Outcome.FAILED.value + assert "1 of 2" in events["reason"], events + + +def test_every_capability_the_report_mentions_can_explain_itself(run: Run) -> None: + """The rule the schema enforces, checked here so a change fails in this package.""" + for tag, result in run.document["capabilities"].items(): + if result["state"] != Outcome.PASSED.value: + assert result.get("reason"), f"{tag} is {result['state']} with no reason" + + +def test_a_capability_no_scenario_exercises_is_not_reported_as_passed( + narrow_run: Run, +) -> None: + """The vacuous pass the capability vocabulary exists to eliminate. + + ``@targeting`` is declared by this suite and carried by no scenario, because + asserting that an evaluation context reached the backend needs an echo + operation the control API does not have. The suite asked no question, so it + has no answer: the tag is absent rather than green, and a consumer sees the + absence rather than a pass it cannot rely on. + """ + capabilities = narrow_run.document["capabilities"] + exercised = {tag for s in narrow_run.scenarios for tag in s.get("tags", ())} + + assert Capability.TARGETING.tag not in exercised, "the premise has changed" + assert Capability.TARGETING.tag not in capabilities, capabilities.get( + Capability.TARGETING.tag + ) + + # Specific rather than a general failure to report: the other declared + # capability is exercised, and is still reported. + numeric = Capability.STRICT_NUMERIC_TYPING.tag + assert numeric in exercised + assert capabilities[numeric]["state"] == Outcome.PASSED.value + # And an undeclared capability is still reported, with its reason, whether or + # not any scenario carries it: that is a fact about the provider. + assert capabilities[Capability.OBJECT.tag]["state"] == Outcome.NOT_DECLARED.value def test_the_provider_name_falls_back_to_the_suite_name() -> None: From b9588b0d48bc6fbd8b7736e5d999e11bf5d33324 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 21:35:05 +0200 Subject: [PATCH 04/11] fix(provider-tck): read the tags of the Examples block a row came from Gherkin lets an Examples block carry its own tags, so two rows of one Scenario Outline can differ in which capability gates them. The capability gate already handled that correctly -- pytest-bdd attaches an Examples block's tags as marks on that block's parameter sets, and the gate reads the node's markers -- but the report did not. A scenario's tags were read from the scenario, the feature and the rule, which is everywhere those tags are not. The consequence was a misreport of exactly the kind the format exists to rule out. A row skipped because its Examples block was tagged with an undeclared capability appeared with no tags at all, so it was classified not-applicable rather than not-declared -- the run had a reason not to execute it, said the report, when the reason was a capability the provider does not have. The capability rollup did not count it either. The row's tags are now resolved by intersecting the tags the scenario's Examples blocks declare with the markers pytest put on the node. That names this row's blocks without having to work out which block a row came from, and admits nothing that is not a Gherkin tag of this scenario. No canonical feature file uses per-Examples tags today, so this is latent. It was found while checking a defect the Go implementation hit in the same area, where per-scenario bookkeeping keyed by scenario name let one gated row suppress the accounting for every row of its outline. Nothing here is keyed by name -- the collector, the durations and the records are all keyed by pytest node id, which is unique per row -- and the test added here confirms that every row of an outline is still reported when one of them is gated. Signed-off-by: Simon Schrottner --- .../contrib/tools/provider_tck/emitter.py | 30 ++++++ .../tests/test_report.py | 98 +++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py index 35fc1028..808bbb1a 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py @@ -74,6 +74,7 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: rule = getattr(scenario, "rule", None) if rule is not None: tags |= set(getattr(rule, "tags", None) or ()) + tags |= _examples_tags(node, scenario) return ScenarioIdentity( feature=Path(str(getattr(feature, "filename", ""))).stem, @@ -83,6 +84,35 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: ) +def _examples_tags(node: pytest.Item, scenario: object) -> set[str]: + """The tags of the Examples block *this row* came from. + + Gherkin allows an Examples block to carry its own tags, so two rows of one + Scenario Outline can differ in which capability gates them. Those tags are not + on the scenario, the feature or the rule, so a report built from those three + alone would show a row the capability gate skipped as carrying no capability + at all -- and it would then be classified ``not-applicable`` rather than + ``not-declared``, which is precisely the distinction Appendix F asks a report + to keep. It would also not count towards the capability rollup. + + Resolved by intersecting the tags the scenario's Examples blocks declare with + the markers pytest actually put on this node: pytest-bdd attaches an Examples + block's tags as marks on that block's parameter sets, so the intersection + names this row's blocks without having to work out which block a row came + from, and admits nothing that is not a Gherkin tag of this scenario. + + No canonical feature file uses per-Examples tags today, so this is latent -- + but it is latent in the direction of under-reporting a skip, which is the one + failure mode the format exists to rule out. + """ + declared: set[str] = set() + for examples in getattr(scenario, "examples", None) or (): + declared |= set(getattr(examples, "tags", None) or ()) + if not declared: + return set() + return declared & {marker.name for marker in node.iter_markers()} + + def _example_of(node: pytest.Item) -> tuple[tuple[str, str], ...]: """The Examples row this node came from, keyed by column header. diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py index d00dac57..c27aaa68 100644 --- a/tools/openfeature-provider-tck/tests/test_report.py +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -116,6 +116,64 @@ def pytest_collection_modifyitems(items): """ +# A Scenario Outline whose second Examples block carries a tag of its own, which +# no canonical feature file does yet. Written here so that the one case where two +# rows of an outline are gated differently is covered. +_TAGGED_FEATURE = """\ +Feature: Per-Examples tags + + Background: + Given a stable provider + + Scenario Outline: Requesting the wrong type returns the code default + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Examples: ungated + | key | requested | default | + | string-flag | Boolean | false | + | string-flag | Integer | 1 | + + @object + Examples: gated behind a capability this suite does not declare + | key | requested | default | + | string-flag | Float | 0.1 | +""" + +_TAGGED_SUITE = '''\ +"""A suite over the feature file beside it, which tags one Examples block.""" + +import pathlib + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="per-examples", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS}, + ) + + +scenarios(str(pathlib.Path(__file__).parent)) +''' + + @dataclasses.dataclass(frozen=True) class Run: """One subprocess run of the generated suite.""" @@ -442,6 +500,46 @@ def test_a_capability_skipped_outline_row_still_carries_its_example( ) +def test_a_row_gated_by_its_examples_block_is_a_capability_skip( + tmp_path: Path, +) -> None: + """Gherkin lets one Examples block of an outline carry its own tags. + + Two rows of one Scenario Outline can therefore differ in which capability + gates them. Those tags are on neither the scenario, the feature nor the rule, + and a report that read only those three would show the skipped row as + carrying no capability -- reporting a capability skip as ``not-applicable``, + which is exactly the distinction Appendix F asks a report to keep, and + leaving the capability out of the rollup. + + No canonical feature file does this yet, so the feature file is written here. + """ + directory = tmp_path / "suite" + directory.mkdir(parents=True) + (directory / "tagged.feature").write_text(_TAGGED_FEATURE, encoding="utf-8") + (directory / "test_tagged.py").write_text(_TAGGED_SUITE, encoding="utf-8") + + reports = tmp_path / "reports" + result = _pytest(str(directory), report_dir=reports) + path = reports / "per-examples.json" + assert path.exists(), f"pytest exited {result.returncode}\n{result.stdout}" + + document = json.loads(path.read_text(encoding="utf-8")) + by_row = {row["example"]["requested"]: row for row in document["scenarios"]} + # Every row is still reported: nothing about gating one row of an outline may + # drop its siblings from the document. + assert set(by_row) == {"Boolean", "Integer", "Float"}, document["scenarios"] + assert by_row["Boolean"]["outcome"] == Outcome.PASSED.value + assert by_row["Integer"]["outcome"] == Outcome.PASSED.value + + gated = by_row["Float"] + assert gated["outcome"] == Outcome.NOT_DECLARED.value, gated + assert gated["tags"] == [Capability.OBJECT.tag], gated + assert document["capabilities"][Capability.OBJECT.tag]["state"] == ( + Outcome.NOT_DECLARED.value + ) + + # -- identity ---------------------------------------------------------------- From f165522069b0432fdd072332d3e9b2a65e9e67bc Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 10 Sep 2026 19:55:59 +0200 Subject: [PATCH 05/11] refactor(provider-tck): carry results in Cucumber Messages The emitter defined its own per-scenario result list: a four-value outcome enum, a tag list, a reason, and a field naming which Scenario Outline row an entry came from. All of it already exists in Cucumber Messages, which is maintained, cross-language, schema'd, and emitted natively by cucumber-jvm. The report schema was reshaped to reference a Messages payload rather than define one (open-feature/spec#425); this follows it. A run now writes two files per suite: .json, the envelope, and .ndjson, the results it points at, with results.digest over the exact bytes written. Deleted, because Messages carries them: scenarios[] - now TestCase/TestCaseStarted/TestStepFinished/TestCaseFinished. the outcome enum - Cucumber's own seven statuses. The declared/not-applicable distinction was never a property of the run: it follows from the declaration and the scenario's tags, so it is stated once in the envelope instead of once per scenario. example - a pickle's astNodeIds are [scenario id, table row id], and the row id resolves in the GherkinDocument to the cells the feature file wrote. Four implementations were each reinventing this field by hand. tck.assetsTree - the payload carries the executed feature Source verbatim, which answers "did two runs ask the same questions" directly rather than by proxy. Two things Messages cannot carry, so they stay. The declaration is an input to reading the results, not a summary of them. And no standard results format has a slot for the tested subject: Messages records the runtime and the OS, not what was being asked about. pytest-bdd emits no Messages -- it ships the legacy Cucumber JSON format -- so messages.py assembles the stream. Two dependencies, each doing the half it owns: cucumber-messages, the official Python types from the protocol's own repository, for the execution messages; gherkin-official, already a transitive dependency of pytest-bdd, for the gherkinDocument and pickle payloads, which are used as it produces them rather than round-tripped through another representation. The feature files are parsed again because pytest-bdd's own dataclasses drop the AST node ids a pickle refers to. Step results come from pytest-bdd's step hooks rather than from the scenario's verdict, because a stream that marked all eight steps of a scenario failed would be saying something untrue about the seven that passed and the ones never reached. Each test case also carries a before- and after-hook TestStep: pytest runs three phases and only the middle one executes steps, so that is where a capability skip's reason and a teardown failure belong. A verdict no step accounts for -- a strict xfail that passes -- is attached to the after-hook, so it survives a consumer computing the test case's status as the worst of its steps. An expected failure is still a failure in the payload. The acknowledgement moved to the envelope's knownDeviations, declared by TckConfig.known_deviations, where it records the gap without softening the result. TckConfig also gains not_applicable, for a capability that cannot hold rather than one the provider declines. Verified locally; CI does not run on this branch, which targets the report branch rather than main. Both suites' envelopes validate against the reshaped schema with a Draft 2020-12 validator and their digests match; both streams validate clean against the Cucumber Messages JSON schema at v34.2.0 (661 messages each, zero errors). The stream accounts for all 29 collected scenarios; the five the capability gate stopped are SKIPPED for every step, none PASSED, and the one row the SDK fails is FAILED while pytest exits zero. Signed-off-by: Simon Schrottner --- tools/openfeature-provider-tck/README.md | 125 ++- .../hatch_build_sync.py | 18 +- tools/openfeature-provider-tck/pyproject.toml | 30 + .../contrib/tools/provider_tck/__init__.py | 8 +- .../contrib/tools/provider_tck/config.py | 108 ++- .../contrib/tools/provider_tck/emitter.py | 354 +++++-- .../contrib/tools/provider_tck/messages.py | 624 +++++++++++++ .../contrib/tools/provider_tck/report.py | 450 ++++----- .../tests/conftest.py | 49 +- .../tests/test_controllable_conformance.py | 6 +- .../tests/test_in_memory_conformance.py | 9 +- .../tests/test_report.py | 870 +++++++++++++----- uv.lock | 15 +- 13 files changed, 1979 insertions(+), 687 deletions(-) create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md index 7017f0db..5704c7d5 100644 --- a/tools/openfeature-provider-tck/README.md +++ b/tools/openfeature-provider-tck/README.md @@ -128,6 +128,11 @@ of, so a provider that wrongly rejects `10.0` as an integer still passes; adding set for every language at once. Appendix F records that as an open gap, together with a second one: the width of a language's integer accessor — 64-bit against 32-bit — is not modelled at all. +For a capability that *cannot* hold rather than one you chose not to declare, use +`not_applicable={Capability.X: "why"}`. The suite treats it identically — the scenarios are skipped +either way — but the report keeps the two apart, because collapsing them misrepresents a provider: +declining an optional feature is a choice, and an impossibility is not. + ## Controlling the backend `BackendControl` is the single seam between the scenarios and whatever manipulates the backend. Step @@ -177,8 +182,10 @@ This is **Python-specific** — the identical scenario passes in every other lan is a fair advertisement for having more than one implementation. Tracked as [open-feature/python-sdk#619](https://github.com/open-feature/python-sdk/issues/619). -The self-test marks that one row `xfail(strict=True)` with a pointer to the issue, so it stays -visible in the report and un-hides itself automatically once the SDK is fixed. +The self-test marks that one row `xfail(strict=True)` with a pointer to the issue, so the run +un-hides itself automatically once the SDK is fixed, and declares it in +`TckConfig.known_deviations`, so the report acknowledges it. The results payload still reports the +scenario as `FAILED`: the acknowledgement records the gap, it does not soften it. ### 2. The in-memory provider cannot update its flag set @@ -220,59 +227,90 @@ This mirrors what `openfeature-flagd-api-testkit` already does for the flagd tes ## Conformance reports -Set `PROVIDER_TCK_REPORT_DIR` and each suite writes a machine-readable record of its run to -`/.json`, conforming to the [report schema][report-schema] in the specification. +Set `PROVIDER_TCK_REPORT_DIR` and each suite writes **two** files: an envelope at `/.json`, +conforming to the [report schema][report-schema] in the specification, and the results it points at +at `/.ndjson`, which is a [Cucumber Messages][messages] stream. ```console $ PROVIDER_TCK_REPORT_DIR=./reports pytest -provider-tck [in-memory]: report written to reports/in-memory.json (1 failed, 5 not-declared, 23 passed) - -$ jq '.scenarios | group_by(.outcome) | map({(.[0].outcome): length}) | add' reports/in-memory.json -{ - "failed": 1, - "not-declared": 5, - "passed": 23 -} +provider-tck [in-memory]: report written to reports/in-memory.json with results in in-memory.ndjson (1 failed, 23 passed, 5 skipped) + +$ jq -c .results reports/in-memory.json +{"format":"cucumber-messages","location":"in-memory.ndjson","digest":"sha256:c7e12a…"} + +$ jq -r 'select(.testStepFinished) | .testStepFinished.testStepResult.status' \ + reports/in-memory.ndjson | sort | uniq -c + 1 FAILED + 220 PASSED + 45 SKIPPED ``` +Statuses are per step, not per scenario. Of the 45 skipped, 42 belong to the five scenarios the +capability gate stopped — their before-hooks included, which is where the reason is — and three are +the steps of the failing scenario that were never reached. + It is an environment variable rather than a `TckConfig` field so that emitting a report is a property of the *run* and not of the code: CI sets it, a developer running the suite locally does not, and no adopter changes a line to publish one. Unset means no report, which is not an error. Several suites -in one pytest session each write their own file, so flagd's two resolvers would not collide. +in one pytest session each write their own pair, so flagd's two resolvers would not collide. + +### Why the results are not our format -### Why every scenario is listed +Per-scenario outcomes, tags, Scenario Outline row identity and the executed feature source are all +already specified by Cucumber Messages, which is maintained, cross-language, schema'd, and emitted +natively by cucumber-jvm. Defining them again in the report schema created a second format to +maintain and version, and two places for the same fact to disagree. So the envelope says what was +tested and what the provider claims; the payload says what happened. + +The results are referenced rather than inlined because the stream carries the feature sources and is +far larger than the envelope, and a consumer deciding whether it cares about a report should not have +to fetch a whole run to find out. `results.digest` is a SHA-256 over the exact bytes written, so a +consumer can tell that what it fetched is what the envelope described. + +Two things Messages cannot carry, so they stay in the envelope. `declaration` is an *input* to +reading the results rather than a summary of them: a skipped scenario says the question was not put +to this provider, and only the declaration says whether that is because the provider declines the +capability. And no standard results format has a slot for the tested subject — Messages records the +runtime and the OS, not what was being asked about. + +### Reading the payload Appendix F requires that a scenario skipped for an undeclared capability is reported as skipped **with the reason** and never as passed. A consumer cannot check that against a summary line, so the -report records the outcome of *every* scenario individually — and is required to be complete, because -a document that quietly dropped what it skipped would satisfy the letter of the rule and still -mislead whoever read it. +stream carries every scenario the run collected, including the ones the capability gate skipped +before their first step, and Cucumber's own `SKIPPED` is what it reports them as. + +Each scenario is a `TestCase` referring to a `Pickle`, and a test case is as bad as its worst step, +which is Cucumber's rule. Every test case carries two hook steps as well as its Gherkin steps: pytest +runs a scenario in three phases and only the middle one executes steps, so the before-hook is where a +capability skip's reason lands and the after-hook is where a teardown failure does. + +Given a scenario's tags — in its pickle — and the envelope's `declaration`, the capability +responsible for a skip follows, which is why it is no longer transported once per scenario. -Which also means the report is not a transcription of pytest's summary. The run above finishes green: -the one scenario the Python SDK cannot satisfy is marked `xfail` (finding 1), so pytest counts it as -expected and exits zero. The provider still did not satisfy it, and the document says `failed` with -the reason — an expected failure is a recorded deviation, not an excused one. +Which also means the payload is not a transcription of pytest's summary. The run above finishes +green: the one scenario the Python SDK cannot satisfy is marked `xfail` (finding 1), so pytest counts +it as expected and exits zero. The provider still did not satisfy it, and the stream says `FAILED`. +The acknowledgement goes in the envelope's `knownDeviations` instead — an expected failure is a +recorded deviation, not an excused one — which an adoption declares with `TckConfig.known_deviations`. -Four outcomes rather than two, because "did not run" is not one thing: +### Which row of a Scenario Outline -| Outcome | Means | -| --- | --- | -| `passed` | the scenario ran and passed | -| `failed` | the scenario ran and failed, including a known deviation marked `xfail` | -| `not-declared` | skipped because the provider did not declare a capability the scenario is tagged with | -| `not-applicable` | skipped for any other reason — a marker an adopter applied, a step calling `pytest.skip` | +A pickle's `astNodeIds` are `[scenario id, table row id]`, and the row id resolves in the +`GherkinDocument` to exactly the cells the feature file wrote. That is what tells the eleven rows of +the type-mismatch matrix apart — one of which differs in outcome from its ten siblings — and it is +exact rather than a naming convention every implementation has to reproduce byte-for-byte. ### What identifies a report -`tck.specRevision` and `tck.assetsTree` come from `spec_revision.json`, which `hatch_build_sync.py` -generates from the submodule alongside the copied assets. It has to be captured at build time: the -submodule is not in the wheel, so an installed copy has nothing left to ask. A build that cannot -reach git — an unpacked sdist, say — warns and records `unknown` rather than inventing a commit. +`tck.specRevision` comes from `spec_revision.json`, which `hatch_build_sync.py` generates from the +submodule alongside the copied assets. It has to be captured at build time: the submodule is not in +the wheel, so an installed copy has nothing left to ask. A build that cannot reach git — an unpacked +sdist, say — warns and records `unknown` rather than inventing a commit. -The tree hash is carried as well as the commit because it identifies the assets alone. It is -unchanged by unrelated edits elsewhere in the specification, so two runs that executed identical -assets report the same value even when pinned to different commits — and it is checkable, since -`git rev-parse :specification/assets/provider-tck` must reproduce it. +No asset tree hash. It was carried so a consumer could tell whether two runs executed the same +questions; the payload's `Source` messages carry the executed feature files verbatim, which answers +that directly rather than by proxy. `provider.name` is what the provider reports through its own metadata, not `TckConfig.name`. `TckConfig.name` is chosen to read well in a failure message — `flagd-rpc` — which makes it the @@ -291,10 +329,10 @@ the field. | `test_in_memory_conformance` | the SDK's `InMemoryProvider` | reference adoption for a backend-less provider | | `test_controllable_conformance` | `ControllableInMemoryProvider` | the only suite that exercises the configuration-change path — see finding 2 | | `test_in_process_control` | `InProcessControl` | pins what the Gherkin cannot assert about itself | -| `test_report` | the conformance report | checks the two properties a consumer is entitled to assume | +| `test_report` | the conformance report | checks the two properties a consumer is entitled to assume, against the emitted Messages stream | ``` -78 passed, 9 skipped, 2 xfailed +94 passed, 9 skipped, 2 xfailed ``` No Docker and no network. The conformance suites take under a second; `test_report` takes most of a @@ -311,14 +349,15 @@ what they did while the feature was gated on `@events`. cannot assert one *reached* the backend. That needs an echo operation on the control API. - **No HTTP control client yet.** It arrives with the first containerised adopter. - **Caching, hooks and flag metadata** are not covered. -- **A report cannot name a Scenario Outline row portably.** Every row of an outline shares one - scenario name, and the report schema has nowhere to put the row, so several entries would be - indistinguishable — including, here, one that differs in outcome from its siblings. This - implementation qualifies the name with pytest's example id (`... [boolean-flag-Integer-1]`), which - is unambiguous but is not what another language would produce for the same row. Raised on +- **The results payload is assembled here.** pytest-bdd emits no Cucumber Messages — it ships the + legacy Cucumber JSON format and nothing for the ndjson protocol — so `messages.py` builds the + stream from the official types and re-parses the feature files to get the AST node ids a pickle + refers to. If pytest-bdd ever emits Messages itself, that module should shrink to a shim. Whether + a report belongs inside a provider's released artifact is still open on [open-feature/spec#424](https://github.com/open-feature/spec/issues/424). [report-schema]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/report/conformance-report.schema.json +[messages]: https://github.com/cucumber/messages [appendix-a]: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md [appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md [spec]: https://github.com/open-feature/spec diff --git a/tools/openfeature-provider-tck/hatch_build_sync.py b/tools/openfeature-provider-tck/hatch_build_sync.py index acee9bf0..8e0a5270 100644 --- a/tools/openfeature-provider-tck/hatch_build_sync.py +++ b/tools/openfeature-provider-tck/hatch_build_sync.py @@ -81,19 +81,17 @@ def sync() -> None: def write_revision() -> None: - """Record the spec commit and the asset tree these copies came from. - - The tree hash is carried as well as the commit because it identifies the - assets alone: it does not change when an unrelated part of the specification - does, so two runs that executed identical assets report the same value even - when pinned to different commits. It is also checkable rather than merely - asserted, since ``git rev-parse :specification/assets/provider-tck`` - must reproduce it. + """Record the spec commit these copies came from. + + The asset tree hash that used to accompany it is gone. It was carried so a + consumer could tell whether two runs executed the same questions; the + conformance report's results are now a Cucumber Messages stream, which + carries the executed feature source itself and answers that directly rather + than by proxy. """ commit = _git("rev-parse", "HEAD") or UNKNOWN_REVISION - tree = _git("rev-parse", f"HEAD:{ASSETS_PATH_IN_SPEC}") or "" (DEST_BASE / REVISION_FILE).write_text( - json.dumps({"specRevision": commit, "assetsTree": tree}, indent=2) + "\n", + json.dumps({"specRevision": commit}, indent=2) + "\n", encoding="utf-8", ) diff --git a/tools/openfeature-provider-tck/pyproject.toml b/tools/openfeature-provider-tck/pyproject.toml index 5e23849a..d8b98dc1 100644 --- a/tools/openfeature-provider-tck/pyproject.toml +++ b/tools/openfeature-provider-tck/pyproject.toml @@ -22,6 +22,21 @@ dependencies = [ # Same runner the flagd provider and the flagd testkit already use, so an # adopting module gains no new test framework. "pytest-bdd>=8.1.0,<9.0.0", + # The conformance report's results are a Cucumber Messages stream rather + # than a format this package defines. These two are the reference + # implementations of the halves of that protocol: cucumber-messages is the + # official Python types, published from the same repository as the protocol + # itself, and gherkin-official is the parser that produces the + # gherkinDocument and pickle messages. pytest-bdd already depends on + # gherkin-official, so only the first is genuinely new -- and it has no + # dependencies of its own. + # + # pytest-bdd ships no Messages emitter (its cucumber_json.py is the legacy + # JSON format), so the stream is assembled here; assembling it from typed + # messages rather than hand-written dicts is what keeps it from drifting + # away from the protocol. + "cucumber-messages>=34.0.0,<35.0.0", + "gherkin-official>=29.0.0", ] requires-python = ">=3.10" @@ -78,6 +93,21 @@ fixed_format_cache = true pretty = true strict = true disallow_any_generics = false +# cucumber-messages and gherkin-official ship no py.typed. Both are annotated +# internally, so following them gives real types for the messages this package +# builds rather than the Any a plain `ignore_missing_imports` would hand back -- +# which is the point of using the typed library at all. +follow_untyped_imports = true + +[[tool.mypy.overrides]] +# gherkin-official has no annotations at all, so following it turns every call +# into a `no-untyped-call` error rather than into a type. pytest-bdd silences +# the same import the same way. cucumber-messages is the opposite case -- fully +# annotated, only missing py.typed -- and is followed, which is where the value +# of using it rather than hand-written dicts actually lands. +module = ["gherkin.*"] +follow_untyped_imports = false +ignore_missing_imports = true [tool.coverage.run] omit = ["tests/**"] diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py index 8f4e651b..e770538e 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -50,23 +50,25 @@ def tck_config(): import importlib.resources from .capability import ALL_CAPABILITIES, Capability -from .config import TckConfig +from .config import KnownDeviation, TckConfig from .control import ( BackendControl, ConnectionControl, UnsupportedControlError, ) from .inprocess import InProcessControl +from .messages import MESSAGES_FORMAT from .provider import ( CHANGING_FLAG_KEY, ControllableInMemoryProvider, canonical_flag_set, ) -from .report import REPORT_DIR_ENV, SCHEMA_VERSION, Outcome +from .report import REPORT_DIR_ENV, SCHEMA_VERSION __all__ = [ "ALL_CAPABILITIES", "CHANGING_FLAG_KEY", + "MESSAGES_FORMAT", "REPORT_DIR_ENV", "SCHEMA_VERSION", "BackendControl", @@ -74,7 +76,7 @@ def tck_config(): "ConnectionControl", "ControllableInMemoryProvider", "InProcessControl", - "Outcome", + "KnownDeviation", "TckConfig", "UnsupportedControlError", "canonical_flag_set", diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py index 77b783cd..1f533ce1 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py @@ -2,7 +2,8 @@ from __future__ import annotations -from collections.abc import Callable, Collection, Iterable +import typing +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass, field from openfeature.provider import FeatureProvider @@ -10,7 +11,7 @@ from .capability import ALL_CAPABILITIES, Capability from .control import BackendControl -__all__ = ["ProviderFactory", "TckConfig"] +__all__ = ["KnownDeviation", "ProviderFactory", "TckConfig"] ProviderFactory = Callable[[], FeatureProvider] """Creates the provider under test. @@ -24,6 +25,46 @@ DEFAULT_READY_TIMEOUT = 30.0 +@dataclass(frozen=True) +class KnownDeviation: + """A gap the provider is known to have, acknowledged rather than hidden. + + Distinct from an undeclared capability, which is a choice, and from a + not-applicable one, which is impossible: this is a defect against something + the specification does not treat as optional, with the gap tracked + somewhere. + + It changes nothing about how the suite runs. The scenario still fails, and + the results payload still reports it as failed -- a report that softened a + failure into a footnote would hide exactly what the acknowledgement exists to + keep visible. What this adds is the acknowledgement itself, in the envelope, + so that a consumer can tell a known and tracked gap from a surprise. + """ + + issue: str + """Where the gap is tracked. A URI, because the schema requires one.""" + + summary: str + """What is wrong, for a person reading a comparison page.""" + + capability: Capability | None = None + """The capability the deviation concerns, when it maps to one. + + Left out for a deviation against a mandatory scenario, which belongs to no + capability -- which is the common case, since a capability a provider fails + is usually one it should not have declared. + """ + + def as_json(self) -> dict[str, typing.Any]: + document: dict[str, typing.Any] = { + "issue": self.issue, + "summary": self.summary, + } + if self.capability is not None: + document["capability"] = self.capability.tag + return document + + @dataclass(frozen=True) class TckConfig: """Everything the TCK needs to test one provider. @@ -92,6 +133,32 @@ class TckConfig: widening it. """ + not_applicable: Mapping[Capability, str] = field(default_factory=dict) + """Capabilities that cannot hold for this provider, each with a reason. + + Kept apart from simply leaving a capability out of :attr:`capabilities`, + because the two are different claims and collapsing them misrepresents whole + languages: ``@strict-numeric-typing`` is unsatisfiable in JavaScript because + the language has no integer type, and reporting that as a choice would show + every JavaScript provider as missing something none of them can have. + + Scenarios behind a not-applicable capability are skipped exactly as an + undeclared one's are -- the gate makes no distinction, and neither does the + results payload. The difference is recorded once, here, and reaches the + report's declaration. + + Where the impossibility is a property of the language rather than of the + provider it belongs in the capability documentation rather than in every + report, so this is for provider-specific cases. + """ + + known_deviations: Sequence[KnownDeviation] = () + """Gaps this provider is known to have, with each one tracked somewhere. + + An acknowledgement, not an excuse: the scenarios still fail and the results + payload still says so. See :class:`KnownDeviation`. + """ + event_timeout: float = DEFAULT_EVENT_TIMEOUT """Seconds to wait for a provider event. @@ -138,6 +205,43 @@ def __post_init__(self) -> None: f"the Capability enum" ) + # Normalised the same way, so a dict literal keyed by Capability is what + # an adopter writes and a plain mapping is what everything else reads. + object.__setattr__(self, "not_applicable", dict(self.not_applicable)) + object.__setattr__(self, "known_deviations", tuple(self.known_deviations)) + + stray = [c for c in self.not_applicable if not isinstance(c, Capability)] + if stray: + problems.append( + f"unknown capabilities {stray!r} in not_applicable: capabilities are " + f"the members of the Capability enum" + ) + + both = sorted( + capability.tag + for capability in self.not_applicable + if isinstance(capability, Capability) and capability in self.capabilities + ) + if both: + problems.append( + f"capabilities and not_applicable both claim {' '.join(both)}: a " + f"capability is either declared or impossible, and a report saying " + f"both leaves a consumer to guess which" + ) + + unreasoned = sorted( + capability.tag + for capability, reason in self.not_applicable.items() + if isinstance(capability, Capability) + and (not isinstance(reason, str) or not reason.strip()) + ) + if unreasoned: + problems.append( + f"not_applicable gives no reason for {' '.join(unreasoned)}: " + f"'impossible for this provider' is only useful to a reader who is " + f"told why, and the report schema requires the reason" + ) + if ( Capability.UNAVAILABLE_INIT in self.capabilities and self.new_unavailable_provider is None diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py index 808bbb1a..1589b308 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py @@ -1,39 +1,71 @@ -"""The pytest half of the conformance report: turning a run into the document. - -Kept apart from :mod:`report`, which knows what a report *is* and nothing about -pytest. Everything here is translation -- a pytest node into a scenario, a -:class:`pytest.TestReport` into an :class:`~.report.Outcome`, the end of a -session into a file on disk. - -The translation that matters is the one for skips. pytest reports a skip -honestly, unlike some runners, but "skipped" alone does not distinguish a -capability the provider never declared from a scenario the run had some other -reason not to execute, and the report format does. So the decision is made -against the scenario's own tags and the suite's declared capabilities rather than -against the wording of a skip message. +"""The pytest half of the conformance report: turning a run into the two documents. + +Kept apart from :mod:`report`, which knows what an envelope *is*, and from +:mod:`messages`, which knows what a Cucumber Messages stream is; neither knows +anything about pytest. Everything here is translation -- a pytest node into a +scenario, a :class:`pytest.TestReport` into a step status, the end of a session +into a pair of files on disk. + +Two translations matter. + +**Skips.** pytest reports a skip honestly, unlike some runners, and Cucumber's +``SKIPPED`` says the same thing, so a capability-gated scenario reaches the +stream as skipped without anything having to be decided. What the stream does not +say is *why* -- and it does not need to, because the envelope carries the +provider's declaration and the stream carries the scenario's tags, so the reason +for the skip follows from the two. The skip message is carried anyway, on the +setup hook's result, because a person reading the stream should not have to +perform that derivation. + +**Expected failures.** A scenario marked ``xfail`` is one pytest reports as +skipped and finishes green on. The provider still did not satisfy it, so the +stream reports it as failed. The acknowledgement belongs in the envelope's +``knownDeviations``, where it is a claim about the provider rather than a +softening of the result. """ from __future__ import annotations import os +import time import typing from pathlib import Path import pytest from .config import TckConfig +from .messages import ( + FeatureCatalog, + ScenarioIdentity, + ScenarioRun, + Status, + StepRun, + feature_uri, + worse, + write_stream, +) from .report import ( REPORT_DIR_ENV, - Outcome, + TCK_DISTRIBUTION, + TCK_IMPLEMENTATION, PhaseOutcome, ReportCollector, - ScenarioIdentity, + Results, + SuiteReport, + distribution_version, + envelope_file_name, normalise_tags, - report_file_name, - write_report, + stream_file_name, + write_envelope, ) -__all__ = ["COLLECTOR_KEY", "ReportEmitter", "classify_phase", "scenario_identity"] +__all__ = [ + "COLLECTOR_KEY", + "ReportEmitter", + "classify_phase", + "scenario_identity", + "scenario_run", +] COLLECTOR_KEY = pytest.StashKey[ReportCollector]() """Where the session's collector lives, so a fixture can reach it from a request.""" @@ -48,13 +80,16 @@ """ _MAX_REASON = 500 -"""How much of a failure message the report carries. +"""How much of a failure message the stream carries. -A reason is for a person reading a comparison page, not for debugging: whoever +A message is for a person reading a comparison page, not for debugging: whoever ran the suite has the traceback. Whole tracebacks in a published document also leak local paths. """ +_SKIPPED = pytest.skip.Exception +"""What ``pytest.skip`` raises, named so a step hook can recognise it.""" + def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: """Describe a pytest node as a Gherkin scenario, or return ``None``. @@ -76,8 +111,12 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: tags |= set(getattr(rule, "tags", None) or ()) tags |= _examples_tags(node, scenario) + filename = str(getattr(feature, "filename", "")) + relative = str(getattr(feature, "rel_filename", "") or Path(filename).name) + return ScenarioIdentity( - feature=Path(str(getattr(feature, "filename", ""))).stem, + uri=feature_uri(relative), + path=Path(filename), name=str(getattr(scenario, "name", "")), example=_example_of(node), tags=normalise_tags(tags), @@ -88,12 +127,11 @@ def _examples_tags(node: pytest.Item, scenario: object) -> set[str]: """The tags of the Examples block *this row* came from. Gherkin allows an Examples block to carry its own tags, so two rows of one - Scenario Outline can differ in which capability gates them. Those tags are not - on the scenario, the feature or the rule, so a report built from those three - alone would show a row the capability gate skipped as carrying no capability - at all -- and it would then be classified ``not-applicable`` rather than - ``not-declared``, which is precisely the distinction Appendix F asks a report - to keep. It would also not count towards the capability rollup. + Scenario Outline can differ in which capability gates them. Those tags are + not on the scenario, the feature or the rule, so a stream built from those + three alone would show a row the capability gate skipped as carrying no + capability at all -- and the envelope's declaration would then not explain + the skip, which is the one derivation the format asks a consumer to make. Resolved by intersecting the tags the scenario's Examples blocks declare with the markers pytest actually put on this node: pytest-bdd attaches an Examples @@ -116,24 +154,23 @@ def _examples_tags(node: pytest.Item, scenario: object) -> set[str]: def _example_of(node: pytest.Item) -> tuple[tuple[str, str], ...]: """The Examples row this node came from, keyed by column header. - Every row of a Scenario Outline shares one scenario name, so the row is what - tells eleven otherwise identical entries apart -- and in this suite one row - of the type-mismatch matrix genuinely differs in outcome from its ten - siblings. The row goes in its own field rather than into a mangled name - because the parameters *are* the identity and they come from the feature - file, whereas a name format would be a rule about this runner: pytest-bdd's - own id for the row above is ``boolean-flag-Integer-1``, which no other - language's runner has any reason to reproduce. + No longer reported -- Cucumber Messages identifies an outline row by the AST + node id of the table row a pickle was compiled from, which is exact and which + every runner that emits Messages already carries. This survives as the *join + key*: it is the one description of a row that both a pytest-bdd node and a + Gherkin pickle can produce independently, so it is how a node is matched to + its pickle. Matching on the pickle's name would not work, because the + compiler interpolates the row's parameters into it and pytest-bdd does not. pytest-bdd renders an outline by parametrizing the generated test over one dict per row, keyed by the Examples column header, and pytest hangs it on the node's callspec. A scenario that is not an outline is not parametrized and - has no callspec at all, which is why the empty tuple -- and therefore an - omitted field -- is the answer for one. + has no callspec at all, which is why the empty tuple is the answer for one -- + and it matches the empty row of a pickle with a single AST node id. Values are passed through as the parser produced them: Gherkin cells are - strings, and the report says what the table said rather than guessing that - ``1`` was meant as a number. + strings, and both sides of the join have to agree on ``"1"`` rather than one + of them guessing it was meant as a number. """ params = getattr(getattr(node, "callspec", None), "params", None) if not isinstance(params, dict): @@ -168,12 +205,13 @@ class ReportEmitter: def __init__(self, config: pytest.Config) -> None: self.collector = ReportCollector() + self._step_started: dict[str, int] = {} config.stash[COLLECTOR_KEY] = self.collector def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: """Enumerate every TCK scenario the session collected. - At collection rather than as each runs, so that the document accounts for + At collection rather than as each runs, so that the stream accounts for scenarios that never got as far as running a fixture. """ for item in items: @@ -184,6 +222,63 @@ def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: self.collector.observe(report.nodeid, _phase_outcome(report)) + # -- what each Gherkin step did ------------------------------------------ + # + # pytest reports a scenario, not its steps. Cucumber Messages records a + # result per step, and inventing one -- marking all eight steps failed + # because the scenario failed -- would be saying something untrue about the + # seven that passed and the ones that were never reached. pytest-bdd's step + # hooks are the only place the truth is available. + + def pytest_bdd_before_step( + self, request: pytest.FixtureRequest, step: object + ) -> None: + self._step_started[request.node.nodeid] = time.time_ns() + + def pytest_bdd_after_step(self, request: pytest.FixtureRequest) -> None: + self._finish_step(request, Status.passed) + + def pytest_bdd_step_error( + self, request: pytest.FixtureRequest, exception: BaseException + ) -> None: + # A step that calls ``pytest.skip`` raises through the same hook as one + # that failed, and the two are not the same result. Told apart by the + # exception type rather than by the message, which is prose. + if isinstance(exception, _SKIPPED): + self._finish_step(request, Status.skipped, exception) + return + self._finish_step(request, Status.failed, exception) + + def pytest_bdd_step_func_lookup_error( + self, request: pytest.FixtureRequest, exception: BaseException + ) -> None: + # UNDEFINED rather than FAILED: the step was never run, because nothing + # claimed to know how to run it. That is a defect in an adoption rather + # than a finding about the provider, and the stream says which. + self._step_started.setdefault(request.node.nodeid, time.time_ns()) + self._finish_step(request, Status.undefined, exception) + + def _finish_step( + self, + request: pytest.FixtureRequest, + status: Status, + exception: BaseException | None = None, + ) -> None: + node_id = request.node.nodeid + finished = time.time_ns() + self.collector.observe_step( + node_id, + StepRun( + status=status, + message=_reason(str(exception)) if exception is not None else "", + exception_type=type(exception).__name__ + if exception is not None + else "", + started_ns=self._step_started.pop(node_id, finished), + finished_ns=finished, + ), + ) + def pytest_sessionfinish(self, session: pytest.Session) -> None: directory = os.environ.get(REPORT_DIR_ENV, "").strip() if not directory: @@ -191,20 +286,20 @@ def pytest_sessionfinish(self, session: pytest.Session) -> None: self.write(session, Path(directory)) def write(self, session: pytest.Session, directory: Path) -> None: - """Write every suite's report, failing the session if one cannot be written. + """Write every suite's pair of files, failing the session if one cannot be. A run that asked for a report and silently did not get one is how a publishing pipeline ends up serving a stale result forever, so both a write failure and an incomplete document are loud and change the exit status rather than being logged and forgotten. """ - for problem in self.collector.resolve(classify_phase): + for problem in self.collector.resolve(scenario_run): self._fail(session, f"provider-tck: {problem}") written: dict[str, str] = {} for suite in self.collector.suites: name = suite.config.name - file_name = report_file_name(name) + file_name = envelope_file_name(name) if written.get(file_name, name) != name: self._fail( session, @@ -213,23 +308,67 @@ def write(self, session: pytest.Session, directory: Path) -> None: ) continue written[file_name] = name + self._write_suite(session, directory, suite) + + def _write_suite( + self, session: pytest.Session, directory: Path, suite: SuiteReport + ) -> None: + name = suite.config.name + runs = suite.sorted_runs + + catalog = FeatureCatalog() + try: + for run in runs: + catalog.load(run.identity) + except OSError as error: + self._fail( + session, + f"provider-tck [{name}]: could not read the feature files the run " + f"executed, so the results payload cannot name them: {error}", + ) + return - try: - path = write_report(directory, name, suite.build()) - except OSError as error: - self._fail( - session, - f"provider-tck [{name}]: could not write the conformance report " - f"to {directory}: {error}", - ) - continue - counts = ", ".join( - f"{count} {outcome}" - for outcome, count in sorted(suite.counts().items()) + unmatched = [ + run.identity for run in runs if catalog.pickle_for(run.identity) is None + ] + for identity in unmatched: + # The one failure mode this format exists to rule out: a scenario + # that ran and is missing from the results. Reported per scenario + # rather than as a count, because which one it is is the whole point. + self._fail( + session, + f"provider-tck [{name}]: {identity.uri} scenario " + f"{identity.name!r}{_row(identity)} matched no Gherkin pickle, so " + f"the results payload does not account for it", ) - self._say( - session, f"provider-tck [{name}]: report written to {path} ({counts})" + + stream_path = directory / stream_file_name(name) + try: + digest = write_stream( + stream_path, + catalog, + runs, + implementation=TCK_IMPLEMENTATION, + implementation_version=distribution_version(TCK_DISTRIBUTION), + ) + envelope = suite.build(Results(location=stream_path.name, digest=digest)) + path = write_envelope(directory, name, envelope) + except OSError as error: + self._fail( + session, + f"provider-tck [{name}]: could not write the conformance report " + f"to {directory}: {error}", ) + return + + counts = ", ".join( + f"{count} {status}" for status, count in sorted(suite.counts().items()) + ) + self._say( + session, + f"provider-tck [{name}]: report written to {path} with results in " + f"{stream_path.name} ({counts})", + ) def _say(self, session: pytest.Session, message: str) -> None: reporter = session.config.pluginmanager.get_plugin("terminalreporter") @@ -241,6 +380,13 @@ def _fail(self, session: pytest.Session, message: str) -> None: session.exitstatus = pytest.ExitCode.INTERNAL_ERROR +def _row(identity: ScenarioIdentity) -> str: + if not identity.example: + return "" + row = " ".join(f"{header}={cell}" for header, cell in identity.example) + return f" [{row}]" + + def _phase_outcome(report: pytest.TestReport) -> PhaseOutcome: """Reduce a pytest phase report to what the conformance report needs.""" xfail_reason: str | None = getattr(report, "wasxfail", None) @@ -250,54 +396,68 @@ def _phase_outcome(report: pytest.TestReport) -> PhaseOutcome: outcome=report.outcome, xfail_reason=xfail_reason, message=message, - duration=report.duration, + start=getattr(report, "start", 0.0), + stop=getattr(report, "stop", 0.0), ) -def classify_phase( - phase: PhaseOutcome, identity: ScenarioIdentity, config: TckConfig -) -> tuple[Outcome, str] | None: - """Map one phase onto an outcome, or onto nothing. +def scenario_run( + identity: ScenarioIdentity, + phases: list[PhaseOutcome], + steps: list[StepRun], +) -> ScenarioRun: + """Assemble one scenario's execution from what pytest reported about it. - Nothing is the answer for a setup or teardown that simply worked: it says - nothing about the scenario, and letting it speak would overwrite what the - call phase already established. + The scenario's own status is the most serious of its phases', so a scenario + whose steps passed and whose teardown then blew up is a failed scenario: the + phase that reports last must not be the one that decides. + """ + starts = [phase.start for phase in phases if phase.start] + stops = [phase.stop for phase in phases if phase.stop] + run = ScenarioRun( + identity=identity, + steps=list(steps), + started_ns=int(min(starts, default=0.0) * 1_000_000_000), + finished_ns=int(max(stops, default=0.0) * 1_000_000_000), + ) + for phase in phases: + status, message = classify_phase(phase) + result = StepRun( + status=status, + message=message, + started_ns=int(phase.start * 1_000_000_000), + finished_ns=int(phase.stop * 1_000_000_000), + ) + if phase.when == "setup": + run.setup = result + elif phase.when == "teardown": + run.teardown = result + upgraded = worse(run.status, status) + if upgraded is not run.status: + # The message belongs to whichever phase decided the verdict, so a + # teardown failure does not inherit the reason a passing call gave. + run.message = message + run.status = upgraded + return run + + +def classify_phase(phase: PhaseOutcome) -> tuple[Status, str]: + """Map one pytest phase onto a Cucumber status. + + The one decision that is not a rename: an expected failure is still a + failure. pytest reports an ``xfail`` as skipped and exits zero; the provider + did not satisfy the scenario, and a stream calling it anything else would + hide exactly the deviation the marker was added to keep visible. The + acknowledgement goes in the envelope's ``knownDeviations`` instead, which is + where a claim about the provider belongs. """ if phase.outcome == "skipped" and phase.xfail_reason is not None: - # An expected failure is still a failure. The provider did not satisfy - # the scenario, and a report calling it anything else would hide exactly - # the deviation the marker was added to keep visible. - return Outcome.FAILED, _reason(f"expected failure: {phase.xfail_reason}") + return Status.failed, _reason(f"expected failure: {phase.xfail_reason}") if phase.outcome == "failed": - return Outcome.FAILED, phase.message or "failed" + return Status.failed, phase.message or "failed" if phase.outcome == "skipped": - return _skipped(phase, identity, config) - if phase.when == "call": - return Outcome.PASSED, "" - return None - - -def _skipped( - phase: PhaseOutcome, identity: ScenarioIdentity, config: TckConfig -) -> tuple[Outcome, str]: - """Tell a capability skip apart from every other kind. - - Decided from the scenario's tags and the suite's declared capabilities rather - than from the skip message, because the message is prose and the distinction - is not. Anything else that skipped a scenario -- a marker an adopter applied, - a step calling ``pytest.skip`` -- is reported as not applicable: it did not - run, and not because a capability was left undeclared. - """ - undeclared = [ - capability.tag - for capability in identity.capabilities() - if not config.declares(capability) - ] - if undeclared: - return Outcome.NOT_DECLARED, phase.message or ( - f"provider does not declare {' '.join(undeclared)}" - ) - return Outcome.NOT_APPLICABLE, phase.message or "skipped" + return Status.skipped, phase.message or "skipped" + return Status.passed, "" def _skip_reason(report: pytest.TestReport) -> str: diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py new file mode 100644 index 00000000..cad1737a --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py @@ -0,0 +1,624 @@ +"""The results payload: one run of the suite written as Cucumber Messages. + +The conformance report used to define its own per-scenario result list -- an +outcome enum, a tag list, and a field naming which Scenario Outline row an entry +came from. All three already exist in `Cucumber Messages`_, the ndjson protocol +Cucumber itself emits: it is maintained, schema'd, cross-language, and it carries +things a bespoke format would have had to invent, including the executed feature +source. So the report no longer describes results. It points at a stream of them. + +Nothing here knows about pytest. It takes a set of scenario outcomes and a set of +feature files and produces the stream; :mod:`.emitter` is what turns a pytest +session into the former. + +**Where the messages come from.** Two libraries, each doing the half it owns. + +``gherkin-official`` -- the reference Gherkin parser, already a dependency of +pytest-bdd -- produces the ``gherkinDocument`` and ``pickle`` payloads. Those +payloads *are* Messages: emitting Messages ndjson is what that library exists +for, so its output is used as it comes rather than round-tripped through +another representation that could quietly drop a field it does not model. + +``cucumber-messages`` -- the official Python types, from the same repository as +the protocol -- builds the execution half: ``Meta``, ``TestCase``, +``TestCaseStarted``, ``TestStepFinished`` and the rest. Hand-writing those dicts +would work until the protocol moved. + +**Why the feature files are parsed again.** pytest-bdd parses them with +``gherkin-official`` too, but converts the result into dataclasses of its own +that do not carry the AST node ids. Those ids are what a pickle refers to, and +what makes one Scenario Outline row distinguishable from another, so the stream +needs a parse whose ids it owns. Four small files, parsed once per session. + +.. _Cucumber Messages: https://github.com/cucumber/messages +""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import platform +import typing +from dataclasses import dataclass, field +from pathlib import Path, PurePath + +import cucumber_messages as cucumber +from gherkin.ast_builder import AstBuilder +from gherkin.parser import Parser +from gherkin.pickles.compiler import Compiler +from gherkin.stream.id_generator import IdGenerator + +from .capability import Capability, capability_for_tag + +__all__ = [ + "MESSAGES_FORMAT", + "FeatureCatalog", + "ScenarioIdentity", + "ScenarioRun", + "Status", + "StepRun", + "worse", + "write_stream", +] + +MESSAGES_FORMAT = "cucumber-messages" +"""The ``results.format`` value the envelope carries for this payload.""" + +Status = cucumber.TestStepResultStatus +"""The protocol's own status vocabulary, used rather than one of ours. + +Cucumber's seven statuses already draw the distinctions a conformance run needs, +and the four-value outcome enum this replaced drew a different set: it split +"did not run" into a capability the provider did not declare and one that cannot +apply to it, and merged "failed" with "the step was never reached". The first +distinction is not a property of the run at all -- it follows from the report's +declaration and the scenario's tags -- so it belongs in the envelope, once, and +not in every scenario. +""" + +_SEVERITY = { + Status.unknown: 0, + Status.passed: 1, + Status.skipped: 2, + Status.pending: 3, + Status.undefined: 4, + Status.ambiguous: 5, + Status.failed: 6, +} +"""How Cucumber orders its statuses, which is how a test case takes one. + +A test case is as bad as its worst step -- that is the rule a consumer applies to +derive a scenario's outcome from the stream, and this module applies the same one +so that what the stream says and what this package believes cannot diverge. +""" + + +def worse(left: Status, right: Status) -> Status: + """The more serious of two statuses.""" + return right if _SEVERITY[right] > _SEVERITY[left] else left + + +_MEDIA_TYPE = cucumber.SourceMediaType.text_x_cucumber_gherkin_plain + +_SETUP_HOOK_ID = "provider-tck-setup" +_TEARDOWN_HOOK_ID = "provider-tck-teardown" +"""The two hooks every test case carries, as the protocol models them. + +pytest runs a scenario in three phases and only the middle one executes Gherkin +steps: the capability gate skips during setup, and a provider that fails to shut +down fails during teardown. Neither has a pickle step to attach a result to, so +without hooks a gated skip would have to borrow the first step's result and a +teardown failure would be invisible behind a row of passed steps. Cucumber +represents exactly this with a ``Hook`` and a ``TestStep`` that references it, +which is what these are. +""" + +_MESSAGES_DISTRIBUTION = "cucumber-messages" +_UNKNOWN_VERSION = "unknown" + + +@dataclass(frozen=True) +class ScenarioIdentity: + """What a scenario is, independent of how it turned out. + + Established at collection, from the pytest node alone, so that a scenario + skipped before a single step ran is identified exactly as fully as one that + passed. That is what lets the stream account for every scenario rather than + only for the ones that got far enough to be interesting. + """ + + uri: str + """The feature file as Cucumber names it, e.g. ``features/errors.feature``. + + Slash-separated on every platform, and the same string in ``Source``, + ``GherkinDocument`` and ``Pickle``, which is what ties the three together. + """ + + path: Path + """Where that file actually is, so its source can be read and parsed.""" + + name: str + """The scenario name as the feature file spells it. + + For a Scenario Outline this is the template name, shared by every row -- + which is why it is not on its own an identity. + """ + + tags: tuple[str, ...] + + example: tuple[tuple[str, str], ...] = () + """The Examples row, as header/cell pairs, for a scenario from an outline. + + Not reported: Messages carries row identity as the pickle's AST node ids, + which is where four independent implementations of a bespoke ``example`` + field were each converging by hand. It survives here only as the join key + that matches a pytest node to its pickle -- pytest-bdd parametrises the + generated test over the row, and the row is the one thing both sides of that + join can see. + """ + + def capabilities(self) -> tuple[Capability, ...]: + """The capabilities this scenario's tags gate it behind.""" + gated = (capability_for_tag(tag) for tag in self.tags) + return tuple(capability for capability in gated if capability is not None) + + +@dataclass(frozen=True) +class StepRun: + """What happened to one step, as the protocol records it.""" + + status: Status + message: str = "" + exception_type: str = "" + started_ns: int = 0 + finished_ns: int = 0 + + +@dataclass +class ScenarioRun: + """One scenario's execution: its verdict, and what each phase did.""" + + identity: ScenarioIdentity + status: Status = Status.unknown + message: str = "" + setup: StepRun | None = None + teardown: StepRun | None = None + steps: list[StepRun] = field(default_factory=list) + """Step results in execution order, as far as execution got. + + Shorter than the pickle's step list whenever a scenario stopped early, which + is the normal case for a failure and the whole list for a skip. The stream + pads the difference with ``SKIPPED``, which is what Cucumber means by it. + """ + + started_ns: int = 0 + finished_ns: int = 0 + + +@dataclass(frozen=True) +class _Pickle: + """One compiled pickle, reduced to what the stream needs to refer to it.""" + + id: str + step_ids: tuple[str, ...] + payload: dict[str, typing.Any] + + +class FeatureCatalog: + """The feature files a run executed, parsed into Messages and indexed. + + Indexed by what both sides of the join can see: the file, the scenario name + as the feature file spells it, and the Examples row. A pytest-bdd node knows + those three; a pickle can be made to yield them by following its AST node ids + back to the scenario and the table row it was compiled from. Matching on the + pickle's own name would not do, because the compiler interpolates outline + parameters into it and pytest-bdd does not. + """ + + def __init__(self) -> None: + # One generator across the whole session, so ids are unique across + # feature files rather than only within one -- a stream is a single id + # space and two files numbering from zero would collide. + self._ids = IdGenerator() + self._sources: dict[str, str] = {} + self._documents: dict[str, dict[str, typing.Any]] = {} + self._pickles: dict[str, list[_Pickle]] = {} + self._index: dict[tuple[str, str, tuple[tuple[str, str], ...]], _Pickle] = {} + + @property + def uris(self) -> list[str]: + return sorted(self._documents) + + def load(self, identity: ScenarioIdentity) -> None: + """Parse the feature file this scenario came from, once.""" + if identity.uri in self._documents: + return + source = identity.path.read_text(encoding="utf-8") + document: dict[str, typing.Any] = Parser( + ast_builder=AstBuilder(self._ids) + ).parse(source) + document["uri"] = identity.uri + pickles: list[dict[str, typing.Any]] = Compiler(self._ids).compile(document) + + self._sources[identity.uri] = source + self._documents[identity.uri] = document + self._pickles[identity.uri] = [ + _Pickle( + id=str(pickle["id"]), + step_ids=tuple(str(step["id"]) for step in pickle.get("steps") or ()), + payload=pickle, + ) + for pickle in pickles + ] + self._index_pickles(identity.uri, document, pickles) + + def _index_pickles( + self, + uri: str, + document: dict[str, typing.Any], + pickles: list[dict[str, typing.Any]], + ) -> None: + names, rows = _ast_index(document) + for pickle, entry in zip(pickles, self._pickles[uri], strict=True): + ast_node_ids = [str(node) for node in pickle["astNodeIds"]] + name = names.get(ast_node_ids[0], str(pickle["name"])) + row = rows.get(ast_node_ids[1], ()) if len(ast_node_ids) > 1 else () + self._index.setdefault((uri, name, row), entry) + + def pickle_for(self, identity: ScenarioIdentity) -> _Pickle | None: + """The pickle this scenario was compiled from, or ``None`` if unmatched. + + ``None`` is a defect rather than a possibility to tolerate: a scenario + that ran and has no pickle cannot appear in the stream, which is the one + failure mode the report exists to rule out. The caller fails the run. + """ + return self._index.get((identity.uri, identity.name, identity.example)) + + def source_envelopes(self) -> typing.Iterator[dict[str, typing.Any]]: + """The ``Source``, ``GherkinDocument`` and ``Pickle`` messages, in order. + + The source comes first because everything after it refers to it, and it + is the reason this format beats recording an asset revision: a consumer + can read the questions that were actually asked rather than trusting a + commit hash to stand for them. + """ + for uri in self.uris: + yield _envelope( + cucumber.Envelope( + source=cucumber.Source( + data=self._sources[uri], media_type=_MEDIA_TYPE, uri=uri + ) + ) + ) + yield {"gherkinDocument": self._documents[uri]} + for entry in self._pickles[uri]: + yield {"pickle": entry.payload} + + +def _ast_index( + document: dict[str, typing.Any], +) -> tuple[dict[str, str], dict[str, tuple[tuple[str, str], ...]]]: + """Map AST node ids onto scenario names and Examples rows. + + Walks the parsed document rather than the pickles, because the pickle is + where the outline has already been expanded: the scenario name it carries has + the row's parameters substituted into it, and the row itself has become a + list of interpolated step texts. The AST still has both separately, which is + what a pytest-bdd node can be compared against. + """ + names: dict[str, str] = {} + rows: dict[str, tuple[tuple[str, str], ...]] = {} + + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children") or ()) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[str(scenario["id"])] = str(scenario["name"]) + for examples in scenario.get("examples") or (): + header = examples.get("tableHeader") + if header is None: + continue + headers = [str(cell["value"]) for cell in header["cells"]] + for row in examples.get("tableBody") or (): + cells = [str(cell["value"]) for cell in row["cells"]] + rows[str(row["id"])] = tuple(zip(headers, cells, strict=False)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children") or ()) + return names, rows + + +def write_stream( + path: Path, + catalog: FeatureCatalog, + runs: typing.Sequence[ScenarioRun], + implementation: str, + implementation_version: str, +) -> str: + """Write the stream and return its digest as ``sha256:``. + + The digest is returned rather than recomputed by the caller so that what is + hashed is exactly the bytes that were written, which is the only version of + that claim worth putting in the envelope. + """ + digest = hashlib.sha256() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as stream: + for envelope in _stream(catalog, runs, implementation, implementation_version): + line = ( + json.dumps(envelope, ensure_ascii=False, separators=(",", ":")) + "\n" + ) + stream.write(line) + digest.update(line.encode("utf-8")) + return f"sha256:{digest.hexdigest()}" + + +def _stream( + catalog: FeatureCatalog, + runs: typing.Sequence[ScenarioRun], + implementation: str, + implementation_version: str, +) -> typing.Iterator[dict[str, typing.Any]]: + started = min((run.started_ns for run in runs), default=0) + finished = max((run.finished_ns for run in runs), default=started) + + yield _envelope( + cucumber.Envelope(meta=_meta(implementation, implementation_version)) + ) + yield _envelope( + cucumber.Envelope( + test_run_started=cucumber.TestRunStarted( + id=_RUN_ID, timestamp=_timestamp(started) + ) + ) + ) + yield from _hook_envelopes() + yield from catalog.source_envelopes() + + for index, run in enumerate(runs): + entry = catalog.pickle_for(run.identity) + if entry is None: + # Ruled out by the caller before it gets here; skipping rather than + # raising keeps a defect in the report from destroying the run's own + # exit status, which says something the report cannot. + continue + yield from _test_case_envelopes(f"test-case-{index}", entry, run) + + yield _envelope( + cucumber.Envelope( + test_run_finished=cucumber.TestRunFinished( + success=all(run.status is not Status.failed for run in runs), + timestamp=_timestamp(finished), + test_run_started_id=_RUN_ID, + ) + ) + ) + + +_RUN_ID = "provider-tck-run" + + +def _test_case_envelopes( + test_case_id: str, entry: _Pickle, run: ScenarioRun +) -> typing.Iterator[dict[str, typing.Any]]: + """One scenario: its test case, and what each of its steps did.""" + started_id = f"{test_case_id}-started" + setup_step_id = f"{test_case_id}-setup" + teardown_step_id = f"{test_case_id}-teardown" + + test_steps = [ + cucumber.TestStep(id=setup_step_id, hook_id=_SETUP_HOOK_ID), + *( + cucumber.TestStep(id=f"{test_case_id}-{position}", pickle_step_id=step_id) + for position, step_id in enumerate(entry.step_ids) + ), + cucumber.TestStep(id=teardown_step_id, hook_id=_TEARDOWN_HOOK_ID), + ] + yield _envelope( + cucumber.Envelope( + test_case=cucumber.TestCase( + id=test_case_id, + pickle_id=entry.id, + test_run_started_id=_RUN_ID, + test_steps=test_steps, + ) + ) + ) + yield _envelope( + cucumber.Envelope( + test_case_started=cucumber.TestCaseStarted( + attempt=0, + id=started_id, + test_case_id=test_case_id, + timestamp=_timestamp(run.started_ns), + ) + ) + ) + + for step_id, result in zip( + (step.id for step in test_steps), _step_runs(entry, run), strict=True + ): + yield from _step_envelopes(started_id, step_id, result) + + yield _envelope( + cucumber.Envelope( + test_case_finished=cucumber.TestCaseFinished( + test_case_started_id=started_id, + timestamp=_timestamp(run.finished_ns), + will_be_retried=False, + ) + ) + ) + + +def _step_runs(entry: _Pickle, run: ScenarioRun) -> list[StepRun]: + """Every step of the pickle and both hooks, including what never ran. + + A scenario that stopped early -- the ordinary shape of both a failure and a + skip -- has fewer recorded results than the pickle has steps, and the + remainder are reported ``SKIPPED``, which is what Cucumber means by it and + what makes the stream account for the whole scenario rather than the part of + it that executed. + + The last thing this does is make sure the scenario's own verdict survives the + trip. A consumer reads a test case's outcome as the worst of its steps, so a + verdict no step accounts for would be lost: a strict ``xfail`` that passed is + a failed scenario every one of whose steps passed, and there are other ways + for a runner to fail a test case between its steps. Whatever is left over is + attached to the after-hook, which is where a test case failing outside its + own steps belongs. + """ + unreached = StepRun( + status=Status.skipped, + started_ns=run.finished_ns, + finished_ns=run.finished_ns, + ) + steps = list(run.steps[: len(entry.step_ids)]) + steps += [unreached] * (len(entry.step_ids) - len(steps)) + setup = run.setup or StepRun( + status=Status.passed, + started_ns=run.started_ns, + finished_ns=run.started_ns, + ) + teardown = run.teardown or StepRun( + status=Status.passed, + started_ns=run.finished_ns, + finished_ns=run.finished_ns, + ) + + reported = Status.unknown + for result in (setup, *steps, teardown): + reported = worse(reported, result.status) + if worse(reported, run.status) is not reported: + teardown = StepRun( + status=run.status, + message=run.message, + started_ns=teardown.started_ns, + finished_ns=teardown.finished_ns, + ) + return [setup, *steps, teardown] + + +def _step_envelopes( + started_id: str, step_id: str, result: StepRun +) -> typing.Iterator[dict[str, typing.Any]]: + yield _envelope( + cucumber.Envelope( + test_step_started=cucumber.TestStepStarted( + test_case_started_id=started_id, + test_step_id=step_id, + timestamp=_timestamp(result.started_ns), + ) + ) + ) + exception = ( + cucumber.Exception(type=result.exception_type, message=result.message or None) + if result.exception_type + else None + ) + yield _envelope( + cucumber.Envelope( + test_step_finished=cucumber.TestStepFinished( + test_case_started_id=started_id, + test_step_id=step_id, + test_step_result=cucumber.TestStepResult( + duration=_duration(result.finished_ns - result.started_ns), + status=result.status, + message=result.message or None, + exception=exception, + ), + timestamp=_timestamp(result.finished_ns), + ) + ) + ) + + +def _hook_envelopes() -> typing.Iterator[dict[str, typing.Any]]: + for hook_id, hook_type, name in ( + ( + _SETUP_HOOK_ID, + cucumber.HookType.before_test_case, + "provider-tck setup: capability gate, provider registration", + ), + ( + _TEARDOWN_HOOK_ID, + cucumber.HookType.after_test_case, + "provider-tck teardown: provider shutdown", + ), + ): + yield _envelope( + cucumber.Envelope( + hook=cucumber.Hook( + id=hook_id, + name=name, + type=hook_type, + source_reference=cucumber.SourceReference( + uri="openfeature/contrib/tools/provider_tck/plugin.py" + ), + ) + ) + ) + + +def _meta(implementation: str, implementation_version: str) -> cucumber.Meta: + """Who produced the stream, and against which protocol version.""" + return cucumber.Meta( + cpu=cucumber.Product(name=platform.machine() or _UNKNOWN_VERSION), + implementation=cucumber.Product( + name=implementation, version=implementation_version + ), + os=cucumber.Product(name=platform.system() or _UNKNOWN_VERSION), + protocol_version=_protocol_version(), + runtime=cucumber.Product( + name=platform.python_implementation(), version=platform.python_version() + ), + ) + + +def _protocol_version() -> str: + """The Messages version this stream is written against. + + Read from the installed library rather than written down, because the library + is what decides: a version pinned here would go on claiming 34.2.0 after a + dependency bump moved the types underneath it. + """ + try: + return importlib.metadata.version(_MESSAGES_DISTRIBUTION) + except importlib.metadata.PackageNotFoundError: + return _UNKNOWN_VERSION + + +def _envelope(envelope: cucumber.Envelope) -> dict[str, typing.Any]: + converted: dict[str, typing.Any] = cucumber.message_converter.to_dict(envelope) + return converted + + +def _timestamp(nanoseconds: int) -> cucumber.Timestamp: + return cucumber.Timestamp( + seconds=nanoseconds // 1_000_000_000, nanos=nanoseconds % 1_000_000_000 + ) + + +def _duration(nanoseconds: int) -> cucumber.Duration: + nanoseconds = max(nanoseconds, 0) + return cucumber.Duration( + seconds=nanoseconds // 1_000_000_000, nanos=nanoseconds % 1_000_000_000 + ) + + +def feature_uri(relative_filename: str) -> str: + """Normalise pytest-bdd's relative feature path into a Cucumber uri. + + pytest-bdd builds it with ``os.path.join``, so on Windows it arrives + backslash-separated. A uri is slash-separated everywhere, and the same string + has to appear in the ``Source``, the ``GherkinDocument`` and every ``Pickle`` + or nothing ties them together -- so a report emitted on Windows would + otherwise not be comparable with one emitted on Linux. + """ + return PurePath(relative_filename).as_posix() diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py index ab3625b7..b1e9cb6d 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py @@ -1,4 +1,4 @@ -"""The machine-readable conformance report: what a run of the suite claims. +"""The conformance report envelope: what was tested, what it claims, where the results are. A run of the suite produces a pass or a fail on a terminal, which is enough for the person who started it and useless to anyone else. The report is the same run @@ -7,14 +7,25 @@ specification rather than by this package, so that four languages emit the same document. -The load-bearing part is the per-scenario list. Appendix F requires that a -scenario skipped for an undeclared capability is reported as skipped *with the -reason* and never as passed, and a summary line cannot be checked against that -rule by anything downstream. Recording every scenario's outcome individually -makes the rule checkable by the consumer instead of dependent on each runner's -summary being trustworthy -- and the outcomes are required to be complete, -because a report that silently omitted what it skipped would satisfy the letter -of the rule while still misleading its reader. +This document no longer describes the results. It is an envelope that identifies +the subject and points at a :mod:`Cucumber Messages <.messages>` stream beside +it. The per-scenario outcome list, the outcome enum and the field naming which +Scenario Outline row an entry came from have all been deleted, because Messages +already carries every one of them -- along with the executed feature source, +which no bespoke format had. + +Two things stay here, because Messages has no slot for either. + +The **declaration** is an input to reading the results rather than a summary of +them. A skipped scenario in the stream says the question was not put to this +provider; only the declaration says whether that is because the provider +declines the capability. Given the declaration and a scenario's tags -- both +present -- the reason for a skip follows, so it no longer has to be transported +once per scenario. + +The **tested subject**: no standard results format has a slot for "the provider +under test". Messages records the runtime and the OS, which is what produced the +answers, not what was being asked about. See https://github.com/open-feature/spec/issues/424 for the format and ``specification/assets/provider-tck/report/`` for the schema. @@ -28,22 +39,20 @@ import re import typing from dataclasses import dataclass, field -from enum import Enum from pathlib import Path -from .capability import Capability, capability_for_tag from .config import TckConfig +from .messages import MESSAGES_FORMAT, ScenarioIdentity, ScenarioRun, StepRun __all__ = [ "REPORT_DIR_ENV", "SCHEMA_VERSION", - "Outcome", "PhaseOutcome", "ReportCollector", - "ScenarioIdentity", - "ScenarioRecord", + "Results", "SuiteReport", - "report_file_name", + "envelope_file_name", + "stream_file_name", ] REPORT_DIR_ENV = "PROVIDER_TCK_REPORT_DIR" @@ -52,9 +61,10 @@ An environment variable rather than a :class:`~.config.TckConfig` field, so that emitting a report is a property of the *run* and not of the code: CI sets it, a developer running the suite locally does not, and no adopter changes a line to -publish one. Each suite writes ``/.json``, so several suites in one -pytest session -- flagd's RPC and in-process resolvers, say -- each produce their -own file without colliding. +publish one. Each suite writes two files -- ``/.json``, the envelope, +and ``/.ndjson``, the results the envelope points at -- so several +suites in one pytest session, flagd's RPC and in-process resolvers say, each +produce their own pair without colliding. Unset means no report, which is the default and is not an error. """ @@ -100,98 +110,69 @@ _UNSAFE_IN_FILENAME = re.compile(r"[^A-Za-z0-9._-]") -class Outcome(str, Enum): - """The result of one scenario, or of one capability. +@dataclass(frozen=True) +class Results: + """Where the executed results are, and what covers them.""" - Four rather than two, because "did not run" is not one thing. A capability - the provider chose not to declare is a different statement from one the - language makes impossible -- ``@strict-numeric-typing`` cannot hold in a - language with no integer type -- and reporting both as not declared would - show a whole language as missing something none of its providers can have. - """ + location: str + digest: str + format: str = MESSAGES_FORMAT - PASSED = "passed" - FAILED = "failed" - NOT_DECLARED = "not-declared" - NOT_APPLICABLE = "not-applicable" + def as_json(self) -> dict[str, typing.Any]: + document = {"format": self.format, "location": self.location} + if self.digest: + document["digest"] = self.digest + return document @dataclass(frozen=True) -class ScenarioIdentity: - """What a scenario is, independent of how it turned out. +class PhaseOutcome: + """One pytest phase report, reduced to what the conformance report needs. - Established at collection, from the pytest-bdd node alone, so that a scenario - skipped before a single step ran is identified exactly as fully as one that - passed. That is what lets the report account for every scenario rather than - only for the ones that got far enough to be interesting. + Reduced rather than kept, because a :class:`pytest.TestReport` holds a + formatted traceback and holding a session's worth of them to classify at the + end would be a memory leak with a nice name. """ - feature: str - name: str - tags: tuple[str, ...] - example: tuple[tuple[str, str], ...] = () - """The Examples row, as header/cell pairs, for a scenario from an outline. - - Pairs rather than a mapping so that this stays hashable and ordered: the - order is the feature file's column order, and the report carries it through - rather than imposing one of its own. - """ + when: str + """``setup``, ``call`` or ``teardown``.""" - def capabilities(self) -> tuple[Capability, ...]: - """The capabilities this scenario's tags gate it behind.""" - gated = (capability_for_tag(tag) for tag in self.tags) - return tuple(capability for capability in gated if capability is not None) + outcome: str + """``passed``, ``failed`` or ``skipped``, as pytest decided.""" + xfail_reason: str | None = None + """Set when pytest marked this an expected failure.""" -@dataclass -class ScenarioRecord: - """One scenario's outcome, as the report will carry it.""" + message: str = "" + """The skip reason, or the failure's headline, already trimmed.""" - feature: str - """The feature file without its extension, e.g. ``errors``.""" + start: float = 0.0 + stop: float = 0.0 - name: str - tags: tuple[str, ...] - outcome: Outcome - example: tuple[tuple[str, str], ...] = () - """The Examples row this entry came from; empty for a scenario that is not - an outline, in which case the field is omitted rather than emitted empty.""" - reason: str = "" - duration_ms: float = 0.0 +Resolver = typing.Callable[ + [ScenarioIdentity, "list[PhaseOutcome]", "list[StepRun]"], ScenarioRun +] +"""Turns what pytest reported about one scenario into what the stream records. - def as_json(self) -> dict[str, typing.Any]: - document: dict[str, typing.Any] = { - "feature": self.feature, - "name": self.name, - "outcome": self.outcome.value, - } - if self.example: - document["example"] = dict(self.example) - if self.tags: - document["tags"] = list(self.tags) - if self.reason: - document["reason"] = self.reason - if self.duration_ms: - document["durationMs"] = round(self.duration_ms, 3) - return document +A callable rather than a method, because the mapping is entirely about pytest -- +which phase means what, and that an expected failure is still a failure -- and +this module deliberately knows nothing about pytest. +""" @dataclass class SuiteReport: """What one suite -- one :class:`~.config.TckConfig` -- accumulates as it runs. - Records are keyed by pytest node id rather than appended to a list, which is + Runs are keyed by pytest node id rather than appended to a list, which is what makes "every scenario appears exactly once" a property of the structure - instead of a promise made by the code that fills it. A scenario reports - through several phases (setup, call, teardown) and each of them finds the - same entry. + instead of a promise made by the code that fills it. """ config: TckConfig provider_name: str | None = None - records: dict[str, ScenarioRecord] = field(default_factory=dict) - durations: dict[str, float] = field(default_factory=dict) + runs: dict[str, ScenarioRun] = field(default_factory=dict) def observe_provider_name(self, name: str) -> None: """Remember what the provider called itself through its own metadata. @@ -201,75 +182,36 @@ def observe_provider_name(self, name: str) -> None: if name: self.provider_name = name - def add_duration(self, node_id: str, seconds: float) -> None: - """Add one phase's time to a scenario's total. - - Kept apart from the record rather than added to it, because a scenario's - first phase can take time before anything has decided its outcome, and - time spent on a scenario that ended up skipped is still time. - """ - self.durations[node_id] = self.durations.get(node_id, 0.0) + seconds * 1000.0 - - def set_outcome( - self, - node_id: str, - identity: ScenarioIdentity, - outcome: Outcome, - reason: str = "", - ) -> None: - """Record, or revise, one scenario's outcome. - - A failure is never revised away. A scenario whose steps passed and whose - teardown then blew up is a failed scenario, and the phase that reports - last must not be the one that decides. - """ - record = self.records.get(node_id) - if record is None: - self.records[node_id] = ScenarioRecord( - feature=identity.feature, - name=identity.name, - tags=identity.tags, - outcome=outcome, - example=identity.example, - reason=reason, - ) - return - if record.outcome is Outcome.FAILED: - return - record.outcome = outcome - record.reason = reason or record.reason + def record(self, node_id: str, run: ScenarioRun) -> None: + self.runs[node_id] = run @property - def sorted_records(self) -> list[ScenarioRecord]: - for node_id, record in self.records.items(): - record.duration_ms = self.durations.get(node_id, 0.0) - # Sorted by the whole identity, example included, so that two rows of one - # outline come out in a stable order rather than in whichever order the - # dictionary happened to be filled. + def sorted_runs(self) -> list[ScenarioRun]: + """The runs in a stable order: feature, then scenario, then row. + + Sorted by the whole identity, the Examples row included, so that two rows + of one outline reach the stream in the feature file's terms rather than + in whichever order the dictionary happened to be filled. + """ return sorted( - self.records.values(), key=lambda r: (r.feature, r.name, r.example) + self.runs.values(), + key=lambda run: ( + run.identity.uri, + run.identity.name, + run.identity.example, + ), ) def counts(self) -> dict[str, int]: - """Outcome tallies, for a log line and for the tests that check them.""" + """Status tallies, for a log line and for the tests that check them.""" tally: dict[str, int] = {} - for record in self.records.values(): - tally[record.outcome.value] = tally.get(record.outcome.value, 0) + 1 + for run in self.runs.values(): + key = run.status.value.lower() + tally[key] = tally.get(key, 0) + 1 return tally - def build(self) -> dict[str, typing.Any]: - """Assemble the report document.""" - records = self.sorted_records - spec_revision, assets_tree = spec_identity() - - tck: dict[str, typing.Any] = { - "implementation": TCK_IMPLEMENTATION, - "version": distribution_version(TCK_DISTRIBUTION), - "specRevision": spec_revision, - } - if assets_tree: - tck["assetsTree"] = assets_tree - + def build(self, results: Results) -> dict[str, typing.Any]: + """Assemble the envelope around a results payload already written.""" document: dict[str, typing.Any] = { "schemaVersion": SCHEMA_VERSION, "provider": { @@ -286,16 +228,47 @@ def build(self) -> dict[str, typing.Any]: "name": SDK_DISTRIBUTION, "version": distribution_version(SDK_DISTRIBUTION), }, - "tck": tck, - "capabilities": self._capabilities(records), - "scenarios": [record.as_json() for record in records], + "tck": { + "implementation": TCK_IMPLEMENTATION, + "version": distribution_version(TCK_DISTRIBUTION), + "specRevision": spec_revision(), + }, + "declaration": self._declaration(), + "results": results.as_json(), } backend = self._backend() if backend: document["backend"] = backend + deviations = [deviation.as_json() for deviation in self.config.known_deviations] + if deviations: + # Omitted rather than emitted empty: stating no deviations is a + # claim, and an emitter that always emitted the field would make that + # claim on every provider's behalf whether or not it had checked. + document["knownDeviations"] = deviations return document + def _declaration(self) -> dict[str, typing.Any]: + """What the provider claims, which is what makes a skip legible. + + The declared set and the not-applicable set are disjoint and mean + different things -- a choice against a capability, and an impossibility. + :class:`~.config.TckConfig` refuses a configuration that puts a + capability in both, so a consumer never has to decide which one wins. + """ + declaration: dict[str, typing.Any] = { + "declared": self.config.sorted_capabilities + } + not_applicable = { + capability.tag: reason + for capability, reason in sorted( + self.config.not_applicable.items(), key=lambda item: item[0].tag + ) + } + if not_applicable: + declaration["notApplicable"] = not_applicable + return declaration + def _backend(self) -> dict[str, typing.Any]: backend: dict[str, typing.Any] = {} description = getattr(self.config.control, "description", "") @@ -306,97 +279,6 @@ def _backend(self) -> dict[str, typing.Any]: backend["controlApi"] = control_api return backend - def _capabilities( - self, records: list[ScenarioRecord] - ) -> dict[str, dict[str, typing.Any]]: - """Roll the per-scenario outcomes up to one verdict per capability. - - A capability is only reported as passed when everything gating on it - actually passed, and only reported as not declared when the provider did - not declare it -- in which case the reason says so, because "this - provider does not support configuration-change events" is exactly what - someone comparing providers came to find out. - - A capability the provider declared and *no scenario carries* is omitted - rather than reported. ``@targeting`` is reserved: it exists in the - vocabulary but nothing tests it, because asserting that an evaluation - context reached the backend needs an echo operation the control API does - not have. Reporting it as passed would be a green result for a claim - nothing examined -- the vacuous pass the capability vocabulary exists to - eliminate, arriving through the report rather than through the suite. - Omitting beats inventing a fifth outcome: the four the schema allows are - about what the provider did, and "the suite does not test this" is a fact - about the suite. - """ - # Counted rather than flagged, so that a failure can say how much of what - # failed, and so that "no scenario exercises this at all" is a case the - # rollup can see rather than one it silently reads as success. - exercised: dict[Capability, int] = {} - failed: dict[Capability, int] = {} - for record in records: - for tag in record.tags: - capability = capability_for_tag(tag) - if capability is None: - continue - exercised[capability] = exercised.get(capability, 0) + 1 - if record.outcome is Outcome.FAILED: - failed[capability] = failed.get(capability, 0) + 1 - - capabilities: dict[str, dict[str, typing.Any]] = {} - for capability in Capability: - if not self.config.declares(capability): - capabilities[capability.tag] = { - "state": Outcome.NOT_DECLARED.value, - "reason": ( - f"not declared by this provider's configuration; the " - f"{capability.tag} scenarios were skipped and did not " - f"contribute to this result" - ), - } - elif not exercised.get(capability): - continue - elif failed.get(capability): - capabilities[capability.tag] = { - "state": Outcome.FAILED.value, - "reason": ( - f"{failed[capability]} of {exercised[capability]} scenarios " - f"carrying {capability.tag} failed; the per-scenario results " - f"say which, and why" - ), - } - else: - capabilities[capability.tag] = {"state": Outcome.PASSED.value} - return capabilities - - -@dataclass(frozen=True) -class PhaseOutcome: - """One pytest phase report, reduced to what the conformance report needs. - - Reduced rather than kept, because a :class:`pytest.TestReport` holds a - formatted traceback and holding a session's worth of them to classify at the - end would be a memory leak with a nice name. - """ - - when: str - """``setup``, ``call`` or ``teardown``.""" - - outcome: str - """``passed``, ``failed`` or ``skipped``, as pytest decided.""" - - xfail_reason: str | None = None - """Set when pytest marked this an expected failure.""" - - message: str = "" - """The skip reason, or the failure's headline, already trimmed.""" - - duration: float = 0.0 - - -Classifier = typing.Callable[ - [PhaseOutcome, ScenarioIdentity, TckConfig], "tuple[Outcome, str] | None" -] - class ReportCollector: """Session-wide accumulator: which scenario belongs to which suite, and how it went. @@ -404,14 +286,14 @@ class ReportCollector: One pytest session can run several suites -- the TCK's own tests run two, and a provider with more than one resolver runs one per resolver -- so outcomes are attributed to a suite rather than to the session, and each suite writes - its own file. - - Scenarios are enumerated at collection and resolved into records only at the - end of the session. The order matters. A scenario skipped by a marker never - runs a fixture, so a design that learned of a scenario when its fixtures ran - would leave it out of the document entirely -- and a report that silently - omits what it skipped satisfies "a skip is never reported as passed" while - still misleading the person reading it. + its own pair of files. + + Scenarios are enumerated at collection and resolved into runs only at the end + of the session. The order matters. A scenario skipped by a marker never runs + a fixture, so a design that learned of a scenario when its fixtures ran would + leave it out of the stream entirely -- and a report that silently omits what + it skipped satisfies "a skip is never reported as passed" while still + misleading the person reading it. """ def __init__(self) -> None: @@ -422,6 +304,7 @@ def __init__(self) -> None: self._suite_by_group: dict[str, SuiteReport] = {} self._collected: dict[str, tuple[str, ScenarioIdentity]] = {} self._phases: dict[str, list[PhaseOutcome]] = {} + self._steps: dict[str, list[StepRun]] = {} def collect(self, node_id: str, group: str, identity: ScenarioIdentity) -> None: """Note that this scenario exists, and which group of tests it came from. @@ -433,11 +316,26 @@ def collect(self, node_id: str, group: str, identity: ScenarioIdentity) -> None: """ self._collected[node_id] = (group, identity) + @property + def identities(self) -> list[ScenarioIdentity]: + """Every collected scenario, so the feature files can be parsed once.""" + return [identity for _, identity in self._collected.values()] + def observe(self, node_id: str, phase: PhaseOutcome) -> None: """Record one phase's result for a scenario, if it is one of ours.""" if node_id in self._collected: self._phases.setdefault(node_id, []).append(phase) + def observe_step(self, node_id: str, step: StepRun) -> None: + """Record what one Gherkin step did, in execution order. + + Recorded as it happens rather than reconstructed afterwards, because + pytest reports a scenario and not its steps: only the runner knows which + step of eight failed, and a stream that marked all eight failed would be + saying something untrue about seven of them. + """ + self._steps.setdefault(node_id, []).append(step) + def bind(self, node_id: str, config: TckConfig) -> None: """Learn which suite a group of scenarios is testing. @@ -456,12 +354,12 @@ def suite_for(self, config: TckConfig) -> SuiteReport: def suites(self) -> list[SuiteReport]: return list(self._suites.values()) - def resolve(self, classify: Classifier) -> list[str]: - """Turn the collected phases into records, and report what could not be. + def resolve(self, resolver: Resolver) -> list[str]: + """Turn the collected phases into runs, and report what could not be. Returns the problems, one string each, and they are meant to be shouted about rather than logged: a scenario that ran but is missing from the - document is the one failure mode this format exists to rule out. + stream is the one failure mode this format exists to rule out. """ problems: list[str] = [] for node_id, (group, identity) in sorted(self._collected.items()): @@ -479,12 +377,9 @@ def resolve(self, classify: Classifier) -> list[str]: f"{suite.config.name!r} does not account for it" ) continue - for phase in phases: - classified = classify(phase, identity, suite.config) - if classified is not None: - outcome, reason = classified - suite.set_outcome(node_id, identity, outcome, reason) - suite.add_duration(node_id, phase.duration) + suite.record( + node_id, resolver(identity, phases, self._steps.get(node_id, [])) + ) return problems @@ -512,8 +407,8 @@ def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: return tuple(sorted(f"@{tag}" for tag in tags if _TAG_PATTERN.match(tag))) -def report_file_name(suite_name: str) -> str: - """Turn a suite name into a file name. +def report_stem(suite_name: str) -> str: + """Turn a suite name into a file name stem. Suite names are chosen to read well in a failure message rather than to be path-safe, so anything not obviously safe becomes a hyphen. Without this a @@ -521,15 +416,29 @@ def report_file_name(suite_name: str) -> str: given. """ cleaned = _UNSAFE_IN_FILENAME.sub("-", suite_name).strip("-.") - return f"{cleaned or 'report'}.json" + return cleaned or "report" + + +def envelope_file_name(suite_name: str) -> str: + return f"{report_stem(suite_name)}.json" -def write_report( +def stream_file_name(suite_name: str) -> str: + """The results payload, beside the envelope that points at it. + + A sibling rather than a subdirectory so that ``results.location`` is a bare + file name, which is a relative reference that survives the whole pair being + moved, uploaded or served from somewhere other than where it was written. + """ + return f"{report_stem(suite_name)}.ndjson" + + +def write_envelope( directory: Path, suite_name: str, document: dict[str, typing.Any] ) -> Path: - """Write one report, returning where it went.""" + """Write one envelope, returning where it went.""" directory.mkdir(parents=True, exist_ok=True) - path = directory / report_file_name(suite_name) + path = directory / envelope_file_name(suite_name) path.write_text( json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" ) @@ -549,25 +458,26 @@ def distribution_version(distribution: str) -> str: return UNKNOWN -def spec_identity() -> tuple[str, str]: - """Return the spec commit and asset tree these feature files came from. +def spec_revision() -> str: + """Return the spec commit these feature files came from. Captured at build time rather than read here, because the submodule that holds the answer is not in the wheel. A build that could not reach git says so with :data:`UNKNOWN` instead of inventing a commit, and an installation old enough to predate the generated file degrades the same way rather than failing to emit a report at all. + + The asset tree hash that used to accompany it is gone. It was carried so that + a consumer could tell whether two runs executed the same questions; the + results payload now carries the executed feature source itself, which answers + that directly rather than by proxy. """ reference = importlib.resources.files(_PACKAGE) / _REVISION_FILE try: data = json.loads(reference.read_text(encoding="utf-8")) except (OSError, ValueError): - return UNKNOWN, "" + return UNKNOWN if not isinstance(data, dict): - return UNKNOWN, "" + return UNKNOWN revision = data.get("specRevision") - tree = data.get("assetsTree") - return ( - revision if isinstance(revision, str) and len(revision) >= 7 else UNKNOWN, - tree if isinstance(tree, str) and re.fullmatch(r"[0-9a-f]{40}", tree) else "", - ) + return revision if isinstance(revision, str) and len(revision) >= 7 else UNKNOWN diff --git a/tools/openfeature-provider-tck/tests/conftest.py b/tools/openfeature-provider-tck/tests/conftest.py index a5e6726f..cbf775ee 100644 --- a/tools/openfeature-provider-tck/tests/conftest.py +++ b/tools/openfeature-provider-tck/tests/conftest.py @@ -3,35 +3,66 @@ A conformance suite that quietly goes green on scenarios it did not run is worse than no suite at all -- and the same is true of one that quietly goes green on a scenario it *did* run and fail. So the one scenario the Python SDK cannot -currently satisfy is marked ``xfail(strict=True)`` here, which: +currently satisfy is recorded twice, in two forms that answer different +questions. -* keeps it visible in the report, as XFAIL with the reason attached; -* fails the suite if it ever *passes*, so the marker is removed the moment the - SDK is fixed rather than lingering as a lie. +``xfail(strict=True)`` keeps the *run* honest: the scenario is expected to fail, +and the suite fails if it ever passes, so the marker is removed the moment the +SDK is fixed rather than lingering as a lie. -This lives in the TCK's own self-test rather than in the shared package. It is a -fact about the SDK under test, not part of the conformance definition, and -Appendix F deliberately leaves a general "known deviations" concept as an open -question (spec#417, Q4). If that concept lands, this moves into it. +:class:`KnownDeviation` keeps the *report* honest. The results payload reports +the scenario as failed regardless of the marker -- an expected failure is still a +failure, and softening it there would hide exactly what the marker exists to keep +visible -- and the envelope carries the acknowledgement beside it, with the issue +it is tracked under. That is what lets a consumer tell a known and tracked gap +from a surprise without the result itself being weakened. + +The two are declared together here so they cannot drift: the reason on the marker +and the summary in the report are the same sentence. """ from __future__ import annotations import pytest +from openfeature.contrib.tools.provider_tck import KnownDeviation + # The Scenario Outline row that asks for boolean-flag as an Integer. _BOOL_AS_INT = ( "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" ) +_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + _REASON = ( "python-sdk: a boolean satisfies an Integer request. The client type-checks with " "isinstance(value, int) and bool is a subclass of int in Python, so boolean-flag " "requested as an Integer returns True with reason STATIC and no error code, where " "the specification requires the code default and TYPE_MISMATCH. " - "See https://github.com/open-feature/python-sdk/issues/619" + f"See {_ISSUE}" ) +KNOWN_DEVIATIONS = (KnownDeviation(issue=_ISSUE, summary=_REASON),) +"""What the report acknowledges. + +No ``capability``: the scenario carries no capability tag, because returning the +code default on a type mismatch is mandatory. ``@strict-numeric-typing`` is a +neighbouring question -- whether 0.5 satisfies an integer request -- and this +provider satisfies it, so attributing the deviation there would be wrong twice +over. +""" + + +@pytest.fixture(scope="session") +def tck_known_deviations() -> tuple[KnownDeviation, ...]: + """The deviations a suite in this package declares. + + A fixture rather than an import so that the marker below and the report's + acknowledgement are written down once, in one place, and a suite picks it up + the same way it picks up everything else it is given. + """ + return KNOWN_DEVIATIONS + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: for item in items: diff --git a/tools/openfeature-provider-tck/tests/test_controllable_conformance.py b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py index 77b3c3d0..a9f0db5f 100644 --- a/tools/openfeature-provider-tck/tests/test_controllable_conformance.py +++ b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py @@ -20,13 +20,14 @@ from openfeature.contrib.tools.provider_tck import ( Capability, InProcessControl, + KnownDeviation, TckConfig, features_path, ) @pytest.fixture(scope="session") -def tck_config() -> TckConfig: +def tck_config(tck_known_deviations: tuple[KnownDeviation, ...]) -> TckConfig: """Declare the provider under test and what it can do. ``STALE`` and ``UNAVAILABLE_INIT`` stay undeclared: there is still no @@ -50,6 +51,9 @@ def tck_config() -> TckConfig: Capability.OBJECT, Capability.NUMERIC_COERCION, }, + # The same SDK bug, against the same issue: it is a defect in the client + # rather than in either provider, so both suites acknowledge it. + known_deviations=tck_known_deviations, ) diff --git a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py index 9b9e4a19..17d49fd8 100644 --- a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py +++ b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py @@ -22,6 +22,7 @@ from openfeature.contrib.tools.provider_tck import ( Capability, + KnownDeviation, TckConfig, canonical_flag_set, features_path, @@ -70,7 +71,7 @@ def _new_provider() -> FeatureProvider: @pytest.fixture(scope="session") -def tck_config() -> TckConfig: +def tck_config(tck_known_deviations: tuple[KnownDeviation, ...]) -> TckConfig: """Declare the provider under test and what it can do. Each omission is a fact about the provider rather than a convenience: @@ -93,6 +94,11 @@ def tck_config() -> TckConfig: vacuously while the feature was gated on ``EVENTS``, which is precisely the failure mode the split of ``@lifecycle`` from ``@events`` exists to end. A skip with a reason is the honest outcome. + + ``known_deviations`` is the one thing here that is not a claim about what + this provider supports: it is the acknowledgement of a scenario the SDK + fails, which the results payload still reports as a failure. See + ``conftest.py``. """ return TckConfig( name="in-memory", @@ -103,6 +109,7 @@ def tck_config() -> TckConfig: Capability.OBJECT, Capability.NUMERIC_COERCION, }, + known_deviations=tck_known_deviations, ) diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py index c27aaa68..2daa74d7 100644 --- a/tools/openfeature-provider-tck/tests/test_report.py +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -3,25 +3,36 @@ The report exists because a runner's summary cannot be checked by anything downstream. So the tests that matter here are not about JSON shape; they are about the two properties a consumer is entitled to assume, neither of which is -guaranteed by the code that happens to assemble the document: +guaranteed by the code that happens to assemble the documents: * a scenario skipped for an undeclared capability is never reported as passed, - and carries the reason it was skipped; + and the reason it was skipped is recoverable; * every scenario the run collected appears exactly once, which is what makes the first property checkable rather than merely asserted -- a document that quietly dropped what it skipped would satisfy the letter of it and still mislead. +Both are now checked against the *results payload* rather than against the +envelope, because that is where the results moved: a run writes an envelope and a +`Cucumber Messages`_ stream beside it, and the envelope says only what was tested, +what the provider claims and where the results are. The assertions are written +the way a consumer reads the stream -- a test case is as bad as its worst step -- +so that what these tests check is what a consumer would see rather than an +internal representation. + Both are checked against a real pytest session in a subprocess, because both are -properties of how the suite runs rather than of how the document is assembled. -That session is also the only place all four outcomes occur together, and the -only place the document can be seen to disagree with the runner's summary -- -which it does, deliberately, for a known deviation. +properties of how the suite runs rather than of how the documents are assembled. +That session is also the only place a skip, a pass and a failure occur together, +and the only place the payload can be seen to disagree with the runner's summary +-- which it does, deliberately, for a known deviation. + +.. _Cucumber Messages: https://github.com/cucumber/messages """ from __future__ import annotations import collections import dataclasses +import hashlib import json import os import subprocess @@ -33,24 +44,35 @@ from openfeature.contrib.tools.provider_tck import ( Capability, + KnownDeviation, TckConfig, features_path, ) -from openfeature.contrib.tools.provider_tck.emitter import classify_phase +from openfeature.contrib.tools.provider_tck.emitter import ( + classify_phase, + scenario_run, +) +from openfeature.contrib.tools.provider_tck.messages import ( + MESSAGES_FORMAT, + ScenarioIdentity, + Status, + StepRun, + _Pickle, + _step_runs, + feature_uri, +) from openfeature.contrib.tools.provider_tck.report import ( REPORT_DIR_ENV, - Outcome, PhaseOutcome, - ScenarioIdentity, + Results, SuiteReport, control_api_of, + envelope_file_name, normalise_tags, - report_file_name, - spec_identity, + spec_revision, + stream_file_name, ) -OUTCOMES = {outcome.value for outcome in Outcome} - # The generated suite's name is deliberately not path-safe. SUITE_NAME = "report/fixture" SUITE_FILE = "report-fixture.json" @@ -58,13 +80,28 @@ UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" # The type-mismatch matrix: eleven Examples rows under one scenario name, one of -# which the Python SDK fails. It is the case the example field exists for. +# which the Python SDK fails. It is the case row identity exists for. MISMATCH_SCENARIO = "Requesting the wrong type returns the code default" # The row that fails, spelled as the feature file spells it -- strings, because # Gherkin has no types and "1" is not 1. DEVIATING_ROW = {"key": "boolean-flag", "requested": "Integer", "default": "1"} +DEVIATION_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + +# How Cucumber orders its statuses. A test case is as bad as its worst step, and +# this is the rule a consumer applies to derive a scenario's outcome from the +# stream -- so it is the rule these tests apply too. +SEVERITY = [ + "UNKNOWN", + "PASSED", + "SKIPPED", + "PENDING", + "UNDEFINED", + "AMBIGUOUS", + "FAILED", +] + _SUITE_MODULE = '''\ """A one-fixture adoption, generated so the report can be checked end to end.""" @@ -74,6 +111,7 @@ from openfeature.contrib.tools.provider_tck import ( Capability, InProcessControl, + KnownDeviation, TckConfig, features_path, ) @@ -87,6 +125,8 @@ def tck_config(): control=control, new_provider=control.new_provider, capabilities={capabilities}, + not_applicable={not_applicable}, + known_deviations={deviations}, ) @@ -96,10 +136,20 @@ def tck_config(): CAPABILITIES = ( "{Capability.EVENTS, Capability.OBJECT, Capability.STRICT_NUMERIC_TYPING}" ) -"""What the main generated suite declares: enough to produce all four outcomes.""" +"""What the main generated suite declares: enough to produce a skip and a pass.""" + +NOT_APPLICABLE = '{Capability.STALE: "this provider has no connection to lose"}' +"""One capability the provider cannot have rather than merely does not declare.""" + +DEVIATIONS = ( + "(KnownDeviation(" + f'issue="{DEVIATION_ISSUE}", ' + 'summary="a boolean satisfies an Integer request"),)' +) # One scenario skipped outright and one known deviation marked xfail, so the run -# produces all four outcomes and finishes green while the document does not. +# produces a skip, a pass and a failure and finishes green while the payload +# does not. _CONFTEST_MODULE = """\ import pytest @@ -174,18 +224,179 @@ def tck_config(): ''' +# -- reading the payload back the way a consumer would ----------------------- + + +@dataclasses.dataclass(frozen=True) +class Case: + """One scenario as the stream reports it, assembled from its messages.""" + + uri: str + name: str + """The scenario name from the *AST*, so an outline's rows share it.""" + + row: tuple[tuple[str, str], ...] + """The Examples row, resolved from the pickle's AST node ids.""" + + tags: frozenset[str] + status: str + """The worst of the test case's steps, which is Cucumber's rule.""" + + setup_message: str + """What the before-hook said, which is where a skip's reason lands.""" + + step_statuses: tuple[str, ...] + + @property + def identity(self) -> tuple[str, str, tuple[tuple[str, str], ...]]: + return (self.uri, self.name, self.row) + + +@dataclasses.dataclass(frozen=True) +class Stream: + """A parsed Cucumber Messages stream.""" + + kinds: collections.Counter[str] + sources: dict[str, str] + cases: list[Case] + + def named(self, name: str) -> list[Case]: + return [case for case in self.cases if case.name == name] + + @property + def statuses(self) -> collections.Counter[str]: + return collections.Counter(case.status for case in self.cases) + + +@dataclasses.dataclass +class _Index: + """The stream's messages, keyed the way the protocol says they relate.""" + + kinds: collections.Counter[str] = dataclasses.field( + default_factory=collections.Counter + ) + sources: dict[str, str] = dataclasses.field(default_factory=dict) + names: dict[str, str] = dataclasses.field(default_factory=dict) + rows: dict[str, tuple[tuple[str, str], ...]] = dataclasses.field( + default_factory=dict + ) + pickles: dict[str, typing.Any] = dataclasses.field(default_factory=dict) + test_cases: dict[str, typing.Any] = dataclasses.field(default_factory=dict) + hooks: dict[str, str] = dataclasses.field(default_factory=dict) + started: dict[str, str] = dataclasses.field(default_factory=dict) + results: dict[str, list[tuple[str, typing.Any]]] = dataclasses.field( + default_factory=dict + ) + + def add(self, kind: str, body: typing.Any) -> None: + self.kinds[kind] += 1 + if kind == "source": + self.sources[body["uri"]] = body["data"] + elif kind == "gherkinDocument": + _index_document(body, self.names, self.rows) + elif kind == "pickle": + self.pickles[body["id"]] = body + elif kind == "testCase": + self.test_cases[body["id"]] = body + for step in body["testSteps"]: + if "hookId" in step: + self.hooks[step["id"]] = step["hookId"] + elif kind == "testCaseStarted": + self.started[body["id"]] = body["testCaseId"] + elif kind == "testStepFinished": + self.results.setdefault(body["testCaseStartedId"], []).append( + (body["testStepId"], body["testStepResult"]) + ) + + +def _read_stream(path: Path) -> Stream: + """Assemble the stream into test cases the way a consumer has to. + + Deliberately written against the protocol rather than against this package: + a pickle's ``astNodeIds`` are followed back into the ``GherkinDocument`` to + recover the scenario name and the Examples row, and a test case's status is + computed as the worst of its steps. If the stream does not actually support + those two operations, these tests fail -- which is the point. + """ + index = _Index() + for line in path.read_text(encoding="utf-8").splitlines(): + message = json.loads(line) + kind = next(iter(message)) + index.add(kind, message[kind]) + + cases = [ + _case(index, started_id, case_id) + for started_id, case_id in index.started.items() + ] + return Stream(kinds=index.kinds, sources=index.sources, cases=cases) + + +def _case(index: _Index, started_id: str, case_id: str) -> Case: + pickle = index.pickles[index.test_cases[case_id]["pickleId"]] + ast = pickle["astNodeIds"] + steps = index.results[started_id] + setup: dict[str, typing.Any] = next( + ( + result + for step_id, result in steps + if index.hooks.get(step_id, "").endswith("setup") + ), + {}, + ) + return Case( + uri=pickle["uri"], + name=index.names[ast[0]], + row=index.rows.get(ast[1], ()) if len(ast) > 1 else (), + tags=frozenset(tag["name"] for tag in pickle.get("tags", ())), + status=max((result["status"] for _, result in steps), key=SEVERITY.index), + setup_message=setup.get("message", ""), + step_statuses=tuple( + result["status"] for step_id, result in steps if step_id not in index.hooks + ), + ) + + +def _index_document( + document: dict[str, typing.Any], + names: dict[str, str], + rows: dict[str, tuple[tuple[str, str], ...]], +) -> None: + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children", ())) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[scenario["id"]] = scenario["name"] + for examples in scenario.get("examples", ()): + header = examples.get("tableHeader") + if header is None: + continue + headers = [cell["value"] for cell in header["cells"]] + for row in examples.get("tableBody", ()): + cells = [cell["value"] for cell in row["cells"]] + rows[row["id"]] = tuple(zip(headers, cells, strict=True)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children", ())) + + @dataclasses.dataclass(frozen=True) class Run: - """One subprocess run of the generated suite.""" + """One subprocess run of the generated suite: both documents it wrote.""" directory: Path result: subprocess.CompletedProcess[str] - document: dict[str, typing.Any] + envelope: dict[str, typing.Any] + stream: Stream + stream_path: Path @property - def scenarios(self) -> list[dict[str, typing.Any]]: - scenarios: list[dict[str, typing.Any]] = self.document["scenarios"] - return scenarios + def declared(self) -> set[str]: + return set(self.envelope["declaration"]["declared"]) # -- helpers ----------------------------------------------------------------- @@ -222,24 +433,27 @@ def _config(**overrides: typing.Any) -> TckConfig: return TckConfig(**settings) -def _identity(*tags: str) -> ScenarioIdentity: - return ScenarioIdentity(feature="events", name="a scenario", tags=tags) +def _identity(*tags: str, name: str = "a scenario") -> ScenarioIdentity: + return ScenarioIdentity( + uri="features/events.feature", + path=Path(features_path()) / "events.feature", + name=name, + tags=tags, + ) -def _identity_of(scenario: dict[str, typing.Any]) -> tuple[typing.Any, ...]: - """What identifies one entry: feature, name and the Examples row together.""" - example = scenario.get("example") or {} - return (scenario["feature"], scenario["name"], tuple(sorted(example.items()))) +def _results() -> Results: + return Results(location="stub.ndjson", digest="sha256:" + "0" * 64) def _examples_from_the_feature_file(feature: str, outline: str) -> list[dict[str, str]]: """Read an outline's Examples tables straight out of the Gherkin. - Hand-read rather than taken from pytest-bdd's parser, because the parser is - what produced the values under test: asking it what it should have said would - check nothing. It is a small reader for a small shape -- the tables in these - files are plain pipe-delimited rows -- and it exists so that "the report says - what the table said" is checked against the table. + Hand-read rather than taken from a parser, because a parser is what produced + the values under test: asking it what it should have said would check + nothing. It is a small reader for a small shape -- the tables in these files + are plain pipe-delimited rows -- and it exists so that "the stream says what + the table said" is checked against the table. """ source = Path(features_path()) / f"{feature}.feature" lines = source.read_text(encoding="utf-8").splitlines() @@ -291,11 +505,18 @@ def _write_suite( directory: Path, name: str = SUITE_NAME, capabilities: str = CAPABILITIES, + not_applicable: str = NOT_APPLICABLE, deviations: bool = True, ) -> Path: directory.mkdir(parents=True, exist_ok=True) (directory / "test_suite.py").write_text( - _SUITE_MODULE.format(name=name, capabilities=capabilities), encoding="utf-8" + _SUITE_MODULE.format( + name=name, + capabilities=capabilities, + not_applicable=not_applicable, + deviations=DEVIATIONS if deviations else "()", + ), + encoding="utf-8", ) if deviations: (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") @@ -307,7 +528,7 @@ def _run_suite( file_name: str = SUITE_FILE, **suite: typing.Any, ) -> Run: - """Run one generated suite in a subprocess and read the report it wrote.""" + """Run one generated suite in a subprocess and read what it wrote.""" directory = _write_suite(tmp_path_factory.mktemp("suite"), **suite) reports = tmp_path_factory.mktemp("reports") result = _pytest(str(directory), report_dir=reports) @@ -317,10 +538,14 @@ def _run_suite( f"no report at {path}; pytest exited {result.returncode}\n" f"{result.stdout}\n{result.stderr}" ) + envelope = json.loads(path.read_text(encoding="utf-8")) + stream_path = path.parent / envelope["results"]["location"] return Run( directory=directory, result=result, - document=json.loads(path.read_text(encoding="utf-8")), + envelope=envelope, + stream=_read_stream(stream_path), + stream_path=stream_path, ) @@ -332,20 +557,18 @@ def run(tmp_path_factory: pytest.TempPathFactory) -> Run: @pytest.fixture(scope="module") def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: - """A run of a suite that declares one capability the suite never tests. - - ``@targeting`` is reserved -- it is in the vocabulary and no scenario carries - it. ``@object`` is left undeclared so that a whole Scenario Outline is skipped - by the capability gate, which is the case that has to keep saying which row it - skipped. ``@strict-numeric-typing`` is declared and does have a scenario, so - the omission of ``@targeting`` is specific rather than a general failure to - report capabilities. + """A run of a suite that leaves a whole Scenario Outline gated. + + ``@object`` is undeclared so that every row of one outline is skipped by the + capability gate, which is the case that has to keep saying which row it + skipped. """ return _run_suite( tmp_path_factory, file_name="narrow.json", name="narrow", capabilities="{Capability.STRICT_NUMERIC_TYPING, Capability.TARGETING}", + not_applicable="{}", deviations=False, ) @@ -354,35 +577,61 @@ def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: def test_a_capability_skip_is_never_reported_as_passed(run: Run) -> None: - """The rule Appendix F states, checked against the document, not the runner.""" - undeclared = { - tag - for tag, result in run.document["capabilities"].items() - if result["state"] == Outcome.NOT_DECLARED.value - } - assert undeclared, "the generated suite is meant to leave capabilities undeclared" - - gated = [s for s in run.scenarios if undeclared & set(s.get("tags", ()))] + """The rule Appendix F states, checked against the payload, not the runner.""" + gated = [case for case in run.stream.cases if case.tags - run.declared] assert gated, "the generated suite is meant to have scenarios behind those" - for scenario in gated: - assert scenario["outcome"] == Outcome.NOT_DECLARED.value, scenario - assert scenario.get("reason"), f"a skip must say why: {scenario}" + + for case in gated: + assert case.status == "SKIPPED", case + # Cucumber's SKIPPED is per step, so "never as passed" has to hold of + # every step and not merely of the rolled-up verdict. + assert set(case.step_statuses) == {"SKIPPED"}, case + assert case.setup_message, f"a skip must say why: {case}" + + +def test_the_reason_for_a_gated_skip_follows_from_the_two_documents(run: Run) -> None: + """Which is why the per-scenario reason no longer has to be transported. + + The envelope says what the provider declares; the payload says which tags + each scenario carries and that it was skipped. A consumer with both can name + the capability responsible without the emitter having written it down once + per scenario, and that derivation is what the declaration exists for. + """ + skipped = [case for case in run.stream.cases if case.status == "SKIPPED"] + gated = [case for case in skipped if case.tags - run.declared] + assert gated, "no skip was attributable to an undeclared capability" + + # The derivation is checked against the reason the runner actually gave: for + # every skip the two documents attribute to a capability, that capability is + # the one the gate named. If they disagreed, the declaration would be the + # wrong thing to read a skip against. + for case in gated: + responsible = case.tags - run.declared + assert any(tag in case.setup_message for tag in responsible), case + + # And it distinguishes: the one scenario skipped for another reason carries + # no undeclared tag, so it is not attributed to a capability at all. + others = [case for case in skipped if case not in gated] + assert others, "the generated suite is meant to skip one scenario outright" + for case in others: + assert not case.tags - run.declared, case + assert "capability" not in case.setup_message, case def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: """The property that makes the rule above checkable rather than promised. - An entry is identified by feature, name **and example** together. Feature and - name alone are shared by every row of a Scenario Outline, so keying on them + A test case is identified by its uri, its scenario name **and its Examples + row**, all three recovered from the stream by following a pickle's AST node + ids. Name alone is shared by every row of a Scenario Outline, so keying on it would let eleven rows of the type-mismatch matrix collapse into one and this - test would not notice -- which is the ambiguity the example field exists to - remove. + test would not notice. Counted against pytest's own collection rather than against a number written down here, so that adding a scenario to the specification cannot leave this - passing while the report loses one. + passing while the payload loses one. """ - identities = [_identity_of(s) for s in run.scenarios] + identities = [case.identity for case in run.stream.cases] assert len(identities) == len(set(identities)), "a scenario is reported twice" collected = _pytest("--collect-only", str(run.directory)) @@ -391,126 +640,118 @@ def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: ) -def test_the_outcomes_account_for_every_scenario(run: Run) -> None: - counts = collections.Counter(s["outcome"] for s in run.scenarios) - assert set(counts) <= OUTCOMES, "an outcome outside the four the schema allows" - assert sum(counts.values()) == len(run.scenarios) - # All four occur, which is what makes the distinctions worth drawing. - assert set(counts) == OUTCOMES, counts +def test_the_statuses_account_for_every_scenario(run: Run) -> None: + counts = run.stream.statuses + assert set(counts) <= set(SEVERITY), "a status outside the protocol's own" + assert sum(counts.values()) == len(run.stream.cases) + # A skip, a pass and a failure all occur, which is what makes the run worth + # asserting against at all. + assert set(counts) == {"PASSED", "SKIPPED", "FAILED"}, counts -def test_the_document_does_not_repeat_the_runner_summary(run: Run) -> None: - """A known deviation is a failure in the report even when pytest finishes green. +def test_the_payload_does_not_repeat_the_runner_summary(run: Run) -> None: + """A known deviation is a failure in the payload even when pytest is green. The suite marks the one scenario the Python SDK cannot satisfy as an expected failure, so pytest exits zero. The provider still did not satisfy it, and a - document that agreed with the summary would hide exactly what the marker was - added to keep visible. + payload that agreed with the summary would hide exactly what the marker was + added to keep visible. The acknowledgement goes in the envelope instead. """ assert run.result.returncode == 0, run.result.stdout - failed = [s for s in run.scenarios if s["outcome"] == Outcome.FAILED.value] + failed = [case for case in run.stream.cases if case.status == "FAILED"] assert len(failed) == 1 - assert "python-sdk#619" in failed[0]["reason"] + assert failed[0].row == tuple(DEVIATING_ROW.items()) + deviations = run.envelope["knownDeviations"] + assert [deviation["issue"] for deviation in deviations] == [DEVIATION_ISSUE] -def test_a_scenario_skipped_for_another_reason_is_not_a_missing_capability( - run: Run, -) -> None: - """A run that chose not to execute a scenario is a different fact from a gap. - ``not-applicable`` rather than ``not-declared``, because nothing about the - provider's declared capabilities kept it from running -- and it appears at - all, even though a marker skip never runs a fixture. +def test_a_scenario_skipped_for_another_reason_is_still_reported(run: Run) -> None: + """A run that chose not to execute a scenario still accounts for it. + + Skipped, and present, even though a marker skip never runs a fixture -- and + with its own reason rather than the capability gate's, which is what tells + the two apart now that the payload has one status for both. """ - matching = [s for s in run.scenarios if s["name"] == UNKNOWN_KEY_SCENARIO] + matching = run.stream.named(UNKNOWN_KEY_SCENARIO) assert len(matching) == 1 - assert matching[0]["outcome"] == Outcome.NOT_APPLICABLE.value - assert "deliberately not run here" in matching[0]["reason"] + assert matching[0].status == "SKIPPED" + assert "deliberately not run here" in matching[0].setup_message + # Nothing about the provider's declaration explains this one, which is how a + # consumer tells it from a capability skip. + assert not matching[0].tags - run.declared # -- which row of an outline ------------------------------------------------- -def test_an_outline_row_is_named_by_its_example_not_by_its_name(run: Run) -> None: +def test_an_outline_row_is_identified_by_its_ast_node_id(run: Run) -> None: """The eleven rows of the type-mismatch matrix are told apart, and only here. All eleven share one scenario name, which is the feature file's name and must stay that way: it is what a report from Go or JavaScript carries for the same - row, and qualifying it with this runner's id for the row -- which an earlier - version of this emitter did -- makes the three disagree about a scenario they - all ran. + row. What tells them apart is the pickle's second ``astNodeIds`` entry, the + id of the table row it was compiled from, which resolves in the + ``GherkinDocument`` to exactly the cells the feature file wrote. That is the + identity four implementations were each reinventing as a bespoke ``example`` + field before this format carried it. """ - rows = [s for s in run.scenarios if s["name"] == MISMATCH_SCENARIO] + rows = run.stream.named(MISMATCH_SCENARIO) expected = _examples_from_the_feature_file("errors", MISMATCH_SCENARIO) assert len(rows) == len(expected) == 11 - for row in rows: - assert row["name"] == MISMATCH_SCENARIO, "the name carries a runner's id" - - observed = [row["example"] for row in rows] - assert len(observed) == len({tuple(sorted(e.items())) for e in observed}) - assert sorted(map(sorted, (e.items() for e in observed))) == sorted( - map(sorted, (e.items() for e in expected)) + observed = [dict(case.row) for case in rows] + assert len(observed) == len({case.row for case in rows}), "two rows collapsed" + assert sorted(map(sorted, (row.items() for row in observed))) == sorted( + map(sorted, (row.items() for row in expected)) ) + # Verbatim strings, because Gherkin has no types: a "1" in a table is the + # one-character cell the feature file contains. + for row in observed: + assert all(isinstance(value, str) for value in row.values()), row -def test_an_example_says_what_the_table_said(run: Run) -> None: - """Verbatim strings, because Gherkin has no types. - - A ``1`` in a table is the two-character cell the feature file contains, and a - report that emitted it as a number would be saying something the table did - not -- and would not validate, since the schema types the values as strings. - """ - rows = [s for s in run.scenarios if s["name"] == MISMATCH_SCENARIO] - for row in rows: - assert all(isinstance(value, str) for value in row["example"].values()), row - - failed = [row for row in rows if row["outcome"] == Outcome.FAILED.value] - assert len(failed) == 1 - assert failed[0]["example"] == DEVIATING_ROW - -def test_a_scenario_that_is_not_an_outline_has_no_example(run: Run) -> None: - """Omitted rather than empty: there is no row, so there is nothing to say.""" - plain = [s for s in run.scenarios if s["name"] == UNKNOWN_KEY_SCENARIO] +def test_a_scenario_that_is_not_an_outline_has_no_row(run: Run) -> None: + """One AST node id, so there is no row to resolve and nothing to say.""" + plain = run.stream.named(UNKNOWN_KEY_SCENARIO) assert len(plain) == 1 - assert "example" not in plain[0] + assert plain[0].row == () -def test_a_capability_skipped_outline_row_still_carries_its_example( +def test_a_capability_skipped_outline_row_is_still_identified( narrow_run: Run, ) -> None: """A skipped row is exactly as ambiguous as a failed one. - Identity is established at collection, from the node alone, so it does not - depend on the scenario having run -- which is what lets a row the capability - gate stopped before its first step be told apart from its siblings just as - well as one that failed. + Row identity comes from the pickle rather than from the run, so it does not + depend on the scenario having executed -- which is what lets a row the + capability gate stopped before its first step be told apart from its siblings + just as well as one that failed. """ outline = "Requesting a structured flag as a scalar returns the code default" expected = _examples_from_the_feature_file("errors", outline) - rows = [s for s in narrow_run.scenarios if s["name"] == outline] + rows = narrow_run.stream.named(outline) assert len(rows) == len(expected) - for row in rows: - assert row["outcome"] == Outcome.NOT_DECLARED.value, row - assert row.get("example"), f"a skipped outline row must say which row: {row}" - assert sorted(map(sorted, (row["example"].items() for row in rows))) == sorted( - map(sorted, (e.items() for e in expected)) + for case in rows: + assert case.status == "SKIPPED", case + assert case.row, f"a skipped outline row must say which row: {case}" + assert sorted(map(sorted, (dict(case.row).items() for case in rows))) == sorted( + map(sorted, (row.items() for row in expected)) ) -def test_a_row_gated_by_its_examples_block_is_a_capability_skip( +def test_a_row_gated_by_its_examples_block_is_the_only_one_skipped( tmp_path: Path, ) -> None: """Gherkin lets one Examples block of an outline carry its own tags. Two rows of one Scenario Outline can therefore differ in which capability gates them. Those tags are on neither the scenario, the feature nor the rule, - and a report that read only those three would show the skipped row as - carrying no capability -- reporting a capability skip as ``not-applicable``, - which is exactly the distinction Appendix F asks a report to keep, and - leaving the capability out of the rollup. + and a payload built from those three would show the skipped row as carrying + no capability -- leaving the envelope's declaration unable to explain the + skip, which is the one derivation this format asks a consumer to make. No canonical feature file does this yet, so the feature file is written here. """ @@ -524,48 +765,127 @@ def test_a_row_gated_by_its_examples_block_is_a_capability_skip( path = reports / "per-examples.json" assert path.exists(), f"pytest exited {result.returncode}\n{result.stdout}" - document = json.loads(path.read_text(encoding="utf-8")) - by_row = {row["example"]["requested"]: row for row in document["scenarios"]} + envelope = json.loads(path.read_text(encoding="utf-8")) + stream = _read_stream(path.parent / envelope["results"]["location"]) + by_row = {dict(case.row)["requested"]: case for case in stream.cases} + # Every row is still reported: nothing about gating one row of an outline may - # drop its siblings from the document. - assert set(by_row) == {"Boolean", "Integer", "Float"}, document["scenarios"] - assert by_row["Boolean"]["outcome"] == Outcome.PASSED.value - assert by_row["Integer"]["outcome"] == Outcome.PASSED.value + # drop its siblings from the payload. + assert set(by_row) == {"Boolean", "Integer", "Float"}, by_row + assert by_row["Boolean"].status == "PASSED" + assert by_row["Integer"].status == "PASSED" gated = by_row["Float"] - assert gated["outcome"] == Outcome.NOT_DECLARED.value, gated - assert gated["tags"] == [Capability.OBJECT.tag], gated - assert document["capabilities"][Capability.OBJECT.tag]["state"] == ( - Outcome.NOT_DECLARED.value - ) + assert gated.status == "SKIPPED", gated + assert Capability.OBJECT.tag in gated.tags, gated + assert Capability.OBJECT.tag not in envelope["declaration"]["declared"] + + +# -- the payload, and the envelope that points at it ------------------------- + +def test_the_envelope_points_at_the_payload_it_describes(run: Run) -> None: + """Referenced rather than inlined, and covered by a digest. -# -- identity ---------------------------------------------------------------- + A stream carries the feature sources and is far larger than the envelope, so + a consumer deciding whether it cares about a report should not have to fetch + a whole run to find out -- and needs to be able to tell that what it did + fetch is what the envelope described. + """ + results = run.envelope["results"] + assert results["format"] == MESSAGES_FORMAT + # A bare file name, so the reference survives the pair being moved together. + assert results["location"] == run.stream_path.name + assert Path(results["location"]).parent == Path() + + digest = hashlib.sha256(run.stream_path.read_bytes()).hexdigest() + assert results["digest"] == f"sha256:{digest}" + + +def test_the_payload_carries_the_source_of_every_feature_it_ran(run: Run) -> None: + """Which is what replaced recording an asset tree hash. + + A hash said only whether two runs executed the same assets. The source says + what the assets were, so a consumer can read the questions that were actually + asked rather than trusting a recorded revision to stand for them. + """ + uris = {case.uri for case in run.stream.cases} + assert uris, "no test case named a feature file" + assert set(run.stream.sources) == uris + + for uri, data in run.stream.sources.items(): + on_disk = Path(features_path()) / Path(uri).name + assert data == on_disk.read_text(encoding="utf-8"), uri + + assert "assetsTree" not in run.envelope["tck"] + + +def test_the_payload_is_a_well_formed_messages_stream(run: Run) -> None: + """The message types a consumer needs are all present, once each per scenario.""" + kinds = run.stream.kinds + cases = len(run.stream.cases) + assert kinds["meta"] == 1 + assert kinds["testRunStarted"] == 1 + assert kinds["testRunFinished"] == 1 + assert kinds["source"] == kinds["gherkinDocument"] == len(run.stream.sources) + assert kinds["pickle"] == cases + assert kinds["testCase"] == kinds["testCaseStarted"] == cases + assert kinds["testCaseFinished"] == cases + assert kinds["testStepStarted"] == kinds["testStepFinished"] + + +def test_the_declaration_is_an_input_not_a_summary(run: Run) -> None: + """Which is why it cannot be derived from the payload and is stated here. + + Declared and not-applicable are disjoint and mean different things -- a + choice against a capability, and an impossibility -- and the payload can + express neither, because a skip in it says only that the question was not + put to this provider. + """ + declaration = run.envelope["declaration"] + assert declaration["declared"] == [ + Capability.EVENTS.tag, + Capability.OBJECT.tag, + Capability.STRICT_NUMERIC_TYPING.tag, + ] + assert declaration["notApplicable"] == { + Capability.STALE.tag: "this provider has no connection to lose" + } + assert Capability.STALE.tag not in declaration["declared"] def test_the_provider_and_its_configuration_are_reported_separately(run: Run) -> None: - assert run.document["provider"]["name"] == "In-Memory Provider" - assert run.document["provider"]["configuration"] == SUITE_NAME - assert run.document["provider"]["language"] == "python" + assert run.envelope["provider"]["name"] == "In-Memory Provider" + assert run.envelope["provider"]["configuration"] == SUITE_NAME + assert run.envelope["provider"]["language"] == "python" -def test_the_report_names_what_ran_it(run: Run) -> None: - assert run.document["schemaVersion"] == "1" +def test_the_envelope_names_what_ran_it(run: Run) -> None: + assert run.envelope["schemaVersion"] == "1" assert ( - run.document["tck"]["implementation"] + run.envelope["tck"]["implementation"] == "python-sdk-contrib/tools/openfeature-provider-tck" ) - assert run.document["sdk"]["name"] == "openfeature-sdk" - assert run.document["sdk"]["version"] - assert len(run.document["tck"]["specRevision"]) >= 7 - assert run.document["backend"]["controlApi"] == "in-process" + assert run.envelope["sdk"]["name"] == "openfeature-sdk" + assert run.envelope["sdk"]["version"] + assert len(run.envelope["tck"]["specRevision"]) >= 7 + assert run.envelope["backend"]["controlApi"] == "in-process" + + +def test_the_envelope_carries_no_results_of_its_own(run: Run) -> None: + """The fields Cucumber Messages made redundant, checked to be gone. + + Not a shape test for its own sake: while both existed there were two places + for the same fact to disagree, which is the whole reason the per-scenario + list was deleted rather than kept alongside the payload. + """ + assert "scenarios" not in run.envelope + assert "capabilities" not in run.envelope def test_the_spec_revision_comes_from_the_build() -> None: """Generated beside the assets, because the submodule is not in the wheel.""" - revision, tree = spec_identity() - assert len(revision) >= 7 - assert tree == "" or len(tree) == 40 + assert len(spec_revision()) >= 7 # -- opting in --------------------------------------------------------------- @@ -580,6 +900,7 @@ def test_no_report_is_written_without_the_environment_variable( assert result.returncode == 0, result.stdout assert "report written" not in result.stdout assert not list(tmp_path.rglob("*.json")) + assert not list(tmp_path.rglob("*.ndjson")) def test_a_report_that_cannot_be_written_fails_the_run(tmp_path: Path) -> None: @@ -597,80 +918,123 @@ def test_a_report_that_cannot_be_written_fails_the_run(tmp_path: Path) -> None: assert result.returncode != 0 -# -- assembling the document ------------------------------------------------- +# -- assembling the documents ------------------------------------------------ def test_a_failure_is_not_revised_away_by_a_later_phase() -> None: """A scenario whose steps passed and whose teardown blew up is a failure.""" - suite = SuiteReport(config=_config()) - identity = _identity() - suite.set_outcome("node", identity, Outcome.FAILED, "teardown exploded") - suite.set_outcome("node", identity, Outcome.PASSED) - assert suite.records["node"].outcome is Outcome.FAILED - assert suite.records["node"].reason == "teardown exploded" - - -def test_an_undeclared_capability_is_reported_with_a_reason() -> None: - suite = SuiteReport(config=_config()) - suite.set_outcome("node", _identity("@events"), Outcome.PASSED) - document = suite.build() - assert document["capabilities"]["@events"] == {"state": Outcome.PASSED.value} - stale = document["capabilities"]["@stale"] - assert stale["state"] == Outcome.NOT_DECLARED.value - assert "@stale" in stale["reason"] - - -def test_a_capability_whose_scenario_failed_is_not_reported_as_passed() -> None: - """And says how much failed, because the schema requires a reason. - - Reached by driving the builder directly: every self-test suite that runs end - to end passes, so nothing else gets near this branch -- and an entry without a - reason would be rejected by the schema at exactly the moment the report - matters most, when a provider is failing. - """ - suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) - suite.set_outcome("failed", _identity("@events"), Outcome.FAILED, "boom") - suite.set_outcome("passed", _identity("@events"), Outcome.PASSED) - - events = suite.build()["capabilities"]["@events"] - assert events["state"] == Outcome.FAILED.value - assert "1 of 2" in events["reason"], events - + run = scenario_run( + _identity(), + [ + _phase("passed", when="setup"), + _phase("passed", when="call"), + _phase("failed", when="teardown", message="teardown exploded"), + ], + [StepRun(status=Status.passed)], + ) + assert run.status is Status.failed + assert run.message == "teardown exploded" -def test_every_capability_the_report_mentions_can_explain_itself(run: Run) -> None: - """The rule the schema enforces, checked here so a change fails in this package.""" - for tag, result in run.document["capabilities"].items(): - if result["state"] != Outcome.PASSED.value: - assert result.get("reason"), f"{tag} is {result['state']} with no reason" +def test_a_verdict_no_step_accounts_for_reaches_the_stream_anyway() -> None: + """A strict xfail that passes fails a scenario every step of which passed. -def test_a_capability_no_scenario_exercises_is_not_reported_as_passed( - narrow_run: Run, -) -> None: - """The vacuous pass the capability vocabulary exists to eliminate. - - ``@targeting`` is declared by this suite and carried by no scenario, because - asserting that an evaluation context reached the backend needs an echo - operation the control API does not have. The suite asked no question, so it - has no answer: the tag is absent rather than green, and a consumer sees the - absence rather than a pass it cannot rely on. + A consumer reads a test case's outcome as the worst of its steps, so a + verdict left only in this package's own head would be lost on the way out. + It is attached to the after-hook, which is where a test case failing outside + its own steps belongs. """ - capabilities = narrow_run.document["capabilities"] - exercised = {tag for s in narrow_run.scenarios for tag in s.get("tags", ())} + run = scenario_run( + _identity(), + [ + _phase("passed", when="setup"), + _phase("failed", when="call", message="[XPASS(strict)] python-sdk#619"), + _phase("passed", when="teardown"), + ], + [StepRun(status=Status.passed)], + ) + assert run.status is Status.failed + assert run.teardown is not None + assert run.teardown.status is Status.passed, "the phase itself did pass" + + # The stream is where the discrepancy has to be resolved, and it is: + # `_step_runs` upgrades the after-hook when nothing else carries the verdict. + steps = _step_runs(_Pickle(id="p", step_ids=("s",), payload={}), run) + worst = max((step.status.value for step in steps), key=SEVERITY.index) + assert worst == Status.failed.value + assert "XPASS(strict)" in steps[-1].message + + +def test_a_gated_skip_is_skipped_for_every_step() -> None: + """The gate stops a scenario in setup, so no step of it ran.""" + run = scenario_run( + _identity("@stale"), + [_phase("skipped", when="setup", message="provider does not declare @stale")], + [], + ) + assert run.status is Status.skipped + assert run.setup is not None + assert run.setup.message == "provider does not declare @stale" + - assert Capability.TARGETING.tag not in exercised, "the premise has changed" - assert Capability.TARGETING.tag not in capabilities, capabilities.get( - Capability.TARGETING.tag +def test_the_declaration_reports_what_the_configuration_declares() -> None: + suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) + declaration = suite.build(_results())["declaration"] + assert declaration["declared"] == [Capability.EVENTS.tag] + assert "notApplicable" not in declaration + + +def test_a_not_applicable_capability_is_reported_with_its_reason() -> None: + """Impossible is not the same claim as undeclared, and the report keeps both.""" + suite = SuiteReport( + config=_config( + capabilities={Capability.EVENTS}, + not_applicable={Capability.STRICT_NUMERIC_TYPING: "no integer type"}, + ) ) + declaration = suite.build(_results())["declaration"] + assert declaration["notApplicable"] == { + Capability.STRICT_NUMERIC_TYPING.tag: "no integer type" + } + - # Specific rather than a general failure to report: the other declared - # capability is exercised, and is still reported. - numeric = Capability.STRICT_NUMERIC_TYPING.tag - assert numeric in exercised - assert capabilities[numeric]["state"] == Outcome.PASSED.value - # And an undeclared capability is still reported, with its reason, whether or - # not any scenario carries it: that is a fact about the provider. - assert capabilities[Capability.OBJECT.tag]["state"] == Outcome.NOT_DECLARED.value +def test_a_capability_cannot_be_both_declared_and_impossible() -> None: + with pytest.raises(ValueError, match="both claim @events"): + _config( + capabilities={Capability.EVENTS}, + not_applicable={Capability.EVENTS: "a reason"}, + ) + + +def test_a_not_applicable_capability_must_say_why() -> None: + with pytest.raises(ValueError, match="no reason for @stale"): + _config( + capabilities={Capability.EVENTS}, not_applicable={Capability.STALE: " "} + ) + + +def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: + """Stating none is a claim; omitting the field is silence.""" + assert "knownDeviations" not in SuiteReport(config=_config()).build(_results()) + + acknowledged = SuiteReport( + config=_config( + known_deviations=( + KnownDeviation( + issue=DEVIATION_ISSUE, + summary="a boolean satisfies an Integer request", + capability=Capability.STRICT_NUMERIC_TYPING, + ), + ) + ) + ).build(_results())["knownDeviations"] + assert acknowledged == [ + { + "issue": DEVIATION_ISSUE, + "summary": "a boolean satisfies an Integer request", + "capability": Capability.STRICT_NUMERIC_TYPING.tag, + } + ] def test_the_provider_name_falls_back_to_the_suite_name() -> None: @@ -679,12 +1043,14 @@ def test_the_provider_name_falls_back_to_the_suite_name() -> None: Reporting the suite name is more useful than the empty string the schema would reject. """ - assert SuiteReport(config=_config()).build()["provider"]["name"] == "stub" + envelope = SuiteReport(config=_config()).build(_results()) + assert envelope["provider"]["name"] == "stub" def test_the_control_api_is_omitted_when_the_control_does_not_say() -> None: - assert "controlApi" not in SuiteReport(config=_config()).build()["backend"] - http = SuiteReport(config=_config(control=_HttpControl())).build() + plain = SuiteReport(config=_config()).build(_results()) + assert "controlApi" not in plain["backend"] + http = SuiteReport(config=_config(control=_HttpControl())).build(_results()) assert http["backend"]["controlApi"] == "http" @@ -698,55 +1064,59 @@ class Odd(_StubControl): @pytest.mark.parametrize( ("suite_name", "expected"), [ - ("in-memory", "in-memory.json"), - ("flagd/rpc", "flagd-rpc.json"), - ("../escape", "escape.json"), - ("...", "report.json"), + ("in-memory", "in-memory"), + ("flagd/rpc", "flagd-rpc"), + ("../escape", "escape"), + ("...", "report"), ], ) def test_a_suite_name_cannot_write_outside_its_directory( suite_name: str, expected: str ) -> None: """Suite names are chosen to read well in a failure message, not to be paths.""" - assert report_file_name(suite_name) == expected + assert envelope_file_name(suite_name) == f"{expected}.json" + assert stream_file_name(suite_name) == f"{expected}.ndjson" def test_only_tags_the_schema_accepts_are_carried() -> None: assert normalise_tags({"events", "Not A Tag", "stale"}) == ("@events", "@stale") +def test_a_feature_uri_is_slash_separated_on_every_platform() -> None: + """The same string has to appear in the source, the document and the pickles. + + pytest-bdd builds it with ``os.path.join``, so on Windows it arrives + backslash-separated -- and a report emitted there would otherwise not be + comparable with one emitted on Linux. + """ + assert feature_uri("features/errors.feature") == "features/errors.feature" + assert feature_uri(os.path.join("features", "errors.feature")) == ( + "features/errors.feature" + ) + + # -- classifying one phase --------------------------------------------------- def test_an_expected_failure_is_still_a_failure() -> None: """An xfail marker records a known deviation; it does not excuse one.""" - classified = classify_phase( - _phase("skipped", xfail_reason="the SDK coerces a bool to an int"), - _identity(), - _config(), + status, message = classify_phase( + _phase("skipped", xfail_reason="the SDK coerces a bool to an int") ) - assert classified is not None - outcome, reason = classified - assert outcome is Outcome.FAILED - assert "the SDK coerces a bool to an int" in reason - + assert status is Status.failed + assert "the SDK coerces a bool to an int" in message -def test_a_phase_that_merely_worked_says_nothing() -> None: - assert ( - classify_phase(_phase("passed", when="setup"), _identity(), _config()) is None - ) - assert classify_phase(_phase("passed", when="call"), _identity(), _config()) == ( - Outcome.PASSED, - "", - ) +def test_a_phase_that_merely_worked_says_nothing_in_particular() -> None: + assert classify_phase(_phase("passed", when="setup")) == (Status.passed, "") + assert classify_phase(_phase("passed", when="call")) == (Status.passed, "") -def test_a_gated_skip_and_an_ungated_skip_are_different_outcomes() -> None: - config = _config(capabilities={Capability.EVENTS}) - gated = classify_phase(_phase("skipped", when="setup"), _identity("@stale"), config) - assert gated == (Outcome.NOT_DECLARED, "provider does not declare @stale") - ungated = classify_phase( - _phase("skipped", when="setup"), _identity("@events"), config +def test_a_skip_keeps_its_reason() -> None: + assert classify_phase( + _phase("skipped", when="setup", message="provider does not declare @stale") + ) == (Status.skipped, "provider does not declare @stale") + assert classify_phase(_phase("skipped", when="setup")) == ( + Status.skipped, + "skipped", ) - assert ungated == (Outcome.NOT_APPLICABLE, "skipped") diff --git a/uv.lock b/uv.lock index b0ab379c..b52619cf 100644 --- a/uv.lock +++ b/uv.lock @@ -825,6 +825,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] +[[package]] +name = "cucumber-messages" +version = "34.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/b8/16f4f31045776b7b7fc962e72ebc2a8bf4a12df2f18d8c9be4e258cbe3bb/cucumber_messages-34.2.0.tar.gz", hash = "sha256:712102e0a0f7fb7a3d068a2754b31ce9b605fb04ab65f9f37282f9c27f7254d4", size = 11703, upload-time = "2026-07-19T13:07:38.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/81/00577acc873ebce750da0429c6726e111c75e62caa71843166e522e19b80/cucumber_messages-34.2.0-py3-none-any.whl", hash = "sha256:2b20b7a7151b2ccc296b8f101b413b2b6b8a2f90c3af748d5a357b59243ee2fc", size = 13006, upload-time = "2026-07-19T13:07:37.692Z" }, +] + [[package]] name = "docker" version = "7.1.0" @@ -844,7 +853,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1995,6 +2004,8 @@ name = "openfeature-provider-tck" version = "0.1.0" source = { editable = "tools/openfeature-provider-tck" } dependencies = [ + { name = "cucumber-messages" }, + { name = "gherkin-official" }, { name = "openfeature-sdk" }, { name = "pytest" }, { name = "pytest-bdd" }, @@ -2009,6 +2020,8 @@ dev = [ [package.metadata] requires-dist = [ + { name = "cucumber-messages", specifier = ">=34.0.0,<35.0.0" }, + { name = "gherkin-official", specifier = ">=29.0.0" }, { name = "openfeature-sdk", specifier = ">=0.8.2" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, From e2102dc4de6664d87942be8dc07bc60b3bfce18c Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 10 Sep 2026 20:10:23 +0200 Subject: [PATCH 06/11] feat(provider-tck): record which Cucumber Messages release produced the stream The envelope named the results format but not its version, and Messages is versioned. This implementation is on 34.2.0 while the Go TCK builds against v21 and cucumber-jvm ships a different release again, so a consumer holding two reports cannot assume one schema validates both. Guessing is worse than not validating. A later schema accepts messages this producer could not have emitted, and an earlier one rejects messages that are perfectly valid, so a check against the wrong version reports a result that has nothing to do with the stream. It reuses the function that already computes the stream's own Meta protocolVersion rather than adding a second source, so the envelope and the stream cannot disagree about which release produced it. That function reads the version from the installed distribution rather than declaring it, so a dependency bump cannot leave the report claiming the old one. Signed-off-by: Simon Schrottner --- .../contrib/tools/provider_tck/messages.py | 13 ++++++++++++ .../contrib/tools/provider_tck/report.py | 20 +++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py index cad1737a..96877a6b 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py @@ -581,6 +581,19 @@ def _meta(implementation: str, implementation_version: str) -> cucumber.Meta: ) +def messages_protocol_version() -> str: + """The Messages release this stream was produced against. + + Exposed because the report envelope has to record it too. Messages is + versioned and the implementations pin different releases -- this one is on + 34.2.0 while the Go TCK builds against v21 -- so a consumer holding two + reports cannot assume one schema validates both. Sharing this one function + with the stream's own Meta message means the envelope and the stream cannot + disagree about which release produced it. + """ + return _protocol_version() + + def _protocol_version() -> str: """The Messages version this stream is written against. diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py index b1e9cb6d..87c6b1f1 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py @@ -42,7 +42,13 @@ from pathlib import Path from .config import TckConfig -from .messages import MESSAGES_FORMAT, ScenarioIdentity, ScenarioRun, StepRun +from .messages import ( + MESSAGES_FORMAT, + ScenarioIdentity, + ScenarioRun, + StepRun, + messages_protocol_version, +) __all__ = [ "REPORT_DIR_ENV", @@ -119,7 +125,17 @@ class Results: format: str = MESSAGES_FORMAT def as_json(self) -> dict[str, typing.Any]: - document = {"format": self.format, "location": self.location} + # The format's version is recorded alongside its name because Cucumber + # Messages is versioned and the four implementations pin different + # releases. Without it a consumer validating this stream has to guess + # which schema to use, and guessing wrong is worse than not checking: a + # later schema accepts messages this producer could not have emitted, + # and an earlier one rejects messages that are perfectly valid. + document = { + "format": self.format, + "formatVersion": messages_protocol_version(), + "location": self.location, + } if self.digest: document["digest"] = self.digest return document From 639bb83adbcb8eeba961fb3f972bcae42fac10bd Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:02:22 +0200 Subject: [PATCH 07/11] fix(provider-tck): say when a control does not report how it drives the backend Two reports of the same kind of provider disagreed about whether they described an in-process backend: the JavaScript in-memory suite said so, the Python one stayed silent. Not because the backends differ, but because the self-test control never offered the optional attribute that reports it. The field is optional in the report and the attribute is optional here, both so that introducing it left no existing control incomplete. Together they make omission invisible: the suite passes, the report validates, and the field is simply absent. It surfaced only when reports from four languages were compared side by side. PlainMemoryControl now reports in-process, which is what it is -- the in-memory provider is rebuilt in this process for every scenario and there is no backend to drive. More usefully, a control that reports nothing now says so in the run output. Every control either drives a real backend over the normative HTTP API or manipulates one in process, so there is no third case an absent value legitimately describes, and an adopter had no way to discover their report had a hole in it. Written to the terminal rather than failing the run, because a missing optional field is not a conformance problem -- it is a gap in what the report can say. Signed-off-by: Simon Schrottner --- .../contrib/tools/provider_tck/emitter.py | 5 ++++ .../contrib/tools/provider_tck/report.py | 26 +++++++++++++++++++ .../tests/test_in_memory_conformance.py | 9 +++++++ 3 files changed, 40 insertions(+) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py index 1589b308..d3b8e42b 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py @@ -52,6 +52,7 @@ ReportCollector, Results, SuiteReport, + control_api_gap, distribution_version, envelope_file_name, normalise_tags, @@ -370,6 +371,10 @@ def _write_suite( f"{stream_path.name} ({counts})", ) + gap = control_api_gap(suite.config.control) + if gap: + self._say(session, f"provider-tck [{name}]: {gap}") + def _say(self, session: pytest.Session, message: str) -> None: reporter = session.config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py index 87c6b1f1..d66922be 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py @@ -415,6 +415,32 @@ def control_api_of(control: object) -> str: return "" +def control_api_gap(control: object) -> str: + """Describe a control that does not say how it drives the backend, or "". + + The field is optional in the report and the attribute is optional here, both + so that introducing it made no existing control incomplete. Together they + make omission invisible: the suite passes, the report validates, and the + field is simply absent. That went unnoticed until reports from four + languages were compared side by side and two of them were silent about the + same kind of in-process backend. + + Every control either drives a real backend over the normative HTTP API or + manipulates one in this process, so there is no third case an absent value + legitimately describes -- which makes the silence worth breaking, in the run + output where an adopter will see it rather than in the report where they + will not. + """ + if control_api_of(control): + return "" + description = getattr(control, "description", "") or type(control).__name__ + return ( + f"{description} does not offer a control_api attribute, so the conformance " + f"report cannot say whether the backend was driven over HTTP or in process. " + f'Add one, returning "http" or "in-process".' + ) + + def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: """Turn Gherkin tags as pytest-bdd holds them into the form the schema wants. diff --git a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py index 17d49fd8..5d6868c2 100644 --- a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py +++ b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py @@ -48,6 +48,15 @@ class PlainMemoryControl: this error would mean the capability had been declared anyway. """ + @property + def control_api(self) -> str: + """Report that this control manipulates a provider in this process. + + There is no backend to drive: the in-memory provider is rebuilt in + process for every scenario, which is exactly what "in-process" names. + """ + return "in-process" + @property def description(self) -> str: return "the Python SDK's InMemoryProvider, rebuilt per scenario" From cb3b3559b1b9edbc3387a5af2aa107dd5b8901c6 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:26:29 +0200 Subject: [PATCH 08/11] refactor(provider-tck): rename @strict-numeric-typing to @numeric-coercion The tag was named for a stricter rule than the specification asks for, and it was about to collide with a second vocabulary for the same property. flagd is implementing an accepted numeric coercion ADR (open-feature/flagd#1996) whose rule is that coercion is permitted when lossless and must fail with TYPE_MISMATCH only when information would be lost: 10.0 requested as an integer succeeds, 0.5 does not. Appendix F said "does not coerce between integer and float", which forbids the case the ADR requires to work, and flagd's own testbed is gaining @numeric-coercion scenarios -- two names for one property is the drift a shared vocabulary exists to prevent. The specification renamed the tag and corrected the rule in open-feature/spec dc4d7ae8; this follows it. So Capability.STRICT_NUMERIC_TYPING becomes Capability.NUMERIC_COERCION, the marker and tag become numeric-coercion and @numeric-coercion, and the docstring states the rule that now holds rather than the one it was named for. The pytest marker registration needs no change: it iterates the enum. The submodule bump also carries two unrelated spec changes into the executed assets -- a lifecycle scenario renamed, and POST /start required not to return until the seeded flag state is being served. Neither is referenced by name here. Two gaps are recorded rather than closed, in the capability docstring and the README, because closing either is a change to every language at once. The lossless half of the contract has no scenario: the canonical flag set contains no integral float to ask it of, so a provider that wrongly rejects 10.0 as an integer still passes. And accessor width is not modelled at all -- the ADR distinguishes a 64-bit integer accessor from a 32-bit one, and this suite is silent about it. Signed-off-by: Simon Schrottner --- .../contrib/tools/provider_tck/config.py | 2 +- tools/openfeature-provider-tck/tests/conftest.py | 2 +- .../tests/test_report.py | 16 +++++++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py index 1f533ce1..4d07420c 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py @@ -138,7 +138,7 @@ class TckConfig: Kept apart from simply leaving a capability out of :attr:`capabilities`, because the two are different claims and collapsing them misrepresents whole - languages: ``@strict-numeric-typing`` is unsatisfiable in JavaScript because + languages: ``@numeric-coercion`` is unsatisfiable in JavaScript because the language has no integer type, and reporting that as a choice would show every JavaScript provider as missing something none of them can have. diff --git a/tools/openfeature-provider-tck/tests/conftest.py b/tools/openfeature-provider-tck/tests/conftest.py index cbf775ee..47e88028 100644 --- a/tools/openfeature-provider-tck/tests/conftest.py +++ b/tools/openfeature-provider-tck/tests/conftest.py @@ -46,7 +46,7 @@ """What the report acknowledges. No ``capability``: the scenario carries no capability tag, because returning the -code default on a type mismatch is mandatory. ``@strict-numeric-typing`` is a +code default on a type mismatch is mandatory. ``@numeric-coercion`` is a neighbouring question -- whether 0.5 satisfies an integer request -- and this provider satisfies it, so attributing the deviation there would be wrong twice over. diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py index 2daa74d7..0e883899 100644 --- a/tools/openfeature-provider-tck/tests/test_report.py +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -133,9 +133,7 @@ def tck_config(): scenarios(features_path()) ''' -CAPABILITIES = ( - "{Capability.EVENTS, Capability.OBJECT, Capability.STRICT_NUMERIC_TYPING}" -) +CAPABILITIES = "{Capability.EVENTS, Capability.OBJECT, Capability.NUMERIC_COERCION}" """What the main generated suite declares: enough to produce a skip and a pass.""" NOT_APPLICABLE = '{Capability.STALE: "this provider has no connection to lose"}' @@ -567,7 +565,7 @@ def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: tmp_path_factory, file_name="narrow.json", name="narrow", - capabilities="{Capability.STRICT_NUMERIC_TYPING, Capability.TARGETING}", + capabilities="{Capability.NUMERIC_COERCION, Capability.TARGETING}", not_applicable="{}", deviations=False, ) @@ -845,8 +843,8 @@ def test_the_declaration_is_an_input_not_a_summary(run: Run) -> None: declaration = run.envelope["declaration"] assert declaration["declared"] == [ Capability.EVENTS.tag, + Capability.NUMERIC_COERCION.tag, Capability.OBJECT.tag, - Capability.STRICT_NUMERIC_TYPING.tag, ] assert declaration["notApplicable"] == { Capability.STALE.tag: "this provider has no connection to lose" @@ -989,12 +987,12 @@ def test_a_not_applicable_capability_is_reported_with_its_reason() -> None: suite = SuiteReport( config=_config( capabilities={Capability.EVENTS}, - not_applicable={Capability.STRICT_NUMERIC_TYPING: "no integer type"}, + not_applicable={Capability.NUMERIC_COERCION: "no integer type"}, ) ) declaration = suite.build(_results())["declaration"] assert declaration["notApplicable"] == { - Capability.STRICT_NUMERIC_TYPING.tag: "no integer type" + Capability.NUMERIC_COERCION.tag: "no integer type" } @@ -1023,7 +1021,7 @@ def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: KnownDeviation( issue=DEVIATION_ISSUE, summary="a boolean satisfies an Integer request", - capability=Capability.STRICT_NUMERIC_TYPING, + capability=Capability.NUMERIC_COERCION, ), ) ) @@ -1032,7 +1030,7 @@ def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: { "issue": DEVIATION_ISSUE, "summary": "a boolean satisfies an Integer request", - "capability": Capability.STRICT_NUMERIC_TYPING.tag, + "capability": Capability.NUMERIC_COERCION.tag, } ] From 7ec7e4320eed25e0359ce49171008b9fb01bc11c Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:38:03 +0200 Subject: [PATCH 09/11] fix(provider-tck): a reserved capability must not be declarable @targeting and @caching exist in the vocabulary and no scenario carries either. The enum said so in a docstring and left it there, which reads as documentation rather than as the rule it is: a capability nothing carries cannot be verified, cannot produce a skip, and tells a reader of a conformance report only that something was claimed and nothing examined. It is a live defect rather than a hypothetical one. A real Java report asserts both tags as declared -- not by anyone's decision, but because that adoption declares "every capability except X" and picks up every reserved tag on the way past. The report schema now forbids it: see the declaration.declared description in open-feature/spec. So the set is written down once, as RESERVED_CAPABILITIES, and read everywhere else -- by Capability.reserved, by the declare-everything helper, and by the validation in TckConfig -- so the list cannot drift from the rule. ALL_CAPABILITIES becomes DECLARABLE_CAPABILITIES: the vocabulary minus the reserved tags, and named for what it is rather than for "all", because the declare-everything convenience is precisely the route a reserved tag takes into a report by accident. It is also TckConfig.capabilities' default, so a suite that does not narrow its capabilities no longer declares a tag nothing tests. An adopter who names a reserved capability explicitly gets a ValueError from TckConfig rather than a warning or a silent drop. TckConfig already refuses a capability claimed as both declared and not-applicable, and this is the same class of error -- a claim that cannot be true -- caught in the same place, where the adopter's own code is still on the stack. A silent drop would make a rejected configuration look like an accepted one; a warning is a line of CI output nobody reads while an untested capability goes on being asserted in a published report, which is how it got into one. Naming one in not_applicable is refused too: an impossibility recorded about a question never asked reaches the same declaration block. Nothing filters the declaration at emission time, and report.py says why: by the time an envelope is built, a reserved tag cannot be in the TckConfig at all. Signed-off-by: Simon Schrottner --- tools/openfeature-provider-tck/README.md | 19 ++++-- .../contrib/tools/provider_tck/__init__.py | 5 +- .../contrib/tools/provider_tck/capability.py | 44 +++++++++++-- .../contrib/tools/provider_tck/config.py | 52 ++++++++++++++-- .../contrib/tools/provider_tck/report.py | 8 +++ .../tests/test_report.py | 62 ++++++++++++++++++- 6 files changed, 172 insertions(+), 18 deletions(-) diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md index 5704c7d5..64bad41a 100644 --- a/tools/openfeature-provider-tck/README.md +++ b/tools/openfeature-provider-tck/README.md @@ -94,8 +94,8 @@ SKIPPED provider does not declare capability @stale. | `Capability.OBJECT` | `@object` | supports structured flag values | | `Capability.UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging against a dead backend | | `Capability.NUMERIC_COERCION` | `@numeric-coercion` | coerces between integer and float only when lossless, else `TYPE_MISMATCH` | -| `Capability.TARGETING` | `@targeting` | reserved; no scenarios yet | -| `Capability.CACHING` | `@caching` | reserved; no scenarios yet | +| `Capability.TARGETING` | `@targeting` | reserved; **not declarable** — no scenarios yet | +| `Capability.CACHING` | `@caching` | reserved; **not declarable** — no scenarios yet | `@lifecycle` and `@events` are deliberately separate, and the split matters in both directions. An SDK dispatches `PROVIDER_READY` around `initialize` for *any* provider, so a provider declaring only @@ -104,9 +104,18 @@ identically. Meanwhile a stateless provider has a real initialisation to verify of its own to declare `@events` for, and gating on `@events` shut it out of a scenario it should be held to. -Untagged scenarios are mandatory and always run. `capabilities` defaults to everything — narrow it -rather than widening it: start from the default, run the suite, and remove only what your provider -genuinely cannot do. +Untagged scenarios are mandatory and always run. `capabilities` defaults to every *declarable* +capability — `DECLARABLE_CAPABILITIES` — and you should narrow it rather than widen it: start from +the default, run the suite, and remove only what your provider genuinely cannot do. + +A reserved capability is documented so the vocabulary has a place for it once scenarios exist, and +until then it **must not be declared**. Nothing carries the tag, so declaring it cannot be verified, +cannot produce a skip, and tells a reader of a conformance report only that something was claimed and +nothing examined. `TckConfig` raises if you name one in `capabilities` or in `not_applicable`, and +`DECLARABLE_CAPABILITIES` excludes them — which is the case that matters, because "every capability +except X" is how a reserved tag reaches a report by accident rather than by decision. One +implementation's published report asserts `@targeting` and `@caching` as declared for exactly that +reason. `@numeric-coercion` deserves a note, because it is the one capability here that **the specification does not define**. OpenFeature has a single numeric type on purpose — `number` is "a numeric value of diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py index e770538e..90688439 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -49,7 +49,7 @@ def tck_config(): import importlib.resources -from .capability import ALL_CAPABILITIES, Capability +from .capability import DECLARABLE_CAPABILITIES, RESERVED_CAPABILITIES, Capability from .config import KnownDeviation, TckConfig from .control import ( BackendControl, @@ -66,10 +66,11 @@ def tck_config(): from .report import REPORT_DIR_ENV, SCHEMA_VERSION __all__ = [ - "ALL_CAPABILITIES", "CHANGING_FLAG_KEY", + "DECLARABLE_CAPABILITIES", "MESSAGES_FORMAT", "REPORT_DIR_ENV", + "RESERVED_CAPABILITIES", "SCHEMA_VERSION", "BackendControl", "Capability", diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py index c285b276..8be2b240 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py @@ -97,26 +97,58 @@ class Capability(str, Enum): """ TARGETING = "targeting" - """Reserved. No scenario carries this tag: targeting is backend evaluation logic.""" + """Reserved, and **not declarable**. No scenario carries this tag: targeting + is backend evaluation logic.""" CACHING = "caching" - """Reserved; no scenario carries this tag yet.""" + """Reserved, and **not declarable**. No scenario carries this tag yet.""" @property def tag(self) -> str: """Return the Gherkin tag, with its leading at-sign, that gates this capability.""" return f"@{self.value}" + @property + def reserved(self) -> bool: + """Whether this capability exists in the vocabulary but gates no scenario.""" + return self in RESERVED_CAPABILITIES + def __str__(self) -> str: return self.tag -ALL_CAPABILITIES: frozenset[Capability] = frozenset(Capability) -"""Every capability the TCK recognises. +RESERVED_CAPABILITIES: frozenset[Capability] = frozenset( + {Capability.TARGETING, Capability.CACHING} +) +"""Capabilities that exist in the vocabulary and gate no scenario. + +They are documented so the vocabulary has a place for them when scenarios exist, +and until then they **must not be declared** and must not appear in a conformance +report's declaration. Nothing carries the tag, so declaring it cannot be +verified, cannot produce a skip, and tells a reader of the report only that +something was claimed and nothing examined. + +Listed once, here, and read everywhere else -- by +:data:`DECLARABLE_CAPABILITIES`, by :attr:`Capability.reserved` and by the +validation in :class:`~.config.TckConfig` -- so that the set and the rule cannot +drift apart. +""" + +DECLARABLE_CAPABILITIES: frozenset[Capability] = ( + frozenset(Capability) - RESERVED_CAPABILITIES +) +"""Every capability an adoption may declare: the vocabulary minus the reserved tags. A reasonable starting point for a new adoption: declare everything, run the -suite, and remove only what the provider genuinely cannot do. Narrowing from the -full set surfaces gaps; widening towards it hides them. +suite, and remove only what the provider genuinely cannot do. Narrowing from this +set surfaces gaps; widening towards it hides them. + +It excludes the reserved capabilities rather than spanning the whole enum, and it +is named for what it is rather than for "all", because the declare-everything +convenience is exactly how a reserved tag reaches a report by accident: an +adopter writing "every capability except X" picks up every reserved tag on the +way past, which is how one implementation came to report ``@targeting`` and +``@caching`` as declared without anyone deciding to claim them. """ _BY_MARKER: dict[str, Capability] = {c.value: c for c in Capability} diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py index 4d07420c..38470266 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py @@ -8,7 +8,7 @@ from openfeature.provider import FeatureProvider -from .capability import ALL_CAPABILITIES, Capability +from .capability import DECLARABLE_CAPABILITIES, Capability from .control import BackendControl __all__ = ["KnownDeviation", "ProviderFactory", "TckConfig"] @@ -120,7 +120,7 @@ class TckConfig: skipped with the reason reported. """ - capabilities: Collection[Capability] = field(default=ALL_CAPABILITIES) + capabilities: Collection[Capability] = field(default=DECLARABLE_CAPABILITIES) """Which optional parts of the provider contract this provider supports. Typed as a ``Collection`` rather than a ``frozenset`` so that the obvious @@ -129,8 +129,13 @@ class TckConfig: construction, so a list, a set or a generator all behave identically. Scenarios tagged with an undeclared capability are reported as skipped with - the reason, never as passed. Defaults to everything; narrow it rather than - widening it. + the reason, never as passed. Defaults to every *declarable* capability -- + :data:`~.capability.DECLARABLE_CAPABILITIES`, which excludes the reserved + tags no scenario carries -- and narrowing it surfaces gaps where widening + towards it hides them. + + Naming a reserved capability here is rejected at construction rather than + passed into a report. See :data:`~.capability.RESERVED_CAPABILITIES`. """ not_applicable: Mapping[Capability, str] = field(default_factory=dict) @@ -229,6 +234,10 @@ def __post_init__(self) -> None: f"both leaves a consumer to guess which" ) + problems.extend( + reserved_problems(self.capabilities, self.not_applicable.keys()) + ) + unreasoned = sorted( capability.tag for capability, reason in self.not_applicable.items() @@ -278,6 +287,41 @@ def sorted_capabilities(self) -> list[str]: return sorted(c.tag for c in self.capabilities) +def reserved_problems(*named: Iterable[Capability]) -> list[str]: + """Refuse a reserved capability named anywhere in a configuration. + + A reserved capability gates no scenario, so naming it cannot be verified + either way: declaring it claims something nothing examined, and calling it + not-applicable records an impossibility about a question that was never + asked. Either would reach the report's declaration, which the schema + forbids. + + Refused rather than dropped quietly. The adopter wrote it down and meant + something by it, so a configuration silently different from the one they + wrote is worse than one that will not build -- and construction is where + their own code is still on the stack to say which line to fix. The + alternative, a warning, is a line of CI output nobody reads while an + untested capability goes on being asserted in a published report, which is + how this got into one in the first place. + """ + reserved = sorted( + capability.tag + for group in named + for capability in group + if isinstance(capability, Capability) and capability.reserved + ) + if not reserved: + return [] + declarable = " ".join(sorted(c.tag for c in DECLARABLE_CAPABILITIES)) + return [ + f"reserved capabilities {' '.join(sorted(set(reserved)))} cannot be declared " + f"or called not-applicable: no scenario carries them, so the claim cannot be " + f"verified, cannot produce a skip, and would tell a reader of the report only " + f"that something was claimed and nothing examined. The declarable " + f"capabilities, which is what DECLARABLE_CAPABILITIES holds, are {declarable}" + ] + + def capabilities_of(values: Iterable[Capability]) -> frozenset[Capability]: """Convenience for building a capability set from any iterable.""" return frozenset(values) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py index d66922be..b08e2aca 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/report.py @@ -271,6 +271,14 @@ def _declaration(self) -> dict[str, typing.Any]: different things -- a choice against a capability, and an impossibility. :class:`~.config.TckConfig` refuses a configuration that puts a capability in both, so a consumer never has to decide which one wins. + + Neither set is filtered here. A reserved capability -- one no scenario + carries, which the schema forbids in this block -- cannot be in a + ``TckConfig`` at all: it is refused at construction, and the default + capability set excludes it. Dropping one silently at emission time would + make a rejected configuration look like an accepted one, and leave the + adopter who wrote it believing the declaration they read back was the + declaration they asked for. """ declaration: dict[str, typing.Any] = { "declared": self.config.sorted_capabilities diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py index 0e883899..f7475406 100644 --- a/tools/openfeature-provider-tck/tests/test_report.py +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -43,6 +43,8 @@ import pytest from openfeature.contrib.tools.provider_tck import ( + DECLARABLE_CAPABILITIES, + RESERVED_CAPABILITIES, Capability, KnownDeviation, TckConfig, @@ -420,6 +422,23 @@ def control_api(self) -> str: return "http" +def _config_leaving_capabilities_to_their_default() -> TckConfig: + """A config that does not narrow ``capabilities``, so the field default runs. + + Written out rather than routed through ``_config``, which supplies a narrow + set of its own: the default is the whole point of this one. It needs an + unavailable-provider factory because ``@unavailable`` is declarable and the + default therefore declares it. + """ + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "new_unavailable_provider": lambda: None, + } + return TckConfig(**settings) + + def _config(**overrides: typing.Any) -> TckConfig: settings: dict[str, typing.Any] = { "name": "stub", @@ -565,7 +584,7 @@ def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: tmp_path_factory, file_name="narrow.json", name="narrow", - capabilities="{Capability.NUMERIC_COERCION, Capability.TARGETING}", + capabilities="{Capability.NUMERIC_COERCION}", not_applicable="{}", deviations=False, ) @@ -1011,6 +1030,47 @@ def test_a_not_applicable_capability_must_say_why() -> None: ) +def test_a_reserved_capability_cannot_be_declared() -> None: + """A tag no scenario carries is a claim nothing can check, so it is refused. + + Refused at construction rather than dropped at emission time, for the same + reason a capability claimed as both declared and impossible is: the adopter + wrote it down and meant something by it, and a config silently different + from the one they wrote is worse than one that will not build. This is also + where their own code is still on the stack. + """ + for reserved in RESERVED_CAPABILITIES: + with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): + _config(capabilities={Capability.EVENTS, reserved}) + with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): + _config( + capabilities={Capability.EVENTS}, + not_applicable={reserved: "no scenario asks"}, + ) + + +def test_a_reserved_capability_cannot_reach_the_declaration() -> None: + """Including by the route that actually caused it: declaring everything. + + The schema forbids a capability no executed scenario carries from appearing + in ``declaration.declared``, because such a tag cannot produce a skip and so + plays no part in reading the results -- it only invites a reader to believe + something was verified when nothing examined it. A real report from another + implementation asserts ``@targeting`` and ``@caching`` for exactly this + reason: that adoption declares "every capability except X" and collected the + reserved tags on the way past. So the default is the declarable set rather + than the whole enum. + """ + declared = SuiteReport( + config=_config_leaving_capabilities_to_their_default() + ).build(_results())["declaration"]["declared"] + + assert declared == sorted(capability.tag for capability in DECLARABLE_CAPABILITIES) + assert RESERVED_CAPABILITIES, "the rule is vacuous if nothing is reserved" + for reserved in RESERVED_CAPABILITIES: + assert reserved.tag not in declared + + def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: """Stating none is a claim; omitting the field is silence.""" assert "knownDeviations" not in SuiteReport(config=_config()).build(_results()) From 17264e52968a92fcd7df5c1e846ff3228d0a9dd1 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:00:17 +0200 Subject: [PATCH 10/11] feat(provider-tck): let an adopter add their own scenarios to the suite A provider is rarely only a provider. flagd has `fractional` targeting, another vendor has a proprietary rollout rule, and pinning those used to mean a second harness beside the conformance suite: a second backend lifecycle, a second set of fixtures, a second thing to keep working. An adopter's scenarios now run inside the canonical suite instead -- same session, same provider registration, same backend control. Almost nothing was needed to make that happen, because pytest already scans: it collects `conftest.py` on its own and pytest-bdd resolves step definitions through the fixture system, so a step an adopter writes beside their test module is already in scope for the scenarios generated into it. The only thing pytest cannot find by itself is the feature files, because the canonical ones live inside the installed distribution. `feature_paths()` returns both -- the packaged assets, and a `tck-extensions` directory beside the calling module if there is one -- so an adoption gains one call and no configuration: scenarios(*feature_paths()) An extension must never be able to stand in for a canonical scenario. Java's suite found that a same-named feature file in a second classpath root replaced the canonical one outright and the run went green having asked the adopter's questions; Python has a narrower route to the same place, because pytest-bdd names a feature file by its parent directory joined to its own name and `tck-extensions/features/errors.feature` therefore arrives under the uri the canonical `errors.feature` already occupies. So the uri a feature file reaches the results payload under is derived from where the file is: `features/` for the packaged assets and nothing else, `extensions/` for anything below a `tck-extensions` directory -- the same prefix the Go and JavaScript suites mount extensions under, so a consumer holding reports from several languages applies one rule. The two cases the derivation cannot rule out are refused rather than documented, and no report is written for either: a file of the adopter's own that would reach the reserved `features/` prefix, and two feature files that would share one uri, which a Messages stream cannot carry because it holds one source per uri. Signed-off-by: Simon Schrottner --- tools/openfeature-provider-tck/README.md | 81 +- .../contrib/tools/provider_tck/__init__.py | 31 +- .../contrib/tools/provider_tck/emitter.py | 59 +- .../contrib/tools/provider_tck/extensions.py | 286 +++++++ .../contrib/tools/provider_tck/plugin.py | 7 +- .../tests/test_extensions.py | 699 ++++++++++++++++++ 6 files changed, 1136 insertions(+), 27 deletions(-) create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/extensions.py create mode 100644 tools/openfeature-provider-tck/tests/test_extensions.py diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md index 64bad41a..9c7309b9 100644 --- a/tools/openfeature-provider-tck/README.md +++ b/tools/openfeature-provider-tck/README.md @@ -32,7 +32,7 @@ from pytest_bdd import scenarios from openfeature.contrib.tools.provider_tck import ( Capability, TckConfig, - features_path, + feature_paths, ) @@ -47,7 +47,7 @@ def tck_config(): ) -scenarios(features_path()) +scenarios(*feature_paths()) ``` There is **no `conftest.py` to write and nothing to import for the steps**. The step definitions @@ -70,6 +70,74 @@ different timescales — a streaming provider sees a configuration change in mil polls every 30 seconds may need most of a poll interval. Set it to comfortably exceed your worst-case detection latency, or the suite reports timeouts that are really just impatience. +## Adding your own scenarios + +A provider is rarely only a provider. flagd has `fractional` targeting, another vendor has a +proprietary rollout rule, and the behaviour of those is as worth pinning as the contract they sit on +top of. Verifying them used to mean a second harness: a second backend lifecycle, a second set of +fixtures, a second thing to keep working. + +Put them in the same run instead. Create a directory named `tck-extensions` beside the module that +calls `scenarios()`, and write step definitions for whatever is new in a `conftest.py` beside it: + +``` +tests/ +├── conftest.py # your step definitions +├── test_conformance.py # the fixture and the one call, unchanged +└── tck-extensions/ + └── fractional.feature +``` + +```python +# conftest.py +from pytest_bdd import then + +from openfeature.contrib.tools.provider_tck import TckState + + +@then("the fractional rule splits the population") +def fractional_splits(tck_state: TckState) -> None: + ... +``` + +That is the whole of it — **no registration, no option and no new argument**. pytest collects +`conftest.py` on its own, pytest-bdd resolves steps through the fixture system, and the canonical +step vocabulary is in scope in your feature file beside your own steps. `tck_state` is the same +per-scenario state the canonical steps use, so your scenario runs against the provider the suite +registered, in the same backend lifecycle, with the same reset between scenarios. + +The one thing pytest cannot find by itself is the feature files, because the canonical ones are +inside the installed distribution rather than in your repository. `feature_paths()` returns both: + +```python +scenarios(*feature_paths()) +``` + +That line does not change when you add an extension, and it is the only difference from the older +`scenarios(features_path())` — which still works and still sees only the canonical set. An adopter +with no `tck-extensions` directory runs exactly what they ran before: same scenarios, same count, +same report. + +### Your scenarios cannot stand in for ours + +In the report, canonical scenarios are the ones under the `features/` uri prefix and yours are under +`extensions/` — the prefix Go and JavaScript mount theirs under too, so a consumer holding reports +from several languages applies one rule. The prefix is derived from where a file *is*, not from what +the runner called it, and two cases are refused outright rather than documented: + +- **A feature file of yours under the reserved `features/` prefix.** Handing `scenarios()` a + directory of your own named `features` is the one route left to a canonical-looking uri. No report + is written and the run fails. +- **Two feature files that would share one uri.** A Cucumber Messages stream carries one source per + uri, so the second file's scenarios would be reported against the first file's source. + +This is not hypothetical. Java's suite found that a same-named feature file in a second classpath +root *replaced* the canonical one, and the run went green having asked the adopter's questions +instead of the specification's — the worst outcome available to a conformance suite. The Python +route to the same place is narrower and just as quiet: pytest-bdd names a feature file by its parent +directory joined to its own name, so `tck-extensions/features/errors.feature` arrives under the uri +the canonical `errors.feature` already occupies. + ## Capabilities Not every provider implements every optional part of the contract. Each scenario exercising an @@ -339,14 +407,15 @@ the field. | `test_controllable_conformance` | `ControllableInMemoryProvider` | the only suite that exercises the configuration-change path — see finding 2 | | `test_in_process_control` | `InProcessControl` | pins what the Gherkin cannot assert about itself | | `test_report` | the conformance report | checks the two properties a consumer is entitled to assume, against the emitted Messages stream | +| `test_extensions` | an adopter's own scenarios | an extension runs inside the canonical suite, changes nothing for an adopter who has none, and cannot stand in for a canonical scenario | ``` -94 passed, 9 skipped, 2 xfailed +110 passed, 9 skipped, 2 xfailed ``` -No Docker and no network. The conformance suites take under a second; `test_report` takes most of a -minute, because the properties it checks are properties of a whole pytest session and it runs four of -them in subprocesses to check them. +No Docker and no network. The conformance suites take under a second; `test_report` and +`test_extensions` take most of the time, because the properties they check are properties of a whole +pytest session and they run generated adoptions in subprocesses to check them. Neither in-memory suite declares `@lifecycle`, so the three lifecycle scenarios are skipped in both. That is the point: with no backend to reach, they would pass without testing anything — which is diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py index 90688439..d3a1d773 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -16,7 +16,7 @@ Capability, InProcessControl, TckConfig, - features_path, + feature_paths, ) @pytest.fixture(scope="session") @@ -29,11 +29,14 @@ def tck_config(): capabilities={Capability.EVENTS, Capability.OBJECT}, ) - scenarios(features_path()) + scenarios(*feature_paths()) ``scenarios()`` is pytest-bdd's own, called directly rather than wrapped: it injects the generated tests into the *calling module* by walking the stack, so a convenience wrapper around it would deposit them inside this package instead. +:func:`~.extensions.feature_paths` is the canonical assets plus a +``tck-extensions`` directory beside the calling module, if there is one -- see +:mod:`~.extensions`. The step definitions arrive through this package's pytest plugin, so there is nothing to import for them and no ``conftest.py`` to write. Everything else -- @@ -56,6 +59,11 @@ def tck_config(): ConnectionControl, UnsupportedControlError, ) +from .extensions import ( + EXTENSIONS_DIRECTORY, + feature_paths, + features_path, +) from .inprocess import InProcessControl from .messages import MESSAGES_FORMAT from .provider import ( @@ -64,10 +72,12 @@ def tck_config(): canonical_flag_set, ) from .report import REPORT_DIR_ENV, SCHEMA_VERSION +from .state import TckState __all__ = [ "CHANGING_FLAG_KEY", "DECLARABLE_CAPABILITIES", + "EXTENSIONS_DIRECTORY", "MESSAGES_FORMAT", "REPORT_DIR_ENV", "RESERVED_CAPABILITIES", @@ -79,10 +89,12 @@ def tck_config(): "InProcessControl", "KnownDeviation", "TckConfig", + "TckState", "UnsupportedControlError", "canonical_flag_set", "canonical_flags_json", "control_api_spec", + "feature_paths", "features_path", ] @@ -108,21 +120,6 @@ def tck_config(): _PACKAGE = "openfeature.contrib.tools.provider_tck" -def features_path() -> str: - """Return the directory holding the canonical feature files. - - Packaged with this distribution, so a consumer needs no submodule and no - particular directory layout. Hand it to pytest-bdd's ``scenarios()``, which - accepts an absolute path:: - - scenarios(features_path()) - - pytest-bdd generates one test per scenario -- and one per row of a Scenario - Outline -- so failures name a scenario and ``-k`` selects one as usual. - """ - return str(importlib.resources.files(_PACKAGE) / "features") - - def canonical_flags_json() -> str: """Return the canonical flag set as raw JSON, in the flagd flag-definition format. diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py index d3b8e42b..9ca9dab9 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py @@ -34,6 +34,12 @@ import pytest from .config import TckConfig +from .extensions import ( + collision_problem, + reserved_prefix_problem, + uri_collisions, + uri_for, +) from .messages import ( FeatureCatalog, ScenarioIdentity, @@ -113,11 +119,18 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: tags |= _examples_tags(node, scenario) filename = str(getattr(feature, "filename", "")) - relative = str(getattr(feature, "rel_filename", "") or Path(filename).name) + path = Path(filename) + relative = str(getattr(feature, "rel_filename", "") or path.name) return ScenarioIdentity( - uri=feature_uri(relative), - path=Path(filename), + # Derived from where the file is, falling back to what pytest-bdd called + # it. pytest-bdd names a feature by its parent directory joined to its + # own name, which two files can share: an extension at + # tck-extensions/features/errors.feature arrives under exactly the uri + # the canonical errors.feature already occupies, and the payload carries + # one source per uri. + uri=uri_for(path) or feature_uri(relative), + path=path, name=str(getattr(scenario, "name", "")), example=_example_of(node), tags=normalise_tags(tags), @@ -317,6 +330,13 @@ def _write_suite( name = suite.config.name runs = suite.sorted_runs + if not self._identities_are_sound(session, suite): + # Refusing to write is the point: a document that presents an + # adopter's feature file as the specification's -- or reports one + # file's scenarios against another's source -- is worse than no + # document, because it is the one thing a consumer cannot check. + return + catalog = FeatureCatalog() try: for run in runs: @@ -375,6 +395,39 @@ def _write_suite( if gap: self._say(session, f"provider-tck [{name}]: {gap}") + def _identities_are_sound( + self, session: pytest.Session, suite: SuiteReport + ) -> bool: + """Whether every feature file this suite ran is named in the payload as itself. + + Two ways it might not be, and both are silent without this. A file that + is not one of the packaged assets must not be reported under their uri + prefix, or a consumer reading ``features/errors.feature`` in the stream + has no way to tell that the specification did not write it. And two + files must not share a uri, or the stream carries one source for both + and the second file's scenarios are reported against the first's. + """ + name = suite.config.name + runs = suite.sorted_runs + + reserved = [ + problem + for run in runs + if (problem := reserved_prefix_problem(run.identity.uri, run.identity.path)) + ] + for problem in dict.fromkeys(reserved): + self._fail(session, f"provider-tck [{name}]: {problem}") + + collisions = uri_collisions( + (run.identity.uri, run.identity.path) for run in runs + ) + for uri, paths in sorted(collisions.items()): + self._fail( + session, f"provider-tck [{name}]: {collision_problem(uri, paths)}" + ) + + return not reserved and not collisions + def _say(self, session: pytest.Session, message: str) -> None: reporter = session.config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/extensions.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/extensions.py new file mode 100644 index 00000000..b1a667cd --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/extensions.py @@ -0,0 +1,286 @@ +"""Where the scenarios come from: the canonical set, plus whatever an adopter adds. + +A provider is rarely only a provider. flagd has ``fractional`` targeting, another +vendor has a proprietary rollout rule, and the behaviour of those is as worth +pinning as the contract they sit on top of. Verifying them used to mean standing +up a second harness: a second backend lifecycle, a second set of fixtures, a +second thing to keep working. The canonical suite ran, then something else ran, +and nothing tied the two together. + +So an adopter's own scenarios run **inside** the canonical suite instead -- +against the same provider instance, in the same backend lifecycle, with the same +step vocabulary available. Almost nothing is needed to make that happen, because +pytest already scans. It collects ``conftest.py`` on its own and pytest-bdd +resolves step definitions through the fixture system, so a step defined in the +adopter's ``conftest.py`` -- or in the test module itself -- is in scope for the +scenarios ``scenarios()`` generates there. The only thing pytest cannot find by +itself is the feature files, which is what this module finds: a directory named +``tck-extensions`` beside the adopter's test module. + +That leaves one line, and it is the same line whether or not there are +extensions:: + + scenarios(*feature_paths()) + +**An extension can never stand in for a canonical scenario.** The two are told +apart by the uri each feature file reaches the results payload under, and this +module derives that uri from where the file *is* rather than taking what the +runner offers: + +* ``features/…`` is the packaged canonical assets, and nothing else; +* ``extensions/…`` is a discovered extension, whatever the adopter's own + directory layout under ``tck-extensions`` looks like. + +The derivation is not decoration. pytest-bdd names a feature file by its parent +directory joined to its own name, so ``tck-extensions/features/errors.feature`` +arrives as ``features/errors.feature`` -- the same uri as a canonical file, and +the payload can carry only one source per uri. The canonical source is parsed +first, the extension's is never read, and its scenarios are reported against the +canonical file's pickles or against none at all. Java hit the same thing by a +different route: a same-named feature file in a second classpath root replaced +the canonical one outright and the suite went green having run the adopter's +version. +""" + +from __future__ import annotations + +import importlib.resources +import inspect +import typing +from pathlib import Path + +from .messages import feature_uri + +__all__ = [ + "CANONICAL_DIRECTORY", + "EXTENSIONS_DIRECTORY", + "EXTENSIONS_URI_PREFIX", + "canonical_root", + "collision_problem", + "extension_root", + "feature_paths", + "features_path", + "is_canonical", + "is_canonical_uri", + "reserved_prefix_problem", + "uri_collisions", + "uri_for", +] + +_PACKAGE = "openfeature.contrib.tools.provider_tck" + +CANONICAL_DIRECTORY = "features" +"""The packaged directory the canonical feature files live in. + +Also the uri prefix they carry in the results payload, which is why it is +reserved: a consumer reading ``features/errors.feature`` in a stream is entitled +to assume it is reading the specification's file rather than a local one that +happened to land in a directory of that name. +""" + +EXTENSIONS_DIRECTORY = "tck-extensions" +"""Where an adopter puts feature files of their own, beside their test module. + +Deliberately not ``features``: a directory sharing the canonical name is how an +extension comes to occupy a canonical file's identity, and a convention that +cannot collide is worth more than one that reads slightly better. The name is +the one Java's TCK scans for on the classpath, so an adopter who ships a provider +in both languages puts the same directory in both repositories. +""" + +EXTENSIONS_URI_PREFIX = "extensions" +"""The uri prefix an extension's scenarios reach the results payload under. + +The Go and JavaScript suites mount extensions under the same prefix, so a +consumer reading reports from several languages applies one rule to tell an +adopter's scenario from the specification's. +""" + + +def features_path() -> str: + """Return the directory holding the canonical feature files. + + Packaged with this distribution, so a consumer needs no submodule and no + particular directory layout. This is the canonical set on its own; prefer + :func:`feature_paths`, which also picks up an adopter's own scenarios. + """ + return str(importlib.resources.files(_PACKAGE) / CANONICAL_DIRECTORY) + + +def feature_paths() -> tuple[str, ...]: + """Return every feature directory this adoption should run. + + The canonical set, always, and a ``tck-extensions`` directory beside the + calling module if there is one. Hand the result to pytest-bdd's + ``scenarios()``:: + + scenarios(*feature_paths()) + + That line does not change when an adopter adds an extension, which is what + makes adding one a matter of creating a directory rather than of configuring + anything. + + The calling module is located from the caller's frame, which is how + pytest-bdd locates it for ``scenarios()`` itself, so the two agree about + which module is adopting the suite. Call it from the test module rather than + from a helper: a helper's directory is what a helper would find. A caller + with no ``__file__`` -- an interactive session, an exec'd string -- gets the + canonical set alone. + """ + paths = [features_path()] + directory = _caller_directory() + if directory is not None: + extensions = extension_root(directory) + if extensions is not None: + paths.append(str(extensions)) + return tuple(paths) + + +def extension_root(module_directory: Path) -> Path | None: + """The extension directory beside a test module, or ``None`` if there is none. + + ``None`` rather than a path that contributes nothing, so that an adopter + without extensions hands ``scenarios()`` exactly what they handed it before: + same scenarios, same count, same report. + """ + candidate = module_directory / EXTENSIONS_DIRECTORY + return candidate if candidate.is_dir() else None + + +def canonical_root() -> Path | None: + """The packaged canonical features directory, as a real path. + + ``None`` if the assets are not on the filesystem -- an installation from a + zipimport, say. Everything built on this degrades to "cannot tell", which is + the honest answer and never a false accusation. + """ + try: + return _resolve(Path(features_path())) + except (OSError, TypeError): # pragma: no cover - assets outside a filesystem + return None + + +def is_canonical(path: Path) -> bool: + """Whether a feature file is one of the packaged canonical ones.""" + canonical = canonical_root() + return canonical is not None and _resolve(path).is_relative_to(canonical) + + +def is_canonical_uri(uri: str) -> bool: + """Whether a uri names a canonical feature file. + + The discriminator between a canonical scenario and an extension one wherever + it matters -- the report, and a consumer reading the stream. Derived from the + uri rather than carried beside it, so there is no second fact to disagree + with the first. + """ + return uri.startswith(f"{CANONICAL_DIRECTORY}/") + + +def uri_for(path: Path) -> str | None: + """The uri a feature file should reach the results payload under. + + ``None`` when the file is neither canonical nor under an extension + directory, in which case the caller falls back to what pytest-bdd named it. + + Derived from the file's location rather than from pytest-bdd's + ``rel_filename``, which is the parent directory's name joined to the file's + own. That is what let ``tck-extensions/features/errors.feature`` present + itself as ``features/errors.feature``: the same uri as a canonical file, and + the payload can carry only one source per uri. + """ + resolved = _resolve(path) + + canonical = canonical_root() + if canonical is not None and resolved.is_relative_to(canonical): + return feature_uri( + str(Path(CANONICAL_DIRECTORY) / resolved.relative_to(canonical)) + ) + + for parent in resolved.parents: + if parent.name == EXTENSIONS_DIRECTORY: + return feature_uri( + str(Path(EXTENSIONS_URI_PREFIX) / resolved.relative_to(parent)) + ) + return None + + +def reserved_prefix_problem(uri: str, path: Path) -> str | None: + """Report a feature file claiming the canonical uri prefix without being canonical. + + The one thing the naming convention cannot rule out on its own: an adopter + who hands ``scenarios()`` a directory of their own named ``features``. The + file is then named exactly as a canonical one would be, and a consumer + reading the payload has no way to tell that the specification did not write + it. Caught where the report is assembled rather than shipped. + """ + if not is_canonical_uri(uri) or is_canonical(path): + return None + return ( + f"{uri} is not a canonical feature file -- it is {path} -- but it would be " + f"reported under the {CANONICAL_DIRECTORY}/ prefix, which is reserved for " + f"the packaged conformance assets. Move it into a directory named " + f"{EXTENSIONS_DIRECTORY} beside the test module, which feature_paths() " + f"finds on its own" + ) + + +def uri_collisions( + identified: typing.Iterable[tuple[str, Path]], +) -> dict[str, tuple[Path, ...]]: + """Feature files that would reach the payload under one uri, keyed by that uri. + + Deriving the uri from the file's location removes the collision an adopter + is actually likely to hit, but it does not make one impossible. Two + extension roots contributing the same relative path to a single suite -- two + test modules sharing one ``tck_config`` from a conftest, each with a + ``tck-extensions/vendor.feature`` -- still land on ``extensions/vendor.feature`` + twice, and so does a ``tck-extensions`` directory nested inside another one. + + That has to be refused rather than resolved. A Messages stream carries one + ``Source`` per uri, so the second file's source is never read: its scenarios + are reported against the first file's pickles where the names happen to + match, and go missing where they do not. The first is the silent form of + exactly the failure Java measured, and it is the one a consumer cannot + detect from the outside. + + Compared by resolved path, so the same file reached by two routes is one + file rather than a collision. + """ + files: dict[str, dict[Path, None]] = {} + for uri, path in identified: + files.setdefault(uri, {})[_resolve(path)] = None + return {uri: tuple(paths) for uri, paths in files.items() if len(paths) > 1} + + +def collision_problem(uri: str, paths: typing.Sequence[Path]) -> str: + """Say which files collided and what to do about it.""" + listed = ", ".join(str(path) for path in sorted(paths)) + return ( + f"{uri} is the uri of {len(paths)} different feature files -- {listed} -- " + f"and the results payload carries one source per uri, so one of them " + f"would be reported against the other's source. Give them paths that " + f"differ below their {EXTENSIONS_DIRECTORY} directory" + ) + + +def _caller_directory() -> Path | None: + """The directory of the module two frames up, if it has a file.""" + frame = inspect.currentframe() + for _ in range(2): + if frame is None: # pragma: no cover - no Python frames to walk + return None + frame = frame.f_back + if frame is None: # pragma: no cover - called with no caller above + return None + file_name: typing.Any = frame.f_globals.get("__file__") + if not isinstance(file_name, str) or not file_name: + return None + return _resolve(Path(file_name)).parent + + +def _resolve(path: Path) -> Path: + try: + return path.resolve() + except OSError: # pragma: no cover - a path that cannot be resolved at all + return path diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py index 29aeb0d5..c55cb6b9 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py @@ -4,7 +4,12 @@ all it takes for the step definitions to be available. pytest-bdd resolves steps through the fixture system and fixtures from an installed plugin are visible to every test, which is what keeps an adoption down to one fixture and one call to -:func:`tck_scenarios`. +``scenarios(*feature_paths())``. + +The same mechanism is what makes the suite extensible: a step an adopter defines +in their own ``conftest.py`` is resolved by the same fixture lookup as one this +plugin ships, so their scenarios need no glue and no second harness. See +:mod:`~.extensions`. """ from __future__ import annotations diff --git a/tools/openfeature-provider-tck/tests/test_extensions.py b/tools/openfeature-provider-tck/tests/test_extensions.py new file mode 100644 index 00000000..3789c68d --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_extensions.py @@ -0,0 +1,699 @@ +"""What an adopter's own scenarios may and may not do. + +An adopter with provider-specific behaviour -- flagd's ``fractional`` targeting, +a proprietary rollout rule -- has to be able to pin it in the same run as the +contract it sits on top of, or they end up maintaining a second harness beside +the one the TCK gives them. So the properties checked here are the ones that make +that safe rather than merely possible: + +* an extension scenario runs **inside** the canonical suite -- same session, same + provider, same report -- with a step definition the adopter wrote in their own + ``conftest.py`` and nothing else registered; +* an adoption without extensions runs exactly what it ran before, scenario for + scenario and field for field; +* an extension can neither replace a canonical scenario nor be reported as one. + +The last is not hypothetical. Java's suite discovered a same-named feature file +in a second classpath root silently *replacing* the canonical one, and the run +went green having asked the adopter's questions instead of the specification's. +The Python route to the same place is narrower and just as quiet: pytest-bdd +names a feature file by its parent directory joined to its own name, so a file at +``tck-extensions/features/errors.feature`` arrives under the uri the canonical +``errors.feature`` already occupies. + +Most of this is checked against real pytest sessions in subprocesses, because +every one of the properties is about how a whole session runs rather than about +what a function returns. +""" + +from __future__ import annotations + +import collections +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.provider_tck import ( + EXTENSIONS_DIRECTORY, + REPORT_DIR_ENV, + feature_paths, + features_path, +) +from openfeature.contrib.tools.provider_tck.extensions import ( + is_canonical_uri, + reserved_prefix_problem, + uri_collisions, + uri_for, +) + +CANONICAL_FEATURE = "errors.feature" +"""The canonical file the shadowing fixture copies, chosen because it is the one +whose scenarios an extension could most plausibly want to restate.""" + +VENDOR_URI = "extensions/vendor.feature" +VENDOR_SCENARIO = "A vendor rule resolves through the suite's own provider" + + +# -- the generated adoption -------------------------------------------------- + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so extensions can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={{Capability.EVENTS, Capability.OBJECT}}, + ) + + +scenarios({call}) +''' + +_EXTENSION_CALL = "*feature_paths()" +_CANONICAL_CALL = "features_path()" + +# The step the adopter writes, in the adopter's own conftest.py and nowhere else. +# It asks the TCK's own per-scenario state what provider this scenario is running +# against, which is what makes "the same backend lifecycle" checkable rather than +# asserted: a second harness would have a second provider, or none. +_CONFTEST_MODULE = """\ +import pytest +from pytest_bdd import then + +from openfeature.contrib.tools.provider_tck import TckState + +DEVIATION = "[boolean-flag-Integer-1]" + + +@then("the vendor rule ran against the provider the suite registered") +def vendor_rule_ran(tck_state: TckState) -> None: + assert tck_state.client is not None, "no provider was registered" + assert tck_state.provider_name == "In-Memory Provider", tck_state.provider_name + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Deliberately reuses the canonical step vocabulary and adds exactly one step of +# its own, which is the shape an adopter's feature file actually takes. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + And the vendor rule ran against the provider the suite registered +""" + +# The same Feature name and the same Scenario name as the file above, with a +# different step list -- so that a run reporting one against the other's source +# would be wrong in a way nothing downstream could notice. +_NESTED_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +# An adopter's own directory named `features`, which is the one way a +# non-canonical file can still reach the reserved prefix. Its scenario is +# deliberately trivial: the point is the file name, not what it asks. +_RESERVED_FEATURE = """\ +Feature: A feature file in a directory named features + + Scenario: A flag resolves + Given a stable provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +_RESERVED_SUITE = '''\ +"""An adoption that hands scenarios() a directory of its own named features.""" + +import pathlib + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="reserved", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS}, + ) + + +scenarios(str(pathlib.Path(__file__).parent / "features")) +''' + + +# -- reading a run back ------------------------------------------------------ + + +@dataclasses.dataclass(frozen=True) +class Case: + """One scenario as the stream reports it.""" + + uri: str + name: str + row: tuple[tuple[str, str], ...] + status: str + + @property + def identity(self) -> tuple[str, str, tuple[tuple[str, str], ...]]: + return (self.uri, self.name, self.row) + + +_SEVERITY = [ + "UNKNOWN", + "PASSED", + "SKIPPED", + "PENDING", + "UNDEFINED", + "AMBIGUOUS", + "FAILED", +] +"""Cucumber's own ordering: a test case is as bad as its worst step.""" + + +@dataclasses.dataclass(frozen=True) +class Report: + """One suite's pair of documents, read the way a consumer reads them.""" + + envelope: dict[str, typing.Any] + sources: dict[str, str] + cases: list[Case] + + @property + def canonical(self) -> list[Case]: + return [case for case in self.cases if is_canonical_uri(case.uri)] + + @property + def extensions(self) -> list[Case]: + return [case for case in self.cases if not is_canonical_uri(case.uri)] + + def named(self, name: str) -> list[Case]: + return [case for case in self.cases if case.name == name] + + @property + def identities(self) -> set[tuple[str, str, tuple[tuple[str, str], ...]]]: + return {case.identity for case in self.cases} + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of a generated adoption.""" + + directory: Path + reports: Path + result: subprocess.CompletedProcess[str] + + def report(self, name: str) -> Report: + path = self.reports / f"{name}.json" + assert path.exists(), ( + f"no report at {path}; pytest exited {self.result.returncode}\n" + f"{self.result.stdout}\n{self.result.stderr}" + ) + envelope = json.loads(path.read_text(encoding="utf-8")) + return _read(envelope, self.reports / envelope["results"]["location"]) + + +def _read(envelope: dict[str, typing.Any], stream_path: Path) -> Report: + """Assemble a stream into scenarios by following the protocol's own links.""" + sources: dict[str, str] = {} + names: dict[str, str] = {} + rows: dict[str, tuple[tuple[str, str], ...]] = {} + pickles: dict[str, dict[str, typing.Any]] = {} + test_cases: dict[str, dict[str, typing.Any]] = {} + started: dict[str, str] = {} + results: dict[str, list[str]] = collections.defaultdict(list) + + for line in stream_path.read_text(encoding="utf-8").splitlines(): + message = json.loads(line) + kind = next(iter(message)) + body = message[kind] + if kind == "source": + sources[body["uri"]] = body["data"] + elif kind == "gherkinDocument": + _index_document(body, names, rows) + elif kind == "pickle": + pickles[body["id"]] = body + elif kind == "testCase": + test_cases[body["id"]] = body + elif kind == "testCaseStarted": + started[body["id"]] = body["testCaseId"] + elif kind == "testStepFinished": + results[body["testCaseStartedId"]].append(body["testStepResult"]["status"]) + + cases = [] + for started_id, case_id in started.items(): + pickle = pickles[test_cases[case_id]["pickleId"]] + ast = pickle["astNodeIds"] + cases.append( + Case( + uri=pickle["uri"], + name=names[ast[0]], + row=rows.get(ast[1], ()) if len(ast) > 1 else (), + status=max(results[started_id], key=_SEVERITY.index), + ) + ) + return Report(envelope=envelope, sources=sources, cases=cases) + + +def _index_document( + document: dict[str, typing.Any], + names: dict[str, str], + rows: dict[str, tuple[tuple[str, str], ...]], +) -> None: + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children", ())) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[scenario["id"]] = scenario["name"] + for examples in scenario.get("examples", ()): + header = examples.get("tableHeader") + if header is None: + continue + headers = [cell["value"] for cell in header["cells"]] + for row in examples.get("tableBody", ()): + cells = [cell["value"] for cell in row["cells"]] + rows[row["id"]] = tuple(zip(headers, cells, strict=True)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children", ())) + + +def _pytest(directory: Path, reports: Path) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment[REPORT_DIR_ENV] = str(reports) + return subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + str(directory), + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +def _suite(name: str, call: str = _EXTENSION_CALL) -> str: + return _SUITE_MODULE.format(name=name, call=call) + + +def _run( + tmp_path_factory: pytest.TempPathFactory, + modules: dict[str, str], + features: dict[str, str] | None = None, +) -> Run: + """Write an adoption, run it, and hand back what it wrote. + + Both mappings are keyed by a path relative to the adoption directory, so a + fixture can put a module or a feature file wherever the property under test + needs it -- including inside a directory named ``features``, which is the + case that has to be refused. + """ + directory = tmp_path_factory.mktemp("adoption") + for relative, body in {**modules, **(features or {})}.items(): + path = directory / Path(relative) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + reports = tmp_path_factory.mktemp("reports") + return Run(directory=directory, reports=reports, result=_pytest(directory, reports)) + + +@pytest.fixture(scope="module") +def adoption(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One session running three adoptions of the same provider. + + ``before`` is the call an adopter writes today, ``scenarios(features_path())``, + which sees no extension however many are lying beside it. ``after`` is + ``scenarios(*feature_paths())`` with an ordinary extension beside it. + ``shadowed`` is the same again, in a directory of its own, with an extension + that is a verbatim copy of a canonical feature file placed under a directory + named ``features`` -- so that both routes to a canonical identity, the file's + own name and its parent's, are taken at once. + + One session rather than three, because a subprocess pytest run is by far the + most expensive thing in this file and the three suites are independent: each + resolves its own ``TckConfig`` and writes its own pair of documents. + """ + canonical = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") + return _run( + tmp_path_factory, + { + "test_before.py": _suite("before", _CANONICAL_CALL), + "test_after.py": _suite("after", _EXTENSION_CALL), + "conftest.py": _CONFTEST_MODULE, + "shadow/test_shadow.py": _suite("shadowed"), + }, + { + f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, + f"shadow/{EXTENSIONS_DIRECTORY}/features/{CANONICAL_FEATURE}": canonical, + }, + ) + + +# -- an extension runs inside the canonical suite ---------------------------- + + +def test_an_extension_scenario_runs_in_the_canonical_suite(adoption: Run) -> None: + """One suite, one report, both sets of scenarios in it. + + The report is written per ``TckConfig``, so an extension scenario appearing + in the same report as the canonical ones is not a presentational detail: it + is the same suite, which is the same provider registration and the same + backend control. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + report = adoption.report("after") + + vendor = report.named(VENDOR_SCENARIO) + assert len(vendor) == 1, report.extensions + assert vendor[0].status == "PASSED" + assert vendor[0].uri == VENDOR_URI + + assert report.canonical, "the canonical scenarios must have run too" + assert {case.status for case in report.canonical} <= {"PASSED", "SKIPPED", "FAILED"} + + +def test_the_extension_step_came_from_the_adopters_conftest(adoption: Run) -> None: + """Nothing was registered, imported or configured to make that step resolve. + + pytest collects ``conftest.py`` on its own and pytest-bdd resolves steps + through the fixture system, so a step defined beside the test module is in + scope for scenarios generated into it. If it were not, the step would be + ``UNDEFINED`` in the stream rather than absent, which is why this asserts on + the payload rather than on the exit status. + """ + conftest = (adoption.directory / "conftest.py").read_text(encoding="utf-8") + assert "the vendor rule ran against the provider the suite registered" in conftest + + vendor = adoption.report("after").named(VENDOR_SCENARIO)[0] + assert vendor.status == "PASSED", ( + f"an unresolved step is reported UNDEFINED, not missing: {vendor}" + ) + + +def test_the_extension_feature_is_reported_under_its_own_prefix( + adoption: Run, +) -> None: + """Which is what keeps an adopter's claim apart from the specification's. + + ``extensions/`` is the prefix the Go and JavaScript suites mount extensions + under too, so a consumer holding reports from several languages applies one + rule. + """ + report = adoption.report("after") + assert [case.uri for case in report.extensions] == [VENDOR_URI] + assert report.sources[VENDOR_URI] == _VENDOR_FEATURE + assert all(case.uri.startswith("features/") for case in report.canonical) + + +def test_the_envelope_is_unaffected_by_an_extension(adoption: Run) -> None: + """The report schema has no slot for extensions and needs none. + + Everything an extension adds is a scenario in the results payload, where a + uri already distinguishes it. An envelope field would be a second place for + the same fact to live. + """ + envelope = adoption.report("after").envelope + assert envelope["provider"]["configuration"] == "after" + assert set(envelope) == { + "schemaVersion", + "provider", + "sdk", + "tck", + "backend", + "declaration", + "results", + } + + +# -- and changes nothing for an adopter who has none ------------------------- + + +def test_the_canonical_scenarios_are_the_ones_that_always_ran( + adoption: Run, +) -> None: + """An extension adds; it does not alter. + + ``before`` is the call an adopter writes today and sees no extension. Every + scenario it ran, ``after`` ran too -- same rows, same outcomes, same sources + -- and the only difference between the two is what the extension added. An + adopter who has no extensions is the same comparison with the right-hand side + empty, which is what ``feature_paths()`` returning the canonical path alone + makes true by construction rather than by luck. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + before = adoption.report("before") + after = adoption.report("after") + + assert not before.extensions, "features_path() must see no extension" + assert {case.identity: case.status for case in after.canonical} == { + case.identity: case.status for case in before.cases + } + assert after.identities - before.identities == {(VENDOR_URI, VENDOR_SCENARIO, ())} + assert { + uri: source for uri, source in after.sources.items() if is_canonical_uri(uri) + } == before.sources + + +def test_an_extension_does_not_change_the_envelope_a_suite_writes( + adoption: Run, +) -> None: + """Everything but the two fields that necessarily differ. + + ``results`` names a file and digests its bytes, and the bytes carry + timestamps; ``configuration`` is the suite name, which is what tells the two + generated suites apart in the first place. + """ + before = dict(adoption.report("before").envelope) + after = dict(adoption.report("after").envelope) + for envelope in (before, after): + del envelope["results"] + envelope["provider"] = { + key: value + for key, value in envelope["provider"].items() + if key != "configuration" + } + assert after == before + + +# -- and cannot stand in for a canonical scenario ---------------------------- + + +def test_an_extension_cannot_replace_a_canonical_feature_file( + adoption: Run, +) -> None: + """The hazard Java measured, checked at the point it would have bitten. + + A verbatim copy of ``errors.feature`` under ``tck-extensions/features/`` + reaches pytest-bdd as ``features/errors.feature`` -- the canonical uri. The + uri the payload reports is derived from where the file is instead, so the + canonical source is still the packaged one, the copy is reported as an + extension, and both ran. + """ + report = adoption.report("shadowed") + canonical_uri = f"features/{CANONICAL_FEATURE}" + extension_uri = f"extensions/features/{CANONICAL_FEATURE}" + + packaged = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") + assert report.sources[canonical_uri] == packaged + assert report.sources[extension_uri] == packaged + + # Both files ran: the copy neither replaced the canonical one nor was + # silently dropped for colliding with it. + canonical = { + case.identity for case in report.canonical if case.uri == canonical_uri + } + copied = {case.identity for case in report.cases if case.uri == extension_uri} + assert canonical, "the canonical feature file did not run" + assert len(copied) == len(canonical) + assert not any(is_canonical_uri(case.uri) for case in report.extensions) + + +def test_a_shadowing_extension_does_not_disturb_the_canonical_run( + adoption: Run, +) -> None: + """The canonical scenarios are the same ones, with the same outcomes. + + Compared against a run that has no extension at all rather than against a + number written down here, so a change to the canonical assets cannot leave + this passing while the copy quietly displaces something. + """ + shadow = adoption.report("shadowed") + baseline = adoption.report("before") + assert {case.identity: case.status for case in shadow.canonical} == { + case.identity: case.status for case in baseline.cases + } + + +def test_a_directory_named_features_is_refused( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """The one collision the naming convention cannot rule out on its own. + + An adopter can still hand ``scenarios()`` a directory of their own named + ``features``, and its files are then named exactly as canonical ones would + be. No report is written for that suite: a document presenting an adopter's + feature file as the specification's is worse than no document, because it is + the one thing a consumer cannot check. + """ + run = _run( + tmp_path_factory, + {"test_reserved.py": _RESERVED_SUITE}, + {"features/local.feature": _RESERVED_FEATURE}, + ) + assert run.result.returncode != 0, run.result.stdout + assert "features/local.feature is not a canonical feature file" in run.result.stdout + assert EXTENSIONS_DIRECTORY in run.result.stdout, "the message must say the fix" + assert not list(run.reports.glob("*.json")), "no report may be written" + + +def test_two_extension_files_cannot_share_one_uri( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """Deriving the uri from the location narrows the collision; it does not end it. + + A ``tck-extensions`` directory nested inside another one reaches the same + uri as its namesake at the root, and so would two test modules sharing one + ``tck_config``. A Messages stream carries one source per uri, so the second + file's scenarios would be reported against the first file's pickles wherever + the names matched -- which here they do, deliberately. Refused rather than + resolved. + """ + nested = f"{EXTENSIONS_DIRECTORY}/nested/{EXTENSIONS_DIRECTORY}/vendor.feature" + run = _run( + tmp_path_factory, + {"test_collide.py": _suite("collide"), "conftest.py": _CONFTEST_MODULE}, + { + f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, + nested: _NESTED_FEATURE, + }, + ) + assert run.result.returncode != 0, run.result.stdout + assert f"{VENDOR_URI} is the uri of 2 different feature files" in run.result.stdout + assert not list(run.reports.glob("*.json")), "no report may be written" + + +def test_distinct_extension_paths_do_not_collide(tmp_path: Path) -> None: + """The same check, as a function, on the layouts that are fine.""" + root = tmp_path / EXTENSIONS_DIRECTORY + assert not uri_collisions( + [ + ("extensions/vendor.feature", root / "vendor.feature"), + ("extensions/a/vendor.feature", root / "a" / "vendor.feature"), + # One file reached by two routes is one file, not a collision. + ("extensions/vendor.feature", root / "a" / ".." / "vendor.feature"), + ] + ) + + +# -- deriving the uri -------------------------------------------------------- + + +def test_the_canonical_assets_keep_the_reserved_prefix() -> None: + canonical = Path(features_path()) / CANONICAL_FEATURE + assert uri_for(canonical) == f"features/{CANONICAL_FEATURE}" + assert is_canonical_uri(f"features/{CANONICAL_FEATURE}") + assert reserved_prefix_problem(f"features/{CANONICAL_FEATURE}", canonical) is None + + +def test_an_extension_keeps_its_layout_below_the_extensions_prefix( + tmp_path: Path, +) -> None: + """Whatever the adopter's own directory layout under the root looks like. + + Including one that reproduces the canonical name, which is the collision the + derivation exists for: pytest-bdd would have called the second of these + ``features/errors.feature``. + """ + root = tmp_path / EXTENSIONS_DIRECTORY + assert uri_for(root / "vendor.feature") == "extensions/vendor.feature" + assert ( + uri_for(root / "features" / CANONICAL_FEATURE) + == f"extensions/features/{CANONICAL_FEATURE}" + ) + assert ( + uri_for(root / "a" / "b" / "vendor.feature") == "extensions/a/b/vendor.feature" + ) + assert not is_canonical_uri("extensions/features/errors.feature") + + +def test_a_file_that_is_neither_is_left_to_pytest_bdd(tmp_path: Path) -> None: + """``None`` rather than a guess: the caller falls back to what the runner said.""" + assert uri_for(tmp_path / "loose.feature") is None + + +def test_a_local_file_under_the_reserved_prefix_is_a_problem(tmp_path: Path) -> None: + local = tmp_path / "features" / "local.feature" + problem = reserved_prefix_problem("features/local.feature", local) + assert problem is not None + assert EXTENSIONS_DIRECTORY in problem + assert str(local) in problem + + +def test_feature_paths_is_the_canonical_set_when_there_is_no_extension_directory() -> ( + None +): + """This test module has no ``tck-extensions`` beside it, and gets one path.""" + assert not (Path(__file__).parent / EXTENSIONS_DIRECTORY).exists() + assert feature_paths() == (features_path(),) From cabbad80eebfe3bd746340fff4fadb8023e3de54 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:54:56 +0200 Subject: [PATCH 11/11] feat(provider-tck): refuse a report that did not run the canonical set The capability gate rules out the loud way a conformance suite can go green on scenarios it did not run: an undeclared capability is reported as skipped, with its reason, never as passed. Nothing ruled out the quiet way, where the scenarios were never collected at all. `-k`, `-m`, `--deselect`, or a test module that stopped calling `scenarios()` on the canonical path each run less of the suite, and none of them is an error to pytest. Go measured the consequence: `-run` on a single scenario passed green and emitted a well-formed report covering one of twenty-nine canonical scenarios, with nothing in the document saying so. So every run is now checked against the scenarios this distribution ships. The expectation is compiled from the packaged feature files with the same Gherkin compiler that produces the results payload, which makes it one entry per Scenario Outline row -- the granularity the runner generates, and therefore the only one a comparison can be made at. A suite that did not execute all of them fails the run and writes no report, naming the scenarios that are missing. Two things may not close a gap. A scenario the capability gate skipped counts as having run, because it was asked and the report accounts for it with a reason. An adopter's own scenarios do not count at all: they are matched by path against the packaged assets rather than by the uri the emitter derives, so the check does not rest on the same derivation it exists to corroborate. `PROVIDER_TCK_PARTIAL=1` buys a green run for someone working on one scenario, and nothing else -- an incomplete suite writes no report either way. Java's TCK spells the same escape hatch the same way. Two adjustments fall out of it. Scenarios are enumerated `trylast` so that pytest's own deselection has already happened, or a filtered run reports every deselected scenario as collected but never run and drowns the one message that matters. And the self-test for per-Examples tags now runs its feature file as an extension beside the canonical set, because a suite that leaves the canonical set out no longer produces a report to read back. Separable from the extension work by design: it guards a bypass rather than enabling anything, and dropping it leaves the extension point unaffected. Signed-off-by: Simon Schrottner --- tools/openfeature-provider-tck/README.md | 38 +- .../contrib/tools/provider_tck/__init__.py | 2 + .../contrib/tools/provider_tck/canonical.py | 116 ++++++ .../contrib/tools/provider_tck/emitter.py | 115 +++++- .../contrib/tools/provider_tck/messages.py | 46 ++- .../tests/test_canonical_set.py | 342 ++++++++++++++++++ .../tests/test_report.py | 25 +- 7 files changed, 652 insertions(+), 32 deletions(-) create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/canonical.py create mode 100644 tools/openfeature-provider-tck/tests/test_canonical_set.py diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md index 9c7309b9..d15992d3 100644 --- a/tools/openfeature-provider-tck/README.md +++ b/tools/openfeature-provider-tck/README.md @@ -331,6 +331,34 @@ of the *run* and not of the code: CI sets it, a developer running the suite loca adopter changes a line to publish one. Unset means no report, which is not an error. Several suites in one pytest session each write their own pair, so flagd's two resolvers would not collide. +### A partial run is not a conformance run + +The canonical scenario set is fixed by the specification, and a run that executed less of it cannot +support a conformance claim. `-k`, `-m`, `--deselect`, or a test module that stopped calling +`scenarios()` on the canonical path each run fewer scenarios, and none of them is an error to +pytest. Go measured the consequence: `-run` on a single scenario passed green and emitted a +well-formed report covering 1 of 29 canonical scenarios, with nothing in the document saying so. + +So every run is checked against the scenarios this distribution ships, and a suite that did not +execute all of them writes no report: + +```console +$ PROVIDER_TCK_REPORT_DIR=./reports pytest -k "unknown_flag_key" +provider-tck [in-memory]: 28 of 29 canonical scenarios did not run, so this run cannot support a +conformance claim and no report is written for it. … + - features/errors.feature: A float flag is not silently narrowed to an integer + - features/errors.feature: Requesting the wrong type returns the code default [key=float-flag requested=Boolean default=false] + … and 18 more +``` + +A scenario the capability gate skipped **has** run: it was asked, and the report accounts for it +with its reason, so declining a capability never trips this. Your own scenarios are yours — they are +not counted towards the canonical set and cannot close a gap in it. + +Set `PROVIDER_TCK_PARTIAL=1` to work on a single scenario without the guard failing the run. It buys +a green run and nothing else: no report is written for an incomplete suite either way. Java's TCK +spells the same escape hatch the same way. + ### Why the results are not our format Per-scenario outcomes, tags, Scenario Outline row identity and the executed feature source are all @@ -408,14 +436,16 @@ the field. | `test_in_process_control` | `InProcessControl` | pins what the Gherkin cannot assert about itself | | `test_report` | the conformance report | checks the two properties a consumer is entitled to assume, against the emitted Messages stream | | `test_extensions` | an adopter's own scenarios | an extension runs inside the canonical suite, changes nothing for an adopter who has none, and cannot stand in for a canonical scenario | +| `test_canonical_set` | the canonical-set guard | a run that executed less than the canonical set fails and publishes nothing | ``` -110 passed, 9 skipped, 2 xfailed +132 passed, 9 skipped, 2 xfailed ``` -No Docker and no network. The conformance suites take under a second; `test_report` and -`test_extensions` take most of the time, because the properties they check are properties of a whole -pytest session and they run generated adoptions in subprocesses to check them. +No Docker and no network. The conformance suites take under a second; `test_report`, +`test_extensions` and `test_canonical_set` take most of the time, because the properties they check +are properties of a whole pytest session and they run generated adoptions in subprocesses to check +them. Neither in-memory suite declares `@lifecycle`, so the three lifecycle scenarios are skipped in both. That is the point: with no backend to reach, they would pass without testing anything — which is diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py index d3a1d773..299e07fb 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -52,6 +52,7 @@ def tck_config(): import importlib.resources +from .canonical import PARTIAL_ENV from .capability import DECLARABLE_CAPABILITIES, RESERVED_CAPABILITIES, Capability from .config import KnownDeviation, TckConfig from .control import ( @@ -79,6 +80,7 @@ def tck_config(): "DECLARABLE_CAPABILITIES", "EXTENSIONS_DIRECTORY", "MESSAGES_FORMAT", + "PARTIAL_ENV", "REPORT_DIR_ENV", "RESERVED_CAPABILITIES", "SCHEMA_VERSION", diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/canonical.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/canonical.py new file mode 100644 index 00000000..0009b424 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/canonical.py @@ -0,0 +1,116 @@ +"""The canonical scenario set, and whether a run actually executed it. + +A conformance suite that goes green on scenarios it did not run is worse than no +suite. The capability gate already rules out the loud version of that -- an +undeclared capability is reported as skipped, with its reason, never as passed -- +but it says nothing about the quiet version, where the scenarios were never asked +for in the first place. A ``-k`` expression, a ``-m`` filter, a ``--deselect``, a +test module that stopped calling ``scenarios()`` on the canonical path: each of +those runs less of the suite and none of them is an error. The Go implementation +measured it. ``-run`` on a single scenario passed green and emitted a well-formed +report covering one of twenty-nine canonical scenarios, and nothing in the +document said so. + +So the run is checked against what this distribution *ships* rather than against +what it was asked to run. The expectation is compiled from the packaged feature +files with the same Gherkin compiler that produces the results payload, which +means it is one entry per Scenario Outline row -- the granularity the runner +generates and therefore the only one a comparison can be made at. + +Two properties this has to have, and both are about what may close a gap. + +**A skip counts; an absence does not.** A scenario the capability gate skipped +did run: it was asked, and the report accounts for it with a reason. A scenario +that was never collected is missing, and no declaration makes it otherwise. + +**An extension cannot close a gap.** An adopter's own scenarios are matched by +neither uri nor path against the packaged set, and the executed side of the +comparison is filtered to files that are genuinely inside this distribution -- +not merely to files reported under the canonical prefix, so that the check does +not rest on the same derivation it is meant to corroborate. +""" + +from __future__ import annotations + +import functools +import os +import typing + +from .extensions import canonical_root, is_canonical, uri_for +from .messages import FeatureCatalog, ScenarioKey, ScenarioRun + +__all__ = [ + "PARTIAL_ENV", + "canonical_scenarios", + "describe", + "missing_canonical", + "partial_run_allowed", +] + +PARTIAL_ENV = "PROVIDER_TCK_PARTIAL" +"""Set to acknowledge that a run is deliberately not a conformance run. + +For working on one scenario with ``-k`` without the guard failing the run. It +never makes a partial run publishable: no report is written for a suite that did +not execute the canonical set, with or without it. The Java TCK spells the same +escape hatch the same way, so the two are one thing to know rather than two. +""" + +_TRUTHY = {"1", "true", "yes", "on"} + + +def partial_run_allowed(environment: typing.Mapping[str, str] | None = None) -> bool: + """Whether the run has declared itself partial.""" + source = os.environ if environment is None else environment + return source.get(PARTIAL_ENV, "").strip().lower() in _TRUTHY + + +@functools.cache +def canonical_scenarios() -> frozenset[ScenarioKey]: + """Every scenario the packaged feature files define, row by row. + + Empty when the assets are not reachable as files -- an installation from a + zipimport, say. Everything built on this then degrades to "cannot tell", + which is the honest answer and never a false accusation. + + Cached because it is the same answer for the whole process and parsing it is + the same work the results payload already does. + """ + root = canonical_root() + if root is None or not root.is_dir(): + return frozenset() + + catalog = FeatureCatalog() + for path in sorted(root.rglob("*.feature")): + uri = uri_for(path) + if uri is not None: + catalog.load_file(uri, path) + return catalog.scenario_keys + + +def missing_canonical(runs: typing.Iterable[ScenarioRun]) -> tuple[ScenarioKey, ...]: + """The canonical scenarios this suite did not execute, in reporting order. + + ``runs`` is everything the suite accounted for, extensions included; only + the ones whose feature file is genuinely one of the packaged assets are + counted, so an adopter's scenario can neither fill a gap nor be blamed for + one. + """ + expected = canonical_scenarios() + if not expected: + return () + executed = { + (run.identity.uri, run.identity.name, run.identity.example) + for run in runs + if is_canonical(run.identity.path) + } + return tuple(sorted(expected - executed)) + + +def describe(key: ScenarioKey) -> str: + """One missing scenario, named the way a failure message should name it.""" + uri, name, row = key + if not row: + return f"{uri}: {name}" + cells = " ".join(f"{header}={cell}" for header, cell in row) + return f"{uri}: {name} [{cells}]" diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py index 9ca9dab9..85e5c49c 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/emitter.py @@ -33,6 +33,13 @@ import pytest +from .canonical import ( + PARTIAL_ENV, + canonical_scenarios, + describe, + missing_canonical, + partial_run_allowed, +) from .config import TckConfig from .extensions import ( collision_problem, @@ -97,6 +104,14 @@ _SKIPPED = pytest.skip.Exception """What ``pytest.skip`` raises, named so a step hook can recognise it.""" +_MAX_MISSING = 10 +"""How many missing canonical scenarios a failure names before summarising. + +Enough to act on and not so many that the reason is lost above them. A run with +one scenario selected is missing twenty-eight, and listing all of them says +nothing the count did not. +""" + def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: """Describe a pytest node as a Gherkin scenario, or return ``None``. @@ -222,11 +237,19 @@ def __init__(self, config: pytest.Config) -> None: self._step_started: dict[str, int] = {} config.stash[COLLECTOR_KEY] = self.collector + @pytest.hookimpl(trylast=True) def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: - """Enumerate every TCK scenario the session collected. + """Enumerate every TCK scenario the session will run. At collection rather than as each runs, so that the stream accounts for scenarios that never got as far as running a fixture. + + ``trylast`` so that pytest's own deselection -- ``-k``, ``-m``, + ``--deselect`` -- has already removed what it is going to remove. + Enumerating before it does would make a filtered run report every + deselected scenario as collected but never run, which is a true + statement about a list nobody asked for and drowns the one message that + matters: which canonical scenarios are missing. """ for item in items: identity = scenario_identity(item) @@ -294,12 +317,37 @@ def _finish_step( ) def pytest_sessionfinish(self, session: pytest.Session) -> None: + if session.config.getoption("collectonly", False): + # Nothing ran, and nothing was meant to. Every check below asks what + # a run executed, and the answer "nothing" is not a finding here. + return + + problems = self.collector.resolve(scenario_run) + + # Whether a suite may be published is a property of the run rather than + # of the report, so it is established whether or not one was asked for. + # Both checks are made on every suite rather than short-circuited, so a + # suite with two faults hears about both. + unpublishable: set[int] = set() + for suite in self.collector.suites: + sound = self._identities_are_sound(session, suite) + complete = self._canonical_set_ran(session, suite) + if not sound or not complete: + unpublishable.add(id(suite)) + directory = os.environ.get(REPORT_DIR_ENV, "").strip() if not directory: return - self.write(session, Path(directory)) + for problem in problems: + self._fail(session, f"provider-tck: {problem}") + self.write(session, Path(directory), unpublishable) - def write(self, session: pytest.Session, directory: Path) -> None: + def write( + self, + session: pytest.Session, + directory: Path, + unpublishable: typing.AbstractSet[int] = frozenset(), + ) -> None: """Write every suite's pair of files, failing the session if one cannot be. A run that asked for a report and silently did not get one is how a @@ -307,11 +355,10 @@ def write(self, session: pytest.Session, directory: Path) -> None: write failure and an incomplete document are loud and change the exit status rather than being logged and forgotten. """ - for problem in self.collector.resolve(scenario_run): - self._fail(session, f"provider-tck: {problem}") - written: dict[str, str] = {} for suite in self.collector.suites: + if id(suite) in unpublishable: + continue name = suite.config.name file_name = envelope_file_name(name) if written.get(file_name, name) != name: @@ -324,19 +371,56 @@ def write(self, session: pytest.Session, directory: Path) -> None: written[file_name] = name self._write_suite(session, directory, suite) + def _canonical_set_ran(self, session: pytest.Session, suite: SuiteReport) -> bool: + """Whether this suite executed every scenario the TCK ships. + + The check a conformance claim rests on that no amount of reading the + report can supply: the results payload says what happened to the + scenarios that ran, and says nothing at all about the ones that did not. + + A capability-gated skip counts -- it was asked, and the report accounts + for it with a reason. An extension scenario does not count and cannot + close a gap. :data:`~.canonical.PARTIAL_ENV` downgrades the failure to a + note for someone working on a single scenario; it does not make the run + publishable, because the report is withheld either way. + """ + missing = missing_canonical(suite.runs.values()) + if not missing: + return True + + name = suite.config.name + total = len(canonical_scenarios()) + headline = ( + f"provider-tck [{name}]: {len(missing)} of {total} canonical scenarios " + f"did not run, so this run cannot support a conformance claim and no " + f"report is written for it. The canonical set is fixed by the " + f"specification; running less of it is not a configuration. Decline " + f"capabilities your provider does not have through TckConfig instead, " + f"which reports the scenarios as skipped with their reason" + ) + if partial_run_allowed(): + self._say( + session, + f"{headline}. {PARTIAL_ENV} is set, so the run is not failed for it", + ) + else: + self._fail( + session, + f"{headline}. To filter anyway while working on one scenario, set " + f"{PARTIAL_ENV}=1 and accept that the run is not a conformance run", + ) + for key in missing[:_MAX_MISSING]: + self._say(session, f" - {describe(key)}") + if len(missing) > _MAX_MISSING: + self._say(session, f" ... and {len(missing) - _MAX_MISSING} more") + return False + def _write_suite( self, session: pytest.Session, directory: Path, suite: SuiteReport ) -> None: name = suite.config.name runs = suite.sorted_runs - if not self._identities_are_sound(session, suite): - # Refusing to write is the point: a document that presents an - # adopter's feature file as the specification's -- or reports one - # file's scenarios against another's source -- is worse than no - # document, because it is the one thing a consumer cannot check. - return - catalog = FeatureCatalog() try: for run in runs: @@ -406,6 +490,11 @@ def _identities_are_sound( has no way to tell that the specification did not write it. And two files must not share a uri, or the stream carries one source for both and the second file's scenarios are reported against the first's. + + A suite that fails either writes no report. Refusing is the point: a + document that presents an adopter's feature file as the specification's + is worse than no document, because it is the one thing a consumer cannot + check. """ name = suite.config.name runs = suite.sorted_runs diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py index 96877a6b..6d8199ce 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/messages.py @@ -55,6 +55,7 @@ "MESSAGES_FORMAT", "FeatureCatalog", "ScenarioIdentity", + "ScenarioKey", "ScenarioRun", "Status", "StepRun", @@ -62,6 +63,15 @@ "write_stream", ] +ScenarioKey = tuple[str, str, tuple[tuple[str, str], ...]] +"""What names one scenario: its feature file's uri, its name, and its Examples row. + +The one description of a scenario that a pytest-bdd node and a Gherkin pickle +can each produce without consulting the other, which is what makes it usable +both as the join key inside :class:`FeatureCatalog` and as the currency of a +comparison between the scenarios that ran and the ones that were shipped. +""" + MESSAGES_FORMAT = "cucumber-messages" """The ``results.format`` value the envelope carries for this payload.""" @@ -224,26 +234,46 @@ def __init__(self) -> None: self._sources: dict[str, str] = {} self._documents: dict[str, dict[str, typing.Any]] = {} self._pickles: dict[str, list[_Pickle]] = {} - self._index: dict[tuple[str, str, tuple[tuple[str, str], ...]], _Pickle] = {} + self._index: dict[ScenarioKey, _Pickle] = {} @property def uris(self) -> list[str]: return sorted(self._documents) + @property + def scenario_keys(self) -> frozenset[ScenarioKey]: + """Every scenario the loaded feature files define, row by row. + + What a feature file *asks*, as opposed to what a run executed. Taken + from the compiled pickles rather than from the AST so that a Scenario + Outline contributes one entry per row, which is what the runner + generates and therefore what a comparison has to be made in. + """ + return frozenset(self._index) + def load(self, identity: ScenarioIdentity) -> None: """Parse the feature file this scenario came from, once.""" - if identity.uri in self._documents: + self.load_file(identity.uri, identity.path) + + def load_file(self, uri: str, path: Path) -> None: + """Parse one feature file under the uri it will be reported by, once. + + Separate from :meth:`load` because the canonical set has to be parsed + with no run in hand: the question "did every shipped scenario execute" + is asked of feature files, not of outcomes. + """ + if uri in self._documents: return - source = identity.path.read_text(encoding="utf-8") + source = path.read_text(encoding="utf-8") document: dict[str, typing.Any] = Parser( ast_builder=AstBuilder(self._ids) ).parse(source) - document["uri"] = identity.uri + document["uri"] = uri pickles: list[dict[str, typing.Any]] = Compiler(self._ids).compile(document) - self._sources[identity.uri] = source - self._documents[identity.uri] = document - self._pickles[identity.uri] = [ + self._sources[uri] = source + self._documents[uri] = document + self._pickles[uri] = [ _Pickle( id=str(pickle["id"]), step_ids=tuple(str(step["id"]) for step in pickle.get("steps") or ()), @@ -251,7 +281,7 @@ def load(self, identity: ScenarioIdentity) -> None: ) for pickle in pickles ] - self._index_pickles(identity.uri, document, pickles) + self._index_pickles(uri, document, pickles) def _index_pickles( self, diff --git a/tools/openfeature-provider-tck/tests/test_canonical_set.py b/tools/openfeature-provider-tck/tests/test_canonical_set.py new file mode 100644 index 00000000..21a5f4d5 --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_canonical_set.py @@ -0,0 +1,342 @@ +"""That a run which executed less than the canonical set says so, and publishes nothing. + +The capability gate rules out the loud way a conformance suite can go green on +scenarios it did not run: an undeclared capability is reported as skipped, with +its reason. Nothing ruled out the quiet way, where the scenarios were never +collected at all. ``-k``, ``-m``, ``--deselect``, a test module that stopped +calling ``scenarios()`` on the canonical path -- each runs less of the suite, and +none of them is an error to pytest. + +Go measured the consequence: ``-run`` on a single scenario passed green and +emitted a well-formed report covering one of twenty-nine canonical scenarios. +Nothing in that document said so, and nothing reading it could have known. + +So the properties here are about what a report is allowed to be written from. +Everything that has to be checked end to end is, because the question is about a +whole pytest session rather than about what a function returns. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.provider_tck import ( + EXTENSIONS_DIRECTORY, + PARTIAL_ENV, + REPORT_DIR_ENV, +) +from openfeature.contrib.tools.provider_tck.canonical import ( + canonical_scenarios, + describe, + missing_canonical, + partial_run_allowed, +) +from openfeature.contrib.tools.provider_tck.extensions import is_canonical_uri +from openfeature.contrib.tools.provider_tck.messages import ( + ScenarioIdentity, + ScenarioRun, +) + +SUITE_NAME = "guarded" + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so a partial run can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="guarded", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + +scenarios(*feature_paths()) +''' + +_CONFTEST_MODULE = """\ +import pytest + +DEVIATION = "[boolean-flag-Integer-1]" + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Three scenarios of an adopter's own, so that a run which drops one canonical +# scenario still executes more scenarios than the canonical set contains. The +# count is what makes "an extension cannot close a gap" checkable rather than +# asserted. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + + Scenario: A vendor rule resolves again + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + + Scenario: And once more + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +# The canonical scenario the filtered runs below leave out. Named rather than +# counted, so a change to the canonical assets cannot leave these passing while +# they select nothing. +EXCLUDED_SELECTOR = "unknown_flag_key" +EXCLUDED_SCENARIO = "An unknown flag key returns the code default" + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of the generated adoption.""" + + reports: Path + result: subprocess.CompletedProcess[str] + + @property + def stdout(self) -> str: + return self.result.stdout + + @property + def envelopes(self) -> list[Path]: + return sorted(self.reports.glob("*.json")) + + +def _run( + tmp_path: Path, + *arguments: str, + partial: bool = False, + extension: bool = False, +) -> Run: + directory = tmp_path / "adoption" + directory.mkdir(parents=True, exist_ok=True) + (directory / "test_guarded.py").write_text(_SUITE_MODULE, encoding="utf-8") + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + if extension: + features = directory / EXTENSIONS_DIRECTORY + features.mkdir(parents=True, exist_ok=True) + (features / "vendor.feature").write_text(_VENDOR_FEATURE, encoding="utf-8") + + reports = tmp_path / "reports" + environment = dict(os.environ) + environment[REPORT_DIR_ENV] = str(reports) + if partial: + environment[PARTIAL_ENV] = "1" + else: + environment.pop(PARTIAL_ENV, None) + + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + str(directory), + *arguments, + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + return Run(reports=reports, result=result) + + +def _identity(uri: str, name: str, path: Path) -> ScenarioIdentity: + return ScenarioIdentity(uri=uri, path=path, name=name, tags=()) + + +# -- a run that executed the whole set --------------------------------------- + + +@pytest.fixture(scope="module") +def complete(tmp_path_factory: pytest.TempPathFactory) -> Run: + """The unfiltered run, which is what everything else is measured against.""" + return _run(tmp_path_factory.mktemp("complete"), extension=True) + + +def test_the_whole_canonical_set_still_writes_a_report(complete: Run) -> None: + """Including the scenarios the capability gate skipped. + + A gated skip *ran*: it was asked, and the report accounts for it with a + reason. Treating it as missing would make the guard contradict the one rule + the suite is built around. + """ + assert complete.result.returncode == 0, complete.stdout + assert [path.name for path in complete.envelopes] == [f"{SUITE_NAME}.json"] + assert "canonical scenarios did not run" not in complete.stdout + # This adoption declares neither @stale nor @lifecycle, so some canonical + # scenarios were skipped -- which is the case being asserted about. + assert " skipped" in complete.stdout + + +# -- and one that did not ---------------------------------------------------- + + +def test_a_filtered_run_fails_and_writes_no_report(tmp_path: Path) -> None: + """The Go hazard, at the point it would have produced the document. + + A well-formed report covering one scenario of twenty-nine is worse than no + report, because nothing in it says which twenty-eight were never asked. + """ + run = _run(tmp_path, "-k", EXCLUDED_SELECTOR) + + assert run.result.returncode != 0, run.stdout + assert "of 29 canonical scenarios did not run" in run.stdout + assert not run.envelopes, "a partial run must publish nothing" + assert not list(run.reports.glob("*.ndjson")) + # The message names scenarios rather than only counting them, and says how + # to filter deliberately. + assert EXCLUDED_SCENARIO not in run.stdout, "that one is the scenario that ran" + assert PARTIAL_ENV in run.stdout + + +def test_acknowledging_a_partial_run_does_not_make_it_publishable( + tmp_path: Path, +) -> None: + """``PROVIDER_TCK_PARTIAL`` buys a green run, never a document. + + Someone working on one scenario should not have to fight the guard; nobody + should be able to turn a partial run into a conformance claim. Those are + different requests, and only the first is granted. Java's TCK spells the + same escape hatch the same way. + """ + run = _run(tmp_path, "-k", EXCLUDED_SELECTOR, partial=True) + + assert run.result.returncode == 0, run.stdout + assert "canonical scenarios did not run" in run.stdout + assert f"{PARTIAL_ENV} is set" in run.stdout + assert not run.envelopes, "acknowledged or not, it is not a conformance run" + + +def test_an_extension_cannot_close_a_gap(tmp_path: Path) -> None: + """Three extension scenarios do not make up for one canonical one. + + The adoption below runs thirty-one scenarios where the canonical set has + twenty-nine, and is still one short: an adopter's scenarios are theirs, and + counting them towards the specification's set would let any gap be filled by + adding a feature file. + """ + run = _run(tmp_path, "-k", f"not {EXCLUDED_SELECTOR}", extension=True) + + assert run.result.returncode != 0, run.stdout + assert "1 of 29 canonical scenarios did not run" in run.stdout + assert f"features/errors.feature: {EXCLUDED_SCENARIO}" in run.stdout + assert not run.envelopes + + +# -- what the canonical set is ----------------------------------------------- + + +def test_the_canonical_set_is_read_from_the_packaged_assets() -> None: + """One entry per Scenario Outline row, which is what a runner generates.""" + scenarios = canonical_scenarios() + assert len(scenarios) == 29 + assert all(is_canonical_uri(uri) for uri, _, _ in scenarios) + # An outline contributes rows, not a single templated entry. + assert any(row for _, _, row in scenarios) + + +def test_a_run_of_nothing_is_missing_everything() -> None: + assert set(missing_canonical([])) == canonical_scenarios() + + +def test_an_extension_run_is_neither_counted_nor_blamed(tmp_path: Path) -> None: + """Matched by path rather than by uri. + + The uri is derived, and a check that rested on the same derivation it exists + to corroborate would be checking its own arithmetic. + """ + vendor = ScenarioRun( + identity=_identity( + "extensions/vendor.feature", + "A vendor rule resolves", + tmp_path / EXTENSIONS_DIRECTORY / "vendor.feature", + ) + ) + # Even one that claims a canonical uri outright -- which the report refuses + # separately -- cannot reduce the missing set. + impostor = ScenarioRun( + identity=_identity( + "features/errors.feature", + EXCLUDED_SCENARIO, + tmp_path / "features" / "errors.feature", + ) + ) + assert set(missing_canonical([vendor, impostor])) == canonical_scenarios() + + +def test_a_missing_scenario_is_described_by_its_row() -> None: + assert describe(("features/x.feature", "A scenario", ())) == ( + "features/x.feature: A scenario" + ) + assert describe(("features/x.feature", "An outline", (("key", "a"),))) == ( + "features/x.feature: An outline [key=a]" + ) + + +@pytest.mark.parametrize( + ("value", "allowed"), + [("1", True), ("true", True), ("TRUE", True), ("yes", True), ("on", True)], +) +def test_the_acknowledgement_is_read_generously(value: str, allowed: bool) -> None: + assert partial_run_allowed({PARTIAL_ENV: value}) is allowed + + +@pytest.mark.parametrize("value", ["", "0", "false", "no", " ", "maybe"]) +def test_anything_else_is_not_an_acknowledgement(value: str) -> None: + """Including a typo: the default has to be the safe one.""" + assert partial_run_allowed({PARTIAL_ENV: value}) is False + assert partial_run_allowed({}) is False + + +def test_the_envelope_of_a_complete_run_is_unchanged(complete: Run) -> None: + """The guard withholds a report; it does not add anything to one.""" + envelope: dict[str, typing.Any] = json.loads( + complete.envelopes[0].read_text(encoding="utf-8") + ) + assert set(envelope) == { + "schemaVersion", + "provider", + "sdk", + "tck", + "backend", + "declaration", + "results", + } diff --git a/tools/openfeature-provider-tck/tests/test_report.py b/tools/openfeature-provider-tck/tests/test_report.py index f7475406..df3361f0 100644 --- a/tools/openfeature-provider-tck/tests/test_report.py +++ b/tools/openfeature-provider-tck/tests/test_report.py @@ -44,6 +44,7 @@ from openfeature.contrib.tools.provider_tck import ( DECLARABLE_CAPABILITIES, + EXTENSIONS_DIRECTORY, RESERVED_CAPABILITIES, Capability, KnownDeviation, @@ -195,9 +196,11 @@ def pytest_collection_modifyitems(items): """ _TAGGED_SUITE = '''\ -"""A suite over the feature file beside it, which tags one Examples block.""" +"""A suite whose extension feature file tags one Examples block of an outline. -import pathlib +The canonical set runs alongside it, because a suite that leaves the canonical +set out writes no report at all -- see ``_canonical_set_ran``. +""" import pytest from pytest_bdd import scenarios @@ -206,6 +209,7 @@ def pytest_collection_modifyitems(items): Capability, InProcessControl, TckConfig, + feature_paths, ) @@ -220,7 +224,7 @@ def tck_config(): ) -scenarios(str(pathlib.Path(__file__).parent)) +scenarios(*feature_paths()) ''' @@ -770,11 +774,14 @@ def test_a_row_gated_by_its_examples_block_is_the_only_one_skipped( no capability -- leaving the envelope's declaration unable to explain the skip, which is the one derivation this format asks a consumer to make. - No canonical feature file does this yet, so the feature file is written here. + No canonical feature file does this yet, so the feature file is written here + -- as an extension, because a suite that leaves the canonical set out writes + no report to read back. """ directory = tmp_path / "suite" - directory.mkdir(parents=True) - (directory / "tagged.feature").write_text(_TAGGED_FEATURE, encoding="utf-8") + extensions = directory / EXTENSIONS_DIRECTORY + extensions.mkdir(parents=True) + (extensions / "tagged.feature").write_text(_TAGGED_FEATURE, encoding="utf-8") (directory / "test_tagged.py").write_text(_TAGGED_SUITE, encoding="utf-8") reports = tmp_path / "reports" @@ -784,7 +791,11 @@ def test_a_row_gated_by_its_examples_block_is_the_only_one_skipped( envelope = json.loads(path.read_text(encoding="utf-8")) stream = _read_stream(path.parent / envelope["results"]["location"]) - by_row = {dict(case.row)["requested"]: case for case in stream.cases} + by_row = { + dict(case.row)["requested"]: case + for case in stream.cases + if case.uri == "extensions/tagged.feature" + } # Every row is still reported: nothing about gating one row of an outline may # drop its siblings from the payload.