From 04687cda930313263a2677f8abd3947bb35b8b8f Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:04:06 -0700 Subject: [PATCH 1/4] fix: Normalize intstring/floatstring task parameter range elements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Template Schemas 2023-09 §3.4.1.1 (L1110) makes an element ` | `, and §3.4.1.2 (L1184) makes a element ` | `, neither behind an extension gate. §2.3 and §2.4 define / as "a string whose value is the string representation of" a number, so such an element denotes the number rather than its source text. Python was keeping the source text. A `range: ['1', '02', '003']` on an INT parameter rendered FRAME:02 and FRAME:003, and a `range: ['1.5', '02.50']` on a FLOAT parameter rendered W:02.50. These values reach a task command line, so a renderer was invoked with `--frame 02`. The pre-validator on the template models did compute the value -- the element validator evaluates int('02') == 2 -- but validate_list_field returns the original list object, discarding every coercion, and nothing downstream recovered it. Normalize on RangeListTaskParameterDefinition instead, the instantiation target that all three inbound range paths funnel through: a literal list, a range-expression expansion, and an RFC 0006 typed whole-field resolution. Keying off the already-validated `type` field, an INT or CHUNK[INT] element becomes an int and a FLOAT element becomes a normalized Decimal. Decimal's normalize() rewrites Decimal('100') as Decimal('1E+2'), so a positive exponent is quantized away -- exponent notation must never reach a task command line. Only string-form elements are touched. A literal keeps the scale it was written with, so a FLOAT `range: [1.0]` still renders 1.0; openjd-rs renders an integral float the same way and the conformance suite pins it (2023-09/base/jobs/3.4--float-parameter). STRING and PATH ranges are text by definition and are left alone. Job parameter defaults are unaffected: they live on the job-parameter definitions, not the task-parameter definitions, so the verbatim FLOAT default behaviour that 2023-09/EXPR/jobs/expr1.3.4--float-passthrough pins is untouched. Normalizing the stored elements exposed a latent bug in StepParameterSpaceIterator containment. RangeListIdentifierNode compared ParameterValue.value -- the rendered form of an element -- against a set built from the raw elements, so it only ever matched when a range happened to be written as strings. An INT range written as `[1, 2, 3]` already reported every one of its own values as not contained, before this change. The containment set now holds rendered elements. openjd-rs already behaves this way and passes both conformance fixtures; it parses every range element into an i64/f64 and so has no source text to carry. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/model/_step_param_space_iter.py | 9 +- src/openjd/model/v2023_09/_model.py | 57 +++++++++++ .../model_v0/test_step_param_space_iter.py | 99 +++++++++++++++++++ .../model_v0/v2023_09/test_parameter_space.py | 89 +++++++++++++++++ 4 files changed, 253 insertions(+), 1 deletion(-) diff --git a/src/openjd/model/_step_param_space_iter.py b/src/openjd/model/_step_param_space_iter.py index d3aec120..dbcd41d5 100644 --- a/src/openjd/model/_step_param_space_iter.py +++ b/src/openjd/model/_step_param_space_iter.py @@ -338,7 +338,14 @@ def _create_expr_tree( name=name, type=ParameterValueType(parameter.type), range=parameter.range, - range_set=set(parameter.range), + # Non-CHUNK containment compares against ParameterValue.value, + # which is the rendered form of a range element (see + # RangeListIdentifierNode.__getitem__), so the set has to hold + # rendered elements too. A set of the raw elements only matched + # when a range happened to be written as strings — an INT range + # written as `[1, 2, 3]` reported every one of its own values as + # not contained. + range_set={str(v) for v in parameter.range}, ) else: return RangeExpressionIdentifierNode( diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 763fb053..8e709c6e 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -1242,6 +1242,31 @@ def _validate_target_runtime_seconds(cls, value: Any, info: ValidationInfo) -> A return validate_int_fmtstring_field(value, ge=0, context=context) +def _normalized_range_element(elem: str, to_int: bool) -> Any: + """The number an ````/```` range element denotes, or + the element unchanged when it does not denote one.""" + try: + if to_int: + return int(elem) + value = Decimal(elem).normalize() + if not value.is_finite(): + # Decimal accepts 'nan'/'inf', which are not s; leave + # them as text rather than changing how they render. + return elem + exponent = value.as_tuple().exponent + if isinstance(exponent, int) and exponent > 0: + # normalize() rewrites Decimal('100') as Decimal('1E+2'); undo the + # shift so a range element never renders in exponent notation. + value = value.quantize(Decimal(1)) + return value + except (ValueError, ArithmeticError): + # Not a number at all — e.g. a format string that resolved to + # non-numeric text. A literal range is already checked against its + # element type at template parse time, so leave such a value to the + # existing behaviour rather than adding a rejection path here. + return elem + + # Target model for task parameters when instantiating a job. class RangeListTaskParameterDefinition(OpenJDModel_v2023_09): # element type of items in the range @@ -1251,6 +1276,38 @@ class RangeListTaskParameterDefinition(OpenJDModel_v2023_09): # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension chunks: Optional[TaskChunksDefinition] = None + @field_validator("range", mode="before") + @classmethod + def _normalize_numeric_range_elements(cls, value: Any, info: ValidationInfo) -> Any: + # §3.4.1.1/§3.4.1.2: an element is ` | + # ` and a element is ` | + # `, where an / is "a string whose + # value is the string representation of" a number (§2.3, §2.4). Such an + # element therefore denotes that number, not its source text, so '02' + # is the task value 2 and '02.50' is 2.5. Python was keeping the source + # text, which reaches a task command line as `--frame 02`. + # + # Applied on the instantiation target so every inbound range path is + # covered: a literal list, a range-expression expansion, and an RFC 0006 + # typed whole-field resolution all funnel through this model. + # + # Numeric elements are left exactly as parsed. A FLOAT range of [1.0] + # renders 1.0, which openjd-rs also does and the conformance suite pins + # (base/jobs/3.4--float-parameter). STRING and PATH ranges are text by + # definition and are never touched. + param_type = info.data.get("type") + if not isinstance(value, list) or param_type not in ( + TaskParameterType.INT, + TaskParameterType.CHUNK_INT, + TaskParameterType.FLOAT, + ): + return value + to_int = param_type != TaskParameterType.FLOAT + return [ + _normalized_range_element(elem, to_int) if isinstance(elem, str) else elem + for elem in value + ] + @field_validator("range") @classmethod def _validate_range_len(cls, value: Any) -> Any: diff --git a/test/openjd/model_v0/test_step_param_space_iter.py b/test/openjd/model_v0/test_step_param_space_iter.py index e059dd00..d11e277d 100644 --- a/test/openjd/model_v0/test_step_param_space_iter.py +++ b/test/openjd/model_v0/test_step_param_space_iter.py @@ -567,3 +567,102 @@ def test_nested_expr_iteration(self) -> None: "Param3": ParameterValue(type=ParameterValueType.STRING, value="11"), "Param4": ParameterValue(type=ParameterValueType.INT, value="20"), } not in it + + +class TestRangeListElementValues: + """The values a task command line actually receives, end to end from a + template through `create_job`. + + §3.4.1.1/§3.4.1.2 allow a range element to be written in the `` / + `` form, which §2.3/§2.4 define as "a string whose value is the + string representation of" a number. Such an element denotes the number, so + `['1', '02', '003']` on an INT parameter is the task values 1, 2 and 3 — not + the source text, which would reach a renderer as `--frame 02`. + """ + + @staticmethod + def _task_values(param_type: str, range_list: list) -> list[str]: + job_template = parse_model( + model=JobTemplate_2023_09, + obj={ + "specificationVersion": "jobtemplate-2023-09", + "name": "Job", + "steps": [ + { + "name": "step", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "P", "type": param_type, "range": range_list} + ] + }, + "script": {"actions": {"onRun": {"command": "do thing"}}}, + } + ], + }, + ) + job = create_job(job_template=job_template, job_parameter_values=dict()) + return [ + params["P"].value + for params in StepParameterSpaceIterator(space=job.steps[0].parameterSpace) + ] + + def test_int_range_intstring_elements(self) -> None: + # WHEN an INT range is written with elements + # THEN each task gets the integer the element denotes + assert self._task_values("INT", ["1", "02", "003"]) == ["1", "2", "3"] + + def test_float_range_floatstring_elements(self) -> None: + # WHEN a FLOAT range is written with elements + # THEN each task gets the number the element denotes + assert self._task_values("FLOAT", ["1.5", "02.50"]) == ["1.5", "2.5"] + + def test_float_range_numeric_literals_keep_their_scale(self) -> None: + # A literal is not a string representation, so it keeps the scale + # it was written with. openjd-rs renders an integral float as `1.0` too, + # and the conformance suite pins this + # (2023-09/base/jobs/3.4--float-parameter). + assert self._task_values("FLOAT", [0.5, 1.0, 1.5]) == ["0.5", "1.0", "1.5"] + + @pytest.mark.parametrize( + "param_type,range_list", + ( + pytest.param("STRING", ["02", "1.50"], id="STRING"), + pytest.param("PATH", ["/frames/02", "/frames/003"], id="PATH"), + ), + ) + def test_string_and_path_ranges_are_text(self, param_type: str, range_list: list) -> None: + # §3.4.1.3/§3.4.1.4 give STRING and PATH no numeric element form, so + # nothing about their elements is normalizable. + assert self._task_values(param_type, range_list) == range_list + + @pytest.mark.parametrize( + "range_list", + ( + pytest.param([10, 11, 12], id="written as integers"), + pytest.param(["10", "11", "12"], id="written as intstrings"), + ), + ) + def test_containment_matches_the_rendered_values(self, range_list: list) -> None: + # Containment compares a ParameterValue against the parameter space, and + # ParameterValue.value is the *rendered* form of a range element. The + # containment set was built from the raw elements, so an INT range + # written as `[10, 11, 12]` reported its own values as not contained. + + # GIVEN a parameter space over an int range + space = StepParameterSpace_2023_09( + taskParameterDefinitions={ + "Param1": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.INT, range=range_list + ), + }, + ) + + # WHEN it is iterated + it = StepParameterSpaceIterator(space=space) + + # THEN every value it yields is contained in it, however the range was written + values = list(it) + assert [v["Param1"].value for v in values] == ["10", "11", "12"] + for value in values: + assert value in it + assert {"Param1": ParameterValue(type=ParameterValueType.INT, value="13")} not in it diff --git a/test/openjd/model_v0/v2023_09/test_parameter_space.py b/test/openjd/model_v0/v2023_09/test_parameter_space.py index 23f0af9e..7011e159 100644 --- a/test/openjd/model_v0/v2023_09/test_parameter_space.py +++ b/test/openjd/model_v0/v2023_09/test_parameter_space.py @@ -439,6 +439,95 @@ def test_list_form_range_is_still_capped(self, model: Any, obj: dict[str, Any]) assert len(excinfo.value.errors()) > 0 +class TestRangeListElementNormalization: + """§3.4.1.1/§3.4.1.2: an `` element is ` | ` + and a `` element is ` | `. An + ``/`` is "a string whose value is the string + representation of" a number (§2.3, §2.4), so it denotes that number and not + its source text — a range of `['1', '02', '003']` is the task values 1, 2 and + 3. These values reach a task command line, so keeping the text renders + `--frame 02`. + """ + + @pytest.mark.parametrize( + "param_type", + (pytest.param("INT", id="INT"), pytest.param("CHUNK[INT]", id="CHUNK[INT]")), + ) + def test_intstring_elements_carry_their_value(self, param_type: str) -> None: + # GIVEN an int range mixing the and forms + obj: dict[str, Any] = {"type": param_type, "range": [1, "02", "003", 4]} + if param_type == "CHUNK[INT]": + obj["chunks"] = {"defaultTaskCount": 1, "rangeConstraint": "CONTIGUOUS"} + + # WHEN the instantiation target parses it + model = _parse_model(model=RangeListTaskParameterDefinition, obj=obj) + + # THEN every element is the integer it denotes, and renders without the + # leading zeros of its source text + assert model.range == [1, 2, 3, 4] + assert [str(v) for v in model.range] == ["1", "2", "3", "4"] + + @pytest.mark.parametrize( + "element,expected", + ( + pytest.param("1.5", "1.5", id="no normalization needed"), + pytest.param("02.50", "2.5", id="leading and trailing zeros"), + pytest.param("3.500", "3.5", id="trailing zeros"), + pytest.param("007", "7", id="leading zeros on a whole number"), + pytest.param("-02.50", "-2.5", id="negative"), + pytest.param("0.0", "0", id="zero"), + # normalize() rewrites Decimal('100') as Decimal('1E+2'). Exponent + # notation must never reach a task command line. + pytest.param("100", "100", id="whole number that normalize() shifts"), + pytest.param("1E+2", "100", id="exponent notation in the source"), + ), + ) + def test_floatstring_elements_carry_their_value(self, element: str, expected: str) -> None: + # WHEN the instantiation target parses a float range holding a + model = _parse_model( + model=RangeListTaskParameterDefinition, obj={"type": "FLOAT", "range": [element]} + ) + + # THEN the element renders as the number it denotes + assert [str(v) for v in model.range] == [expected] + + @pytest.mark.parametrize( + "param_type,range_list", + ( + # A literal keeps the scale it was written with. openjd-rs + # renders an integral float as `1.0` too, and the conformance suite + # pins it (2023-09/base/jobs/3.4--float-parameter). + pytest.param("FLOAT", [0.5, 1.0, 1.5], id="FLOAT numeric literals"), + pytest.param("INT", [1, 2, 3], id="INT numeric literals"), + # STRING and PATH ranges are text by definition — §3.4.1.3/§3.4.1.4 + # give no numeric form, so nothing about them is normalizable. + pytest.param("STRING", ["02", "003", "1.50"], id="STRING"), + pytest.param("PATH", ["/frames/02", "/frames/003"], id="PATH"), + ), + ) + def test_elements_that_must_not_change(self, param_type: str, range_list: list) -> None: + # WHEN the instantiation target parses a range that normalization must not touch + model = _parse_model( + model=RangeListTaskParameterDefinition, + obj={"type": param_type, "range": range_list}, + ) + + # THEN every element renders exactly as it was written + assert [str(v) for v in model.range] == [str(v) for v in range_list] + + def test_unparseable_element_is_left_alone(self) -> None: + # GIVEN a range element that does not denote a number — reachable when a + # format string resolves to non-numeric text, since the template-layer + # element check only sees literals. + # WHEN the instantiation target parses it + model = _parse_model( + model=RangeListTaskParameterDefinition, obj={"type": "INT", "range": ["notanumber"]} + ) + + # THEN it is carried through unchanged rather than rejected here + assert model.range == ["notanumber"] + + class TestStepParameterSpaceDefinition: @pytest.mark.parametrize( "data", From 5ad0e6110a3ea2112269a210d9e8050167fd306b Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:29:31 -0700 Subject: [PATCH 2/4] chore: Update THIRD-PARTY-LICENSES for pydantic 2.13.5 Regenerated versions for pydantic (2.13.4 -> 2.13.5) and pydantic_core (2.46.4 -> 2.46.5) to match the resolved dependency set, clearing the THIRD-PARTY-LICENSES check. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- THIRD-PARTY-LICENSES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index a89a1057..f749ff3b 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -24,7 +24,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------ -** pydantic; version 2.13.4 -- https://pypi.org/project/pydantic/ +** pydantic; version 2.13.5 -- https://pypi.org/project/pydantic/ The MIT License (MIT) Copyright (c) 2017 to present Pydantic Services Inc. and individual contributors. @@ -48,7 +48,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------ -** pydantic_core; version 2.46.4 -- https://pypi.org/project/pydantic_core/ +** pydantic_core; version 2.46.5 -- https://pypi.org/project/pydantic_core/ The MIT License (MIT) Copyright (c) 2022 Samuel Colvin From 2109e2f58e0e85a0bb87f1a0217edc1f07392933 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 29 Aug 2026 11:03:54 -0700 Subject: [PATCH 3/4] fix: Render floatstring range elements exactly and in plain notation Review found three defects that share one root cause: Decimal.normalize() plus a positive-exponent quantize() guard is both context-sensitive and notation-unstable. - normalize() rounds to getcontext().prec. A 41-significant-digit element came back as 1.234567890123456789012345679, and an embedding application that sets getcontext().prec = 5 changed what this library rendered for the same template (measured: '1.2345678901234567890123456789012345678901' -> '1.2346'). Template output must not depend on host process state. - str(Decimal) switches to exponent notation once the adjusted exponent is below -6, which the positive-exponent guard did not cover, so the element '0.0000001' rendered 1E-7. That was a regression: before normalization was introduced the source text was kept and it rendered 0.0000001. - quantize(Decimal(1)) raises InvalidOperation once the result needs more digits than getcontext().prec, and the except clause swallowed it and returned the raw string. '1E+30' therefore rendered as 1E+30, silently defeating the very invariant the guard existed to hold. '123456.75' under a narrowed context hit the same path. Render with format(value, 'f') instead. With no precision in the format spec it is exact and plain at every magnitude, so it needs neither the rounding of normalize() nor the precision-bounded quantize(): verified identical output under getcontext().prec = 5. Redundant leading and trailing zeros are then stripped from the text, which is not an arithmetic operation and cannot consult the context. An integral now keeps one fractional digit, so '1.0' renders 1.0 and not 1. openjd-rs was measured as the reference for this: it renders '1.0' as 1.0, '0.0' as 0.0, '007' as 7.0 and '100' as 100.0. Dropping the fraction made the same denoted number render two ways depending on whether it was spelled as a or a , since a literal 1.0 renders 1.0. The conformance fixture does not settle it -- 3.4.1.2 uses ['1.5', '02.50'], neither of them integral -- so the Rust implementation was probed directly. '-0.0' renders 0.0, since the number it denotes is zero, which openjd-rs also does. Plain notation is a deliberate divergence from openjd-rs at the extremes, where its f64 Display gives 1e-07 and 1e+30. Exponent notation must not reach a task command line at any magnitude, and Decimal carries the written value exactly where f64 cannot. Scope is unchanged. Numeric literals are still untouched, STRING and PATH ranges are still text, and job parameter defaults are not involved. 2023-09/base/jobs/3.4--float-parameter and 2023-09/EXPR/jobs/expr1.3.4--float-passthrough both still pass, as do both proposed 3.4.1.1/3.4.1.2 normalization fixtures; the full 2023-09 conformance suite is 1172 passed, 0 failed. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/model/v2023_09/_model.py | 36 +++++++++---- .../model_v0/v2023_09/test_parameter_space.py | 51 ++++++++++++++++--- 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 8e709c6e..ddc092e1 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -1243,22 +1243,40 @@ def _validate_target_runtime_seconds(cls, value: Any, info: ValidationInfo) -> A def _normalized_range_element(elem: str, to_int: bool) -> Any: - """The number an ````/```` range element denotes, or - the element unchanged when it does not denote one.""" + """The number an ````/```` range element denotes -- an + ``int`` for ````, and for ```` the plain-notation text + of the value, since ``str(Decimal)`` cannot render every magnitude without an + exponent. Returns the element unchanged when it does not denote a number.""" try: if to_int: return int(elem) - value = Decimal(elem).normalize() + value = Decimal(elem) if not value.is_finite(): # Decimal accepts 'nan'/'inf', which are not s; leave # them as text rather than changing how they render. return elem - exponent = value.as_tuple().exponent - if isinstance(exponent, int) and exponent > 0: - # normalize() rewrites Decimal('100') as Decimal('1E+2'); undo the - # shift so a range element never renders in exponent notation. - value = value.quantize(Decimal(1)) - return value + # format(value, 'f') is exact and plain at every magnitude: with no + # precision in the format spec it neither rounds to getcontext().prec + # nor switches to exponent notation, so an embedding app that sets a + # different context cannot change what this library renders. Neither + # holds of the obvious alternatives -- normalize() rounds to the + # ambient precision, str() switches to exponent notation below 1e-6, + # and quantize() raises InvalidOperation once the result needs more + # digits than the ambient precision allows. + text = format(value, "f") + negative = text.startswith("-") + integer, _, fraction = text.lstrip("-").partition(".") + # A denotes a number, so redundant zeros in its source + # text are not part of the value: '02.50' is 2.5. One fractional digit + # is always kept, matching openjd-rs, which renders the float 1.0 as + # `1.0` and never as `1`. + integer = integer.lstrip("0") or "0" + fraction = fraction.rstrip("0") or "0" + if negative and integer == "0" and fraction == "0": + # The number denoted by '-0.0' is zero, which has no sign; openjd-rs + # renders it `0.0`. + negative = False + return f"{'-' if negative else ''}{integer}.{fraction}" except (ValueError, ArithmeticError): # Not a number at all — e.g. a format string that resolved to # non-numeric text. A literal range is already checked against its diff --git a/test/openjd/model_v0/v2023_09/test_parameter_space.py b/test/openjd/model_v0/v2023_09/test_parameter_space.py index 7011e159..222c8866 100644 --- a/test/openjd/model_v0/v2023_09/test_parameter_space.py +++ b/test/openjd/model_v0/v2023_09/test_parameter_space.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +from decimal import localcontext from typing import Any import pytest @@ -473,13 +474,34 @@ def test_intstring_elements_carry_their_value(self, param_type: str) -> None: pytest.param("1.5", "1.5", id="no normalization needed"), pytest.param("02.50", "2.5", id="leading and trailing zeros"), pytest.param("3.500", "3.5", id="trailing zeros"), - pytest.param("007", "7", id="leading zeros on a whole number"), pytest.param("-02.50", "-2.5", id="negative"), - pytest.param("0.0", "0", id="zero"), - # normalize() rewrites Decimal('100') as Decimal('1E+2'). Exponent - # notation must never reach a task command line. - pytest.param("100", "100", id="whole number that normalize() shifts"), - pytest.param("1E+2", "100", id="exponent notation in the source"), + # An integral float keeps one fractional digit. openjd-rs renders + # the '1.0' as `1.0` and '007' as `7.0`, so dropping + # the fraction would make the same number render two ways depending + # on whether it was written as a or a . + pytest.param("1.0", "1.0", id="integral float"), + pytest.param("0.0", "0.0", id="zero"), + pytest.param("-0.0", "0.0", id="negative zero has no sign"), + pytest.param("007", "7.0", id="leading zeros on a whole number"), + pytest.param("100", "100.0", id="whole number"), + # Exponent notation must never reach a task command line, at any + # magnitude and in either direction. + pytest.param("1E+2", "100.0", id="exponent notation in the source"), + pytest.param( + "0.0000001", + "0.0000001", + id="below 1e-6, where Decimal.__str__ would give 1E-7", + ), + pytest.param( + "1E+30", + "1" + "0" * 30 + ".0", + id="more digits than the default decimal precision", + ), + pytest.param( + "1.2345678901234567890123456789012345678901", + "1.2345678901234567890123456789012345678901", + id="more significant digits than the default decimal precision", + ), ), ) def test_floatstring_elements_carry_their_value(self, element: str, expected: str) -> None: @@ -491,6 +513,23 @@ def test_floatstring_elements_carry_their_value(self, element: str, expected: st # THEN the element renders as the number it denotes assert [str(v) for v in model.range] == [expected] + def test_floatstring_normalization_ignores_the_decimal_context(self) -> None: + # GIVEN an embedding application that has narrowed the process-wide + # decimal context, and a with more significant digits + # than that precision allows + element = "1.2345678901234567890123456789012345678901" + + # WHEN the instantiation target parses it under that context + with localcontext() as ctx: + ctx.prec = 5 + model = _parse_model( + model=RangeListTaskParameterDefinition, obj={"type": "FLOAT", "range": [element]} + ) + + # THEN the value is unrounded: what this library renders is a property of + # the template, not of the host application's decimal context + assert [str(v) for v in model.range] == [element] + @pytest.mark.parametrize( "param_type,range_list", ( From 9a41393db0383d3a78a1c79f76624a633c250084 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 29 Aug 2026 11:39:53 -0700 Subject: [PATCH 4/4] fix: Bound floatstring range element expansion by the field's char cap Rendering a range element with format(value, 'f') is unbounded in the element's exponent, which review found has two reachable faces. Unbounded expansion. Decimal construction from a string is not bounded by the context -- Emax/Emin constrain arithmetic results, not construction -- so Decimal('1e999999999') is finite and the is_finite() guard did not stop it. The plain-notation length is exponent + 1, measured: 10**7 gives 10,000,001 characters. Parsing a FLOAT range of ['1e100000000'] on the instantiation target took 0.37s and 266 MiB peak RSS from 11 characters of template text; the 1e999999999 case is 10x that and killed the probe process. It is reachable from an ordinary template, because the template layer accepts the element and keeps it as a TaskParameterStringValue. Silently exceeding the field cap, far lower down. A TaskRangeList element's str member is TaskParameterStringValueAsJob, capped at 1024 characters. Measured: '1E+1022' -- 7 characters, accepted before this branch -- expands to 1025 and so fails that member, and pydantic then falls through to the numeric members, rendering the element as a 1023-digit int. In the other direction '1E-1023' expands to 1025, fails the same way, and renders 0.0, silently losing the value. Both were valid text-rendering elements on mainline, which rendered '1E+1030' as 1E+1030. Bound the length from value.adjusted() and the exponent before materializing anything. The limit is the field's own 1024-character cap rather than a new number, because an expansion past it cannot be carried here as text at all, so there is nothing to gain by producing one. An element over the bound keeps its source text -- what this function already does for anything it cannot normalize, and how such an element rendered before normalization was introduced -- so there is no new rejection path and no new error type. The bound is exact at the boundary: '1E+1021' and '1E-1022' expand to exactly 1024 and are still normalized. After the fix '1e999999999' parses in 0.01ms at 42 MiB peak RSS. Model suite 5521 -> 5527 passed (6 new tests, 24 skipped, 3 xfailed). The two boundary tests fail against 2109e2f, rendering 1000...000 and 0.0. Conformance 3.4.1.1, 3.4.1.2, 3.4--float-parameter and expr1.3.4--float-passthrough all pass. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/model/v2023_09/_model.py | 34 +++++++++++++- .../model_v0/v2023_09/test_parameter_space.py | 47 +++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index ddc092e1..56caa30c 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -1208,7 +1208,10 @@ def __new__(cls, value: str, *, context: ModelParsingContextInterface = ModelPar list[Union[Decimal, TaskParameterStringValue]], Field(min_length=1, max_length=1024) ] StringRangeList = Annotated[list[TaskParameterStringValue], Field(min_length=1, max_length=1024)] -TaskParameterStringValueAsJob = Annotated[str, StringConstraints(min_length=0, max_length=1024)] +_MAX_TASK_PARAM_VALUE_LEN = 1024 +TaskParameterStringValueAsJob = Annotated[ + str, StringConstraints(min_length=0, max_length=_MAX_TASK_PARAM_VALUE_LEN) +] TaskRangeList = list[Union[TaskParameterStringValueAsJob, int, float, Decimal]] @@ -1246,7 +1249,8 @@ def _normalized_range_element(elem: str, to_int: bool) -> Any: """The number an ````/```` range element denotes -- an ``int`` for ````, and for ```` the plain-notation text of the value, since ``str(Decimal)`` cannot render every magnitude without an - exponent. Returns the element unchanged when it does not denote a number.""" + exponent. Returns the element unchanged when it does not denote a number, or + when its plain-notation form would not fit this field's character cap.""" try: if to_int: return int(elem) @@ -1255,6 +1259,32 @@ def _normalized_range_element(elem: str, to_int: bool) -> Any: # Decimal accepts 'nan'/'inf', which are not s; leave # them as text rather than changing how they render. return elem + # Bound the expansion from the exponent before performing it. Decimal + # construction from a string is unbounded -- the context's Emax/Emin + # constrain arithmetic results, not construction, so Decimal('1e999999999') + # is finite and the guard above does not stop it -- and the plain-notation + # length is linear in the exponent, so 11 characters of template text + # would otherwise materialize ~10**9 characters. + # + # The bound is the character cap this field already enforces, because an + # expansion past it cannot be carried here as text at all: the value is + # stored in a TaskRangeList element, whose str member is + # TaskParameterStringValueAsJob, and pydantic's union falls through to + # the numeric members instead, rendering the element as an integer, or as + # inf, or as 0.0. Such an element keeps its source text -- what this + # function already does for anything it cannot normalize, and how it + # rendered before normalization was introduced. + exponent = cast(int, value.as_tuple().exponent) # finite: never 'n'/'N'/'F' + # Length of the '[-].' text returned below. Stripping + # redundant zeros can only shorten it, so this is an upper bound. + expansion_len = ( + (1 if value.is_signed() else 0) + + max(value.adjusted() + 1, 1) # integer digits, at least one + + 1 # the decimal point + + max(-exponent, 1) # fraction digits, at least one + ) + if expansion_len > _MAX_TASK_PARAM_VALUE_LEN: + return elem # format(value, 'f') is exact and plain at every magnitude: with no # precision in the format spec it neither rounds to getcontext().prec # nor switches to exponent notation, so an embedding app that sets a diff --git a/test/openjd/model_v0/v2023_09/test_parameter_space.py b/test/openjd/model_v0/v2023_09/test_parameter_space.py index 222c8866..25ae76aa 100644 --- a/test/openjd/model_v0/v2023_09/test_parameter_space.py +++ b/test/openjd/model_v0/v2023_09/test_parameter_space.py @@ -513,6 +513,53 @@ def test_floatstring_elements_carry_their_value(self, element: str, expected: st # THEN the element renders as the number it denotes assert [str(v) for v in model.range] == [expected] + @pytest.mark.parametrize( + "element", + ( + # Decimal's context bounds arithmetic results, not construction from + # a string, so these are finite: 11 characters of template text + # whose plain-notation expansion is ~10**9 characters. + pytest.param("1e999999999", id="huge positive exponent"), + pytest.param("1e-999999999", id="huge negative exponent"), + # One character past the cap in each direction: both expand to 1025. + pytest.param("1E+1022", id="one past the cap, positive exponent"), + pytest.param("1E-1023", id="one past the cap, negative exponent"), + ), + ) + def test_element_too_long_to_expand_keeps_its_source_text(self, element: str) -> None: + # WHEN the instantiation target parses a whose plain-notation + # form would not fit TaskParameterStringValueAsJob's 1024-character cap + model = _parse_model( + model=RangeListTaskParameterDefinition, obj={"type": "FLOAT", "range": [element]} + ) + + # THEN it keeps its source text rather than being expanded. Expanding it + # would allocate the whole expansion, and the result could not be carried + # as text here anyway: it fails the str member of TaskRangeList's union, + # and pydantic then falls through to the numeric members, rendering the + # element as an integer, or as inf, or as 0.0. + assert [str(v) for v in model.range] == [element] + + @pytest.mark.parametrize( + "element,expected", + ( + pytest.param("1E+1021", "1" + "0" * 1021 + ".0", id="positive exponent"), + pytest.param("1E-1022", "0." + "0" * 1021 + "1", id="negative exponent"), + ), + ) + def test_element_at_the_cap_is_still_normalized(self, element: str, expected: str) -> None: + # GIVEN a whose plain-notation form is exactly 1024 + # characters — at the cap, not past it + assert len(expected) == 1024 + + # WHEN the instantiation target parses it + model = _parse_model( + model=RangeListTaskParameterDefinition, obj={"type": "FLOAT", "range": [element]} + ) + + # THEN the length bound has not cost it its normalization + assert [str(v) for v in model.range] == [expected] + def test_floatstring_normalization_ignores_the_decimal_context(self) -> None: # GIVEN an embedding application that has narrowed the process-wide # decimal context, and a with more significant digits