From de4916ce01193dfec7df5d7002c385e49e3ac5ba Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 17 Sep 2026 15:34:41 -0600 Subject: [PATCH 1/6] Materialize Any and dict targets like the untyped parse; write Any containers statically Reading: - `StructUtils.make(style, Any, x)` and the `Vector{Any}`, `Object{String,Any}` and `AbstractDict{String,Any}` targets now go through a styled `applyvalue(f, x, style)`, the typed counterpart of the untyped materializer: each value reaches its sink with a concrete type instead of a boxed `(value::Any, pos)` pair. A 137-key object into `Object{String,Any}` drops from 105us/1433 allocations to 9us/428 (untyped: 425), and a 3000-key nested object from 17ms to 0.17ms (the generic dict path filled `Object` through `setindex!`, a linear scan per key). Writing: - `Any`-valued dicts, vectors and pair vectors hand each of the types the untyped parse produces to the writer statically; anything else goes through the new `JSON.applyany(style, f, key, value)` hook, whose default lowers dynamically. A style that overrides it to throw has no dynamic call on its write path under `juliac --trim=safe`. - The output buffer grows by at least doubling, so a large output no longer needs a `sizeguess` method to avoid repeated copies. - Nested writes under `jsonlines=true` keep the caller's `style` and `bufsize`; a custom style was silently replaced by the default for every nested value. Co-Authored-By: Claude Fable 5.1 --- src/parse.jl | 94 +++++++++++++++++++++++++++++++++++++++++++-------- src/write.jl | 62 +++++++++++++++++++++++++++++++-- test/json.jl | 28 +++++++++++++++ test/parse.jl | 21 ++++++++++++ 4 files changed, 188 insertions(+), 17 deletions(-) diff --git a/src/parse.jl b/src/parse.jl index 3628db2..a661a3a 100644 --- a/src/parse.jl +++ b/src/parse.jl @@ -322,10 +322,10 @@ mutable struct ObjectClosure{T} root::Object{String,Any} obj::Object{String,Any} keys::Set{String} - null::T + ctx::T # the null value of an untyped parse, or the style of a typed one end -ObjectClosure(obj, null) = ObjectClosure(obj, obj, sizehint!(Set{String}(), 16), null) +ObjectClosure(obj, ctx) = ObjectClosure(obj, obj, sizehint!(Set{String}(), 16), ctx) @inline function insert_or_overwrite!(oc::ObjectClosure, key, val) # in! does both a hash lookup and also sets the key if not present @@ -339,7 +339,7 @@ ObjectClosure(obj, null) = ObjectClosure(obj, obj, sizehint!(Set{String}(), 16), oc.obj = Object{String,Any}(oc.obj, key, val) # fast append path end -(oc::ObjectClosure)(k, v) = applyvalue(val -> insert_or_overwrite!(oc, convert(String, k), val), v, oc.null) +(oc::ObjectClosure)(k, v) = applyvalue(val -> insert_or_overwrite!(oc, convert(String, k), val), v, oc.ctx) # generic apply `f` to LazyValue, using default types to materialize, depending on type function applyvalue(f, x::LazyValues, null) @@ -392,27 +392,93 @@ function applyvalue(f, x::LazyValues, null) end end -# we overload make! for Any for LazyValues because we can dispatch to more specific -# types base on the LazyValue type -function StructUtils.make(st::StructStyle, ::Type{Any}, x::LazyValues) +# The typed counterparts of the untyped closures above: `Any` slots under a typed parse and +# the container types the untyped parse produces. A value materializes exactly as the untyped +# parse would, except that the style's object type and null value apply. A target that is +# not the matching JSON kind falls through to the generic `StructUtils.make`. +""" + JSON.applyvalue(f, x::LazyValue, style::StructUtils.StructStyle) -> pos + +Materialize `x` as the untyped `JSON.parse` would, with `style`'s object type and null +value, call `f(value)`, and return the position past `x`. `value` reaches `f` with its +concrete type, so no `(value::Any, pos)` pair is boxed; `StructUtils.make(style, Any, x)` +is this with a `Ref` around `f`. +""" +function applyvalue(f, x::LazyValues, st::StructStyle) type = gettype(x) if type == JSONTypes.OBJECT - return StructUtils.make(st, objecttype(st), x) + obj, pos = StructUtils.make(st, objecttype(st), x) + f(obj) + return pos elseif type == JSONTypes.ARRAY - return StructUtils.make(st, Vector{Any}, x) + arr, pos = StructUtils.make(st, Vector{Any}, x) + f(arr) + return pos elseif type == JSONTypes.STRING - return StructUtils.lift(st, String, x) + buf = getbuf(x) + GC.@preserve buf begin + str, pos = parsestring(x) + f(convert(String, str)) + end + return pos elseif type == JSONTypes.NUMBER - return StructUtils.lift(st, Number, x) + num, pos = parsenumber(x) + if isint(num) + f(num.int) + elseif isfloat(num) + f(num.float) + elseif isbigint(num) + f(num.bigint) + else + f(num.bigfloat) + end + return pos elseif type == JSONTypes.NULL - return StructUtils.lift(st, Nothing, x) - elseif type == JSONTypes.TRUE || type == JSONTypes.FALSE - return StructUtils.lift(st, Bool, x) + f(nullvalue(st)) + return getpos(x) + 4 + elseif type == JSONTypes.TRUE + f(true) + return getpos(x) + 4 + elseif type == JSONTypes.FALSE + f(false) + return getpos(x) + 5 else - throw(ArgumentError("cannot parse $x")) + throw(ArgumentError("cannot parse json")) end end +function StructUtils.make(st::StructStyle, ::Type{Any}, x::LazyValues) + box = Ref{Any}() + pos = applyvalue(v -> (box[] = v), x, st) + return box[], pos +end + +function StructUtils.make(st::StructStyle, ::Type{Vector{Any}}, x::LazyValues) + gettype(x) == JSONTypes.ARRAY || + return @invoke StructUtils.make(st::StructStyle, Vector{Any}::Type, x::Any) + arr = sizehint!(Vector{Any}(), 16) + pos = applyarray((_, v) -> applyvalue(val -> push!(arr, val), v, st), x) + return arr, pos +end + +# `ObjectClosure` with the style as its context appends in O(n) with the untyped path's +# duplicate handling; the generic `makedict` went through `setindex!`, a linear scan per key. +function StructUtils.make(st::StructStyle, ::Type{Object{String,Any}}, x::LazyValues) + gettype(x) == JSONTypes.OBJECT || + return @invoke StructUtils.make(st::StructStyle, Object{String,Any}::Type, x::Any) + obj = Object{String,Any}() + pos = applyobject(ObjectClosure(obj, st), x) + return obj, pos +end + +function StructUtils.make(st::StructStyle, ::Type{T}, x::LazyValues) where {T<:AbstractDict{String,Any}} + gettype(x) == JSONTypes.OBJECT || + return @invoke StructUtils.make(st::StructStyle, T::Type, x::Any) + dict = StructUtils.initialize(st, T, x) + pos = applyobject((k, v) -> applyvalue(val -> StructUtils.addkeyval!(dict, convert(String, k), val), v, st), x) + return dict, pos +end + # catch PtrString via lift or make! so we can ensure it never "escapes" to user-level StructUtils.liftkey(st::JSONReadStyle, ::Type{T}, x::PtrString) where {T} = StructUtils.liftkey(st, T, convert(String, x)) diff --git a/src/write.jl b/src/write.jl index 8d81f72..351c557 100644 --- a/src/write.jl +++ b/src/write.jl @@ -140,6 +140,62 @@ StructUtils.lower(::JSONStyle, x::AbstractVector) = x StructUtils.arraylike(::JSONStyle, x::AbstractVector{<:Pair}) = false StructUtils.structlike(::JSONStyle, ::Type{<:NamedTuple}) = true +""" + JSON.applyany(style::JSONStyle, f, key, value) + +Called by the writer for a value whose container element type is `Any` when its runtime +type is none of the types the untyped `JSON.parse` produces (`nothing`, `Bool`, `Int64`, +`Float64`, `String`, `BigInt`, `BigFloat`, `Vector{Any}`, `JSON.Object{String,Any}`), nor +`Dict{String,Any}` or `missing`. The default lowers the value and hands it to the writer, +`f(key, StructUtils.lower(style, value))`, a dynamic call. A custom style whose `Any`-valued +containers only ever hold the types above can overload this to throw, so a program built +with `juliac --trim=safe` has no dynamic call on its write path. +""" +applyany(st::JSONStyle, f, key, @nospecialize(value)) = f(key, StructUtils.lower(st, value)) + +# The value types the untyped parse produces each reach the write closure as one concrete +# type, so writing parsed JSON back out is static dispatch all the way down and needs no +# `lower` call; anything else goes through `applyany`. `v` is not specialized on, so the +# call here is one resolved method rather than a dynamic dispatch on the element type. +@noinline function _applyvalue(st::JSONStyle, f, key, @nospecialize(v)) + v isa String && return f(key, v) + v isa Int64 && return f(key, v) + v isa Float64 && return f(key, v) + v === nothing && return f(key, nothing) + v isa Bool && return f(key, v) + v isa Vector{Any} && return f(key, v) + v isa Object{String,Any} && return f(key, v) + v isa Dict{String,Any} && return f(key, v) + v === missing && return f(key, nothing) + v isa BigInt && return f(key, v) + v isa BigFloat && return f(key, v) + return applyany(st, f, key, v) +end + +function StructUtils.applyeach(st::JSONStyle, f, x::AbstractDict{<:Any,Any}) + for (k, v) in x + ret = _applyvalue(st, f, StructUtils.lowerkey(st, k), v) + ret isa StructUtils.EarlyReturn && return ret + end + return StructUtils.defaultstate(st) +end + +function StructUtils.applyeach(st::JSONStyle, f, x::AbstractVector{Pair{K,Any}}) where {K} + for (k, v) in x + ret = _applyvalue(st, f, StructUtils.lowerkey(st, k), v) + ret isa StructUtils.EarlyReturn && return ret + end + return StructUtils.defaultstate(st) +end + +function StructUtils.applyeach(st::JSONStyle, f, x::AbstractVector{Any}) + for i in eachindex(x) + ret = @inbounds(isassigned(x, i)) ? _applyvalue(st, f, i, @inbounds(x[i])) : f(i, nothing) + ret isa StructUtils.EarlyReturn && return ret + end + return StructUtils.defaultstate(st) +end + # for pre-1.0 compat, which serialized Tuple object keys by default StructUtils.lowerkey(::JSONStyle, x::Tuple) = string(x) @@ -618,7 +674,7 @@ macro checkn(n, force_resize=false) end # Resize buffer if still needed if (pos + $n - 1) > length(buf) - resize!(buf, newlen(pos + $n)) + resize!(buf, max(newlen(pos + $n), 2 * length(buf))) end end end) @@ -697,7 +753,7 @@ function (f::WriteClosure{JS, arraylike, T, I})(key, val) where {JS, arraylike, track_ref && push!(f.ancestor_stack, val) # if jsonlines, we need to recursively set to false if f.opts.jsonlines - opts = WriteOptions(; omit_null=f.opts.omit_null, omit_empty=f.opts.omit_empty, allownan=f.opts.allownan, jsonlines=false, pretty=f.opts.pretty, ninf=f.opts.ninf, inf=f.opts.inf, nan=f.opts.nan, inline_limit=f.opts.inline_limit, float_style=f.opts.float_style, float_precision=f.opts.float_precision, sort_keys=f.opts.sort_keys) + opts = WriteOptions(; omit_null=f.opts.omit_null, omit_empty=f.opts.omit_empty, allownan=f.opts.allownan, jsonlines=false, pretty=f.opts.pretty, ninf=f.opts.ninf, inf=f.opts.inf, nan=f.opts.nan, inline_limit=f.opts.inline_limit, float_style=f.opts.float_style, float_precision=f.opts.float_precision, sort_keys=f.opts.sort_keys, bufsize=f.opts.bufsize, style=f.opts.style) else opts = f.opts end @@ -780,7 +836,7 @@ function json!(buf, pos, x, opts::WriteOptions, ancestor_stack::Union{Nothing, V if _sort_keys && !al && x isa AbstractDict sorted_keys = sort!(collect(keys(x)), by=k -> StructUtils.lowerkey(opts.style, k)) for k in sorted_keys - c(StructUtils.lowerkey(opts.style, k), StructUtils.lower(opts.style, x[k])) + _applyvalue(opts.style, c, StructUtils.lowerkey(opts.style, k), x[k]) end else StructUtils.applyeach(opts.style, c, x) diff --git a/test/json.jl b/test/json.jl index 6901762..552c522 100644 --- a/test/json.jl +++ b/test/json.jl @@ -42,6 +42,11 @@ end @enum JsonFruit Apple Orange +struct ClosedStyle <: JSON.JSONStyle end +struct FloatRationalStyle <: JSON.JSONStyle end +JSON.lower(::FloatRationalStyle, x::Rational) = float(x) +JSON.applyany(::ClosedStyle, f, k, v) = throw(ArgumentError("closed")) + @testset "JSON.json" begin @testset "Basics" begin @@ -781,4 +786,27 @@ end @test parsed_keys == sort(parsed_keys) end +@testset "jsonlines keeps the custom style for nested values" begin + @test JSON.json([Dict("a" => 1//4)]; style=FloatRationalStyle()) == "[{\"a\":0.25}]" + @test JSON.json([Dict("a" => 1//4)]; style=FloatRationalStyle(), jsonlines=true) == "{\"a\":0.25}\n" +end + +@testset "Any-valued containers write each parse type directly; others take applyany" begin + v = Any[1, 1.5, "s", true, nothing, missing, Any[1], JSON.Object{String,Any}("a" => 1), Dict{String,Any}("b" => 2), BigInt(1), BigFloat(1)] + @test JSON.json(v) == "[1,1.5,\"s\",true,null,null,[1],{\"a\":1},{\"b\":2},1,1.0]" + @test JSON.json(Dict{String,Any}("a" => Any[missing])) == "{\"a\":[null]}" + @test JSON.json(Pair{String,Any}["a" => 1, "b" => nothing]) == "{\"a\":1,\"b\":null}" + s = "{\"i\":1,\"a\":[1.5,\"x\",null,{\"b\":[true]}]}" + @test JSON.json(JSON.parse(s)) == s + # any other element type lowers through `applyany`, which a style can close off + @test JSON.json(Any[Int8(1), (a=1,)]) == "[1,{\"a\":1}]" + @test JSON.json(Dict{String,Any}("z" => Int8(1), "a" => (b=1,)); sort_keys=true) == "{\"a\":{\"b\":1},\"z\":1}" + @test JSON.json(Any[1, "x"]; style=ClosedStyle()) == "[1,\"x\"]" + @test_throws ArgumentError JSON.json(Any[Int8(1)]; style=ClosedStyle()) + @test_throws ArgumentError JSON.json(Dict{String,Any}("z" => Int8(1)); style=ClosedStyle(), sort_keys=true) + # outputs far past the initial size guess grow the buffer correctly + strs = fill("y"^1000, 5000) + @test JSON.json(strs) == "[" * join(("\"" * x * "\"" for x in strs), ",") * "]" +end + end # @testset "JSON.json" diff --git a/test/parse.jl b/test/parse.jl index 0feec0a..b907ffb 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -988,3 +988,24 @@ end @test !JSON.isvalidjson(input; duplicate_keys=:error) @test_throws ArgumentError JSON.parse("{}"; duplicate_keys=:keep_first) end + +@testset "Any and Object targets materialize like the untyped parse" begin + s = "{\"i\":1,\"f\":1.5,\"s\":\"x\",\"t\":true,\"n\":null,\"a\":[1,{\"b\":2}],\"big\":123456789012345678901234567890,\"bf\":1e400}" + x = JSON.parse(s, Any) + @test x == JSON.parse(s) + @test x isa JSON.Object{String,Any} + @test map(k -> typeof(x[k]), ["i", "f", "s", "t", "n", "a", "big", "bf"]) == + [Int64, Float64, String, Bool, Nothing, Vector{Any}, BigInt, BigFloat] + @test x["a"][2] isa JSON.Object{String,Any} + # the style's object type and null value apply below an `Any` slot + @test JSON.parse(s, Any; dicttype=Dict{String,Any}) == JSON.parse(s; dicttype=Dict{String,Any}) + @test JSON.parse(s, Dict{String,Any}; dicttype=Dict{String,Any})["a"][2] isa Dict{String,Any} + @test JSON.parse("[null]", Vector{Any}; null=missing)[1] === missing + # an Object target appends in order and keeps the duplicate-key policy + @test JSON.parse(s, JSON.Object{String,Any}) == JSON.parse(s) + @test JSON.parse("{\"a\":1,\"a\":2}", JSON.Object{String,Any}) == JSON.Object("a" => 2) + @test_throws JSON.DuplicateKeyError JSON.parse("{\"a\":1,\"a\":2}", JSON.Object{String,Any}; duplicate_keys=:error) + @test JSON.parse("{\"a\":1,\"a\":2}", Dict{String,Any}) == Dict("a" => 2) + @test_throws JSON.DuplicateKeyError JSON.parse("{\"a\":1,\"a\":2}", Dict{String,Any}; duplicate_keys=:error) + @test JSON.parse("[{\"a\":[{\"b\":null}]}]", Vector{JSON.Object{String,Any}}) == JSON.parse("[{\"a\":[{\"b\":null}]}]") +end From 4640f2af45a03d49ef4c22599dda8a67e7f435d0 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 17 Sep 2026 19:42:47 -0600 Subject: [PATCH 2/6] Test: write the closed-style integer as Int64 so the test holds on 32-bit Co-Authored-By: Claude Fable 5.1 --- test/json.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/json.jl b/test/json.jl index 552c522..57dce22 100644 --- a/test/json.jl +++ b/test/json.jl @@ -801,7 +801,8 @@ end # any other element type lowers through `applyany`, which a style can close off @test JSON.json(Any[Int8(1), (a=1,)]) == "[1,{\"a\":1}]" @test JSON.json(Dict{String,Any}("z" => Int8(1), "a" => (b=1,)); sort_keys=true) == "{\"a\":{\"b\":1},\"z\":1}" - @test JSON.json(Any[1, "x"]; style=ClosedStyle()) == "[1,\"x\"]" + # the parse produces `Int64` on every platform, so the literal is written as one (`Int32` on x86 would take `applyany`) + @test JSON.json(Any[Int64(1), "x"]; style=ClosedStyle()) == "[1,\"x\"]" @test_throws ArgumentError JSON.json(Any[Int8(1)]; style=ClosedStyle()) @test_throws ArgumentError JSON.json(Dict{String,Any}("z" => Int8(1)); style=ClosedStyle(), sort_keys=true) # outputs far past the initial size guess grow the buffer correctly From c61d257c2d1915dd68727137682b68d7a675ad1f Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Fri, 18 Sep 2026 11:01:11 -0600 Subject: [PATCH 3/6] Preserve conversion hooks and avoid fast-path performance regressions --- benchmarks/geojson_regressions.jl | 118 +++++++++++++++++++++++++++ src/parse.jl | 81 ++++++++++++++---- src/write.jl | 57 ++++++++----- test/json_trim_public_entrypoints.jl | 18 ++++ test/runtests.jl | 1 + test/style_fastpaths.jl | 101 +++++++++++++++++++++++ 6 files changed, 338 insertions(+), 38 deletions(-) create mode 100644 benchmarks/geojson_regressions.jl create mode 100644 test/style_fastpaths.jl diff --git a/benchmarks/geojson_regressions.jl b/benchmarks/geojson_regressions.jl new file mode 100644 index 0000000..efce816 --- /dev/null +++ b/benchmarks/geojson_regressions.jl @@ -0,0 +1,118 @@ +# Run each mode in a fresh process with the JSON/StructUtils versions under test: +# julia --startup-file=no --project= benchmarks/geojson_regressions.jl +# Modes: runtime (needs Chairmarks), cold, load, precompile. +# TSV output: runtime name/seconds/allocations/bytes; cold name/seconds/bytes/compile_seconds. +# Precompile measures rebuilding these packages with dependencies already cached. +# Repeat in alternating version order; do not run performance jobs concurrently. +mode = isempty(ARGS) ? "runtime" : only(ARGS) +if mode in ("load", "precompile") + using UUIDs + for (name, uuid) in (("StructUtils", "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42"), + ("JSON", "682c06a0-de6a-54ab-a142-c8b1cf79cde6")) + result = if mode == "load" + @timed Core.eval(Main, Expr(:using, Expr(:., Symbol(name)))) + else + @timed Base.compilecache(Base.PkgId(UUID(uuid), name)) + end + println(join((name, result.time, result.bytes, result.compile_time), '\t')) + flush(stdout) + end + exit() +end + +using JSON, Dates +struct SmallRecord + a::Int + b::String +end +struct RecursiveRecord + value::Int + children::Vector{RecursiveRecord} +end +struct ConversionStyle <: JSON.JSONStyle end +JSON.lower(::ConversionStyle, x::String) = uppercase(x) +JSON.lift(::ConversionStyle, ::Type{String}, x::String) = uppercase(x) + +if mode == "cold" + for (name, expression) in ( + ("write_small", :(JSON.json(SmallRecord(1, "x")))), + ("read_small", :(JSON.parse("{\"a\":1,\"b\":\"x\"}", SmallRecord))), + ("write_any", :(JSON.json(JSON.Object{String,Any}("a" => Any[1, "x", nothing])))), + ("read_any", :(JSON.parse("{\"a\":[1,\"x\",null]}", JSON.Object{String,Any}))), + ("write_tree", :(JSON.json(RecursiveRecord(1, [RecursiveRecord(2, RecursiveRecord[])]))))) + result = @timed Core.eval(Main, expression) + println(join((name, result.time, result.bytes, result.compile_time), '\t')) + flush(stdout) + end + for width in (10, 30, 60) + name = Symbol(:WideRecord, width) + @eval struct $name + $([:($(Symbol(:field, i))::Union{Nothing,Int,String,Float64,Bool}) for i in 1:width]...) + end + value = Core.eval(Main, Expr(:call, name, fill(1, width)...)) + result = @timed Core.eval(Main, :(JSON.json($value))) + println(join((name, result.time, result.bytes, result.compile_time), '\t')) + flush(stdout) + end + exit() +end +mode == "runtime" || error("unknown benchmark mode: $mode") +using Chairmarks, Statistics +function measure(name, f) + f() # separate compilation from steady-state measurements + result = median(@be f() seconds=0.3) + println(join((name, result.time, result.allocs, result.bytes), '\t')) + flush(stdout) +end +for n in (4, 137, 10000) + for (name, values) in (("Int32", Int32.(1:n)), ("Float32", Float32.(1:n)), + ("Float64", Float64.(1:n)), ("Date", fill(Date(2026, 1, 1), n)), + ("struct", fill(SmallRecord(1, "x"), n)), + ("Any", Any[isodd(i) ? i : "x" for i in 1:n]), + ("union", Union{Nothing,Int,String,Float64,Bool}[isodd(i) ? i : "x" for i in 1:n])) + measure("write_array_$(name)_$n", () -> JSON.json(values)) + dict = Dict(string(i) => value for (i, value) in enumerate(values)) + measure("write_dict_$(name)_$n", () -> JSON.json(dict)) + measure("write_unsorted_$(name)_$n", () -> JSON.json(dict; sort_keys=false)) + end + source = "{" * join(("\"k$i\":$i" for i in 1:n), ",") * "}" + for T in (Any, JSON.Object{String,Any}, Dict{String,Any}) + measure("read_object_$(T)_$n", () -> JSON.parse(source, T)) + end +end +for depth in (1, 16, 64, 256) + value = Any[1, "x", true, nothing] + for level in 1:depth + value = isodd(level) ? Any[value] : + level % 4 == 0 ? Dict{String,Any}("a" => value) : JSON.Object{String,Any}("a" => value) + end + source = JSON.json(value) + measure("write_nested_$depth", () -> JSON.json(value)) + measure("read_nested_$depth", () -> JSON.parse(source)) + io = IOBuffer() + measure("write_io_nested_$depth", () -> (truncate(io, 0); seekstart(io); JSON.json(io, value; bufsize=64))) +end +for depth in (1, 16, 64, 256) + tree = RecursiveRecord(1, RecursiveRecord[]) + for _ in 1:depth + tree = RecursiveRecord(1, [tree]) + end + measure("write_tree_$depth", () -> JSON.json(tree)) +end +for n in (1, 1000, 100000) + source = "[" * join(fill("{\"a\":1,\"b\":\"x\"}", n), ",") * "]" + measure("read_bulk_any_$n", () -> JSON.parse(source, Vector{Any})) + measure("read_bulk_struct_$n", () -> JSON.parse(source, Vector{SmallRecord})) +end +for width in (10, 30, 60) + name = Symbol(:WideRecord, width) + @eval struct $name + $([:($(Symbol(:field, i))::Union{Nothing,Int,String,Float64,Bool}) for i in 1:width]...) + end + value = Core.eval(Main, Expr(:call, name, fill(1, width)...)) + measure("write_wide_$width", () -> JSON.json(value)) +end +strings = fill("secret", 1000) +measure("write_custom_style", () -> JSON.json(strings; style=ConversionStyle())) +source = JSON.json(strings) +measure("read_custom_style", () -> JSON.parse(source, Vector{Any}; style=ConversionStyle())) diff --git a/src/parse.jl b/src/parse.jl index a661a3a..7b327c5 100644 --- a/src/parse.jl +++ b/src/parse.jl @@ -167,6 +167,9 @@ objecttype(::JSONReadStyle{OT}) where {OT} = OT nullvalue(::StructStyle) = nothing nullvalue(st::JSONReadStyle) = st.null +# Custom styles retain the generic make/lift hooks, including lazy-source lifts. +const _DefaultReadStyle = JSONReadStyle{O,N,StructUtils.DefaultStyle} where {O,N} + StructUtils.initialize(::JSONReadStyle, ::Type{Object}, source) = DEFAULT_OBJECT_TYPE() # this allows struct fields to specify tags under the json key specifically to override JSON behavior @@ -321,16 +324,35 @@ end mutable struct ObjectClosure{T} root::Object{String,Any} obj::Object{String,Any} - keys::Set{String} + keys::Union{Nothing,Set{String}} + count::Int ctx::T # the null value of an untyped parse, or the style of a typed one end -ObjectClosure(obj, ctx) = ObjectClosure(obj, obj, sizehint!(Set{String}(), 16), ctx) +ObjectClosure(obj, ctx) = ObjectClosure(obj, obj, nothing, 0, ctx) @inline function insert_or_overwrite!(oc::ObjectClosure, key, val) - # in! does both a hash lookup and also sets the key if not present - if _in!(key, oc.keys) - # slow path for dups; does a linear scan from our root object + # Scan at most four entries before allocating a hash table for larger objects. + # Once the object grows, use the set so distinct-key insertion stays O(n). + keys = oc.keys + if keys === nothing + node = find_node_by_key(oc.root, key) + if node !== nothing + setfield!(node, :value, val) + return + end + oc.count += 1 + if oc.count == 5 + keys = sizehint!(Set{String}(), 16) + node = _ch(oc.root) + while node isa Object{String,Any} + push!(keys, _k(node)::String) + node = _ch(node) + end + push!(keys, key) + oc.keys = keys + end + elseif _in!(key, keys) setindex!(oc.root, val, key) return end @@ -339,7 +361,9 @@ ObjectClosure(obj, ctx) = ObjectClosure(obj, obj, sizehint!(Set{String}(), 16), oc.obj = Object{String,Any}(oc.obj, key, val) # fast append path end -(oc::ObjectClosure)(k, v) = applyvalue(val -> insert_or_overwrite!(oc, convert(String, k), val), v, oc.ctx) +_objectkey(ctx, k) = convert(String, k) +_objectkey(st::StructStyle, k) = StructUtils.liftkey(st, String, k) +(oc::ObjectClosure)(k, v) = applyvalue(val -> insert_or_overwrite!(oc, _objectkey(oc.ctx, k), val), v, oc.ctx) # generic apply `f` to LazyValue, using default types to materialize, depending on type function applyvalue(f, x::LazyValues, null) @@ -400,11 +424,11 @@ end JSON.applyvalue(f, x::LazyValue, style::StructUtils.StructStyle) -> pos Materialize `x` as the untyped `JSON.parse` would, with `style`'s object type and null -value, call `f(value)`, and return the position past `x`. `value` reaches `f` with its -concrete type, so no `(value::Any, pos)` pair is boxed; `StructUtils.make(style, Any, x)` -is this with a `Ref` around `f`. +value, call `f(value)`, and return the position past `x`. With the default read style, +`value` reaches `f` with its concrete type, so no `(value::Any, pos)` pair is boxed. +Custom styles use their `StructUtils.make` and `StructUtils.lift` hooks instead. """ -function applyvalue(f, x::LazyValues, st::StructStyle) +function applyvalue(f, x::LazyValues, st::_DefaultReadStyle) type = gettype(x) if type == JSONTypes.OBJECT obj, pos = StructUtils.make(st, objecttype(st), x) @@ -448,34 +472,59 @@ function applyvalue(f, x::LazyValues, st::StructStyle) end function StructUtils.make(st::StructStyle, ::Type{Any}, x::LazyValues) + type = gettype(x) + if type == JSONTypes.OBJECT + return StructUtils.make(st, objecttype(st), x) + elseif type == JSONTypes.ARRAY + return StructUtils.make(st, Vector{Any}, x) + elseif type == JSONTypes.STRING + return StructUtils.lift(st, String, x) + elseif type == JSONTypes.NUMBER + return StructUtils.lift(st, Number, x) + elseif type == JSONTypes.NULL + return StructUtils.lift(st, Nothing, x) + elseif type == JSONTypes.TRUE || type == JSONTypes.FALSE + return StructUtils.lift(st, Bool, x) + else + throw(ArgumentError("cannot parse $x")) + end +end + +function applyvalue(f, x::LazyValues, st::StructStyle) + val, pos = StructUtils.make(st, Any, x) + f(val) + return pos +end + +function StructUtils.make(st::_DefaultReadStyle, ::Type{Any}, x::LazyValues) box = Ref{Any}() pos = applyvalue(v -> (box[] = v), x, st) return box[], pos end -function StructUtils.make(st::StructStyle, ::Type{Vector{Any}}, x::LazyValues) +function StructUtils.make(st::_DefaultReadStyle, ::Type{Vector{Any}}, x::LazyValues) gettype(x) == JSONTypes.ARRAY || return @invoke StructUtils.make(st::StructStyle, Vector{Any}::Type, x::Any) - arr = sizehint!(Vector{Any}(), 16) + arr = sizehint!(StructUtils.initialize(st, Vector{Any}, x), 16) pos = applyarray((_, v) -> applyvalue(val -> push!(arr, val), v, st), x) return arr, pos end # `ObjectClosure` with the style as its context appends in O(n) with the untyped path's # duplicate handling; the generic `makedict` went through `setindex!`, a linear scan per key. -function StructUtils.make(st::StructStyle, ::Type{Object{String,Any}}, x::LazyValues) +function StructUtils.make(st::_DefaultReadStyle, ::Type{Object{String,Any}}, x::LazyValues) gettype(x) == JSONTypes.OBJECT || return @invoke StructUtils.make(st::StructStyle, Object{String,Any}::Type, x::Any) - obj = Object{String,Any}() + obj = StructUtils.initialize(st, Object{String,Any}, x) pos = applyobject(ObjectClosure(obj, st), x) return obj, pos end -function StructUtils.make(st::StructStyle, ::Type{T}, x::LazyValues) where {T<:AbstractDict{String,Any}} +function StructUtils.make(st::_DefaultReadStyle, ::Type{T}, x::LazyValues) where {T<:AbstractDict{String,Any}} gettype(x) == JSONTypes.OBJECT || return @invoke StructUtils.make(st::StructStyle, T::Type, x::Any) dict = StructUtils.initialize(st, T, x) - pos = applyobject((k, v) -> applyvalue(val -> StructUtils.addkeyval!(dict, convert(String, k), val), v, st), x) + pos = applyobject((k, v) -> applyvalue(val -> StructUtils.addkeyval!(dict, StructUtils.liftkey(st, String, k), val), v, st), x) return dict, pos end diff --git a/src/write.jl b/src/write.jl index 351c557..a1dc5b3 100644 --- a/src/write.jl +++ b/src/write.jl @@ -149,29 +149,17 @@ type is none of the types the untyped `JSON.parse` produces (`nothing`, `Bool`, `Dict{String,Any}` or `missing`. The default lowers the value and hands it to the writer, `f(key, StructUtils.lower(style, value))`, a dynamic call. A custom style whose `Any`-valued containers only ever hold the types above can overload this to throw, so a program built -with `juliac --trim=safe` has no dynamic call on its write path. +with `juliac --trim=safe` can resolve the write path statically. Its `lower` methods +must also return statically known types. Keep the fallback unspecialized, for example: + +```julia +struct ClosedJSONStyle <: JSON.JSONStyle end +JSON.applyany(::ClosedJSONStyle, f, key, @nospecialize(value)) = + throw(ArgumentError("unsupported JSON value")) +``` """ applyany(st::JSONStyle, f, key, @nospecialize(value)) = f(key, StructUtils.lower(st, value)) -# The value types the untyped parse produces each reach the write closure as one concrete -# type, so writing parsed JSON back out is static dispatch all the way down and needs no -# `lower` call; anything else goes through `applyany`. `v` is not specialized on, so the -# call here is one resolved method rather than a dynamic dispatch on the element type. -@noinline function _applyvalue(st::JSONStyle, f, key, @nospecialize(v)) - v isa String && return f(key, v) - v isa Int64 && return f(key, v) - v isa Float64 && return f(key, v) - v === nothing && return f(key, nothing) - v isa Bool && return f(key, v) - v isa Vector{Any} && return f(key, v) - v isa Object{String,Any} && return f(key, v) - v isa Dict{String,Any} && return f(key, v) - v === missing && return f(key, nothing) - v isa BigInt && return f(key, v) - v isa BigFloat && return f(key, v) - return applyany(st, f, key, v) -end - function StructUtils.applyeach(st::JSONStyle, f, x::AbstractDict{<:Any,Any}) for (k, v) in x ret = _applyvalue(st, f, StructUtils.lowerkey(st, k), v) @@ -190,7 +178,7 @@ end function StructUtils.applyeach(st::JSONStyle, f, x::AbstractVector{Any}) for i in eachindex(x) - ret = @inbounds(isassigned(x, i)) ? _applyvalue(st, f, i, @inbounds(x[i])) : f(i, nothing) + ret = @inbounds(isassigned(x, i)) ? _applyvalue(st, f, i, @inbounds(x[i])) : f(i, StructUtils.lower(st, nothing)) ret isa StructUtils.EarlyReturn && return ret end return StructUtils.defaultstate(st) @@ -692,6 +680,27 @@ struct WriteClosure{JS, arraylike, T, I} # T is the type of the parent object/ar bufsize::Int end +# Keep each parsed value concrete at the write closure, while preserving custom +# lowering. Dict parents need a separate method: otherwise inference widens the +# recursive Object -> array -> Dict -> array cycle and safe trimming cannot +# resolve the Vector{Any} write. Generate both bodies here so they stay identical. +for F in (Any, WriteClosure{JS,A,Dict{String,Any},I} where {JS,A,I}) + @eval @noinline function _applyvalue(st::JSONStyle, f::$F, key, @nospecialize(v)) + v isa String && return f(key, StructUtils.lower(st, v)) + v isa Int64 && return f(key, StructUtils.lower(st, v)) + v isa Float64 && return f(key, StructUtils.lower(st, v)) + v === nothing && return f(key, StructUtils.lower(st, v)) + v isa Bool && return f(key, StructUtils.lower(st, v)) + v isa Vector{Any} && return f(key, StructUtils.lower(st, v)) + v isa Object{String,Any} && return f(key, StructUtils.lower(st, v)) + v isa Dict{String,Any} && return f(key, StructUtils.lower(st, v)) + v === missing && return f(key, StructUtils.lower(st, v)) + v isa BigInt && return f(key, StructUtils.lower(st, v)) + v isa BigFloat && return f(key, StructUtils.lower(st, v)) + return applyany(st, f, key, v) + end +end + function indent(buf, pos, ind, depth, io, bufsize) if ind > 0 n = ind * depth + 1 @@ -836,7 +845,11 @@ function json!(buf, pos, x, opts::WriteOptions, ancestor_stack::Union{Nothing, V if _sort_keys && !al && x isa AbstractDict sorted_keys = sort!(collect(keys(x)), by=k -> StructUtils.lowerkey(opts.style, k)) for k in sorted_keys - _applyvalue(opts.style, c, StructUtils.lowerkey(opts.style, k), x[k]) + if valtype(x) === Any && opts.style isa JSONStyle + _applyvalue(opts.style, c, StructUtils.lowerkey(opts.style, k), x[k]) + else + c(StructUtils.lowerkey(opts.style, k), StructUtils.lower(opts.style, x[k])) + end end else StructUtils.applyeach(opts.style, c, x) diff --git a/test/json_trim_public_entrypoints.jl b/test/json_trim_public_entrypoints.jl index eb5bd0f..6fb870f 100644 --- a/test/json_trim_public_entrypoints.jl +++ b/test/json_trim_public_entrypoints.jl @@ -37,6 +37,23 @@ struct TrimTemporal tick::Dates.Time end +# Close the extension point for the types produced by untyped parsing. +struct TrimJSONStyle <: JSON.JSONStyle end +JSON.applyany(::TrimJSONStyle, f, key, @nospecialize(value)) = + throw(ArgumentError("unsupported JSON value")) + +function exercise_nested_any()::Nothing + text = "{\"a\":[1,\"x\",null,{\"b\":[true,1.5]}]}" + object = JSON.parse(text, JSON.Object{String,Any}) + checked(JSON.json(object; style=TrimJSONStyle()) == text, "nested Object write failed") + dict = JSON.parse(text, Dict{String,Any}) + checked(JSON.json(dict; style=TrimJSONStyle()) == text, "nested Dict write failed") + io = IOBuffer() + JSON.json(io, object; style=TrimJSONStyle(), bufsize=16) + checked(String(take!(io)) == text, "nested IO write failed") + return nothing +end + function checked(cond::Bool, msg::String)::Nothing cond || error(msg) return nothing @@ -144,6 +161,7 @@ function run_json_trim_public_entrypoints()::Nothing exercise_lazy_entrypoints() exercise_parse_entrypoints() exercise_write_entrypoints() + exercise_nested_any() return nothing end diff --git a/test/runtests.jl b/test/runtests.jl index f70ddac..ec1f848 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,6 +6,7 @@ include(joinpath(dirname(pathof(JSON)), "../test/parse.jl")) include(joinpath(dirname(pathof(JSON)), "../test/escaped_keys.jl")) include(joinpath(dirname(pathof(JSON)), "../test/inbound_tags.jl")) include(joinpath(dirname(pathof(JSON)), "../test/json.jl")) +include(joinpath(dirname(pathof(JSON)), "../test/style_fastpaths.jl")) # Arrow.jl is broken on 32 bit systems for now :( if Sys.WORD_SIZE == 64 include(joinpath(dirname(pathof(JSON)), "../test/arrow.jl")) diff --git a/test/style_fastpaths.jl b/test/style_fastpaths.jl new file mode 100644 index 0000000..3520054 --- /dev/null +++ b/test/style_fastpaths.jl @@ -0,0 +1,101 @@ +module StyleFastpathTests +using JSON, StructUtils, Test + +struct CustomStyle <: JSON.JSONStyle end +JSON.lower(::CustomStyle, x::String) = uppercase(x) +JSON.lower(::CustomStyle, ::Missing) = JSON.Omit() +JSON.lower(::CustomStyle, ::Nothing) = "NULL" +JSON.lift(::CustomStyle, ::Type{String}, x::String) = uppercase(x) +JSON.lift(::CustomStyle, ::Type{Number}, x::Number) = 2x +StructUtils.liftkey(::JSON.JSONReadStyle{O,N,CustomStyle}, ::Type{String}, x::String) where {O,N} = uppercase(x) + +struct BareStyle <: StructUtils.StructStyle end +StructUtils.lower(::BareStyle, x::String) = uppercase(x) + +struct ValueStyle <: JSON.JSONStyle end +for T in (Nothing, Missing, Bool, Int64, Float64, BigInt, BigFloat) + @eval JSON.lower(::ValueStyle, x::$T) = string(typeof(x)) +end + +struct ContainerStyle <: JSON.JSONStyle end +JSON.lower(::ContainerStyle, x::Vector{Any}) = (length=length(x),) +JSON.lower(::ContainerStyle, x::Dict{String,Any}) = (length=length(x),) +JSON.lower(::ContainerStyle, x::JSON.Object{String,Any}) = (length=length(x),) +StructUtils.initialize(::JSON.JSONReadStyle{O,N,ContainerStyle}, ::Type{Vector{Any}}, source) where {O,N} = Any["prefix"] + +@testset "Fast paths preserve custom styles" begin + for sort_keys in (true, false, nothing) + @test JSON.json(Dict{String,Any}("k" => "v"); style=BareStyle(), sort_keys, omit_null=false, omit_empty=false) == "{\"k\":\"V\"}" + end + for value in (nothing, missing, true, Int64(1), 1.5, big(1), big"1.5") + lowered = JSON.lower(ValueStyle(), value) + @test JSON.json(Any[value]; style=ValueStyle()) == JSON.json([lowered]) + @test JSON.json(Dict{String,Any}("k" => value); style=ValueStyle()) == JSON.json(Dict("k" => lowered)) + end + for x in (["secret"], Any["secret"]) + @test JSON.json(x; style=CustomStyle()) == "[\"SECRET\"]" + end + for x in (Dict("k" => "secret"), Dict{String,Any}("k" => "secret"), + JSON.Object{String,Any}("k" => "secret"), Pair{String,Any}["k" => "secret"]) + for sort_keys in (true, false, nothing) + @test JSON.json(x; style=CustomStyle(), sort_keys) == "{\"k\":\"SECRET\"}" + end + end + @test JSON.json(Any[missing, nothing]; style=CustomStyle()) == "[\"NULL\"]" + @test JSON.json(Vector{Any}(undef, 1); style=CustomStyle()) == "[\"NULL\"]" + for x in (Any[1, 2], Dict{String,Any}("a" => 1), JSON.Object{String,Any}("a" => 1)) + @test JSON.json((value=x,); style=ContainerStyle()) == "{\"value\":{\"length\":$(length(x))}}" + @test JSON.json(Pair{String,Any}["value" => x]; style=ContainerStyle()) == "{\"value\":{\"length\":$(length(x))}}" + @test JSON.json(Dict{String,Any}("value" => x); style=CustomStyle()) isa String + end + for T in (Dict{String,Any}, JSON.Object{String,Any}) + x = JSON.parse("{\"key\":\"secret\",\"n\":2}", T; style=CustomStyle()) + @test x["KEY"] == "SECRET" + @test x["N"] == 4 + y = T() + JSON.parse!("{\"key\":\"secret\",\"n\":2}", y; style=CustomStyle()) + @test x == y + end + @test JSON.parse("[\"secret\",2]", Vector{Any}; style=CustomStyle()) == Any["SECRET", 4] + @test JSON.parse("[1]", Vector{Any}; style=ContainerStyle()) == Any["prefix", 1] +end + +@testset "Nested Any containers" begin + # Exercise alternating Object/Dict/array cycles in the call graph, without + # requiring a different Julia type for every nesting level. + for depth in (1, 16, 64, 256) + value = Any[1, "x", true, nothing] + text = "[1,\"x\",true,null]" + for i in 1:depth + if isodd(i) + value = Any[value] + text = "[" * text * "]" + else + value = i % 4 == 0 ? Dict{String,Any}("a" => value) : JSON.Object{String,Any}("a" => value) + text = "{\"a\":" * text * "}" + end + end + @test JSON.json(value) == text + @test JSON.json(JSON.parse(text)) == text + end + cycle = Any[] + push!(cycle, cycle) + @test JSON.json(cycle) == "[null]" + dict = Dict{String,Any}() + dict["self"] = dict + @test JSON.json(dict) == "{\"self\":null}" +end +@testset "duplicate keys across small-object threshold" begin + for n in (0, 1, 4, 5, 6, 16, 137), T in (Any, JSON.Object{String,Any}, Dict{String,Any}) + members = ["\"k$i\":$i" for i in 1:n] + push!(members, "\"k1\":-1") + result = JSON.parse("{" * join(members, ",") * "}", T) + @test length(result) == max(n, 1) + @test result["k1"] == -1 + for i in 2:n + @test result["k$i"] == i + end + end +end + +end From 3f0d1e3e35ad13d8b677f6e7e8141cc3c873bf4d Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Fri, 18 Sep 2026 11:10:56 -0600 Subject: [PATCH 4/6] Exercise custom materialization fallbacks and support Julia 1.10 benchmarks --- benchmarks/geojson_regressions.jl | 6 +++--- test/style_fastpaths.jl | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/benchmarks/geojson_regressions.jl b/benchmarks/geojson_regressions.jl index efce816..f212952 100644 --- a/benchmarks/geojson_regressions.jl +++ b/benchmarks/geojson_regressions.jl @@ -14,7 +14,7 @@ if mode in ("load", "precompile") else @timed Base.compilecache(Base.PkgId(UUID(uuid), name)) end - println(join((name, result.time, result.bytes, result.compile_time), '\t')) + println(join((name, result.time, result.bytes, get(result, :compile_time, NaN)), '\t')) flush(stdout) end exit() @@ -41,7 +41,7 @@ if mode == "cold" ("read_any", :(JSON.parse("{\"a\":[1,\"x\",null]}", JSON.Object{String,Any}))), ("write_tree", :(JSON.json(RecursiveRecord(1, [RecursiveRecord(2, RecursiveRecord[])]))))) result = @timed Core.eval(Main, expression) - println(join((name, result.time, result.bytes, result.compile_time), '\t')) + println(join((name, result.time, result.bytes, get(result, :compile_time, NaN)), '\t')) flush(stdout) end for width in (10, 30, 60) @@ -51,7 +51,7 @@ if mode == "cold" end value = Core.eval(Main, Expr(:call, name, fill(1, width)...)) result = @timed Core.eval(Main, :(JSON.json($value))) - println(join((name, result.time, result.bytes, result.compile_time), '\t')) + println(join((name, result.time, result.bytes, get(result, :compile_time, NaN)), '\t')) flush(stdout) end exit() diff --git a/test/style_fastpaths.jl b/test/style_fastpaths.jl index 3520054..c1a2bba 100644 --- a/test/style_fastpaths.jl +++ b/test/style_fastpaths.jl @@ -98,4 +98,31 @@ end end end +JSON.lift(::CustomStyle, ::Type{Bool}, x::Bool) = !x +JSON.lift(::CustomStyle, ::Type{Nothing}, ::Nothing) = "nullvalue" + +@testset "Custom materialization and mismatched container shapes" begin + source = "{\"arr\":[true,false,null,1,\"x\",{}]}" + value = JSON.parse(source, JSON.Object{String,Any}; style=CustomStyle()) + @test value["ARR"] == Any[false, true, "nullvalue", 2, "X", JSON.Object{String,Any}()] + @test only(JSON.parse("[null]", Vector{Any}; style=CustomStyle())) == "nullvalue" + @test only(JSON.parse("[false]", Vector{Any})) === false + + # The callback API must use the same custom hooks as typed parsing. + style = JSON.JSONReadStyle{JSON.Object{String,Any}}(nothing, CustomStyle()) + values = Any[] + pos = JSON.applyvalue(x -> push!(values, x), JSON.lazy(source), style) + @test only(values) == value + @test pos == ncodeunits(source) + 1 + + # Fast container methods must preserve the generic shape fallback. + @test JSON.parse("{}", Vector{Any}) == Any[] + for T in (JSON.Object{String,Any}, Dict{String,Any}) + @test isempty(JSON.parse("[]", T)) + end + for T in (Vector{Any}, JSON.Object{String,Any}, Dict{String,Any}), source in ("1", "null") + @test_throws ArgumentError JSON.parse(source, T) + end +end + end From fc4a655eab9e8c95a27a074d3d052cb108f854ff Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Fri, 18 Sep 2026 12:05:26 -0600 Subject: [PATCH 5/6] Defer numeric key strings until object serialization --- benchmarks/geojson_regressions.jl | 5 ++++ docs/src/writing.md | 4 +++- src/write.jl | 24 ++++++++++++------- test/json_trim_public_entrypoints.jl | 1 + test/style_fastpaths.jl | 36 ++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/benchmarks/geojson_regressions.jl b/benchmarks/geojson_regressions.jl index f212952..ea062ea 100644 --- a/benchmarks/geojson_regressions.jl +++ b/benchmarks/geojson_regressions.jl @@ -65,6 +65,11 @@ function measure(name, f) flush(stdout) end for n in (4, 137, 10000) + # Numeric keys must retain string ordering without allocating for indices. + for T in (Int, Float64), sort_keys in (true, false) + dict = Dict(T(i) => i for i in 1:n) + measure("write_numeric_keys_$(T)_$(sort_keys)_$n", () -> JSON.json(dict; sort_keys)) + end for (name, values) in (("Int32", Int32.(1:n)), ("Float32", Float32.(1:n)), ("Float64", Float64.(1:n)), ("Date", fill(Date(2026, 1, 1), n)), ("struct", fill(SmallRecord(1, "x"), n)), diff --git a/docs/src/writing.md b/docs/src/writing.md index 9678a03..3de4290 100644 --- a/docs/src/writing.md +++ b/docs/src/writing.md @@ -371,7 +371,9 @@ JSON.json(node; omit_null=false) ## Custom Dictionary Key Serialization -For dictionaries with non-string keys, [`JSON.json`](@ref) has a few default `lowerkey` definitions to convert keys to strings: +Keys and array indices pass through `StructUtils.lowerkey`. JSON accepts a string or a real number from this hook. Its default numeric method returns the number unchanged: object keys are converted to strings when written or sorted, while unused array indices need no string allocation. Custom `lowerkey` methods still run for every key and index. + +For dictionaries with non-string keys, [`JSON.json`](@ref) writes quoted keys: ```julia # Integer keys diff --git a/src/write.jl b/src/write.jl index a1dc5b3..d31660c 100644 --- a/src/write.jl +++ b/src/write.jl @@ -178,7 +178,8 @@ end function StructUtils.applyeach(st::JSONStyle, f, x::AbstractVector{Any}) for i in eachindex(x) - ret = @inbounds(isassigned(x, i)) ? _applyvalue(st, f, i, @inbounds(x[i])) : f(i, StructUtils.lower(st, nothing)) + key = StructUtils.lowerkey(st, i) + ret = @inbounds(isassigned(x, i)) ? _applyvalue(st, f, key, @inbounds(x[i])) : f(key, StructUtils.lower(st, nothing)) ret isa StructUtils.EarlyReturn && return ret end return StructUtils.defaultstate(st) @@ -395,7 +396,10 @@ end StructUtils.lowerkey(::JSONStyle, s::AbstractString) = s StructUtils.lowerkey(::JSONStyle, sym::Symbol) = String(sym) -StructUtils.lowerkey(::JSONStyle, s::Union{StringLike, Real}) = string(s) +# Array indices also pass through this hook. Convert numeric object keys only +# when writing or sorting them, so discarded array indices do not allocate. +StructUtils.lowerkey(::JSONStyle, s::Real) = s +StructUtils.lowerkey(::JSONStyle, s::StringLike) = string(s) StructUtils.lowerkey(::JSONStyle, x) = throw(ArgumentError("No key representation for $(typeof(x)). Define StructUtils.lowerkey(::JSON.JSONStyle, ::$(typeof(x)))")) """ JSON.json(x) -> String @@ -498,7 +502,10 @@ Circular references are tracked automatically and cycles are broken by writing ` For pre-formatted JSON data as a String, use `JSONText(json)` to write the string out as-is. -For `AbstractDict` objects with non-string keys, `StructUtils.lowerkey` will be called before serializing. This allows aggregate +Keys and array indices pass through `StructUtils.lowerkey` before serializing. +JSON accepts strings or real numbers from this hook. Numeric object keys are converted +to strings when written or sorted; array indices are discarded without conversion. +For `AbstractDict` objects with non-string keys, this allows aggregate or other types of dict keys to be converted to an appropriate string representation. See `StructUtils.liftkey` for the reverse operation, which is called when parsing JSON data back into a dict type. @@ -714,7 +721,9 @@ function indent(buf, pos, ind, depth, io, bufsize) return pos end -checkkey(s) = s isa AbstractString || throw(ArgumentError("Value returned from `StructUtils.lowerkey` must be a string: $(typeof(s))")) +checkkey(s::AbstractString) = s +checkkey(s::Real) = string(s) +checkkey(s) = throw(ArgumentError("Value returned from `StructUtils.lowerkey` must be a string or real number: $(typeof(s))")) _sort_keys_by_default(x) = x isa Dict @@ -741,10 +750,7 @@ function (f::WriteClosure{JS, arraylike, T, I})(key, val) where {JS, arraylike, pos = indent(buf, pos, ind, f.depth, io, bufsize) # if not an array, we need to write the key + ':' if !arraylike - # skey = StructUtils.lowerkey(f.opts, key) - # check if the key is a string - checkkey(key) - pos = _string(buf, pos, key, io, bufsize) + pos = _string(buf, pos, checkkey(key), io, bufsize) @checkn 1 buf[pos] = UInt8(':') pos += 1 @@ -843,7 +849,7 @@ function json!(buf, pos, x, opts::WriteOptions, ancestor_stack::Union{Nothing, V c = WriteClosure{typeof(opts), al, typeof(x), typeof(io)}(buf, Base.unsafe_convert(Ptr{Int}, ref), Base.unsafe_convert(Ptr{Bool}, wroteanyref), local_ind, depth + 1, opts, ancestor_stack, io, bufsize) _sort_keys = opts.sort_keys === true || (opts.sort_keys === nothing && !al && _sort_keys_by_default(x)) if _sort_keys && !al && x isa AbstractDict - sorted_keys = sort!(collect(keys(x)), by=k -> StructUtils.lowerkey(opts.style, k)) + sorted_keys = sort!(collect(keys(x)), by=k -> checkkey(StructUtils.lowerkey(opts.style, k))) for k in sorted_keys if valtype(x) === Any && opts.style isa JSONStyle _applyvalue(opts.style, c, StructUtils.lowerkey(opts.style, k), x[k]) diff --git a/test/json_trim_public_entrypoints.jl b/test/json_trim_public_entrypoints.jl index 6fb870f..f138f87 100644 --- a/test/json_trim_public_entrypoints.jl +++ b/test/json_trim_public_entrypoints.jl @@ -43,6 +43,7 @@ JSON.applyany(::TrimJSONStyle, f, key, @nospecialize(value)) = throw(ArgumentError("unsupported JSON value")) function exercise_nested_any()::Nothing + checked(JSON.json(Dict(2 => 20, 10 => 100)) == "{\"10\":100,\"2\":20}", "numeric key sorting failed") text = "{\"a\":[1,\"x\",null,{\"b\":[true,1.5]}]}" object = JSON.parse(text, JSON.Object{String,Any}) checked(JSON.json(object; style=TrimJSONStyle()) == text, "nested Object write failed") diff --git a/test/style_fastpaths.jl b/test/style_fastpaths.jl index c1a2bba..6fe740a 100644 --- a/test/style_fastpaths.jl +++ b/test/style_fastpaths.jl @@ -12,6 +12,42 @@ StructUtils.liftkey(::JSON.JSONReadStyle{O,N,CustomStyle}, ::Type{String}, x::St struct BareStyle <: StructUtils.StructStyle end StructUtils.lower(::BareStyle, x::String) = uppercase(x) +struct KeyStyle <: JSON.JSONStyle + seen::Vector{Int} +end +function StructUtils.lowerkey(st::KeyStyle, x::Int) + push!(st.seen, x) + return "key$x" +end + +@testset "Every index is lowered; numeric object keys stay quoted and sorted" begin + for x in ([10, 20], Any[10, 20], (10, 20), (v for v in [10, 20]), + Union{Int,String}[10, "a"], Vector{Any}(undef, 2), Core.svec(10, 20)) + st = KeyStyle(Int[]) + @test JSON.json(x; style=st) == JSON.json(x) + @test st.seen == [1, 2] + end + st = KeyStyle(Int[]) + @test JSON.json(Set([10]); style=st) == "[10]" + @test st.seen == [1] + for x in (Dict(2 => 20, 10 => 100), Dict{Int,Any}(2 => 20, 10 => 100)) + @test JSON.json(x) == "{\"10\":100,\"2\":20}" + @test JSON.json(x; style=KeyStyle(Int[])) == "{\"key10\":100,\"key2\":20}" + for sort_keys in (true, false) + @test JSON.parse(JSON.json(x; sort_keys), typeof(x)) == x + end + end + for k in (true, Int32(2), big(2), 1.5, big"1.5", 1//2, Inf, NaN) + @test JSON.json(Dict(k => 1)) == "{" * JSON.json(string(k)) * ":1}" + @test JSON.json([k => 1]) == "{" * JSON.json(string(k)) * ":1}" + end + # Mixed numeric/string keys must still sort by their serialized spelling. + @test JSON.json(Dict{Any,Any}(2 => 1, "10" => 2, :a => 3)) == "{\"10\":2,\"2\":1,\"a\":3}" + substring = SubString("_key_", 2, 4) + @test JSON.checkkey(substring) === substring + @test JSON.json(Dict(substring => 1)) == "{\"key\":1}" +end + struct ValueStyle <: JSON.JSONStyle end for T in (Nothing, Missing, Bool, Int64, Float64, BigInt, BigFloat) @eval JSON.lower(::ValueStyle, x::$T) = string(typeof(x)) From 1d1822b0a163ca741ad0ec7864b615f700310383 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Fri, 18 Sep 2026 14:41:16 -0600 Subject: [PATCH 6/6] Remove redundant lowerkey documentation sentence --- docs/src/writing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/writing.md b/docs/src/writing.md index 3de4290..079c1d8 100644 --- a/docs/src/writing.md +++ b/docs/src/writing.md @@ -371,7 +371,7 @@ JSON.json(node; omit_null=false) ## Custom Dictionary Key Serialization -Keys and array indices pass through `StructUtils.lowerkey`. JSON accepts a string or a real number from this hook. Its default numeric method returns the number unchanged: object keys are converted to strings when written or sorted, while unused array indices need no string allocation. Custom `lowerkey` methods still run for every key and index. +Keys and array indices pass through `StructUtils.lowerkey`. JSON accepts a string or a real number from this hook. Its default numeric method returns the number unchanged: object keys are converted to strings when written or sorted, while unused array indices need no string allocation. For dictionaries with non-string keys, [`JSON.json`](@ref) writes quoted keys: