Materialize Any and dict targets like the untyped parse; write Any containers statically

benchmarks/geojson_regressions.jl

@@ -0,0 +1,123 @@
+# Run each mode in a fresh process with the JSON/StructUtils versions under test:
+# julia --startup-file=no --project=<env> benchmarks/geojson_regressions.jl <mode>
+# Modes: runtime (needs Chairmarks), cold, load, precompile.
+# TSV output: runtime name/seconds/allocations/bytes; cold name/seconds/bytes/compile_seconds.
+# Precompile measures rebuilding these packages with dependencies already cached.
+# Repeat in alternating version order; do not run performance jobs concurrently.
+mode = isempty(ARGS) ? "runtime" : only(ARGS)
+if mode in ("load", "precompile")
+    using UUIDs
+    for (name, uuid) in (("StructUtils", "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42"),
+                         ("JSON", "682c06a0-de6a-54ab-a142-c8b1cf79cde6"))
+        result = if mode == "load"
+            @timed Core.eval(Main, Expr(:using, Expr(:., Symbol(name))))
+        else
+            @timed Base.compilecache(Base.PkgId(UUID(uuid), name))
+        end
+        println(join((name, result.time, result.bytes, get(result, :compile_time, NaN)), '\t'))
+        flush(stdout)
+    end
+    exit()
+end
+
+using JSON, Dates
+struct SmallRecord
+    a::Int
+    b::String
+end
+struct RecursiveRecord
+    value::Int
+    children::Vector{RecursiveRecord}
+end
+struct ConversionStyle <: JSON.JSONStyle end
+JSON.lower(::ConversionStyle, x::String) = uppercase(x)
+JSON.lift(::ConversionStyle, ::Type{String}, x::String) = uppercase(x)
+
+if mode == "cold"
+    for (name, expression) in (
+        ("write_small", :(JSON.json(SmallRecord(1, "x")))),
+        ("read_small", :(JSON.parse("{\"a\":1,\"b\":\"x\"}", SmallRecord))),
+        ("write_any", :(JSON.json(JSON.Object{String,Any}("a" => Any[1, "x", nothing])))),
+        ("read_any", :(JSON.parse("{\"a\":[1,\"x\",null]}", JSON.Object{String,Any}))),
+        ("write_tree", :(JSON.json(RecursiveRecord(1, [RecursiveRecord(2, RecursiveRecord[])])))))
+        result = @timed Core.eval(Main, expression)
+        println(join((name, result.time, result.bytes, get(result, :compile_time, NaN)), '\t'))
+        flush(stdout)
+    end
+    for width in (10, 30, 60)
+        name = Symbol(:WideRecord, width)
+        @eval struct $name
+            $([:($(Symbol(:field, i))::Union{Nothing,Int,String,Float64,Bool}) for i in 1:width]...)
+        end
+        value = Core.eval(Main, Expr(:call, name, fill(1, width)...))
+        result = @timed Core.eval(Main, :(JSON.json($value)))
+        println(join((name, result.time, result.bytes, get(result, :compile_time, NaN)), '\t'))
+        flush(stdout)
+    end
+    exit()
+end
+mode == "runtime" || error("unknown benchmark mode: $mode")
+using Chairmarks, Statistics
+function measure(name, f)
+    f() # separate compilation from steady-state measurements
+    result = median(@be f() seconds=0.3)
+    println(join((name, result.time, result.allocs, result.bytes), '\t'))
+    flush(stdout)
+end
+for n in (4, 137, 10000)
+    # Numeric keys must retain string ordering without allocating for indices.
+    for T in (Int, Float64), sort_keys in (true, false)
+        dict = Dict(T(i) => i for i in 1:n)
+        measure("write_numeric_keys_$(T)_$(sort_keys)_$n", () -> JSON.json(dict; sort_keys))
+    end
+    for (name, values) in (("Int32", Int32.(1:n)), ("Float32", Float32.(1:n)),
+                           ("Float64", Float64.(1:n)), ("Date", fill(Date(2026, 1, 1), n)),
+                           ("struct", fill(SmallRecord(1, "x"), n)),
+                           ("Any", Any[isodd(i) ? i : "x" for i in 1:n]),
+                           ("union", Union{Nothing,Int,String,Float64,Bool}[isodd(i) ? i : "x" for i in 1:n]))
+        measure("write_array_$(name)_$n", () -> JSON.json(values))
+        dict = Dict(string(i) => value for (i, value) in enumerate(values))
+        measure("write_dict_$(name)_$n", () -> JSON.json(dict))
+        measure("write_unsorted_$(name)_$n", () -> JSON.json(dict; sort_keys=false))
+    end
+    source = "{" * join(("\"k$i\":$i" for i in 1:n), ",") * "}"
+    for T in (Any, JSON.Object{String,Any}, Dict{String,Any})
+        measure("read_object_$(T)_$n", () -> JSON.parse(source, T))
+    end
+end
+for depth in (1, 16, 64, 256)
+    value = Any[1, "x", true, nothing]
+    for level in 1:depth
+        value = isodd(level) ? Any[value] :
+            level % 4 == 0 ? Dict{String,Any}("a" => value) : JSON.Object{String,Any}("a" => value)
+    end
+    source = JSON.json(value)
+    measure("write_nested_$depth", () -> JSON.json(value))
+    measure("read_nested_$depth", () -> JSON.parse(source))
+    io = IOBuffer()
+    measure("write_io_nested_$depth", () -> (truncate(io, 0); seekstart(io); JSON.json(io, value; bufsize=64)))
+end
+for depth in (1, 16, 64, 256)
+    tree = RecursiveRecord(1, RecursiveRecord[])
+    for _ in 1:depth
+        tree = RecursiveRecord(1, [tree])
+    end
+    measure("write_tree_$depth", () -> JSON.json(tree))
+end
+for n in (1, 1000, 100000)
+    source = "[" * join(fill("{\"a\":1,\"b\":\"x\"}", n), ",") * "]"
+    measure("read_bulk_any_$n", () -> JSON.parse(source, Vector{Any}))
+    measure("read_bulk_struct_$n", () -> JSON.parse(source, Vector{SmallRecord}))
+end
+for width in (10, 30, 60)
+    name = Symbol(:WideRecord, width)
+    @eval struct $name
+        $([:($(Symbol(:field, i))::Union{Nothing,Int,String,Float64,Bool}) for i in 1:width]...)
+    end
+    value = Core.eval(Main, Expr(:call, name, fill(1, width)...))
+    measure("write_wide_$width", () -> JSON.json(value))
+end
+strings = fill("secret", 1000)
+measure("write_custom_style", () -> JSON.json(strings; style=ConversionStyle()))
+source = JSON.json(strings)
+measure("read_custom_style", () -> JSON.parse(source, Vector{Any}; style=ConversionStyle()))

docs/src/writing.md

@@ -371,7 +371,9 @@ JSON.json(node; omit_null=false)
 
 ## Custom Dictionary Key Serialization
 
-For dictionaries with non-string keys, [`JSON.json`](@ref) has a few default `lowerkey` definitions to convert keys to strings:
+Keys and array indices pass through `StructUtils.lowerkey`. JSON accepts a string or a real number from this hook. Its default numeric method returns the number unchanged: object keys are converted to strings when written or sorted, while unused array indices need no string allocation.
+
+For dictionaries with non-string keys, [`JSON.json`](@ref) writes quoted keys:
 
 ```julia
 # Integer keys

src/parse.jl

@@ -167,6 +167,9 @@ objecttype(::JSONReadStyle{OT}) where {OT} = OT
 nullvalue(::StructStyle) = nothing
 nullvalue(st::JSONReadStyle) = st.null
 
+# Custom styles retain the generic make/lift hooks, including lazy-source lifts.
+const _DefaultReadStyle = JSONReadStyle{O,N,StructUtils.DefaultStyle} where {O,N}
+
 StructUtils.initialize(::JSONReadStyle, ::Type{Object}, source) = DEFAULT_OBJECT_TYPE()
 
 # this allows struct fields to specify tags under the json key specifically to override JSON behavior
@@ -321,16 +324,35 @@ end
 mutable struct ObjectClosure{T}
     root::Object{String,Any}
     obj::Object{String,Any}
-    keys::Set{String}
-    null::T
+    keys::Union{Nothing,Set{String}}
+    count::Int
+    ctx::T # the null value of an untyped parse, or the style of a typed one
 end
 
-ObjectClosure(obj, null) = ObjectClosure(obj, obj, sizehint!(Set{String}(), 16), null)
+ObjectClosure(obj, ctx) = ObjectClosure(obj, obj, nothing, 0, ctx)
 
 @inline function insert_or_overwrite!(oc::ObjectClosure, key, val)
-    # in! does both a hash lookup and also sets the key if not present
-    if _in!(key, oc.keys)
-        # slow path for dups; does a linear scan from our root object
+    # Scan at most four entries before allocating a hash table for larger objects.
+    # Once the object grows, use the set so distinct-key insertion stays O(n).
+    keys = oc.keys
+    if keys === nothing
+        node = find_node_by_key(oc.root, key)
+        if node !== nothing
+            setfield!(node, :value, val)
+            return
+        end
+        oc.count += 1
+        if oc.count == 5
+            keys = sizehint!(Set{String}(), 16)
+            node = _ch(oc.root)
+            while node isa Object{String,Any}
+                push!(keys, _k(node)::String)
+                node = _ch(node)
+            end
+            push!(keys, key)
+            oc.keys = keys
+        end
+    elseif _in!(key, keys)
         setindex!(oc.root, val, key)
         return
     end
@@ -339,7 +361,9 @@ ObjectClosure(obj, null) = ObjectClosure(obj, obj, sizehint!(Set{String}(), 16),
     oc.obj = Object{String,Any}(oc.obj, key, val) # fast append path
 end
 
-(oc::ObjectClosure)(k, v) = applyvalue(val -> insert_or_overwrite!(oc, convert(String, k), val), v, oc.null)
+_objectkey(ctx, k) = convert(String, k)
+_objectkey(st::StructStyle, k) = StructUtils.liftkey(st, String, k)
+(oc::ObjectClosure)(k, v) = applyvalue(val -> insert_or_overwrite!(oc, _objectkey(oc.ctx, k), val), v, oc.ctx)
 
 # generic apply `f` to LazyValue, using default types to materialize, depending on type
 function applyvalue(f, x::LazyValues, null)
@@ -392,8 +416,61 @@ function applyvalue(f, x::LazyValues, null)
     end
 end
 
-# we overload make! for Any for LazyValues because we can dispatch to more specific
-# types base on the LazyValue type
+# The typed counterparts of the untyped closures above: `Any` slots under a typed parse and
+# the container types the untyped parse produces. A value materializes exactly as the untyped
+# parse would, except that the style's object type and null value apply. A target that is
+# not the matching JSON kind falls through to the generic `StructUtils.make`.
+"""
+    JSON.applyvalue(f, x::LazyValue, style::StructUtils.StructStyle) -> pos
+
+Materialize `x` as the untyped `JSON.parse` would, with `style`'s object type and null
+value, call `f(value)`, and return the position past `x`. With the default read style,
+`value` reaches `f` with its concrete type, so no `(value::Any, pos)` pair is boxed.
+Custom styles use their `StructUtils.make` and `StructUtils.lift` hooks instead.
+"""
+function applyvalue(f, x::LazyValues, st::_DefaultReadStyle)
+    type = gettype(x)
+    if type == JSONTypes.OBJECT
+        obj, pos = StructUtils.make(st, objecttype(st), x)
+        f(obj)
+        return pos
+    elseif type == JSONTypes.ARRAY
+        arr, pos = StructUtils.make(st, Vector{Any}, x)
+        f(arr)
+        return pos
+    elseif type == JSONTypes.STRING
+        buf = getbuf(x)
+        GC.@preserve buf begin
+            str, pos = parsestring(x)
+            f(convert(String, str))
+        end
+        return pos
+    elseif type == JSONTypes.NUMBER
+        num, pos = parsenumber(x)
+        if isint(num)
+            f(num.int)
+        elseif isfloat(num)
+            f(num.float)
+        elseif isbigint(num)
+            f(num.bigint)
+        else
+            f(num.bigfloat)
+        end
+        return pos
+    elseif type == JSONTypes.NULL
+        f(nullvalue(st))
+        return getpos(x) + 4
+    elseif type == JSONTypes.TRUE
+        f(true)
+        return getpos(x) + 4
+    elseif type == JSONTypes.FALSE
+        f(false)
+        return getpos(x) + 5
+    else
+        throw(ArgumentError("cannot parse json"))
+    end
+end
+
 function StructUtils.make(st::StructStyle, ::Type{Any}, x::LazyValues)
     type = gettype(x)
     if type == JSONTypes.OBJECT
@@ -413,6 +490,44 @@ function StructUtils.make(st::StructStyle, ::Type{Any}, x::LazyValues)
     end
 end
 
+function applyvalue(f, x::LazyValues, st::StructStyle)
+    val, pos = StructUtils.make(st, Any, x)
+    f(val)
+    return pos
+end
+
+function StructUtils.make(st::_DefaultReadStyle, ::Type{Any}, x::LazyValues)
+    box = Ref{Any}()
+    pos = applyvalue(v -> (box[] = v), x, st)
+    return box[], pos
+end
+
+function StructUtils.make(st::_DefaultReadStyle, ::Type{Vector{Any}}, x::LazyValues)
+    gettype(x) == JSONTypes.ARRAY ||
+        return @invoke StructUtils.make(st::StructStyle, Vector{Any}::Type, x::Any)
+    arr = sizehint!(StructUtils.initialize(st, Vector{Any}, x), 16)
+    pos = applyarray((_, v) -> applyvalue(val -> push!(arr, val), v, st), x)
+    return arr, pos
+end
+
+# `ObjectClosure` with the style as its context appends in O(n) with the untyped path's
+# duplicate handling; the generic `makedict` went through `setindex!`, a linear scan per key.
+function StructUtils.make(st::_DefaultReadStyle, ::Type{Object{String,Any}}, x::LazyValues)
+    gettype(x) == JSONTypes.OBJECT ||
+        return @invoke StructUtils.make(st::StructStyle, Object{String,Any}::Type, x::Any)
+    obj = StructUtils.initialize(st, Object{String,Any}, x)
+    pos = applyobject(ObjectClosure(obj, st), x)
+    return obj, pos
+end
+
+function StructUtils.make(st::_DefaultReadStyle, ::Type{T}, x::LazyValues) where {T<:AbstractDict{String,Any}}
+    gettype(x) == JSONTypes.OBJECT ||
+        return @invoke StructUtils.make(st::StructStyle, T::Type, x::Any)
+    dict = StructUtils.initialize(st, T, x)
+    pos = applyobject((k, v) -> applyvalue(val -> StructUtils.addkeyval!(dict, StructUtils.liftkey(st, String, k), val), v, st), x)
+    return dict, pos
+end
+
 # catch PtrString via lift or make! so we can ensure it never "escapes" to user-level
 StructUtils.liftkey(st::JSONReadStyle, ::Type{T}, x::PtrString) where {T} =
     StructUtils.liftkey(st, T, convert(String, x))