From 66cb8ab183cfbe972c0fa06c4629c1f5f24228fc Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 7 Aug 2026 10:10:19 +0200 Subject: [PATCH 1/9] Speed up loading bundles with many included files Each file in `include` was applied as its own mutator. Entering a mutator converts the whole configuration between its typed and dynamic representations, so file N re-converted the N-1 resources already merged, making load quadratic in the number of included files. A bundle with 6000 job files spent ~20 minutes in configuration load before any API call. Apply the per-file includes within the enclosing mutator scope instead, so the configuration is converted once per load. `ProcessInclude` merges through `config.Root.Merge`, which keeps both representations in sync, so it does not need a scope of its own. The expanded include list now goes through `Mutate` for the same reason: nothing converts the typed field back afterwards. Measured with `bundle validate` on generated bundles of 8-task jobs, one job per file: 3000 files 301s -> 35s, 5000 files 1237s -> 93s, 6000 files 1208s -> 131s. Merging itself is still quadratic: `mergeMap` allocates a new mapping and copies every existing entry, so file N also copies the N-1 resources already merged. That is untouched here and becomes the dominant term, so load is much faster but not linear. Co-authored-by: Isaac --- .../bundles/include-load-performance.md | 1 + bundle/config/loader/process_root_includes.go | 29 ++++- .../loader/process_root_includes_test.go | 100 ++++++++++++++++++ bundle/mutator.go | 28 +++++ 4 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 .nextchanges/bundles/include-load-performance.md diff --git a/.nextchanges/bundles/include-load-performance.md b/.nextchanges/bundles/include-load-performance.md new file mode 100644 index 00000000000..9702f8b802d --- /dev/null +++ b/.nextchanges/bundles/include-load-performance.md @@ -0,0 +1 @@ +Improved configuration load time for bundles with many included files. Loading a bundle with 6000 job files in `include` took ~20 minutes and now takes ~2 minutes. diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index e1dda00e8a3..655ca27162b 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) type processRootIncludes struct{} @@ -130,13 +131,35 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. } } - // Swap out the original includes list with the expanded globs. - b.Config.Include = files + // Swap out the original includes list with the expanded globs. This goes through + // Mutate so the dynamic tree is updated too: the includes below are applied without + // their own mutator scope, so nothing converts the typed field back into the dynamic + // tree afterwards, and the next ToTyped would otherwise restore the raw patterns. + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + // Include is omitempty in the typed configuration, so an empty list must stay + // absent from the dynamic tree rather than be written as []. + if len(files) == 0 { + return dyn.DropKeys(root, []string{"include"}) + } + + includeValues := make([]dyn.Value, 0, len(files)) + for _, file := range files { + includeValues = append(includeValues, dyn.V(file)) + } + return dyn.Set(root, "include", dyn.NewValue(includeValues, root.Get("include").Locations())) + }) + if err != nil { + return diag.FromErr(err) + } // Track number of bundle YAML (or JSON) files in the configuration. The +1 is there // to account for the root databricks.yaml file. b.Metrics.ConfigurationFileCount = int64(len(files)) + 1 - bundle.ApplySeqContext(ctx, b, out...) + // ProcessInclude merges into the configuration via [config.Root.Merge], so it does + // not need its own mutator scope. Giving each included file one would re-convert the + // whole accumulated configuration per file, making load quadratic in the number of + // included files (~20 minutes for 6000 files). + bundle.ApplySeqInScopeContext(ctx, b, out...) return nil } diff --git a/bundle/config/loader/process_root_includes_test.go b/bundle/config/loader/process_root_includes_test.go index cb98037507f..1b083da4e88 100644 --- a/bundle/config/loader/process_root_includes_test.go +++ b/bundle/config/loader/process_root_includes_test.go @@ -1,6 +1,7 @@ package loader_test import ( + "path/filepath" "runtime" "testing" @@ -9,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config/loader" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -98,6 +100,104 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { assert.Equal(t, []string{"a.yml"}, b.Config.Include) } +// The expanded include list must be visible in both the typed and the dynamic +// configuration: the per-file includes are applied without their own mutator scope, so +// nothing converts the typed field back into the dynamic tree afterwards. +func TestProcessRootIncludesUpdatesDynamicValue(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Include: []string{ + "*.yml", + }, + }, + } + + testutil.Touch(t, b.BundleRootPath, "databricks.yml") + testutil.Touch(t, b.BundleRootPath, "a.yml") + + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) + require.NoError(t, diags.Error()) + assert.Equal(t, []string{"a.yml"}, b.Config.Include) + + assert.Equal(t, []any{"a.yml"}, b.Config.Value().Get("include").AsAny()) +} + +// An empty include list must stay absent from the dynamic tree: the typed field is +// omitempty, so writing [] would add an empty "include" to `bundle validate -o json`. +func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Include: []string{ + "*.yml", + }, + }, + } + + testutil.Touch(t, b.BundleRootPath, "databricks.yml") + + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) + require.NoError(t, diags.Error()) + assert.Empty(t, b.Config.Include) + assert.Equal(t, dyn.KindInvalid, b.Config.Value().Get("include").Kind()) +} + +// Merge semantics across included files must be unaffected by how the per-file includes +// are applied: maps merge per key with the later file winning, sequences concatenate, and +// locations accumulate (UniqueResourceKeys reports duplicates by counting locations). +func TestProcessRootIncludesMergesAcrossFiles(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Include: []string{ + "*.yml", + }, + }, + } + + testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "a.yml"), ` +resources: + jobs: + shared: + max_concurrent_runs: 1 + tags: + from_a: yes_a + tasks: + - task_key: task_a +`) + + testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "b.yml"), ` +resources: + jobs: + shared: + tags: + from_b: yes_b + tasks: + - task_key: task_b +`) + + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) + require.NoError(t, diags.Error()) + + job := b.Config.Value().Get("resources").Get("jobs").Get("shared") + + // Set only in a.yml: a per-key map merge must not drop it. + assert.Equal(t, int64(1), job.Get("max_concurrent_runs").MustInt()) + + // Maps merge per key across both files. + assert.Equal(t, map[string]any{"from_a": "yes_a", "from_b": "yes_b"}, job.Get("tags").AsAny()) + + // Sequences concatenate rather than overwrite. + assert.Equal(t, []any{ + map[string]any{"task_key": "task_a"}, + map[string]any{"task_key": "task_b"}, + }, job.Get("tasks").AsAny()) + + // Both definitions must remain visible, otherwise duplicate keys go unreported. + assert.Len(t, job.Locations(), 2) +} + func TestProcessRootIncludesNotExists(t *testing.T) { b := &bundle.Bundle{ BundleRootPath: t.TempDir(), diff --git a/bundle/mutator.go b/bundle/mutator.go index 90fdba28c9b..41ef8fda48c 100644 --- a/bundle/mutator.go +++ b/bundle/mutator.go @@ -104,6 +104,34 @@ func ApplySeqContext(ctx context.Context, b *Bundle, mutators ...Mutator) { } } +// ApplySeqInScopeContext applies mutators without opening a mutator scope per mutator, +// reusing the caller's scope instead. +// +// [ApplyContext] converts the whole configuration tree between its typed and dynamic +// representations on entry and exit (see [config.Root.MarkMutatorEntry]). That cost is +// proportional to the size of the accumulated configuration, so applying N mutators this +// way is quadratic in N. For a bundle with thousands of included files that dominates +// load time, hence this variant. +// +// Only use it for mutators that modify the configuration through [config.Root.Mutate] +// (which keeps both representations in sync). A mutator that assigns to a typed field +// directly relies on the scope entry to carry that value into the dynamic tree, and +// would lose it here. +func ApplySeqInScopeContext(ctx context.Context, b *Bundle, mutators ...Mutator) { + for _, m := range mutators { + mctx := log.NewContext(ctx, log.GetLogger(ctx).With("mutator", m.Name())) + log.Debugf(mctx, "Apply") + + for _, d := range m.Apply(mctx, b) { + logdiag.LogDiag(mctx, d) + } + + if logdiag.HasError(ctx) { + break + } + } +} + type funcMutator struct { fn func(context.Context, *Bundle) } From f448601c2ed8a1daa754698d79c6c5fe81dca731 Mon Sep 17 00:00:00 2001 From: Jan N Rose Date: Fri, 7 Aug 2026 10:48:27 +0200 Subject: [PATCH 2/9] PR link --- .nextchanges/bundles/include-load-performance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/include-load-performance.md b/.nextchanges/bundles/include-load-performance.md index 9702f8b802d..50fb3601ee3 100644 --- a/.nextchanges/bundles/include-load-performance.md +++ b/.nextchanges/bundles/include-load-performance.md @@ -1 +1 @@ -Improved configuration load time for bundles with many included files. Loading a bundle with 6000 job files in `include` took ~20 minutes and now takes ~2 minutes. +Improved configuration load time for bundles with many included files ([#6195](https://github.com/databricks/cli/pull/6195)). From 65edb5f521d199b893c98629bc5d0d704afd5761 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 12 Aug 2026 11:24:57 +0200 Subject: [PATCH 3/9] acc: plan a bundle whose resources span multiple included files Adds an end-to-end acceptance test that plans a bundle whose resources are defined across two include globs (resources/*.yml, resources/*/*.yml) and the root databricks.yml, and asserts every resource appears in the plan exactly once. A target override for one job lives in a separate file from its base definition, verifying that a cross-file merge lands in the plan rather than being dropped or duplicated. This guards the include loading and merge path exercised by the switch to ApplySeqInScopeContext: a regression that dropped or duplicated a resource, or lost a cross-file override, would change the plan summary. Co-authored-by: Isaac --- .../includes/plan_multifile/databricks.yml | 17 +++++++++++++++++ .../includes/plan_multifile/out.test.toml | 3 +++ .../bundle/includes/plan_multifile/output.txt | 14 ++++++++++++++ .../includes/plan_multifile/resources/job_a.yml | 8 ++++++++ .../includes/plan_multifile/resources/job_b.yml | 8 ++++++++ .../includes/plan_multifile/resources/nb.py | 2 ++ .../plan_multifile/resources/nested/job_d.yml | 8 ++++++++ .../plan_multifile/resources/override_job_a.yml | 9 +++++++++ .../plan_multifile/resources/pipeline_c.yml | 7 +++++++ .../bundle/includes/plan_multifile/script | 7 +++++++ .../bundle/includes/plan_multifile/test.toml | 3 +++ 11 files changed, 86 insertions(+) create mode 100644 acceptance/bundle/includes/plan_multifile/databricks.yml create mode 100644 acceptance/bundle/includes/plan_multifile/out.test.toml create mode 100644 acceptance/bundle/includes/plan_multifile/output.txt create mode 100644 acceptance/bundle/includes/plan_multifile/resources/job_a.yml create mode 100644 acceptance/bundle/includes/plan_multifile/resources/job_b.yml create mode 100644 acceptance/bundle/includes/plan_multifile/resources/nb.py create mode 100644 acceptance/bundle/includes/plan_multifile/resources/nested/job_d.yml create mode 100644 acceptance/bundle/includes/plan_multifile/resources/override_job_a.yml create mode 100644 acceptance/bundle/includes/plan_multifile/resources/pipeline_c.yml create mode 100644 acceptance/bundle/includes/plan_multifile/script create mode 100644 acceptance/bundle/includes/plan_multifile/test.toml diff --git a/acceptance/bundle/includes/plan_multifile/databricks.yml b/acceptance/bundle/includes/plan_multifile/databricks.yml new file mode 100644 index 00000000000..ccd037dc93a --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: plan_multifile + +# Resources are spread across separate files matched by two globs, plus one job +# defined directly in the root file. The plan must account for every one of them. +include: + - resources/*.yml + - resources/*/*.yml + +resources: + jobs: + root_job: + name: root_job + +targets: + dev: + default: true diff --git a/acceptance/bundle/includes/plan_multifile/out.test.toml b/acceptance/bundle/includes/plan_multifile/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/plan_multifile/output.txt b/acceptance/bundle/includes/plan_multifile/output.txt new file mode 100644 index 00000000000..1a5b6801b5d --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/output.txt @@ -0,0 +1,14 @@ + +>>> [CLI] bundle plan +create jobs.job_a +create jobs.job_b +create jobs.job_d +create jobs.root_job +create pipelines.pipeline_c + +Plan: 5 to add, 0 to change, 0 to delete, 0 unchanged + +=== job_a merged max_concurrent_runs from the override file + +>>> [CLI] bundle validate -o json +4 diff --git a/acceptance/bundle/includes/plan_multifile/resources/job_a.yml b/acceptance/bundle/includes/plan_multifile/resources/job_a.yml new file mode 100644 index 00000000000..63491130c8e --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/resources/job_a.yml @@ -0,0 +1,8 @@ +resources: + jobs: + job_a: + name: job_a + tasks: + - task_key: main + notebook_task: + notebook_path: ./nb.py diff --git a/acceptance/bundle/includes/plan_multifile/resources/job_b.yml b/acceptance/bundle/includes/plan_multifile/resources/job_b.yml new file mode 100644 index 00000000000..55b1adf5799 --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/resources/job_b.yml @@ -0,0 +1,8 @@ +resources: + jobs: + job_b: + name: job_b + tasks: + - task_key: main + notebook_task: + notebook_path: ./nb.py diff --git a/acceptance/bundle/includes/plan_multifile/resources/nb.py b/acceptance/bundle/includes/plan_multifile/resources/nb.py new file mode 100644 index 00000000000..4914a7436d9 --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/resources/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("hello") diff --git a/acceptance/bundle/includes/plan_multifile/resources/nested/job_d.yml b/acceptance/bundle/includes/plan_multifile/resources/nested/job_d.yml new file mode 100644 index 00000000000..49e07ecb119 --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/resources/nested/job_d.yml @@ -0,0 +1,8 @@ +resources: + jobs: + job_d: + name: job_d + tasks: + - task_key: main + notebook_task: + notebook_path: ../nb.py diff --git a/acceptance/bundle/includes/plan_multifile/resources/override_job_a.yml b/acceptance/bundle/includes/plan_multifile/resources/override_job_a.yml new file mode 100644 index 00000000000..b12a4602c48 --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/resources/override_job_a.yml @@ -0,0 +1,9 @@ +# Target override for job_a lives in a different file than its base definition. +# UniqueResourceKeys allows a resource to span files inside a target block, so this +# must merge into job_a rather than error or produce a second resource. +targets: + dev: + resources: + jobs: + job_a: + max_concurrent_runs: 4 diff --git a/acceptance/bundle/includes/plan_multifile/resources/pipeline_c.yml b/acceptance/bundle/includes/plan_multifile/resources/pipeline_c.yml new file mode 100644 index 00000000000..636e2dbbba1 --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/resources/pipeline_c.yml @@ -0,0 +1,7 @@ +resources: + pipelines: + pipeline_c: + name: pipeline_c + libraries: + - notebook: + path: ./nb.py diff --git a/acceptance/bundle/includes/plan_multifile/script b/acceptance/bundle/includes/plan_multifile/script new file mode 100644 index 00000000000..77a11a7989e --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/script @@ -0,0 +1,7 @@ +# Every resource defined across the included files (two globs + the root file) must +# appear in the plan exactly once, and the cross-file target override must merge into +# job_a rather than be dropped. +trace $CLI bundle plan + +title "job_a merged max_concurrent_runs from the override file\n" +trace $CLI bundle validate -o json | jq '.resources.jobs.job_a.max_concurrent_runs' diff --git a/acceptance/bundle/includes/plan_multifile/test.toml b/acceptance/bundle/includes/plan_multifile/test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/includes/plan_multifile/test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] From 398a950c3e6741205fb4d716e50fe6057cc12b3e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 12 Aug 2026 13:07:26 +0200 Subject: [PATCH 4/9] acc: drop redundant test.toml from plan_multifile The leaf test.toml only repeated Local/Cloud/EnvMatrix, which are inherited from the root acceptance/test.toml. The `Local` key was since removed from TestConfig on main (tests always run locally), so decoding the leaf now errors with an undecoded key. Remove the leaf entirely and inherit from the root instead. Co-authored-by: Isaac --- acceptance/bundle/includes/plan_multifile/out.test.toml | 1 - acceptance/bundle/includes/plan_multifile/test.toml | 3 --- 2 files changed, 4 deletions(-) delete mode 100644 acceptance/bundle/includes/plan_multifile/test.toml diff --git a/acceptance/bundle/includes/plan_multifile/out.test.toml b/acceptance/bundle/includes/plan_multifile/out.test.toml index f784a183258..98ea5040486 100644 --- a/acceptance/bundle/includes/plan_multifile/out.test.toml +++ b/acceptance/bundle/includes/plan_multifile/out.test.toml @@ -1,3 +1,2 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/plan_multifile/test.toml b/acceptance/bundle/includes/plan_multifile/test.toml deleted file mode 100644 index f784a183258..00000000000 --- a/acceptance/bundle/includes/plan_multifile/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] From e10cb5e4c48a19f3f88fd8cc6f83c74039159fd2 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 12 Aug 2026 13:17:32 +0200 Subject: [PATCH 5/9] Add unit tests for ApplySeqInScopeContext Cover the three properties the include loading path relies on: mutators are applied once in order, application stops at the first mutator that logs an error (later mutators do not run), and changes made through Root.Mutate survive without a per-mutator scope, becoming visible in both the typed and dynamic configuration once the enclosing scope exits. Co-authored-by: Isaac --- bundle/mutator_test.go | 80 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/bundle/mutator_test.go b/bundle/mutator_test.go index e4504d657b0..45473634119 100644 --- a/bundle/mutator_test.go +++ b/bundle/mutator_test.go @@ -6,12 +6,16 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type testMutator struct { applyCalled int nestedMutators []Mutator + // fn, if set, runs inside Apply after the call is counted. + fn func(ctx context.Context, b *Bundle) diag.Diagnostics } func (t *testMutator) Name() string { @@ -20,6 +24,9 @@ func (t *testMutator) Name() string { func (t *testMutator) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { t.applyCalled++ + if t.fn != nil { + return t.fn(ctx, b) + } return ApplySeq(ctx, b, t.nestedMutators...) } @@ -45,6 +52,79 @@ func TestMutator(t *testing.T) { assert.Equal(t, 1, nested[1].applyCalled) } +// ApplySeqInScopeContext applies each mutator once, in order. +func TestApplySeqInScopeContext(t *testing.T) { + var order []string + makeMutator := func(name string) *testMutator { + return &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { + order = append(order, name) + return nil + }} + } + first := makeMutator("first") + second := makeMutator("second") + + b := &Bundle{} + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + ApplySeqInScopeContext(ctx, b, first, second) + + assert.Equal(t, 1, first.applyCalled) + assert.Equal(t, 1, second.applyCalled) + assert.Equal(t, []string{"first", "second"}, order) + assert.Empty(t, logdiag.FlushCollected(ctx)) +} + +// ApplySeqInScopeContext collects the diagnostics returned by each mutator and +// stops at the first one that logs an error, without applying later mutators. +func TestApplySeqInScopeContextStopsOnError(t *testing.T) { + failing := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { + return diag.Diagnostics{diag.Diagnostic{Severity: diag.Error, Summary: "boom"}} + }} + later := &testMutator{} + + b := &Bundle{} + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + ApplySeqInScopeContext(ctx, b, failing, later) + + assert.Equal(t, 1, failing.applyCalled) + assert.Equal(t, 0, later.applyCalled, "mutator after an error must not run") + + diags := logdiag.FlushCollected(ctx) + require.Len(t, diags, 1) + assert.Equal(t, "boom", diags[0].Summary) +} + +// Unlike ApplySeqContext, ApplySeqInScopeContext does not open a mutator scope per +// mutator, so it must be called from within one. Changes made through Root.Mutate +// keep the typed and dynamic configuration in sync and survive without a per-mutator +// scope. This is the property ProcessRootIncludes relies on. +func TestApplySeqInScopeContextPreservesMutateChanges(t *testing.T) { + setHost := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "bundle", dyn.V(map[string]dyn.Value{"name": dyn.V("set-in-scope")})) + }) + require.NoError(t, err) + return nil + }} + + // Enclosing mutator that applies setHost in its own scope, mirroring how + // ProcessRootIncludes applies the per-file includes. + outer := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { + ApplySeqInScopeContext(ctx, b, setHost) + return nil + }} + + b := &Bundle{} + diags := Apply(t.Context(), b, outer) + require.NoError(t, diags.Error()) + + // Visible in both representations once the enclosing scope exits. + assert.Equal(t, "set-in-scope", b.Config.Bundle.Name) + assert.Equal(t, "set-in-scope", b.Config.Value().Get("bundle").Get("name").MustString()) +} + func TestSafeMutatorName(t *testing.T) { tests := []struct { name string From e6bcc9e825be7c3a5b5c39ac7c88448d46600c96 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 15 Sep 2026 20:14:17 +0000 Subject: [PATCH 6/9] acc: verify include locations survive the in-scope merge An unknown field in a nested included file makes bundle validate report its location; asserting that location proves source locations still point to the right included file after includes are applied within the caller's mutator scope. Co-authored-by: Isaac --- acceptance/bundle/includes/plan_multifile/output.txt | 6 ++++-- .../includes/plan_multifile/resources/nested/job_d.yml | 4 ++++ acceptance/bundle/includes/plan_multifile/script | 8 ++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/includes/plan_multifile/output.txt b/acceptance/bundle/includes/plan_multifile/output.txt index 1a5b6801b5d..f9f563d462b 100644 --- a/acceptance/bundle/includes/plan_multifile/output.txt +++ b/acceptance/bundle/includes/plan_multifile/output.txt @@ -1,5 +1,9 @@ >>> [CLI] bundle plan +Warning: unknown field: unknown_field + at resources.jobs.job_d + in resources/nested/job_d.yml:8:7 + create jobs.job_a create jobs.job_b create jobs.job_d @@ -9,6 +13,4 @@ create pipelines.pipeline_c Plan: 5 to add, 0 to change, 0 to delete, 0 unchanged === job_a merged max_concurrent_runs from the override file - ->>> [CLI] bundle validate -o json 4 diff --git a/acceptance/bundle/includes/plan_multifile/resources/nested/job_d.yml b/acceptance/bundle/includes/plan_multifile/resources/nested/job_d.yml index 49e07ecb119..b3838b32e2a 100644 --- a/acceptance/bundle/includes/plan_multifile/resources/nested/job_d.yml +++ b/acceptance/bundle/includes/plan_multifile/resources/nested/job_d.yml @@ -2,6 +2,10 @@ resources: jobs: job_d: name: job_d + # Unknown field in a nested included file. `bundle validate` must report its + # location as this file, which verifies that source locations survive the + # in-scope include merge this PR introduces for load performance. + unknown_field: true tasks: - task_key: main notebook_task: diff --git a/acceptance/bundle/includes/plan_multifile/script b/acceptance/bundle/includes/plan_multifile/script index 77a11a7989e..08350775a73 100644 --- a/acceptance/bundle/includes/plan_multifile/script +++ b/acceptance/bundle/includes/plan_multifile/script @@ -1,7 +1,11 @@ # Every resource defined across the included files (two globs + the root file) must # appear in the plan exactly once, and the cross-file target override must merge into -# job_a rather than be dropped. +# job_a rather than be dropped. The unknown field in resources/nested/job_d.yml is +# reported against that file, verifying source locations survive the in-scope include +# merge this PR introduces for load performance. trace $CLI bundle plan title "job_a merged max_concurrent_runs from the override file\n" -trace $CLI bundle validate -o json | jq '.resources.jobs.job_a.max_concurrent_runs' +# The located warning is already asserted by the plan above; drop the duplicate here so +# this step only shows the merged value. +trace $CLI bundle validate -o json 2>/dev/null | jq '.resources.jobs.job_a.max_concurrent_runs' From 0781196bdb0a776e3f781906c8ec29a128b258e2 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 15 Sep 2026 20:14:27 +0000 Subject: [PATCH 7/9] Drop redundant cross-file merge unit test TestProcessRootIncludesMergesAcrossFiles duplicated merge coverage that the plan_multifile acceptance test already exercises (two globs, a root-file resource, and a cross-file target override), including the location-preservation assertion now covered there. Co-authored-by: Isaac --- .../loader/process_root_includes_test.go | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/bundle/config/loader/process_root_includes_test.go b/bundle/config/loader/process_root_includes_test.go index 1b083da4e88..e7a850a835b 100644 --- a/bundle/config/loader/process_root_includes_test.go +++ b/bundle/config/loader/process_root_includes_test.go @@ -1,7 +1,6 @@ package loader_test import ( - "path/filepath" "runtime" "testing" @@ -143,61 +142,6 @@ func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) { assert.Equal(t, dyn.KindInvalid, b.Config.Value().Get("include").Kind()) } -// Merge semantics across included files must be unaffected by how the per-file includes -// are applied: maps merge per key with the later file winning, sequences concatenate, and -// locations accumulate (UniqueResourceKeys reports duplicates by counting locations). -func TestProcessRootIncludesMergesAcrossFiles(t *testing.T) { - b := &bundle.Bundle{ - BundleRootPath: t.TempDir(), - Config: config.Root{ - Include: []string{ - "*.yml", - }, - }, - } - - testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "a.yml"), ` -resources: - jobs: - shared: - max_concurrent_runs: 1 - tags: - from_a: yes_a - tasks: - - task_key: task_a -`) - - testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "b.yml"), ` -resources: - jobs: - shared: - tags: - from_b: yes_b - tasks: - - task_key: task_b -`) - - diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) - require.NoError(t, diags.Error()) - - job := b.Config.Value().Get("resources").Get("jobs").Get("shared") - - // Set only in a.yml: a per-key map merge must not drop it. - assert.Equal(t, int64(1), job.Get("max_concurrent_runs").MustInt()) - - // Maps merge per key across both files. - assert.Equal(t, map[string]any{"from_a": "yes_a", "from_b": "yes_b"}, job.Get("tags").AsAny()) - - // Sequences concatenate rather than overwrite. - assert.Equal(t, []any{ - map[string]any{"task_key": "task_a"}, - map[string]any{"task_key": "task_b"}, - }, job.Get("tasks").AsAny()) - - // Both definitions must remain visible, otherwise duplicate keys go unreported. - assert.Len(t, job.Locations(), 2) -} - func TestProcessRootIncludesNotExists(t *testing.T) { b := &bundle.Bundle{ BundleRootPath: t.TempDir(), From efe6d6b3f7e0ed7a9129594b5f5f6b03ddba6f21 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 15 Sep 2026 20:30:05 +0000 Subject: [PATCH 8/9] regenerate out.test.toml --- acceptance/bundle/includes/plan_multifile/out.test.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/bundle/includes/plan_multifile/out.test.toml b/acceptance/bundle/includes/plan_multifile/out.test.toml index 98ea5040486..e1af1a235ad 100644 --- a/acceptance/bundle/includes/plan_multifile/out.test.toml +++ b/acceptance/bundle/includes/plan_multifile/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DMS = ["", "true"] From bfd593de708850a7dba4db2c0fe2f3cb3305a20b Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 15 Sep 2026 20:34:32 +0000 Subject: [PATCH 9/9] fix nextchanges --- .nextchanges/bundles/include-load-performance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/include-load-performance.md b/.nextchanges/bundles/include-load-performance.md index 50fb3601ee3..bfdd04dbfbc 100644 --- a/.nextchanges/bundles/include-load-performance.md +++ b/.nextchanges/bundles/include-load-performance.md @@ -1 +1 @@ -Improved configuration load time for bundles with many included files ([#6195](https://github.com/databricks/cli/pull/6195)). +* Improved configuration load time for bundles with many included files. ([#6195](https://github.com/databricks/cli/pull/6195))