From ef7d04d099d9916d29fd75132790de2f483e7a87 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Sep 2026 15:38:56 +0000 Subject: [PATCH 01/10] Add opt-in DMS Litebox acceptance smoke test --- acceptance/README.md | 22 +++ acceptance/acceptance_test.go | 10 ++ .../bundle/dms-litebox-smoke/out.test.toml | 2 + .../cmd/bundle/dms-litebox-smoke/output.txt | 3 + .../cmd/bundle/dms-litebox-smoke/script | 1 + .../cmd/bundle/dms-litebox-smoke/test.toml | 3 + acceptance/internal/dms_litebox.go | 128 ++++++++++++++ acceptance/internal/dms_litebox_test.go | 162 ++++++++++++++++++ acceptance/internal/prepare_server.go | 2 + 9 files changed, 333 insertions(+) create mode 100644 acceptance/cmd/bundle/dms-litebox-smoke/out.test.toml create mode 100644 acceptance/cmd/bundle/dms-litebox-smoke/output.txt create mode 100644 acceptance/cmd/bundle/dms-litebox-smoke/script create mode 100644 acceptance/cmd/bundle/dms-litebox-smoke/test.toml create mode 100644 acceptance/internal/dms_litebox.go create mode 100644 acceptance/internal/dms_litebox_test.go diff --git a/acceptance/README.md b/acceptance/README.md index 9cd4d2ed0a0..d76ae7d6345 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -15,6 +15,28 @@ The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/accep To run tests against a real workspace: `deco env run -i -n aws-prod-ucws -- ` (requires the `deco` tool and access to a test env). +## Local DMS in Litebox (POC) + +Set `DMS_LITEBOX_URL` to the HTTPS loopback origin of a fresh Deployment Metadata +Service Litebox, and `DMS_LITEBOX_CERT` / `DMS_LITEBOX_KEY` to LITE's test client +certificate and key, plus `DMS_LITEBOX_CA` to LITE's `ca.crt`. Then run: + +```sh +go test ./acceptance -run '^TestDMSLitebox$' -count=1 -v +``` + +This runs `cmd/bundle/dms-litebox-smoke` through the existing in-process CLI +runner, without building Terraform, Python bundles, yamlfmt, or old CLI versions. +The runner's normal Go, Python, uv, jq, and ruff prerequisites still apply. +Only `/api/2.0/bundle/deployments` and its descendants go to DMS; other workspace +APIs stay mocked. HTTP failures never fall back to the DMS mock. TLS verifies +the LITE service certificate, and the adapter supplies a fixed local test identity. +Do not set `CLOUD_ENV`; no workspace token is needed. + +The smoke test lists an empty deployment collection. Bundle deployment tests +also need the runner's fake workspace and DMS's workspace fake to share TreeNodes; +this adapter does not yet provide that synchronization or per-test DMS isolation. + ## Authoring To author a test, diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index b23346b5cc8..778a534adb9 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -154,6 +154,16 @@ func TestAccept(t *testing.T) { testAccept(t, InprocessMode, nil, false) } +// TestDMSLitebox runs one acceptance script against a fresh local DMS. Running +// the CLI in-process avoids building unrelated tools and downloading old CLIs. +func TestDMSLitebox(t *testing.T) { + if os.Getenv("DMS_LITEBOX_URL") == "" { + t.Skip("DMS_LITEBOX_URL is not set") + } + require.Empty(t, os.Getenv("CLOUD_ENV"), "Litebox must not use a cloud workspace") + require.Equal(t, 1, testAccept(t, true, []string{"cmd/bundle/dms-litebox-smoke"}, true)) +} + func TestInprocessMode(t *testing.T) { if InprocessMode && !Forcerun { t.Skip("Already tested by TestAccept") diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/out.test.toml b/acceptance/cmd/bundle/dms-litebox-smoke/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/cmd/bundle/dms-litebox-smoke/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/output.txt b/acceptance/cmd/bundle/dms-litebox-smoke/output.txt new file mode 100644 index 00000000000..cdbb06d3ef3 --- /dev/null +++ b/acceptance/cmd/bundle/dms-litebox-smoke/output.txt @@ -0,0 +1,3 @@ + +>>> [CLI] bundle-deployments list-deployments +[] diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/script b/acceptance/cmd/bundle/dms-litebox-smoke/script new file mode 100644 index 00000000000..95a8f0ad679 --- /dev/null +++ b/acceptance/cmd/bundle/dms-litebox-smoke/script @@ -0,0 +1 @@ +trace $CLI bundle-deployments list-deployments | jq -c . diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/test.toml b/acceptance/cmd/bundle/dms-litebox-smoke/test.toml new file mode 100644 index 00000000000..a8ff9877d56 --- /dev/null +++ b/acceptance/cmd/bundle/dms-litebox-smoke/test.toml @@ -0,0 +1,3 @@ +# Runs against the local mock by default, or a fresh DMS Litebox when configured. +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/internal/dms_litebox.go b/acceptance/internal/dms_litebox.go new file mode 100644 index 00000000000..80b702d8b47 --- /dev/null +++ b/acceptance/internal/dms_litebox.go @@ -0,0 +1,128 @@ +package internal + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/testserver" + "github.com/stretchr/testify/require" +) + +const dmsAPIPath = "/api/2.0/bundle/deployments" + +// ConfigureDMSLitebox routes DMS requests to an opt-in local service, retaining +// the test server's request recording and mocks for other workspace APIs. +func ConfigureDMSLitebox(t *testing.T, server *testserver.Server) { + endpoint := env.Get(t.Context(), "DMS_LITEBOX_URL") + if endpoint == "" { + return + } + certFile, keyFile := env.Get(t.Context(), "DMS_LITEBOX_CERT"), env.Get(t.Context(), "DMS_LITEBOX_KEY") + caFile := env.Get(t.Context(), "DMS_LITEBOX_CA") + require.NotEmpty(t, certFile, "DMS_LITEBOX_CERT is required with DMS_LITEBOX_URL") + require.NotEmpty(t, keyFile, "DMS_LITEBOX_KEY is required with DMS_LITEBOX_URL") + require.NotEmpty(t, caFile, "DMS_LITEBOX_CA is required with DMS_LITEBOX_URL") + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + require.NoError(t, err) + pem, err := os.ReadFile(caFile) + require.NoError(t, err) + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(pem), "invalid Litebox certificate") + // LITE's checked-in test certificate covers *.svc.cluster.local. + // https://github.com/databricks-eng/universe/blob/master/test-foundation/lite/tech-docs/lite-manual-inspection/litebox/litebox-user-guide.md + handler, closeIdle, err := newDMSLiteboxHandler(endpoint, &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + ServerName: "deployment-metadata-service.svc.cluster.local", + }) + require.NoError(t, err) + t.Cleanup(closeIdle) + routeDMS(server, handler) +} + +func isDMSPath(path string) bool { + return path == dmsAPIPath || strings.HasPrefix(path, dmsAPIPath+"/") +} + +func routeDMS(server *testserver.Server, handler testserver.HandlerFunc) { + dispatch := server.Dispatch + server.Dispatch = func(w http.ResponseWriter, r *http.Request, h testserver.HandlerFunc, vars map[string]string) { + if isDMSPath(r.URL.Path) { + h = handler + } + dispatch(w, r, h, vars) + } + notFound := server.NotFound + server.NotFound = func(w http.ResponseWriter, r *http.Request) { + if isDMSPath(r.URL.Path) { + dispatch(w, r, handler, nil) + return + } + notFound(w, r) + } +} + +func newDMSLiteboxHandler(endpoint string, tlsConfig *tls.Config) (testserver.HandlerFunc, func(), error) { + u, err := url.Parse(endpoint) + if err != nil { + return nil, nil, err + } + ip := net.ParseIP(u.Hostname()) + if u.Scheme != "https" || (u.Hostname() != "localhost" && (ip == nil || !ip.IsLoopback())) || + u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") { + return nil, nil, errors.New("DMS_LITEBOX_URL must be an HTTPS loopback origin") + } + transport := &http.Transport{TLSClientConfig: tlsConfig, ForceAttemptHTTP2: true} + client := &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }, + } + handler := func(r testserver.Request) any { + target := *u + target.Path, target.RawPath, target.RawQuery = r.URL.Path, r.URL.RawPath, r.URL.RawQuery + req, err := http.NewRequestWithContext(r.Context, r.Method, target.String(), bytes.NewReader(r.Body)) + if err != nil { + return liteboxFailure(err) + } + req.Header = r.Headers.Clone() + req.Header.Del("Authorization") + req.Header.Del("Accept-Encoding") + // Fixed test identity; this adapter only connects to a local LITE fixture. + req.Header.Set("X-Databricks-Org-Id", "456") + req.Header.Set("X-Databricks-User-Id", "123") + req.Header.Set("X-Databricks-User-Name", "alice@databricks.com") + resp, err := client.Do(req) + if err != nil { + return liteboxFailure(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return liteboxFailure(err) + } + return testserver.Response{StatusCode: resp.StatusCode, Headers: resp.Header, Body: body} + } + return handler, transport.CloseIdleConnections, nil +} + +func liteboxFailure(err error) testserver.Response { + return testserver.Response{StatusCode: http.StatusBadGateway, Body: map[string]string{ + "error_code": "LITEBOX_CONNECTION_ERROR", + "message": err.Error(), + }} +} diff --git a/acceptance/internal/dms_litebox_test.go b/acceptance/internal/dms_litebox_test.go new file mode 100644 index 00000000000..e19bad3ca92 --- /dev/null +++ b/acceptance/internal/dms_litebox_test.go @@ -0,0 +1,162 @@ +package internal_test + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "io" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/acceptance/internal" + "github.com/databricks/cli/libs/testserver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const dmsPath = "/api/2.0/bundle/deployments" + +// Generate a short-lived fixture instead of distributing LITE's private key. +func liteboxCertificate(t *testing.T) (tls.Certificate, string, string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1), + DNSNames: []string{"deployment-metadata-service.svc.cluster.local"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + IsCA: true, BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, cert, cert, &key.PublicKey, key) + require.NoError(t, err) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + certFile, keyFile := filepath.Join(t.TempDir(), "cert.pem"), filepath.Join(t.TempDir(), "key.pem") + require.NoError(t, os.WriteFile(certFile, certPEM, 0o600)) + require.NoError(t, os.WriteFile(keyFile, keyPEM, 0o600)) + pair, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + return pair, certFile, keyFile +} + +func TestDMSLiteboxRoutesAndRecords(t *testing.T) { + cert, certFile, keyFile := liteboxCertificate(t) + roots := x509.NewCertPool() + parsed, err := x509.ParseCertificate(cert.Certificate[0]) + require.NoError(t, err) + roots.AddCert(parsed) + var calls atomic.Int32 + upstream := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + assert.Equal(t, "POST", r.Method) + assert.Equal(t, "page_token=a%2Fb", r.URL.RawQuery) + assert.Empty(t, r.Header.Get("Authorization")) + assert.Equal(t, "456", r.Header.Get("X-Databricks-Org-Id")) + assert.Equal(t, "123", r.Header.Get("X-Databricks-User-Id")) + assert.Equal(t, "alice@databricks.com", r.Header.Get("X-Databricks-User-Name")) + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + assert.Len(t, r.TLS.PeerCertificates, 1) + body, err := io.ReadAll(r.Body) + assert.NoError(t, err) + assert.JSONEq(t, `{"id":9007199254740993}`, string(body)) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "2") + w.WriteHeader(http.StatusServiceUnavailable) + _, err = w.Write(body) + assert.NoError(t, err) + })) + upstream.TLS = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: roots} + upstream.StartTLS() + t.Cleanup(upstream.Close) + t.Setenv("DMS_LITEBOX_URL", upstream.URL) + t.Setenv("DMS_LITEBOX_CERT", certFile) + t.Setenv("DMS_LITEBOX_CA", certFile) + t.Setenv("DMS_LITEBOX_KEY", keyFile) + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + var recorded atomic.Int32 + server.ResponseCallback = func(_ *testserver.Request, response *testserver.EncodedResponse) { + if response.StatusCode == http.StatusServiceUnavailable { + recorded.Add(1) + } + } + internal.ConfigureDMSLitebox(t, server) + // Cover a registered fake route and an unknown DMS route (no mock fallback). + for _, path := range []string{dmsPath, dmsPath + "/unknown/route"} { + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, server.URL+path+"?page_token=a%2Fb", strings.NewReader(`{"id":9007199254740993}`)) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer fake-token") + req.Header.Set("Content-Type", "application/json") + resp, err := server.Client().Do(req) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, err) + assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + assert.Equal(t, "2", resp.Header.Get("Retry-After")) + assert.JSONEq(t, `{"id":9007199254740993}`, string(body)) + } + resp, err := server.Client().Get(server.URL + "/.well-known/databricks-config") + require.NoError(t, err) + resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.EqualValues(t, 2, calls.Load()) + assert.EqualValues(t, 2, recorded.Load()) +} + +func TestDMSLiteboxFailsClosed(t *testing.T) { + for _, failure := range []string{"untrusted certificate", "unreachable server"} { + t.Run(failure, func(t *testing.T) { + _, certFile, keyFile := liteboxCertificate(t) + // The server's unrelated certificate must not be trusted by the adapter. + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("request reached an untrusted upstream") + })) + t.Cleanup(upstream.Close) + if failure == "unreachable server" { + upstream.Close() + } + t.Setenv("DMS_LITEBOX_URL", upstream.URL) + t.Setenv("DMS_LITEBOX_CERT", certFile) + t.Setenv("DMS_LITEBOX_CA", certFile) + t.Setenv("DMS_LITEBOX_KEY", keyFile) + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + internal.ConfigureDMSLitebox(t, server) + resp, err := server.Client().Get(server.URL + dmsPath) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, http.StatusBadGateway, resp.StatusCode) + assert.Contains(t, string(body), "LITEBOX_CONNECTION_ERROR") + }) + } +} + +func TestDMSLiteboxDisabled(t *testing.T) { + t.Setenv("DMS_LITEBOX_URL", "") + server := testserver.New(t) + server.Handle("GET", dmsPath, func(testserver.Request) any { return "mock" }) + internal.ConfigureDMSLitebox(t, server) + resp, err := server.Client().Get(server.URL + dmsPath) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, "mock", string(body)) +} diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 392cfc3ae77..0d54f192438 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -28,6 +28,7 @@ import ( func StartDefaultServer(t *testing.T, logRequests bool) { s := testserver.New(t) testserver.AddDefaultHandlers(s) + ConfigureDMSLitebox(t, s) // Log API responses if the -logrequests flag is set. if logRequests { @@ -270,6 +271,7 @@ func startLocalServer(t *testing.T, // The first handler registered for a given pattern wins, so default // handlers registered last serve as fallbacks. testserver.AddDefaultHandlers(s) + ConfigureDMSLitebox(t, s) return s.URL } From 2c42230150b60300a09775f6d91f265badd39346 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 21 Sep 2026 15:51:02 +0000 Subject: [PATCH 02/10] Keep the DMS smoke runnable in the normal mock suite --- acceptance/cmd/bundle/dms-litebox-smoke/test.toml | 6 ++++++ acceptance/internal/dms_litebox_test.go | 1 + 2 files changed, 7 insertions(+) diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/test.toml b/acceptance/cmd/bundle/dms-litebox-smoke/test.toml index a8ff9877d56..9970923d080 100644 --- a/acceptance/cmd/bundle/dms-litebox-smoke/test.toml +++ b/acceptance/cmd/bundle/dms-litebox-smoke/test.toml @@ -1,3 +1,9 @@ # Runs against the local mock by default, or a fresh DMS Litebox when configured. Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The fake does not model ListDeployments yet. Litebox routing overrides this +# empty-workspace fixture, so it is only used by the normal local suite. +[[Server]] +Pattern = "GET /api/2.0/bundle/deployments" +Response.Body = '{"deployments": []}' diff --git a/acceptance/internal/dms_litebox_test.go b/acceptance/internal/dms_litebox_test.go index e19bad3ca92..7509ab72878 100644 --- a/acceptance/internal/dms_litebox_test.go +++ b/acceptance/internal/dms_litebox_test.go @@ -136,6 +136,7 @@ func TestDMSLiteboxFailsClosed(t *testing.T) { t.Setenv("DMS_LITEBOX_KEY", keyFile) server := testserver.New(t) testserver.AddDefaultHandlers(server) + server.Handle(http.MethodGet, dmsPath, func(testserver.Request) any { return "mock" }) internal.ConfigureDMSLitebox(t, server) resp, err := server.Client().Get(server.URL + dmsPath) require.NoError(t, err) From b584b1765736c266e5dae33a6973b2c77706763c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 22 Sep 2026 12:02:57 +0000 Subject: [PATCH 03/10] Simplify DMS acceptance routing and select tests with go test -run --- acceptance/README.md | 22 --- acceptance/acceptance_test.go | 8 +- .../bundle/dms-litebox-smoke/out.test.toml | 2 - .../cmd/bundle/dms-litebox-smoke/output.txt | 3 - .../cmd/bundle/dms-litebox-smoke/script | 1 - .../cmd/bundle/dms-litebox-smoke/test.toml | 9 - acceptance/internal/dms_litebox.go | 94 +++++----- acceptance/internal/dms_litebox_test.go | 163 ------------------ acceptance/internal/prepare_server.go | 4 +- 9 files changed, 50 insertions(+), 256 deletions(-) delete mode 100644 acceptance/cmd/bundle/dms-litebox-smoke/out.test.toml delete mode 100644 acceptance/cmd/bundle/dms-litebox-smoke/output.txt delete mode 100644 acceptance/cmd/bundle/dms-litebox-smoke/script delete mode 100644 acceptance/cmd/bundle/dms-litebox-smoke/test.toml delete mode 100644 acceptance/internal/dms_litebox_test.go diff --git a/acceptance/README.md b/acceptance/README.md index d76ae7d6345..9cd4d2ed0a0 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -15,28 +15,6 @@ The cloud run is keyed off `CLOUD_ENV`; see `getSkipReason` in `acceptance/accep To run tests against a real workspace: `deco env run -i -n aws-prod-ucws -- ` (requires the `deco` tool and access to a test env). -## Local DMS in Litebox (POC) - -Set `DMS_LITEBOX_URL` to the HTTPS loopback origin of a fresh Deployment Metadata -Service Litebox, and `DMS_LITEBOX_CERT` / `DMS_LITEBOX_KEY` to LITE's test client -certificate and key, plus `DMS_LITEBOX_CA` to LITE's `ca.crt`. Then run: - -```sh -go test ./acceptance -run '^TestDMSLitebox$' -count=1 -v -``` - -This runs `cmd/bundle/dms-litebox-smoke` through the existing in-process CLI -runner, without building Terraform, Python bundles, yamlfmt, or old CLI versions. -The runner's normal Go, Python, uv, jq, and ruff prerequisites still apply. -Only `/api/2.0/bundle/deployments` and its descendants go to DMS; other workspace -APIs stay mocked. HTTP failures never fall back to the DMS mock. TLS verifies -the LITE service certificate, and the adapter supplies a fixed local test identity. -Do not set `CLOUD_ENV`; no workspace token is needed. - -The smoke test lists an empty deployment collection. Bundle deployment tests -also need the runner's fake workspace and DMS's workspace fake to share TreeNodes; -this adapter does not yet provide that synchronization or per-test DMS isolation. - ## Authoring To author a test, diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 778a534adb9..2189fd59406 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -154,14 +154,14 @@ func TestAccept(t *testing.T) { testAccept(t, InprocessMode, nil, false) } -// TestDMSLitebox runs one acceptance script against a fresh local DMS. Running -// the CLI in-process avoids building unrelated tools and downloading old CLIs. -func TestDMSLitebox(t *testing.T) { +// TestDmsAcc runs acceptance tests against local DMS, selected with go test -run. +// The in-process runner skips unrelated tool builds and old CLI downloads. +func TestDmsAcc(t *testing.T) { if os.Getenv("DMS_LITEBOX_URL") == "" { t.Skip("DMS_LITEBOX_URL is not set") } require.Empty(t, os.Getenv("CLOUD_ENV"), "Litebox must not use a cloud workspace") - require.Equal(t, 1, testAccept(t, true, []string{"cmd/bundle/dms-litebox-smoke"}, true)) + require.Positive(t, testAccept(t, true, nil, true), "-run did not select any acceptance tests") } func TestInprocessMode(t *testing.T) { diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/out.test.toml b/acceptance/cmd/bundle/dms-litebox-smoke/out.test.toml deleted file mode 100644 index 0938e678987..00000000000 --- a/acceptance/cmd/bundle/dms-litebox-smoke/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/output.txt b/acceptance/cmd/bundle/dms-litebox-smoke/output.txt deleted file mode 100644 index cdbb06d3ef3..00000000000 --- a/acceptance/cmd/bundle/dms-litebox-smoke/output.txt +++ /dev/null @@ -1,3 +0,0 @@ - ->>> [CLI] bundle-deployments list-deployments -[] diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/script b/acceptance/cmd/bundle/dms-litebox-smoke/script deleted file mode 100644 index 95a8f0ad679..00000000000 --- a/acceptance/cmd/bundle/dms-litebox-smoke/script +++ /dev/null @@ -1 +0,0 @@ -trace $CLI bundle-deployments list-deployments | jq -c . diff --git a/acceptance/cmd/bundle/dms-litebox-smoke/test.toml b/acceptance/cmd/bundle/dms-litebox-smoke/test.toml deleted file mode 100644 index 9970923d080..00000000000 --- a/acceptance/cmd/bundle/dms-litebox-smoke/test.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Runs against the local mock by default, or a fresh DMS Litebox when configured. -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -# The fake does not model ListDeployments yet. Litebox routing overrides this -# empty-workspace fixture, so it is only used by the normal local suite. -[[Server]] -Pattern = "GET /api/2.0/bundle/deployments" -Response.Body = '{"deployments": []}' diff --git a/acceptance/internal/dms_litebox.go b/acceptance/internal/dms_litebox.go index 80b702d8b47..ac41c093873 100644 --- a/acceptance/internal/dms_litebox.go +++ b/acceptance/internal/dms_litebox.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/tls" "crypto/x509" - "errors" "io" "net" "net/http" @@ -21,70 +20,43 @@ import ( const dmsAPIPath = "/api/2.0/bundle/deployments" -// ConfigureDMSLitebox routes DMS requests to an opt-in local service, retaining +// configureDMSLitebox routes DMS requests to an opt-in local service, retaining // the test server's request recording and mocks for other workspace APIs. -func ConfigureDMSLitebox(t *testing.T, server *testserver.Server) { +func configureDMSLitebox(t *testing.T, server *testserver.Server) { + t.Helper() endpoint := env.Get(t.Context(), "DMS_LITEBOX_URL") if endpoint == "" { return } + u, err := url.Parse(endpoint) + require.NoError(t, err, "invalid DMS_LITEBOX_URL") + ip := net.ParseIP(u.Hostname()) + require.True(t, u.Scheme == "https" && (u.Hostname() == "localhost" || (ip != nil && ip.IsLoopback())) && + u.User == nil && u.RawQuery == "" && u.Fragment == "" && (u.Path == "" || u.Path == "/"), + "DMS_LITEBOX_URL must be an HTTPS loopback origin") + certFile, keyFile := env.Get(t.Context(), "DMS_LITEBOX_CERT"), env.Get(t.Context(), "DMS_LITEBOX_KEY") caFile := env.Get(t.Context(), "DMS_LITEBOX_CA") require.NotEmpty(t, certFile, "DMS_LITEBOX_CERT is required with DMS_LITEBOX_URL") require.NotEmpty(t, keyFile, "DMS_LITEBOX_KEY is required with DMS_LITEBOX_URL") require.NotEmpty(t, caFile, "DMS_LITEBOX_CA is required with DMS_LITEBOX_URL") cert, err := tls.LoadX509KeyPair(certFile, keyFile) - require.NoError(t, err) + require.NoError(t, err, "load Litebox client certificate and key") pem, err := os.ReadFile(caFile) - require.NoError(t, err) + require.NoError(t, err, "read DMS_LITEBOX_CA") roots := x509.NewCertPool() require.True(t, roots.AppendCertsFromPEM(pem), "invalid Litebox certificate") // LITE's checked-in test certificate covers *.svc.cluster.local. - // https://github.com/databricks-eng/universe/blob/master/test-foundation/lite/tech-docs/lite-manual-inspection/litebox/litebox-user-guide.md - handler, closeIdle, err := newDMSLiteboxHandler(endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - Certificates: []tls.Certificate{cert}, - RootCAs: roots, - ServerName: "deployment-metadata-service.svc.cluster.local", - }) - require.NoError(t, err) - t.Cleanup(closeIdle) - routeDMS(server, handler) -} - -func isDMSPath(path string) bool { - return path == dmsAPIPath || strings.HasPrefix(path, dmsAPIPath+"/") -} - -func routeDMS(server *testserver.Server, handler testserver.HandlerFunc) { - dispatch := server.Dispatch - server.Dispatch = func(w http.ResponseWriter, r *http.Request, h testserver.HandlerFunc, vars map[string]string) { - if isDMSPath(r.URL.Path) { - h = handler - } - dispatch(w, r, h, vars) - } - notFound := server.NotFound - server.NotFound = func(w http.ResponseWriter, r *http.Request) { - if isDMSPath(r.URL.Path) { - dispatch(w, r, handler, nil) - return - } - notFound(w, r) - } -} - -func newDMSLiteboxHandler(endpoint string, tlsConfig *tls.Config) (testserver.HandlerFunc, func(), error) { - u, err := url.Parse(endpoint) - if err != nil { - return nil, nil, err - } - ip := net.ParseIP(u.Hostname()) - if u.Scheme != "https" || (u.Hostname() != "localhost" && (ip == nil || !ip.IsLoopback())) || - u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") { - return nil, nil, errors.New("DMS_LITEBOX_URL must be an HTTPS loopback origin") + transport := &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + ServerName: "deployment-metadata-service.svc.cluster.local", + }, + ForceAttemptHTTP2: true, } - transport := &http.Transport{TLSClientConfig: tlsConfig, ForceAttemptHTTP2: true} + t.Cleanup(transport.CloseIdleConnections) client := &http.Client{ Transport: transport, Timeout: 30 * time.Second, @@ -117,7 +89,29 @@ func newDMSLiteboxHandler(endpoint string, tlsConfig *tls.Config) (testserver.Ha } return testserver.Response{StatusCode: resp.StatusCode, Headers: resp.Header, Body: body} } - return handler, transport.CloseIdleConnections, nil + routeDMS(server, handler) +} + +func isDMSPath(path string) bool { + return path == dmsAPIPath || strings.HasPrefix(path, dmsAPIPath+"/") +} + +func routeDMS(server *testserver.Server, handler testserver.HandlerFunc) { + dispatch := server.Dispatch + server.Dispatch = func(w http.ResponseWriter, r *http.Request, h testserver.HandlerFunc, vars map[string]string) { + if isDMSPath(r.URL.Path) { + h = handler + } + dispatch(w, r, h, vars) + } + notFound := server.NotFound + server.NotFound = func(w http.ResponseWriter, r *http.Request) { + if isDMSPath(r.URL.Path) { + dispatch(w, r, handler, nil) + return + } + notFound(w, r) + } } func liteboxFailure(err error) testserver.Response { diff --git a/acceptance/internal/dms_litebox_test.go b/acceptance/internal/dms_litebox_test.go deleted file mode 100644 index 7509ab72878..00000000000 --- a/acceptance/internal/dms_litebox_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package internal_test - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "io" - "math/big" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/databricks/cli/acceptance/internal" - "github.com/databricks/cli/libs/testserver" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const dmsPath = "/api/2.0/bundle/deployments" - -// Generate a short-lived fixture instead of distributing LITE's private key. -func liteboxCertificate(t *testing.T) (tls.Certificate, string, string) { - t.Helper() - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, err) - cert := &x509.Certificate{ - SerialNumber: big.NewInt(1), - DNSNames: []string{"deployment-metadata-service.svc.cluster.local"}, - NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - IsCA: true, BasicConstraintsValid: true, - } - der, err := x509.CreateCertificate(rand.Reader, cert, cert, &key.PublicKey, key) - require.NoError(t, err) - keyDER, err := x509.MarshalPKCS8PrivateKey(key) - require.NoError(t, err) - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) - certFile, keyFile := filepath.Join(t.TempDir(), "cert.pem"), filepath.Join(t.TempDir(), "key.pem") - require.NoError(t, os.WriteFile(certFile, certPEM, 0o600)) - require.NoError(t, os.WriteFile(keyFile, keyPEM, 0o600)) - pair, err := tls.X509KeyPair(certPEM, keyPEM) - require.NoError(t, err) - return pair, certFile, keyFile -} - -func TestDMSLiteboxRoutesAndRecords(t *testing.T) { - cert, certFile, keyFile := liteboxCertificate(t) - roots := x509.NewCertPool() - parsed, err := x509.ParseCertificate(cert.Certificate[0]) - require.NoError(t, err) - roots.AddCert(parsed) - var calls atomic.Int32 - upstream := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - assert.Equal(t, "POST", r.Method) - assert.Equal(t, "page_token=a%2Fb", r.URL.RawQuery) - assert.Empty(t, r.Header.Get("Authorization")) - assert.Equal(t, "456", r.Header.Get("X-Databricks-Org-Id")) - assert.Equal(t, "123", r.Header.Get("X-Databricks-User-Id")) - assert.Equal(t, "alice@databricks.com", r.Header.Get("X-Databricks-User-Name")) - assert.Equal(t, "application/json", r.Header.Get("Content-Type")) - assert.Len(t, r.TLS.PeerCertificates, 1) - body, err := io.ReadAll(r.Body) - assert.NoError(t, err) - assert.JSONEq(t, `{"id":9007199254740993}`, string(body)) - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Retry-After", "2") - w.WriteHeader(http.StatusServiceUnavailable) - _, err = w.Write(body) - assert.NoError(t, err) - })) - upstream.TLS = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: roots} - upstream.StartTLS() - t.Cleanup(upstream.Close) - t.Setenv("DMS_LITEBOX_URL", upstream.URL) - t.Setenv("DMS_LITEBOX_CERT", certFile) - t.Setenv("DMS_LITEBOX_CA", certFile) - t.Setenv("DMS_LITEBOX_KEY", keyFile) - server := testserver.New(t) - testserver.AddDefaultHandlers(server) - var recorded atomic.Int32 - server.ResponseCallback = func(_ *testserver.Request, response *testserver.EncodedResponse) { - if response.StatusCode == http.StatusServiceUnavailable { - recorded.Add(1) - } - } - internal.ConfigureDMSLitebox(t, server) - // Cover a registered fake route and an unknown DMS route (no mock fallback). - for _, path := range []string{dmsPath, dmsPath + "/unknown/route"} { - req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, server.URL+path+"?page_token=a%2Fb", strings.NewReader(`{"id":9007199254740993}`)) - require.NoError(t, err) - req.Header.Set("Authorization", "Bearer fake-token") - req.Header.Set("Content-Type", "application/json") - resp, err := server.Client().Do(req) - require.NoError(t, err) - body, err := io.ReadAll(resp.Body) - resp.Body.Close() - require.NoError(t, err) - assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) - assert.Equal(t, "2", resp.Header.Get("Retry-After")) - assert.JSONEq(t, `{"id":9007199254740993}`, string(body)) - } - resp, err := server.Client().Get(server.URL + "/.well-known/databricks-config") - require.NoError(t, err) - resp.Body.Close() - assert.Equal(t, http.StatusOK, resp.StatusCode) - assert.EqualValues(t, 2, calls.Load()) - assert.EqualValues(t, 2, recorded.Load()) -} - -func TestDMSLiteboxFailsClosed(t *testing.T) { - for _, failure := range []string{"untrusted certificate", "unreachable server"} { - t.Run(failure, func(t *testing.T) { - _, certFile, keyFile := liteboxCertificate(t) - // The server's unrelated certificate must not be trusted by the adapter. - upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Error("request reached an untrusted upstream") - })) - t.Cleanup(upstream.Close) - if failure == "unreachable server" { - upstream.Close() - } - t.Setenv("DMS_LITEBOX_URL", upstream.URL) - t.Setenv("DMS_LITEBOX_CERT", certFile) - t.Setenv("DMS_LITEBOX_CA", certFile) - t.Setenv("DMS_LITEBOX_KEY", keyFile) - server := testserver.New(t) - testserver.AddDefaultHandlers(server) - server.Handle(http.MethodGet, dmsPath, func(testserver.Request) any { return "mock" }) - internal.ConfigureDMSLitebox(t, server) - resp, err := server.Client().Get(server.URL + dmsPath) - require.NoError(t, err) - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - assert.Equal(t, http.StatusBadGateway, resp.StatusCode) - assert.Contains(t, string(body), "LITEBOX_CONNECTION_ERROR") - }) - } -} - -func TestDMSLiteboxDisabled(t *testing.T) { - t.Setenv("DMS_LITEBOX_URL", "") - server := testserver.New(t) - server.Handle("GET", dmsPath, func(testserver.Request) any { return "mock" }) - internal.ConfigureDMSLitebox(t, server) - resp, err := server.Client().Get(server.URL + dmsPath) - require.NoError(t, err) - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - assert.Equal(t, "mock", string(body)) -} diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 0d54f192438..35aeeb429be 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -28,7 +28,7 @@ import ( func StartDefaultServer(t *testing.T, logRequests bool) { s := testserver.New(t) testserver.AddDefaultHandlers(s) - ConfigureDMSLitebox(t, s) + configureDMSLitebox(t, s) // Log API responses if the -logrequests flag is set. if logRequests { @@ -271,7 +271,7 @@ func startLocalServer(t *testing.T, // The first handler registered for a given pattern wins, so default // handlers registered last serve as fallbacks. testserver.AddDefaultHandlers(s) - ConfigureDMSLitebox(t, s) + configureDMSLitebox(t, s) return s.URL } From 91d1fb367f3f9d42ee2efd482058c3cf6de5b880 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 22 Sep 2026 12:31:50 +0000 Subject: [PATCH 04/10] Let the Universe wrapper check for empty DMS acceptance runs --- acceptance/acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 2189fd59406..a42143b3d25 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -161,7 +161,7 @@ func TestDmsAcc(t *testing.T) { t.Skip("DMS_LITEBOX_URL is not set") } require.Empty(t, os.Getenv("CLOUD_ENV"), "Litebox must not use a cloud workspace") - require.Positive(t, testAccept(t, true, nil, true), "-run did not select any acceptance tests") + testAccept(t, true, nil, true) } func TestInprocessMode(t *testing.T) { From ff7b0875dae20316e450d90fb371c39db21e554e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 22 Sep 2026 12:48:50 +0000 Subject: [PATCH 05/10] Route workspace files and workspace-object ACLs to Litebox fakes --- acceptance/internal/dms_litebox.go | 62 +++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/acceptance/internal/dms_litebox.go b/acceptance/internal/dms_litebox.go index ac41c093873..c0f513edaae 100644 --- a/acceptance/internal/dms_litebox.go +++ b/acceptance/internal/dms_litebox.go @@ -20,20 +20,19 @@ import ( const dmsAPIPath = "/api/2.0/bundle/deployments" -// configureDMSLitebox routes DMS requests to an opt-in local service, retaining -// the test server's request recording and mocks for other workspace APIs. +// Deployment TreeNodes use the generic workspace-objects permissions API. +var liteboxPermissionTypes = []string{"directories", "files", "notebooks", "workspace-objects"} + +// configureDMSLitebox routes DMS to the local service and workspace files/ACLs +// to its shared fakes, retaining request recording and mocks for other APIs. func configureDMSLitebox(t *testing.T, server *testserver.Server) { t.Helper() endpoint := env.Get(t.Context(), "DMS_LITEBOX_URL") if endpoint == "" { return } - u, err := url.Parse(endpoint) - require.NoError(t, err, "invalid DMS_LITEBOX_URL") - ip := net.ParseIP(u.Hostname()) - require.True(t, u.Scheme == "https" && (u.Hostname() == "localhost" || (ip != nil && ip.IsLoopback())) && - u.User == nil && u.RawQuery == "" && u.Fragment == "" && (u.Path == "" || u.Path == "/"), - "DMS_LITEBOX_URL must be an HTTPS loopback origin") + dmsURL := liteboxURL(t, "DMS_LITEBOX_URL") + workspaceURL := liteboxURL(t, "DMS_LITEBOX_WORKSPACE_URL") certFile, keyFile := env.Get(t.Context(), "DMS_LITEBOX_CERT"), env.Get(t.Context(), "DMS_LITEBOX_KEY") caFile := env.Get(t.Context(), "DMS_LITEBOX_CA") @@ -46,7 +45,7 @@ func configureDMSLitebox(t *testing.T, server *testserver.Server) { require.NoError(t, err, "read DMS_LITEBOX_CA") roots := x509.NewCertPool() require.True(t, roots.AppendCertsFromPEM(pem), "invalid Litebox certificate") - // LITE's checked-in test certificate covers *.svc.cluster.local. + // Both upstreams must use LITE's test certificate for *.svc.cluster.local. transport := &http.Transport{ TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, @@ -65,7 +64,10 @@ func configureDMSLitebox(t *testing.T, server *testserver.Server) { }, } handler := func(r testserver.Request) any { - target := *u + target := *workspaceURL + if isDMSPath(r.URL.Path) { + target = *dmsURL + } target.Path, target.RawPath, target.RawQuery = r.URL.Path, r.URL.RawPath, r.URL.RawQuery req, err := http.NewRequestWithContext(r.Context, r.Method, target.String(), bytes.NewReader(r.Body)) if err != nil { @@ -76,6 +78,7 @@ func configureDMSLitebox(t *testing.T, server *testserver.Server) { req.Header.Del("Accept-Encoding") // Fixed test identity; this adapter only connects to a local LITE fixture. req.Header.Set("X-Databricks-Org-Id", "456") + req.Header.Set("X-Databricks-Workspace-Id", "456") req.Header.Set("X-Databricks-User-Id", "123") req.Header.Set("X-Databricks-User-Name", "alice@databricks.com") resp, err := client.Do(req) @@ -89,24 +92,55 @@ func configureDMSLitebox(t *testing.T, server *testserver.Server) { } return testserver.Response{StatusCode: resp.StatusCode, Headers: resp.Header, Body: body} } - routeDMS(server, handler) + routeLitebox(server, handler) +} + +func liteboxURL(t *testing.T, name string) *url.URL { + t.Helper() + endpoint := env.Get(t.Context(), name) + require.NotEmpty(t, endpoint, "%s is required with DMS_LITEBOX_URL", name) + u, err := url.Parse(endpoint) + require.NoError(t, err, "invalid %s", name) + ip := net.ParseIP(u.Hostname()) + require.True(t, u.Scheme == "https" && (u.Hostname() == "localhost" || (ip != nil && ip.IsLoopback())) && + u.User == nil && u.RawQuery == "" && u.Fragment == "" && (u.Path == "" || u.Path == "/"), + "%s must be an HTTPS loopback origin", name) + return u } func isDMSPath(path string) bool { return path == dmsAPIPath || strings.HasPrefix(path, dmsAPIPath+"/") } -func routeDMS(server *testserver.Server, handler testserver.HandlerFunc) { +func isLiteboxPath(path string) bool { + if isDMSPath(path) || + strings.HasPrefix(path, "/api/2.0/workspace/") || + strings.HasPrefix(path, "/api/2.0/workspace-files/") { + return true + } + for _, objectType := range liteboxPermissionTypes { + if strings.HasPrefix(path, "/api/2.0/permissions/"+objectType+"/") { + return true + } + } + return false +} + +func routeLitebox(server *testserver.Server, handler testserver.HandlerFunc) { + // Without a PATCH route, ServeMux returns 405 before Dispatch or NotFound. + for _, objectType := range liteboxPermissionTypes { + server.Handle(http.MethodPatch, "/api/2.0/permissions/"+objectType+"/{object_id}", handler) + } dispatch := server.Dispatch server.Dispatch = func(w http.ResponseWriter, r *http.Request, h testserver.HandlerFunc, vars map[string]string) { - if isDMSPath(r.URL.Path) { + if isLiteboxPath(r.URL.Path) { h = handler } dispatch(w, r, h, vars) } notFound := server.NotFound server.NotFound = func(w http.ResponseWriter, r *http.Request) { - if isDMSPath(r.URL.Path) { + if isLiteboxPath(r.URL.Path) { dispatch(w, r, handler, nil) return } From cc5f397dafcb4697693e213275abb7a750671d7a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 22 Sep 2026 14:21:44 +0000 Subject: [PATCH 06/10] Use standard environment reads for DMS acceptance configuration --- acceptance/acceptance_test.go | 4 ++++ acceptance/internal/dms_litebox.go | 14 +++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index a42143b3d25..19313e3b23d 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -495,6 +495,10 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo repls.SetPath(cwd, "[TESTROOT]") repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"}) + if os.Getenv("DMS_LITEBOX_URL") != "" { + // LITE's short TreeIDs need normalization without replacing version numbers. + repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile(`\bdeployments/[0-9]+\b`), New: "deployments/[NUMID]"}) + } // Matches defaultSparkVersion in ../integration/bundle/helpers_test.go t.Setenv("DEFAULT_SPARK_VERSION", "13.3.x-snapshot-scala2.12") diff --git a/acceptance/internal/dms_litebox.go b/acceptance/internal/dms_litebox.go index c0f513edaae..955eb288c4f 100644 --- a/acceptance/internal/dms_litebox.go +++ b/acceptance/internal/dms_litebox.go @@ -1,3 +1,4 @@ +//nolint:forbidigo // Acceptance infrastructure reads the test process environment. package internal import ( @@ -13,7 +14,6 @@ import ( "testing" "time" - "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testserver" "github.com/stretchr/testify/require" ) @@ -27,15 +27,15 @@ var liteboxPermissionTypes = []string{"directories", "files", "notebooks", "work // to its shared fakes, retaining request recording and mocks for other APIs. func configureDMSLitebox(t *testing.T, server *testserver.Server) { t.Helper() - endpoint := env.Get(t.Context(), "DMS_LITEBOX_URL") + endpoint := os.Getenv("DMS_LITEBOX_URL") if endpoint == "" { return } dmsURL := liteboxURL(t, "DMS_LITEBOX_URL") workspaceURL := liteboxURL(t, "DMS_LITEBOX_WORKSPACE_URL") - certFile, keyFile := env.Get(t.Context(), "DMS_LITEBOX_CERT"), env.Get(t.Context(), "DMS_LITEBOX_KEY") - caFile := env.Get(t.Context(), "DMS_LITEBOX_CA") + certFile, keyFile := os.Getenv("DMS_LITEBOX_CERT"), os.Getenv("DMS_LITEBOX_KEY") + caFile := os.Getenv("DMS_LITEBOX_CA") require.NotEmpty(t, certFile, "DMS_LITEBOX_CERT is required with DMS_LITEBOX_URL") require.NotEmpty(t, keyFile, "DMS_LITEBOX_KEY is required with DMS_LITEBOX_URL") require.NotEmpty(t, caFile, "DMS_LITEBOX_CA is required with DMS_LITEBOX_URL") @@ -79,8 +79,8 @@ func configureDMSLitebox(t *testing.T, server *testserver.Server) { // Fixed test identity; this adapter only connects to a local LITE fixture. req.Header.Set("X-Databricks-Org-Id", "456") req.Header.Set("X-Databricks-Workspace-Id", "456") - req.Header.Set("X-Databricks-User-Id", "123") - req.Header.Set("X-Databricks-User-Name", "alice@databricks.com") + req.Header.Set("X-Databricks-User-Id", testserver.TestUser.Id) + req.Header.Set("X-Databricks-User-Name", testserver.TestUser.UserName) resp, err := client.Do(req) if err != nil { return liteboxFailure(err) @@ -97,7 +97,7 @@ func configureDMSLitebox(t *testing.T, server *testserver.Server) { func liteboxURL(t *testing.T, name string) *url.URL { t.Helper() - endpoint := env.Get(t.Context(), name) + endpoint := os.Getenv(name) require.NotEmpty(t, endpoint, "%s is required with DMS_LITEBOX_URL", name) u, err := url.Parse(endpoint) require.NoError(t, err, "invalid %s", name) From 3fc3fae366b86c2c9c70ac7aaf08358c7dad158d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 22 Sep 2026 16:05:52 +0000 Subject: [PATCH 07/10] Use context environment lookup in Litebox acceptance helper --- acceptance/internal/dms_litebox.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/internal/dms_litebox.go b/acceptance/internal/dms_litebox.go index 955eb288c4f..10014dd0c01 100644 --- a/acceptance/internal/dms_litebox.go +++ b/acceptance/internal/dms_litebox.go @@ -1,4 +1,3 @@ -//nolint:forbidigo // Acceptance infrastructure reads the test process environment. package internal import ( @@ -14,6 +13,7 @@ import ( "testing" "time" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testserver" "github.com/stretchr/testify/require" ) @@ -27,15 +27,15 @@ var liteboxPermissionTypes = []string{"directories", "files", "notebooks", "work // to its shared fakes, retaining request recording and mocks for other APIs. func configureDMSLitebox(t *testing.T, server *testserver.Server) { t.Helper() - endpoint := os.Getenv("DMS_LITEBOX_URL") + endpoint := env.Get(t.Context(), "DMS_LITEBOX_URL") if endpoint == "" { return } dmsURL := liteboxURL(t, "DMS_LITEBOX_URL") workspaceURL := liteboxURL(t, "DMS_LITEBOX_WORKSPACE_URL") - certFile, keyFile := os.Getenv("DMS_LITEBOX_CERT"), os.Getenv("DMS_LITEBOX_KEY") - caFile := os.Getenv("DMS_LITEBOX_CA") + certFile, keyFile := env.Get(t.Context(), "DMS_LITEBOX_CERT"), env.Get(t.Context(), "DMS_LITEBOX_KEY") + caFile := env.Get(t.Context(), "DMS_LITEBOX_CA") require.NotEmpty(t, certFile, "DMS_LITEBOX_CERT is required with DMS_LITEBOX_URL") require.NotEmpty(t, keyFile, "DMS_LITEBOX_KEY is required with DMS_LITEBOX_URL") require.NotEmpty(t, caFile, "DMS_LITEBOX_CA is required with DMS_LITEBOX_URL") @@ -97,7 +97,7 @@ func configureDMSLitebox(t *testing.T, server *testserver.Server) { func liteboxURL(t *testing.T, name string) *url.URL { t.Helper() - endpoint := os.Getenv(name) + endpoint := env.Get(t.Context(), name) require.NotEmpty(t, endpoint, "%s is required with DMS_LITEBOX_URL", name) u, err := url.Parse(endpoint) require.NoError(t, err, "invalid %s", name) From b90b646ac9c63be198f774c0785c95b077ad98af Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 22 Sep 2026 17:01:09 +0000 Subject: [PATCH 08/10] Normalize Litebox deployment IDs inside recorded state --- acceptance/acceptance_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 19313e3b23d..98d91fad8e0 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -497,7 +497,10 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"}) if os.Getenv("DMS_LITEBOX_URL") != "" { // LITE's short TreeIDs need normalization without replacing version numbers. - repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile(`\bdeployments/[0-9]+\b`), New: "deployments/[NUMID]"}) + repls.Repls = append(repls.Repls, + testdiff.Replacement{Old: regexp.MustCompile(`\bdeployments/[0-9]+\b`), New: "deployments/[NUMID]"}, + testdiff.Replacement{Old: regexp.MustCompile(`(\\*"deployment_id\\*":\s*\\*")[0-9]+(\\*")`), New: "${1}[NUMID]${2}"}, + ) } // Matches defaultSparkVersion in ../integration/bundle/helpers_test.go From 1c4b6d4e8cf45dc16d05918faa56229f58e48a98 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 23 Sep 2026 10:48:52 +0000 Subject: [PATCH 09/10] Remove Litebox recorded-state normalization --- acceptance/acceptance_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 98d91fad8e0..aa25c369039 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -499,7 +499,6 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo // LITE's short TreeIDs need normalization without replacing version numbers. repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile(`\bdeployments/[0-9]+\b`), New: "deployments/[NUMID]"}, - testdiff.Replacement{Old: regexp.MustCompile(`(\\*"deployment_id\\*":\s*\\*")[0-9]+(\\*")`), New: "${1}[NUMID]${2}"}, ) } From c642700ec91694d5da72a7ec186460f628d6d6c0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 23 Sep 2026 11:02:57 +0000 Subject: [PATCH 10/10] Remove Litebox-specific acceptance replacements --- acceptance/acceptance_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index aa25c369039..a42143b3d25 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -495,12 +495,6 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo repls.SetPath(cwd, "[TESTROOT]") repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"}) - if os.Getenv("DMS_LITEBOX_URL") != "" { - // LITE's short TreeIDs need normalization without replacing version numbers. - repls.Repls = append(repls.Repls, - testdiff.Replacement{Old: regexp.MustCompile(`\bdeployments/[0-9]+\b`), New: "deployments/[NUMID]"}, - ) - } // Matches defaultSparkVersion in ../integration/bundle/helpers_test.go t.Setenv("DEFAULT_SPARK_VERSION", "13.3.x-snapshot-scala2.12")