From c4244ae39467cc1c59638031568cb8d3500ae24b Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:34:27 +0000 Subject: [PATCH] ssh: wait up to 5 minutes for the server to start After the bootstrap task reaches RUNNING, the client allowed only 60 seconds (30 retries x 2s) for the server to write metadata.json. A first-time bootstrap on a cold cluster regularly overran that and failed with a server metadata error, while an immediate retry succeeded. The wait already ends early when the run terminates, so the longer cap only applies while the run is alive. Co-authored-by: Isaac --- experimental/ssh/internal/client/client.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/experimental/ssh/internal/client/client.go b/experimental/ssh/internal/client/client.go index 2e651e68d8a..3ed72bbb266 100644 --- a/experimental/ssh/internal/client/client.go +++ b/experimental/ssh/internal/client/client.go @@ -53,6 +53,9 @@ const ( sshServerTaskKey = "start_ssh_server" serverlessEnvironmentKey = "ssh_tunnel_serverless" minEnvironmentVersion = 4 + + serverMetadataTimeout = 5 * time.Minute + serverMetadataPollInterval = 2 * time.Second ) // acceleratorProvisioningNotice maps a GPU accelerator type to the upfront notice @@ -1350,7 +1353,7 @@ func ensureSSHServerIsRunning(ctx context.Context, client *databricks.WorkspaceC sp := cmdio.NewSpinner(ctx, cmdio.WithElapsedTime()) defer sp.Close() sp.Update("Waiting for the SSH server to start...") - maxRetries := 30 + maxRetries := int(serverMetadataTimeout / serverMetadataPollInterval) for retries := range maxRetries { if ctx.Err() != nil { return "", 0, "", ctx.Err() @@ -1376,7 +1379,7 @@ func ensureSSHServerIsRunning(ctx context.Context, client *databricks.WorkspaceC return "", 0, "", fmt.Errorf("ssh server bootstrap job failed:\n%s", failure) } if retries < maxRetries-1 { - time.Sleep(2 * time.Second) + time.Sleep(serverMetadataPollInterval) } else { return "", 0, "", fmt.Errorf("failed to start the ssh server: %w\n%s", err, describeRunFailure(ctx, client, runID)) }