direct: auto-generate RemapState copiers from resource types

bundle/direct/dresources/README.md

@@ -75,8 +75,15 @@ Declaring a field under `hashed_fields` in `resources.yml` makes the engine pers
 
 ## RemapState is a dumb copy; DoRead owns all remapping
 
-`RemapState` converts `RemoteType` to `StateType` only because `StateType` is typically a
-subset of `RemoteType`. It must be a field-by-field copy (or no-op), never a place for
+New resources should not define `RemapState`. When `StateType` is a subset of `RemoteType`,
+the framework copies it automatically: `buildCopiers` (in `all.go`) compiles a copier per
+resource at package init, matching fields by JSON name, filtering `ForceSendFields`, and
+applying only lossless conversions. Anything it cannot copy safely (a kind-changing conversion,
+a mismatched struct shape) fails at load rather than as silent drift. Add a `RemapState` method
+only for logic the copier cannot express — derived fields, renames, per-field `ForceSendFields`
+rules — and the rest of this section governs those overrides.
+
+Such a `RemapState` must still be a field-by-field copy (or no-op), never a place for arbitrary
 logic. In particular, do not remap a differently-named field there (e.g. `state.x = remote.status.x`).
 Any remapping the API requires belongs in `DoRead`: add `x` directly to `RemoteType` and
 populate it from `status.x` inside `DoRead`.

bundle/direct/dresources/adapter.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/databricks/cli/bundle/deployplan"
 	"github.com/databricks/cli/libs/calladapt"
+	"github.com/databricks/cli/libs/structs/structcopy"
 	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/cli/libs/structs/structvar"
 	"github.com/databricks/databricks-sdk-go"
@@ -130,6 +131,7 @@ type Adapter struct {
 	doCreate     *calladapt.BoundCaller
 
 	// Optional:
+	copier             *structcopy.Copier
 	doDelete           *calladapt.BoundCaller
 	prepareInputConfig *calladapt.BoundCaller
 	isEmptyState       *calladapt.BoundCaller
@@ -169,6 +171,7 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC
 	adapter := &Adapter{
 		prepareState:            nil,
 		remapState:              nil,
+		copier:                  nil,
 		doRefresh:               nil,
 		doDelete:                nil,
 		doCreate:                nil,
@@ -233,11 +236,15 @@ func (a *Adapter) initMethods(resource any) error {
 		return err
 	}
 
-	// RemapState is optional when remote type already matches state type.
+	// RemapState is optional when remote type already matches state type, or when an
+	// auto-generated copier exists for this resource (see buildCopiers).
 	a.remapState, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "RemapState")
 	if err != nil {
 		return err
 	}
+	if a.remapState == nil {
+		a.copier = copiers[reflect.TypeOf(resource)]
+	}
 
 	a.doRefresh, err = prepareCallRequired(resource, "DoRead")
 	if err != nil {
@@ -363,15 +370,16 @@ func (a *Adapter) validate() error {
 		validations = append(validations, "DoDelete state", a.doDelete.InTypes[2], stateType)
 	}
 
-	// If RemapState is implemented, validate its signature.
-	// Otherwise require remote type to equal state type so remapping isn't needed.
+	// If RemapState is implemented, validate its signature. Otherwise the remote type
+	// must equal the state type (no remapping needed) or an auto-generated copier must
+	// exist for this resource (built and validated in buildCopiers).
 	if a.remapState != nil {
 		validations = append(
 			validations,
 			"RemapState input", a.remapState.InTypes[0], remoteType,
 			"RemapState return", a.remapState.OutTypes[0], stateType,
 		)
-	} else if remoteType != stateType {
+	} else if remoteType != stateType && a.copier == nil {
 		return fmt.Errorf("RemapState method not found and remote type %v must match state type %v", remoteType, stateType)
 	}
 
@@ -517,15 +525,19 @@ func (a *Adapter) PrepareState(input any) (any, error) {
 }
 
 func (a *Adapter) RemapState(remoteState any) (any, error) {
-	if a.remapState == nil {
-		return remoteState, nil
+	if a.remapState != nil {
+		outs, err := a.remapState.Call(remoteState)
+		if err != nil {
+			return nil, err
+		}
+		return outs[0], nil
 	}
-
-	outs, err := a.remapState.Call(remoteState)
-	if err != nil {
-		return nil, err
+	if a.copier != nil {
+		return a.copier.Copy(remoteState), nil
 	}
-	return outs[0], nil
+	// No custom method and no copier: validate() only allows this when
+	// remoteType == stateType, so the remote is already the state type.
+	return remoteState, nil
 }
 
 func (a *Adapter) DoRead(ctx context.Context, id string) (any, error) {

bundle/direct/dresources/all.go

@@ -2,7 +2,12 @@ package dresources
 
 import (
 	"fmt"
+	"reflect"
+	"slices"
+	"strings"
 
+	"github.com/databricks/cli/libs/calladapt"
+	"github.com/databricks/cli/libs/structs/structcopy"
 	"github.com/databricks/databricks-sdk-go"
 )
 
@@ -81,6 +86,66 @@ var SupportedResources = map[string]any{
 	"internal_immutable_snapshots": (*ResourceSnapshot)(nil),
 }
 
+// copiers holds an auto-generated RemapState copier for every resource whose
+// remote type differs from its state type and that does not supply a custom RemapState.
+// It is built once, at package initialization, from SupportedResources — so a resource
+// whose types cannot be safely copied fails at load, not at deploy time, and no separate
+// test can forget to cover it. Resources that need real remapping logic keep a custom
+// RemapState method and are skipped here.
+var copiers = buildCopiers()
+
+func buildCopiers() map[reflect.Type]*structcopy.Copier {
+	iface := reflect.TypeFor[IResource]()
+	out := make(map[reflect.Type]*structcopy.Copier)
+	var errs []string
+
+	for resourceType, resource := range SupportedResources {
+		implType := reflect.TypeOf(resource)
+		if _, done := out[implType]; done {
+			continue // same implementation registered under several keys (permissions, grants)
+		}
+
+		remap, err := calladapt.PrepareCall(resource, iface, "RemapState")
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("%s: RemapState: %v", resourceType, err))
+			continue
+		}
+		if remap != nil {
+			continue // custom override
+		}
+
+		prepareState, err := calladapt.PrepareCall(resource, iface, "PrepareState")
+		if err != nil || prepareState == nil {
+			errs = append(errs, fmt.Sprintf("%s: PrepareState: %v", resourceType, err))
+			continue
+		}
+		doRead, err := calladapt.PrepareCall(resource, iface, "DoRead")
+		if err != nil || doRead == nil {
+			errs = append(errs, fmt.Sprintf("%s: DoRead: %v", resourceType, err))
+			continue
+		}
+
+		stateType := prepareState.OutTypes[0]
+		remoteType := doRead.OutTypes[0]
+		if remoteType == stateType {
+			continue // identity: the adapter returns the remote unchanged, no copier needed
+		}
+
+		copier, err := structcopy.Compile(remoteType, stateType)
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("%s: %v (implement RemapState for this resource)", resourceType, err))
+			continue
+		}
+		out[implType] = copier
+	}
+
+	if len(errs) > 0 {
+		slices.Sort(errs)
+		panic("dresources: cannot build RemapState copiers:\n" + strings.Join(errs, "\n"))
+	}
+	return out
+}
+
 func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) {
 	result := make(map[string]*Adapter)
 	for resourceType, resource := range SupportedResources {

bundle/direct/dresources/app.go

@@ -73,18 +73,8 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState {
 	return s
 }
 
-// RemapState maps the remote AppRemote to AppState for diff comparison.
 // DoRead populates config, git_source, and source_code_path from the active
 // deployment when one exists, enabling drift detection for out-of-band redeploys.
-// Started is derived from compute status so the planner can detect start/stop changes.
-func (*ResourceApp) RemapState(remote *AppRemote) *AppState {
-	return &AppState{
-		App:       remote.App,
-		Config:    remote.Config,
-		Lifecycle: remote.Lifecycle,
-	}
-}
-
 func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) {
 	app, err := r.client.Apps.GetByName(ctx, id)
 	if err != nil {

bundle/direct/dresources/catalog.go

@@ -22,22 +22,6 @@ func (*ResourceCatalog) PrepareState(input *resources.Catalog) *catalog.CreateCa
 	return &input.CreateCatalog
 }
 
-func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalog.CreateCatalog {
-	return &catalog.CreateCatalog{
-		Comment:                   info.Comment,
-		ConnectionName:            info.ConnectionName,
-		CustomMaxRetentionHours:   info.CustomMaxRetentionHours,
-		ManagedEncryptionSettings: info.ManagedEncryptionSettings,
-		Name:                      info.Name,
-		Options:                   info.Options,
-		Properties:                info.Properties,
-		ProviderName:              info.ProviderName,
-		ShareName:                 info.ShareName,
-		StorageRoot:               info.StorageRoot,
-		ForceSendFields:           utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields),
-	}
-}
-
 func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.CatalogInfo, error) {
 	return r.client.Catalogs.GetByName(ctx, id)
 }

bundle/direct/dresources/cluster.go

@@ -103,54 +103,6 @@ func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState {
 	return s
 }
 
-// RemapState maps the remote ClusterRemote to ClusterState for diff comparison.
-// Started is derived from cluster state so the planner can detect start/stop changes.
-func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState {
-	spec := &ClusterState{
-		ClusterSpec: compute.ClusterSpec{
-			ApplyPolicyDefaultValues:   input.ApplyPolicyDefaultValues,
-			Autoscale:                  input.Autoscale,
-			AutoterminationMinutes:     input.AutoterminationMinutes,
-			AwsAttributes:              input.AwsAttributes,
-			AzureAttributes:            input.AzureAttributes,
-			ClusterLogConf:             input.ClusterLogConf,
-			ClusterName:                input.ClusterName,
-			CustomTags:                 input.CustomTags,
-			DataSecurityMode:           input.DataSecurityMode,
-			DependencyMode:             input.DependencyMode,
-			DockerImage:                input.DockerImage,
-			DriverInstancePoolId:       input.DriverInstancePoolId,
-			DriverNodeTypeId:           input.DriverNodeTypeId,
-			DriverNodeTypeFlexibility:  input.DriverNodeTypeFlexibility,
-			EnableElasticDisk:          input.EnableElasticDisk,
-			EnableLocalDiskEncryption:  input.EnableLocalDiskEncryption,
-			GcpAttributes:              input.GcpAttributes,
-			InitScripts:                input.InitScripts,
-			InstancePoolId:             input.InstancePoolId,
-			IsSingleNode:               input.IsSingleNode,
-			Kind:                       input.Kind,
-			NodeTypeId:                 input.NodeTypeId,
-			NumWorkers:                 input.NumWorkers,
-			PolicyId:                   input.PolicyId,
-			RemoteDiskThroughput:       input.RemoteDiskThroughput,
-			RuntimeEngine:              input.RuntimeEngine,
-			SingleUserName:             input.SingleUserName,
-			SparkConf:                  input.SparkConf,
-			SparkEnvVars:               input.SparkEnvVars,
-			SparkVersion:               input.SparkVersion,
-			SshPublicKeys:              input.SshPublicKeys,
-			TotalInitialRemoteDiskSize: input.TotalInitialRemoteDiskSize,
-			UseMlRuntime:               input.UseMlRuntime,
-			WorkloadType:               input.WorkloadType,
-			WorkerNodeTypeFlexibility:  input.WorkerNodeTypeFlexibility,
-			ForceSendFields:            utils.FilterFields[compute.ClusterSpec](input.ForceSendFields),
-		},
-		Lifecycle: input.Lifecycle,
-		Libraries: input.Libraries,
-	}
-	return spec
-}
-
 func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote, error) {
 	var details *compute.ClusterDetails
 	var libraries []compute.Library

bundle/direct/dresources/cluster_policy.go

@@ -30,19 +30,6 @@ func (*ResourceClusterPolicy) PrepareState(input *resources.ClusterPolicy) *comp
 }
 
 // RemapState copies the config fields shared by Policy and CreatePolicy;
-// output-only fields (policy_id, created_at_timestamp, creator_user_name, is_default) are not in the state.
-func (*ResourceClusterPolicy) RemapState(remote *compute.Policy) *compute.CreatePolicy {
-	return &compute.CreatePolicy{
-		Definition:                      remote.Definition,
-		Description:                     remote.Description,
-		Libraries:                       remote.Libraries,
-		MaxClustersPerUser:              remote.MaxClustersPerUser,
-		Name:                            remote.Name,
-		PolicyFamilyDefinitionOverrides: remote.PolicyFamilyDefinitionOverrides,
-		PolicyFamilyId:                  remote.PolicyFamilyId,
-		ForceSendFields:                 utils.FilterFields[compute.CreatePolicy](remote.ForceSendFields),
-	}
-}
 
 func (r *ResourceClusterPolicy) DoRead(ctx context.Context, id string) (*compute.Policy, error) {
 	return r.client.ClusterPolicies.GetByPolicyId(ctx, id)

bundle/direct/dresources/copier_test.go

@@ -0,0 +1,88 @@
+package dresources
+
+import (
+	"reflect"
+	"slices"
+	"testing"
+
+	"github.com/databricks/cli/libs/structs/structcopy"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestNoRedundantRemapState enforces the "only override when needed" rule: a resource that
+// keeps a hand-written RemapState must actually do something the auto-generated copier cannot.
+// For every resource that still has a RemapState whose remote and state types differ, it fully
+// populates a remote value and fails if the copier would produce an identical result — meaning
+// the method is dead boilerplate that should be deleted (see buildCopiers).
+func TestNoRedundantRemapState(t *testing.T) {
+	seen := map[reflect.Type]bool{}
+	var redundant []string
+
+	for resourceType, resource := range SupportedResources {
+		implType := reflect.TypeOf(resource)
+		if seen[implType] {
+			continue
+		}
+		seen[implType] = true
+
+		m := reflect.ValueOf(resource).MethodByName("RemapState")
+		if !m.IsValid() {
+			continue
+		}
+		remoteType := m.Type().In(0)
+		stateType := m.Type().Out(0)
+		if remoteType == stateType {
+			continue // identity, the copier is not involved
+		}
+
+		copier, err := structcopy.Compile(remoteType, stateType)
+		if err != nil {
+			continue // copier cannot handle it, so the override is required
+		}
+
+		remote := reflect.New(remoteType.Elem())
+		fillNonZero(remote.Elem(), 0)
+		setForceSendFields(remote.Elem())
+
+		want := m.Call([]reflect.Value{remote})[0].Interface()
+		got := copier.Copy(remote.Interface())
+		if reflect.DeepEqual(want, got) {
+			redundant = append(redundant, resourceType)
+		}
+	}
+
+	slices.Sort(redundant)
+	assert.Empty(t, redundant, "these resources have a RemapState the auto-copier reproduces exactly; delete the method and let buildCopiers handle it")
+}
+
+// setForceSendFields recursively sets every struct's ForceSendFields to its own field names,
+// so the copier's ForceSendFields-filtering path is exercised at every level. fillNonZero
+// deliberately leaves ForceSendFields empty (round-trip tests need that), so we populate it
+// here. Map values are not addressable and cannot hold a relevant FSF source, so they are skipped.
+func setForceSendFields(v reflect.Value) {
+	switch v.Kind() {
+	case reflect.Pointer:
+		if !v.IsNil() {
+			setForceSendFields(v.Elem())
+		}
+	case reflect.Slice:
+		for i := range v.Len() {
+			setForceSendFields(v.Index(i))
+		}
+	case reflect.Struct:
+		var names []string
+		for i := range v.NumField() {
+			sf := v.Type().Field(i)
+			if !sf.IsExported() || sf.Name == "ForceSendFields" {
+				continue
+			}
+			names = append(names, sf.Name)
+			setForceSendFields(v.Field(i))
+		}
+		if f := v.FieldByName("ForceSendFields"); f.IsValid() && f.Kind() == reflect.Slice && f.Type().Elem().Kind() == reflect.String {
+			f.Set(reflect.ValueOf(names))
+		}
+	default:
+		// scalars and other kinds have no nested ForceSendFields to set
+	}
+}