diff --git a/api/v1alpha1/nodeclaim_types.go b/api/v1alpha1/nodeclaim_types.go index e7478cf..c938f54 100644 --- a/api/v1alpha1/nodeclaim_types.go +++ b/api/v1alpha1/nodeclaim_types.go @@ -41,6 +41,17 @@ type NodeClaimSpec struct { // +optional Region string `json:"region,omitempty"` + // AcceleratorID is the provider's own identifier for what actually serves this + // claim (e.g. AWS "p5.48xlarge", RunPod "NVIDIA H100 80GB HBM3"), resolved at + // placement time from the Pod's requested accelerator type + count via the + // provider's MapAccelerator. Unlike Provider/CapacityType/Region it is NOT a + // provisioning input (the provider re-derives it from the Pod) — it is recorded + // for reporting, like PoolRef, so `kubectl get nc` shows the concrete SKU behind + // each instance without cross-referencing the Pod. Empty for a CPU-only claim, + // which requests no accelerator. + // +optional + AcceleratorID string `json:"acceleratorID,omitempty"` + // PoolRef is the NodePool whose policy produced this claim, for reporting. // +optional PoolRef string `json:"poolRef,omitempty"` @@ -139,9 +150,10 @@ type NodeClaimStatus struct { // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Provider",type=string,JSONPath=`.spec.provider` // +kubebuilder:printcolumn:name="Region",type=string,JSONPath=`.spec.region` -// +kubebuilder:printcolumn:name="Instance",type=string,JSONPath=`.status.instanceID` -// +kubebuilder:printcolumn:name="CapacityType",type=string,JSONPath=`.spec.capacityType` +// +kubebuilder:printcolumn:name="ACCELERATOR_ID",type=string,JSONPath=`.spec.acceleratorID` +// +kubebuilder:printcolumn:name="CAPACITY_TYPE",type=string,JSONPath=`.spec.capacityType` // +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Instance",type=string,JSONPath=`.status.instanceID` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // NodeClaim represents one external GPU instance and its lifecycle. diff --git a/config/crd/bases/nebula.inftyai.com_nodeclaims.yaml b/config/crd/bases/nebula.inftyai.com_nodeclaims.yaml index 595a39a..5cba3db 100644 --- a/config/crd/bases/nebula.inftyai.com_nodeclaims.yaml +++ b/config/crd/bases/nebula.inftyai.com_nodeclaims.yaml @@ -23,15 +23,18 @@ spec: - jsonPath: .spec.region name: Region type: string - - jsonPath: .status.instanceID - name: Instance + - jsonPath: .spec.acceleratorID + name: ACCELERATOR_ID type: string - jsonPath: .spec.capacityType - name: CapacityType + name: CAPACITY_TYPE type: string - jsonPath: .status.phase name: Phase type: string + - jsonPath: .status.instanceID + name: Instance + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -67,6 +70,17 @@ spec: ledger, not a spec: its reason to exist is to survive the Node so teardown can reclaim the external instance and never leak a paid GPU. properties: + acceleratorID: + description: |- + AcceleratorID is the provider's own identifier for what actually serves this + claim (e.g. AWS "p5.48xlarge", RunPod "NVIDIA H100 80GB HBM3"), resolved at + placement time from the Pod's requested accelerator type + count via the + provider's MapAccelerator. Unlike Provider/CapacityType/Region it is NOT a + provisioning input (the provider re-derives it from the Pod) — it is recorded + for reporting, like PoolRef, so `kubectl get nc` shows the concrete SKU behind + each instance without cross-referencing the Pod. Empty for a CPU-only claim, + which requests no accelerator. + type: string capacityType: description: |- CapacityType is the purchase tier the placement optimizer selected diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml index 1fac45a..f45239b 100644 --- a/config/samples/deployment.yaml +++ b/config/samples/deployment.yaml @@ -23,7 +23,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: gpu-workload-sample-l4 + name: gpu-workload-sample namespace: default labels: app.kubernetes.io/managed-by: nebula @@ -37,8 +37,8 @@ spec: labels: app: gpu-workload-sample nebula.inftyai.com/enabled: "true" - nebula.inftyai.com/nodepool: aws - nebula.inftyai.com/accelerator-type: l4 + nebula.inftyai.com/nodepool: sample + nebula.inftyai.com/accelerator-type: a100-40gb spec: # Do NOT set nodeName or a provider nodeSelector yourself — the placement # controller fills the nodeSelector in when it ungates the Pod. Setting @@ -65,4 +65,4 @@ spec: # GPU count. Standard extended resource, so the scheduler's fit check # and provisioning read the same number. 8 => 8x the accelerator-type # above. - nvidia.com/gpu: "1" + nvidia.com/gpu: "8" diff --git a/config/samples/nebula_v1alpha1_nodepool_neocloud.yaml b/config/samples/nebula_v1alpha1_nodepool.yaml similarity index 57% rename from config/samples/nebula_v1alpha1_nodepool_neocloud.yaml rename to config/samples/nebula_v1alpha1_nodepool.yaml index e9aafd5..e33ced8 100644 --- a/config/samples/nebula_v1alpha1_nodepool_neocloud.yaml +++ b/config/samples/nebula_v1alpha1_nodepool.yaml @@ -3,16 +3,25 @@ kind: NodePool metadata: labels: app.kubernetes.io/managed-by: nebula - name: neocloud + name: sample spec: - # The NeoClouds this pool may place onto. Order matters only for the Ordered # strategy (the inner, provider-ranking axis). providers: - name: modal + - name: aws + regions: + - us-east-1 + - us-west-1 + - ap-south-1 + - ap-northeast-1 + - eu-central-1 + - eu-west-1 + - ca-central-1 + - sa-east-1 + - name: modal # - name: runpod - # Outer axis: try Reserved on every provider first, fall back to on-demand, then spot. + # Outer axis: try OnDemand on every provider first, fall back to Spot. capacityTypes: - - Reserved - OnDemand - Spot # Inner axis: within the active capacity tier. diff --git a/config/samples/nebula_v1alpha1_nodepool_aws.yaml b/config/samples/nebula_v1alpha1_nodepool_aws.yaml deleted file mode 100644 index 87400d7..0000000 --- a/config/samples/nebula_v1alpha1_nodepool_aws.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: nebula.inftyai.com/v1alpha1 -kind: NodePool -metadata: - labels: - app.kubernetes.io/managed-by: nebula - name: aws -spec: - # Place onto AWS EC2. The provider name must match the ProviderLabel value on - # the AWS virtual node ("aws"); the adapter registers whenever the manager has - # credentials (IRSA / instance role / keys) — no region env is needed, the regions - # come from the `regions` field below. It self-configures the GPU AMI and subnets, - # so no other setup is needed. Otherwise the pool reports Ready=False/UnknownProvider. - providers: - - name: aws - # regions is per-provider and in AWS's OWN vocabulary. Unlike region-simple - # providers (Modal), AWS is region-aware with no meaningful default, so at - # least one region is REQUIRED here (enforced at admission). List more to - # widen the candidate set; prices are similar across regions, so which types - # are actually offered is resolved by the live probe. - regions: - - us-east-1 - - us-west-1 - - ap-south-1 - - ap-northeast-1 - - eu-central-1 - - eu-west-1 - - ca-central-1 - - sa-east-1 - # Outer axis: prefer cheap interruptible Spot, fall back to OnDemand when Spot - # capacity is exhausted. (EC2 has a real Spot tier with a ~2-minute notice; a - # Spot no-capacity failure blocks only Spot in that region, so OnDemand is still - # tried.) Reserved is omitted — EC2 Reserved is a billing construct, not a - # distinct provisioning path here. - capacityTypes: - - OnDemand - - Spot - # Inner axis: within the active capacity tier. A single-provider pool makes this - # a no-op, but LowestPrice is the natural choice once more AWS regions are listed. - strategy: Ordered - failover: - blocklistTTL: 10m diff --git a/internal/controller/pod_placement_controller.go b/internal/controller/pod_placement_controller.go index 09f8383..c6aafea 100644 --- a/internal/controller/pod_placement_controller.go +++ b/internal/controller/pod_placement_controller.go @@ -186,6 +186,10 @@ type placement struct { // Empty means the provider's configured default region; region-simple // providers leave it empty. region string + // acceleratorID is the provider's own identifier for what serves this request + // (e.g. AWS "p5.48xlarge"), resolved via MapAccelerator(type, count). Empty for + // a CPU-only Pod, which requests no accelerator. + acceleratorID string } // needsPlacement reports whether the Pod is an opted-in workload still held by diff --git a/internal/controller/pod_placement_controller_test.go b/internal/controller/pod_placement_controller_test.go index 23bb667..70faad7 100644 --- a/internal/controller/pod_placement_controller_test.go +++ b/internal/controller/pod_placement_controller_test.go @@ -181,6 +181,11 @@ func TestPlacement_UngatesAndRoutesAndCreatesClaim(t *testing.T) { if nc.Spec.Provider != provider.ProviderModal || nc.Spec.PodRef.UID != "uid-1" || nc.Spec.PoolRef != "pool-a" { t.Fatalf("unexpected claim spec: %+v", nc.Spec) } + // The provider's resolved accelerator id is recorded for reporting (the fake's + // MapAccelerator echoes the canonical type, so H100 -> "H100"). + if nc.Spec.AcceleratorID != "H100" { + t.Fatalf("expected claim to record acceleratorID H100, got %q", nc.Spec.AcceleratorID) + } } func TestPlacement_FirstMatchingProviderWins(t *testing.T) { @@ -246,6 +251,14 @@ func TestPlacement_CPUOnlyPodMatchesAnyProvider(t *testing.T) { if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != provider.ProviderModal { t.Fatalf("expected a CPU-only Pod to place on modal, got %v", got.Spec.NodeSelector) } + // A CPU-only claim requests no accelerator, so the id is left empty. + var nc nebulav1alpha1.NodeClaim + if err := c.Get(context.Background(), types.NamespacedName{Name: "default-p1"}, &nc); err != nil { + t.Fatalf("expected NodeClaim default-p1: %v", err) + } + if nc.Spec.AcceleratorID != "" { + t.Fatalf("expected empty acceleratorID for a CPU-only claim, got %q", nc.Spec.AcceleratorID) + } } func TestPlacement_SkipsPodWithoutOptInLabel(t *testing.T) { diff --git a/internal/controller/pod_placement_helpers.go b/internal/controller/pod_placement_helpers.go index 5e95a5d..9efd171 100644 --- a/internal/controller/pod_placement_helpers.go +++ b/internal/controller/pod_placement_helpers.go @@ -139,9 +139,10 @@ func (r *PodPlacementReconciler) selectPlacement(ctx context.Context, pod *corev log.Info("selected placement candidate", "provider", ref.Name, "capacityType", tier, "region", region) return placement{ - provider: ref.Name, - capacityType: tier, - region: region, + provider: ref.Name, + capacityType: tier, + region: region, + acceleratorID: acceleratorID, }, true, 0 } } @@ -234,10 +235,11 @@ func (r *PodPlacementReconciler) ensureClaim(ctx context.Context, pod *corev1.Po Name: pod.Name, UID: string(pod.UID), }, - Provider: p.provider, - CapacityType: p.capacityType, - Region: p.region, - PoolRef: pool.Name, + Provider: p.provider, + CapacityType: p.capacityType, + Region: p.region, + AcceleratorID: p.acceleratorID, + PoolRef: pool.Name, }, } err = r.Create(ctx, claim)