Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions api/v1alpha1/nodeclaim_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,17 @@ type NodeClaimSpec struct {
// +optional
Region string `json:"region,omitempty"`

// AcceleratorID is the provider's own identifier for what actually serves this
// claim (e.g. AWS "p5.48xlarge", RunPod "NVIDIA H100 80GB HBM3"), resolved at
// placement time from the Pod's requested accelerator type + count via the
// provider's MapAccelerator. Unlike Provider/CapacityType/Region it is NOT a
// provisioning input (the provider re-derives it from the Pod) — it is recorded
// for reporting, like PoolRef, so `kubectl get nc` shows the concrete SKU behind
// each instance without cross-referencing the Pod. Empty for a CPU-only claim,
// which requests no accelerator.
// +optional
AcceleratorID string `json:"acceleratorID,omitempty"`

// PoolRef is the NodePool whose policy produced this claim, for reporting.
// +optional
PoolRef string `json:"poolRef,omitempty"`
Expand Down Expand Up @@ -139,9 +150,10 @@ type NodeClaimStatus struct {
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="Provider",type=string,JSONPath=`.spec.provider`
// +kubebuilder:printcolumn:name="Region",type=string,JSONPath=`.spec.region`
// +kubebuilder:printcolumn:name="Instance",type=string,JSONPath=`.status.instanceID`
// +kubebuilder:printcolumn:name="CapacityType",type=string,JSONPath=`.spec.capacityType`
// +kubebuilder:printcolumn:name="ACCELERATOR_ID",type=string,JSONPath=`.spec.acceleratorID`
// +kubebuilder:printcolumn:name="CAPACITY_TYPE",type=string,JSONPath=`.spec.capacityType`
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
// +kubebuilder:printcolumn:name="Instance",type=string,JSONPath=`.status.instanceID`
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`

// NodeClaim represents one external GPU instance and its lifecycle.
Expand Down
20 changes: 17 additions & 3 deletions config/crd/bases/nebula.inftyai.com_nodeclaims.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,18 @@ spec:
- jsonPath: .spec.region
name: Region
type: string
- jsonPath: .status.instanceID
name: Instance
- jsonPath: .spec.acceleratorID
name: ACCELERATOR_ID
type: string
- jsonPath: .spec.capacityType
name: CapacityType
name: CAPACITY_TYPE
type: string
- jsonPath: .status.phase
name: Phase
type: string
- jsonPath: .status.instanceID
name: Instance
type: string
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
Expand Down Expand Up @@ -67,6 +70,17 @@ spec:
ledger, not a spec: its reason to exist is to survive the Node so teardown can
reclaim the external instance and never leak a paid GPU.
properties:
acceleratorID:
description: |-
AcceleratorID is the provider's own identifier for what actually serves this
claim (e.g. AWS "p5.48xlarge", RunPod "NVIDIA H100 80GB HBM3"), resolved at
placement time from the Pod's requested accelerator type + count via the
provider's MapAccelerator. Unlike Provider/CapacityType/Region it is NOT a
provisioning input (the provider re-derives it from the Pod) — it is recorded
for reporting, like PoolRef, so `kubectl get nc` shows the concrete SKU behind
each instance without cross-referencing the Pod. Empty for a CPU-only claim,
which requests no accelerator.
type: string
capacityType:
description: |-
CapacityType is the purchase tier the placement optimizer selected
Expand Down
8 changes: 4 additions & 4 deletions config/samples/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: gpu-workload-sample-l4
name: gpu-workload-sample
namespace: default
labels:
app.kubernetes.io/managed-by: nebula
Expand All @@ -37,8 +37,8 @@ spec:
labels:
app: gpu-workload-sample
nebula.inftyai.com/enabled: "true"
nebula.inftyai.com/nodepool: aws
nebula.inftyai.com/accelerator-type: l4
nebula.inftyai.com/nodepool: sample
nebula.inftyai.com/accelerator-type: a100-40gb
spec:
# Do NOT set nodeName or a provider nodeSelector yourself — the placement
# controller fills the nodeSelector in when it ungates the Pod. Setting
Expand All @@ -65,4 +65,4 @@ spec:
# GPU count. Standard extended resource, so the scheduler's fit check
# and provisioning read the same number. 8 => 8x the accelerator-type
# above.
nvidia.com/gpu: "1"
nvidia.com/gpu: "8"
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,25 @@ kind: NodePool
metadata:
labels:
app.kubernetes.io/managed-by: nebula
name: neocloud
name: sample
spec:
# The NeoClouds this pool may place onto. Order matters only for the Ordered
# strategy (the inner, provider-ranking axis).
providers:
- name: modal
- name: aws
regions:
- us-east-1
- us-west-1
- ap-south-1
- ap-northeast-1
- eu-central-1
- eu-west-1
- ca-central-1
- sa-east-1
- name: modal
# - name: runpod
# Outer axis: try Reserved on every provider first, fall back to on-demand, then spot.
# Outer axis: try OnDemand on every provider first, fall back to Spot.
capacityTypes:
- Reserved
- OnDemand
- Spot
# Inner axis: within the active capacity tier.
Expand Down
41 changes: 0 additions & 41 deletions config/samples/nebula_v1alpha1_nodepool_aws.yaml

This file was deleted.

4 changes: 4 additions & 0 deletions internal/controller/pod_placement_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,10 @@ type placement struct {
// Empty means the provider's configured default region; region-simple
// providers leave it empty.
region string
// acceleratorID is the provider's own identifier for what serves this request
// (e.g. AWS "p5.48xlarge"), resolved via MapAccelerator(type, count). Empty for
// a CPU-only Pod, which requests no accelerator.
acceleratorID string
}

// needsPlacement reports whether the Pod is an opted-in workload still held by
Expand Down
13 changes: 13 additions & 0 deletions internal/controller/pod_placement_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,11 @@ func TestPlacement_UngatesAndRoutesAndCreatesClaim(t *testing.T) {
if nc.Spec.Provider != provider.ProviderModal || nc.Spec.PodRef.UID != "uid-1" || nc.Spec.PoolRef != "pool-a" {
t.Fatalf("unexpected claim spec: %+v", nc.Spec)
}
// The provider's resolved accelerator id is recorded for reporting (the fake's
// MapAccelerator echoes the canonical type, so H100 -> "H100").
if nc.Spec.AcceleratorID != "H100" {
t.Fatalf("expected claim to record acceleratorID H100, got %q", nc.Spec.AcceleratorID)
}
}

func TestPlacement_FirstMatchingProviderWins(t *testing.T) {
Expand Down Expand Up @@ -246,6 +251,14 @@ func TestPlacement_CPUOnlyPodMatchesAnyProvider(t *testing.T) {
if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != provider.ProviderModal {
t.Fatalf("expected a CPU-only Pod to place on modal, got %v", got.Spec.NodeSelector)
}
// A CPU-only claim requests no accelerator, so the id is left empty.
var nc nebulav1alpha1.NodeClaim
if err := c.Get(context.Background(), types.NamespacedName{Name: "default-p1"}, &nc); err != nil {
t.Fatalf("expected NodeClaim default-p1: %v", err)
}
if nc.Spec.AcceleratorID != "" {
t.Fatalf("expected empty acceleratorID for a CPU-only claim, got %q", nc.Spec.AcceleratorID)
}
}

func TestPlacement_SkipsPodWithoutOptInLabel(t *testing.T) {
Expand Down
16 changes: 9 additions & 7 deletions internal/controller/pod_placement_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,10 @@ func (r *PodPlacementReconciler) selectPlacement(ctx context.Context, pod *corev
log.Info("selected placement candidate",
"provider", ref.Name, "capacityType", tier, "region", region)
return placement{
provider: ref.Name,
capacityType: tier,
region: region,
provider: ref.Name,
capacityType: tier,
region: region,
acceleratorID: acceleratorID,
}, true, 0
}
}
Expand Down Expand Up @@ -234,10 +235,11 @@ func (r *PodPlacementReconciler) ensureClaim(ctx context.Context, pod *corev1.Po
Name: pod.Name,
UID: string(pod.UID),
},
Provider: p.provider,
CapacityType: p.capacityType,
Region: p.region,
PoolRef: pool.Name,
Provider: p.provider,
CapacityType: p.capacityType,
Region: p.region,
AcceleratorID: p.acceleratorID,
PoolRef: pool.Name,
},
}
err = r.Create(ctx, claim)
Expand Down
Loading