Support corss-regions provisioning in aws

api/v1alpha1/groupversion_info.go

@@ -100,6 +100,17 @@ const (
 	// default TTL.
 	BlocklistTTLAnnotation = "nebula.inftyai.com/blocklist-ttl"
 
+	// EndpointAnnotation carries the reachable address of the external instance
+	// once it is running (a public DNS name or IP, in the provider's own form).
+	// It is the ONLY way to reach the workload, so it must be visible on the Pod:
+	// PodIP cannot hold it because the API server validates PodIP as a literal IP
+	// and rejects a DNS name (the common AWS case), so the endpoint rides an
+	// annotation instead. Written by the virtual kubelet when it first observes the
+	// instance running; absent until then. Unlike the provisioning-input
+	// annotations above (which the placement controller stamps and VK reads), this
+	// flows the other way — VK writes it for operators/tooling to read.
+	EndpointAnnotation = "nebula.inftyai.com/endpoint"
+
 	// TerminateInstanceFinalizer is held by every NodeClaim to guarantee teardown.
 	// The virtual kubelet owns the happy path (DeletePod → provider.Terminate,
 	// keyed on the Pod-derived claim name), but its teardown is edge-triggered and

api/v1alpha1/nodeclaim_types.go

@@ -59,23 +59,35 @@ type PodReference struct {
 //
 // The NodeClaim is a passive teardown ledger, not a status mirror: it does NOT
 // track finer workload runtime status (CPU/logs/restarts) — the Pod is the
-// source of truth for that (see pkg/vnode/status.go). It tracks only the three
-// coarse states that matter to its own job as a ledger, keyed off the served
-// Pod's phase: Provisioning (instance coming up), Bound (instance running — the
-// guard the teardown backstop trusts), and Terminated (instance gone). Finer
-// states (e.g. Preempted) are deliberately absent: preemption cannot be detected
-// — the provider contract's InstanceState has no Preempted value, and an absent
-// instance only tells us it is gone, not why. Reintroduce a phase only when
-// something actually sets it.
+// source of truth for that (see pkg/vnode/status.go). It tracks only the coarse
+// states that matter to its own job as a ledger, keyed off the served Pod's
+// phase/reason: Provisioning (allocating — instance does not exist yet),
+// Initializing (instance exists and is booting but not yet reachable), Bound
+// (instance running — the guard the teardown backstop trusts), and Terminated
+// (instance gone). Finer states (e.g. Preempted) are deliberately absent:
+// preemption cannot be detected — the provider contract's InstanceState has no
+// Preempted value, and an absent instance only tells us it is gone, not why.
+// Reintroduce a phase only when something actually sets it.
+//
+// Only Bound is a teardown guard: neither Provisioning nor Initializing earns the
+// "trust a later disappearance" trust, because until the instance is confirmed up
+// an absent Pod may be cache lag rather than a real teardown.
 type NodeClaimPhase string
 
 const (
-	// NodeClaimProvisioning: the served Pod has been observed but is not yet
-	// running — the external instance is still being provisioned. The claim does
-	// NOT earn the Bound teardown guard here: a Pod that vanishes while still
+	// NodeClaimProvisioning: the served Pod has been observed but the external
+	// instance does not yet exist — provisioning is still allocating it. The claim
+	// does NOT earn the Bound teardown guard here: a Pod that vanishes while still
 	// provisioning is treated as possible cache lag (grace window), not a real
 	// teardown, because we never confirmed the instance was actually up.
 	NodeClaimProvisioning NodeClaimPhase = "Provisioning"
+	// NodeClaimInitializing: the external instance EXISTS at the provider but is not
+	// yet reachable (e.g. EC2 is "pending", or "running" but its 2/2 status checks
+	// have not passed). The served Pod is Pending with reason Initializing (see
+	// pkg/vnode/status.go). Like Provisioning it does NOT earn the Bound guard — the
+	// instance is not yet confirmed up — but it is surfaced as a distinct phase so
+	// "allocating" and "booting" are distinguishable on the ledger.
+	NodeClaimInitializing NodeClaimPhase = "Initializing"
 	// NodeClaimBound: the served Pod has been observed running (present and not in
 	// a terminal phase). This is the durable guard the backstop trusts — a Bound
 	// claim whose Pod later disappears is a real teardown, not cache lag. The claim
@@ -116,7 +128,9 @@ type NodeClaimStatus struct {
 // +kubebuilder:resource:scope=Cluster,shortName=nc
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:name="Provider",type=string,JSONPath=`.spec.provider`
+// +kubebuilder:printcolumn:name="Region",type=string,JSONPath=`.spec.region`
 // +kubebuilder:printcolumn:name="Instance",type=string,JSONPath=`.status.instanceID`
+// +kubebuilder:printcolumn:name="CapacityType",type=string,JSONPath=`.spec.capacityType`
 // +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
 

cmd/main.go

@@ -32,6 +32,7 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -218,8 +219,10 @@ func main() {
 	// Register provider backends into the process-wide registry that both
 	// reconcilers resolve through (their Providers field defaults to
 	// provider.Get). Done before SetupWithManager so a pool/claim reconciled at
-	// startup already sees its provider.
-	registerProviders(context.Background())
+	// startup already sees its provider. The manager's client backs the AWS region
+	// source (regions are read from NodePools at call time, not env), so it is
+	// threaded in; the client is only queried at runtime, after the cache has synced.
+	registerProviders(context.Background(), mgr.GetClient())
 
 	// One shared failover blocklist, written by the virtual kubelet handlers on a
 	// Provision failure and read by the placement controller to skip a candidate
@@ -329,23 +332,26 @@ func setupVirtualNodes(mgr ctrl.Manager, blocklist vnode.Blocklister) error {
 // still run for the providers that ARE configured, and a pool referencing an
 // unregistered provider surfaces as a clear NodePool condition rather than a
 // crash loop.
-func registerProviders(ctx context.Context) {
+func registerProviders(ctx context.Context, c client.Client) {
 	if p, err := modal.NewSDKClient(ctx, os.Getenv("MODAL_APP_NAME")); err != nil {
 		setupLog.Info("skipping Modal provider registration", "reason", err.Error())
 	} else {
 		provider.Register(p)
 		setupLog.Info("registered provider", "provider", p.Name())
 	}
 
-	// AWS. Region is the only NON-SECRET config, read from AWS_REGION (empty => the
-	// SDK resolves its default from shared config / IMDS). The adapter is otherwise
-	// self-configuring: it resolves the GPU AMI and default-VPC subnets itself, so
-	// no launch template or pre-created infra is needed. Credentials are secrets and
-	// are NEVER read here: the SDK client uses the default credential chain (IRSA /
-	// instance-role / AWS_ACCESS_KEY_ID delivered via a Secret), so nothing
-	// sensitive touches a flag or this call. When AWS is not configured (no
-	// resolvable region, no GPU AMI in the region), registration is a non-fatal skip.
-	if p, err := awsprovider.NewSDKClient(ctx, os.Getenv("AWS_REGION")); err != nil {
+	// AWS. There is NO region env/flag: the regions this provider may use are declared
+	// per-pool in the NodePool (ProviderSpec.Regions) and read at call time via the
+	// region source below, so a pool added at runtime widens the fan-out without a
+	// restart. One AWS provider spans every such region (per-region clients are built
+	// lazily). The adapter is otherwise self-configuring: it resolves each region's
+	// GPU AMI and default-VPC subnets itself, so no launch template or pre-created
+	// infra is needed. Credentials are secrets and are NEVER read here: the SDK client
+	// uses the default credential chain (IRSA / instance-role / AWS_ACCESS_KEY_ID
+	// delivered via a Secret), and one account-global credential authorizes every
+	// region. Registration only fails (and is a non-fatal skip) if the price catalog
+	// cannot load — region config can no longer make it fail.
+	if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c)); err != nil {
 		setupLog.Info("skipping AWS provider registration", "reason", err.Error())
 	} else {
 		provider.Register(p)
@@ -362,3 +368,36 @@ func registerProviders(ctx context.Context) {
 		setupLog.Info("registered provider", "provider", p.Name())
 	}
 }
+
+// awsRegionSource returns the AWS adapter's RegionSource: the union of
+// ProviderSpec.Regions across every NodePool that references the "aws" provider. It
+// needs no env/flag — regions are the operator's per-pool declaration — and a pool
+// added/edited at runtime widens the swept set on the next List tick, no restart
+// required.
+//
+// It is evaluated on each List/Offerings tick. The underlying List is served from
+// the manager's informer cache (no API call), so scanning the pools per tick is
+// cheap even though it is O(pools); sweepRegions dedupes the result. On a list error
+// (cache not yet synced at startup, a transient failure) it returns nil and
+// sweepRegions falls back to the regions already provisioned into. It uses a
+// background context, not the registration ctx, since it runs long after
+// registration returns.
+func awsRegionSource(c client.Client) awsprovider.RegionSource {
+	return func() []string {
+		var pools nebulav1alpha1.NodePoolList
+		if err := c.List(context.Background(), &pools); err != nil {
+			setupLog.V(1).Info("aws region source: list NodePools failed; sweeping provisioned regions only",
+				"reason", err.Error())
+			return nil
+		}
+		var regions []string
+		for i := range pools.Items {
+			for _, ps := range pools.Items[i].Spec.Providers {
+				if ps.Name == provider.ProviderAWS {
+					regions = append(regions, ps.Regions...)
+				}
+			}
+		}
+		return regions
+	}
+}

config/crd/bases/nebula.inftyai.com_nodeclaims.yaml

@@ -20,9 +20,15 @@ spec:
     - jsonPath: .spec.provider
       name: Provider
       type: string
+    - jsonPath: .spec.region
+      name: Region
+      type: string
     - jsonPath: .status.instanceID
       name: Instance
       type: string
+    - jsonPath: .spec.capacityType
+      name: CapacityType
+      type: string
     - jsonPath: .status.phase
       name: Phase
       type: string

config/manager/manager.yaml

@@ -72,14 +72,6 @@ spec:
           # embedded in the binary.
           - name: NEBULA_CATALOG_DIR
             value: /etc/nebula/catalog
-          # AWS NON-SECRET config (never a credential): the region to operate in.
-          # The adapter is otherwise self-configuring — it resolves the GPU AMI and
-          # the default VPC's subnets itself — so no launch template or other infra
-          # config is needed. This is a plain value here, NOT in a Secret. Empty =>
-          # the SDK resolves its default region (shared config / IMDS); with no
-          # resolvable region AWS registration is a non-fatal skip.
-          - name: AWS_REGION
-            value: "us-east-1"
         envFrom:
           # Provider credentials live in a per-provider Secret, one secretRef per
           # provider — NOT a single shared secret. This matches the "creds-absent →
@@ -96,8 +88,8 @@ spec:
           # AWS credentials (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY), read by
           # the SDK's default chain. optional=true so an absent Secret is fine —
           # in production this is the norm: prefer IRSA / instance role and skip
-          # this Secret entirely. Non-secret region/launch-template config is set
-          # as plain env above, never here.
+          # this Secret entirely. There is no non-secret AWS env: regions come from
+          # the NodePool, so this Secret is the ONLY AWS config here.
           - secretRef:
               name: nebula-aws-credentials
               optional: true

config/samples/deployment.yaml

@@ -23,7 +23,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: gpu-workload-sample
+  name: gpu-workload-sample-l4-8
   namespace: default
   labels:
     app.kubernetes.io/managed-by: nebula
@@ -65,4 +65,4 @@ spec:
             # GPU count. Standard extended resource, so the scheduler's fit check
             # and provisioning read the same number. 8 => 8x the accelerator-type
             # above.
-            nvidia.com/gpu: "1"
+            nvidia.com/gpu: "8"

config/samples/nebula_v1alpha1_nodepool_aws.yaml

@@ -6,10 +6,10 @@ metadata:
   name: aws
 spec:
   # Place onto AWS EC2. The provider name must match the ProviderLabel value on
-  # the AWS virtual node ("aws"); the adapter registers whenever the manager has a
-  # resolvable region (AWS_REGION or the SDK default chain) and credentials (IRSA /
-  # instance role / keys) — it self-configures the GPU AMI and subnets, so no other
-  # setup is needed. Otherwise the pool reports Ready=False/UnknownProvider.
+  # the AWS virtual node ("aws"); the adapter registers whenever the manager has
+  # credentials (IRSA / instance role / keys) — no region env is needed, the regions
+  # come from the `regions` field below. It self-configures the GPU AMI and subnets,
+  # so no other setup is needed. Otherwise the pool reports Ready=False/UnknownProvider.
   providers:
   - name: aws
     # regions is per-provider and in AWS's OWN vocabulary. Unlike region-simple
@@ -19,17 +19,23 @@ spec:
     # are actually offered is resolved by the live probe.
     regions:
     - us-east-1
-    # - us-west-2
+    - us-west-1
+    - ap-south-1
+    - ap-northeast-1
+    - eu-central-1
+    - eu-west-1
+    - ca-central-1
+    - sa-east-1
   # Outer axis: prefer cheap interruptible Spot, fall back to OnDemand when Spot
   # capacity is exhausted. (EC2 has a real Spot tier with a ~2-minute notice; a
   # Spot no-capacity failure blocks only Spot in that region, so OnDemand is still
   # tried.) Reserved is omitted — EC2 Reserved is a billing construct, not a
   # distinct provisioning path here.
   capacityTypes:
-  - Spot
   - OnDemand
+  - Spot
   # Inner axis: within the active capacity tier. A single-provider pool makes this
   # a no-op, but LowestPrice is the natural choice once more AWS regions are listed.
-  strategy: LowestPrice
+  strategy: Ordered
   failover:
     blocklistTTL: 10m