feat: support aws

.dockerignore

@@ -1,3 +1,11 @@
 # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
 # Ignore build and test binaries.
 bin/
+
+# Secrets must never enter the build context, so they can never be baked into an
+# image layer even if the Dockerfile later switches to a broad `COPY . .`. The
+# manager reads credentials from the environment at runtime (per-provider
+# Secrets), never from a file in the image.
+.env
+.env.*
+!.env.example

.env.example

@@ -16,6 +16,19 @@
 MODAL_TOKEN_ID=
 MODAL_TOKEN_SECRET=
 
+# --- AWS provider ----------------------------------------------------------
+# SECRETS ONLY. In production prefer IRSA / instance role and leave these blank
+# (the SDK's default credential chain finds the role) — the AWS secret is then
+# skipped, which is fine. Set them only for local/dev without a role. Both keys
+# are required together; leave both blank to skip.
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+# OPTIONAL. Required ONLY for temporary STS/SSO credentials (ASIA... access keys),
+# which are invalid without it; leave blank for long-lived IAM user keys
+# (AKIA...). Note temporary creds expire (minutes–hours) — fine for a quick test,
+# not a long-running deployment.
+AWS_SESSION_TOKEN=
+
 # --- Additional providers (add as adapters land) ---------------------------
 # Each provider gets its OWN secret (see hack/deploy.sh PROVIDER_SECRETS), e.g.:
 # RUNPOD_API_KEY=

api/v1alpha1/groupversion_info.go

@@ -82,6 +82,24 @@ const (
 	// default" (e.g. Modal is OnDemand-only and ignores it).
 	CapacityTypeAnnotation = "nebula.inftyai.com/capacity-type"
 
+	// RegionAnnotation carries the optimizer-chosen provider region. Like
+	// CapacityTypeAnnotation it is a provisioning input absent from the Pod's own
+	// spec, so the placement controller writes it when it ungates the Pod and the
+	// virtual kubelet reads it back on CreatePod into ProvisionRequest.Region.
+	// Empty/absent means "the provider's configured default region" — region-simple
+	// providers (Modal, RunPod) ignore it.
+	RegionAnnotation = "nebula.inftyai.com/region"
+
+	// BlocklistTTLAnnotation carries the pool's FailoverPolicy.BlocklistTTL down to
+	// the virtual kubelet. Like the two annotations above it is a provisioning-time
+	// input the Pod cannot otherwise express: the TTL is a NodePool policy, but the
+	// VK handler (which provisions per-Pod and never sees the pool) needs it to know
+	// how long to blocklist a placement that just failed. The placement controller
+	// stamps it when it ungates the Pod; the handler reads it on a Provision failure
+	// to bound the block it records. Absent/unparseable means the handler's built-in
+	// default TTL.
+	BlocklistTTLAnnotation = "nebula.inftyai.com/blocklist-ttl"
+
 	// TerminateInstanceFinalizer is held by every NodeClaim to guarantee teardown.
 	// The virtual kubelet owns the happy path (DeletePod → provider.Terminate,
 	// keyed on the Pod-derived claim name), but its teardown is edge-triggered and

api/v1alpha1/nodeclaim_types.go

@@ -31,6 +31,16 @@ type NodeClaimSpec struct {
 	// +optional
 	CapacityType CapacityType `json:"capacityType,omitempty"`
 
+	// Region is the provider region the placement optimizer selected, in the
+	// provider's own vocabulary (e.g. AWS "us-east-1"). Stored durably alongside
+	// Provider/CapacityType because it is a provisioning input that cannot be read
+	// off the Pod, and Provision needs it to re-issue the request in the same
+	// region after a controller restart. Immutable, like Provider. Empty means
+	// "the provider's configured default region" — region-simple providers (Modal,
+	// RunPod) leave it empty.
+	// +optional
+	Region string `json:"region,omitempty"`
+
 	// PoolRef is the NodePool whose policy produced this claim, for reporting.
 	// +optional
 	PoolRef string `json:"poolRef,omitempty"`

api/v1alpha1/nodepool_types.go

@@ -12,12 +12,18 @@ import (
 // fixed: capacity type first, provider second.
 //
 //	FOR each capacityType in CapacityTypes (in listed order):   // outer: hard tier
-//	    candidates = Providers x {this capacityType}, available now, minus blocklist
+//	    candidates = Providers x (each provider's Regions) x {this capacityType},
+//	                 available now, minus blocklist                // region nests per provider
 //	    IF candidates non-empty:
-//	        pick one via Strategy (LowestPrice | Ordered | Weighted) // inner: rank providers
+//	        pick one via Strategy (LowestPrice | Ordered | Weighted) // inner: rank candidates
 //	        DONE
 //	    // else fall through to the next capacity tier
 //
+// Region is a per-provider axis (see ProviderSpec.Regions), nested under each
+// provider because a region name only means something to one provider. It
+// widens the candidate key to {provider, region, accelerator, capacityType}
+// without changing the tier-first ordering above.
+//
 // So CapacityTypes is a hard preference: every provider's Spot is tried before
 // ANY provider's OnDemand. This is deliberate — "spot everywhere before any
 // on-demand" is the least surprising behaviour, even if a different provider's
@@ -28,13 +34,14 @@ import (
 // static property of the spec, so it is enforced at admission by the CEL rule
 // below rather than surfaced as a status condition after the fact.
 // +kubebuilder:validation:XValidation:rule="self.strategy != 'Weighted' || self.providers.all(p, has(p.weight))",message="strategy Weighted requires a weight on every provider"
+// +kubebuilder:validation:XValidation:rule="self.providers.all(p, p.name != 'aws' || (has(p.regions) && size(p.regions) > 0))",message="provider aws requires at least one region"
 type NodePoolSpec struct {
 	// Providers is the ordered set of NeoClouds this pool is allowed to use.
 	// A Pod bound to this pool can only ever be placed on a provider in this
 	// list. Order is significant only for the Ordered strategy (it is the
 	// inner, provider-ranking axis).
 	// +kubebuilder:validation:MinItems=1
-	Providers []ProviderRef `json:"providers"`
+	Providers []ProviderSpec `json:"providers"`
 
 	// CapacityTypes is the OUTER axis: the purchase models to try, in fallback
 	// order. e.g. [Spot, OnDemand] means "use spot on any provider first; only
@@ -57,8 +64,10 @@ type NodePoolSpec struct {
 	Failover *FailoverPolicy `json:"failover,omitempty"`
 }
 
-// ProviderRef names a provider and, for the Weighted strategy, its share.
-type ProviderRef struct {
+// ProviderSpec is one provider's entry in a pool: which provider, and the
+// per-provider placement policy (Weighted share, allowed regions). It is not a
+// mere reference — it carries config — so it is a Spec, not a Ref.
+type ProviderSpec struct {
 	// Name is the provider identifier, matching the ProviderLabel value on that
 	// provider's virtual node (e.g. "runpod", "modal", "kubernetes").
 	Name string `json:"name"`
@@ -68,6 +77,29 @@ type ProviderRef struct {
 	// +kubebuilder:validation:Minimum=1
 	// +optional
 	Weight *int32 `json:"weight,omitempty"`
+
+	// Regions constrains this provider to a subset of its regions, in the
+	// provider's OWN vocabulary (e.g. ["us-east-1","eu-west-2"] for AWS). Region
+	// is provider-namespaced — there is no cross-provider region vocabulary — so
+	// it lives here per provider, not on the pool. Two cases:
+	//   - omitted/empty => the provider's configured default region (the region
+	//     its client resolved from env/config/instance metadata at startup). This
+	//     is the no-surprise default for region-simple providers (Modal, RunPod),
+	//     which have a single region and ignore this field.
+	//   - explicit list => exactly those regions.
+	// AWS is the exception: it is region-aware with no meaningful single default,
+	// so a `- name: aws` entry MUST list at least one region. That is enforced at
+	// admission by the CEL rule on NodePoolSpec (a per-provider requirement, so it
+	// belongs on the spec where all provider entries are visible, not as a blanket
+	// MinItems that would burden region-simple providers). An "all regions"
+	// wildcard is intentionally NOT supported yet: it only makes sense once the
+	// price-ranking optimizer can expand it against the provider's catalog and
+	// choose among the results, so it is reserved for then. At most 8 regions;
+	// maxLength bounds each entry.
+	// +optional
+	// +kubebuilder:validation:MaxItems=8
+	// +kubebuilder:validation:items:MaxLength=32
+	Regions []string `json:"regions,omitempty"`
 }
 
 // PlacementStrategy ranks providers WITHIN a capacity tier (the inner axis).

cmd/main.go

@@ -45,7 +45,9 @@ import (
 	nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1"
 	"github.com/InftyAI/Nebula/internal/controller"
 	webhookv1 "github.com/InftyAI/Nebula/internal/webhook/v1"
+	"github.com/InftyAI/Nebula/pkg/failover"
 	"github.com/InftyAI/Nebula/pkg/provider"
+	awsprovider "github.com/InftyAI/Nebula/pkg/provider/aws"
 	"github.com/InftyAI/Nebula/pkg/provider/fake"
 	"github.com/InftyAI/Nebula/pkg/provider/modal"
 	"github.com/InftyAI/Nebula/pkg/vnode"
@@ -219,6 +221,13 @@ func main() {
 	// startup already sees its provider.
 	registerProviders(context.Background())
 
+	// One shared failover blocklist, written by the virtual kubelet handlers on a
+	// Provision failure and read by the placement controller to skip a candidate
+	// that just failed (zone → region → tier). It is in-memory, process-wide state
+	// (empty on restart, self-refilling), so a single instance is threaded into
+	// both sides rather than persisted.
+	blocklist := failover.New()
+
 	if err := (&controller.NodePoolReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
@@ -234,8 +243,9 @@ func main() {
 		os.Exit(1)
 	}
 	if err := (&controller.PodPlacementReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:    mgr.GetClient(),
+		Scheme:    mgr.GetScheme(),
+		Blocklist: blocklist,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "PodPlacement")
 		os.Exit(1)
@@ -246,7 +256,7 @@ func main() {
 	// provider.Terminate on DeletePod, so an ungated Pod bound to a provider's
 	// virtual node materializes an external instance. Each Runner is a
 	// manager.Runnable, so it shares the manager's lifecycle and leader election.
-	if err := setupVirtualNodes(mgr); err != nil {
+	if err := setupVirtualNodes(mgr, blocklist); err != nil {
 		setupLog.Error(err, "unable to set up virtual nodes")
 		os.Exit(1)
 	}
@@ -295,7 +305,7 @@ func main() {
 // provider. The Runner needs a typed clientset (the virtual kubelet's node/pod
 // controllers use client-go directly, not the controller-runtime client), built
 // from the same rest.Config the manager uses.
-func setupVirtualNodes(mgr ctrl.Manager) error {
+func setupVirtualNodes(mgr ctrl.Manager, blocklist vnode.Blocklister) error {
 	clientset, err := kubernetes.NewForConfig(mgr.GetConfig())
 	if err != nil {
 		return err
@@ -305,7 +315,7 @@ func setupVirtualNodes(mgr ctrl.Manager) error {
 		if !ok {
 			continue
 		}
-		if err := mgr.Add(vnode.NewRunner(prov, clientset)); err != nil {
+		if err := mgr.Add(vnode.NewRunner(prov, clientset, blocklist)); err != nil {
 			return err
 		}
 		setupLog.Info("registered virtual node", "provider", name, "node", vnode.NodeName(name))
@@ -327,6 +337,21 @@ func registerProviders(ctx context.Context) {
 		setupLog.Info("registered provider", "provider", p.Name())
 	}
 
+	// AWS. Region is the only NON-SECRET config, read from AWS_REGION (empty => the
+	// SDK resolves its default from shared config / IMDS). The adapter is otherwise
+	// self-configuring: it resolves the GPU AMI and default-VPC subnets itself, so
+	// no launch template or pre-created infra is needed. Credentials are secrets and
+	// are NEVER read here: the SDK client uses the default credential chain (IRSA /
+	// instance-role / AWS_ACCESS_KEY_ID delivered via a Secret), so nothing
+	// sensitive touches a flag or this call. When AWS is not configured (no
+	// resolvable region, no GPU AMI in the region), registration is a non-fatal skip.
+	if p, err := awsprovider.NewSDKClient(ctx, os.Getenv("AWS_REGION")); err != nil {
+		setupLog.Info("skipping AWS provider registration", "reason", err.Error())
+	} else {
+		provider.Register(p)
+		setupLog.Info("registered provider", "provider", p.Name())
+	}
+
 	// The fake provider is an in-memory backend used only by the e2e suite to
 	// exercise the full control-plane loop without cloud credentials. It ships in
 	// the binary but registers ONLY when explicitly enabled, so it can never place

config/catalog/kustomization.yaml

@@ -16,6 +16,7 @@ configMapGenerator:
 - name: catalog
   files:
   - modal.csv=../../pkg/provider/catalog/data/modal.csv
+  - aws.csv=../../pkg/provider/catalog/data/aws.csv
 
 generatorOptions:
   # Stable name (no content-hash suffix) so `kubectl edit` and the volume

config/crd/bases/nebula.inftyai.com_nodeclaims.yaml

@@ -104,6 +104,16 @@ spec:
                   Recovery from preemption is delete-and-recreate. Held durably so teardown
                   knows which provider API to call even after status is lost.
                 type: string
+              region:
+                description: |-
+                  Region is the provider region the placement optimizer selected, in the
+                  provider's own vocabulary (e.g. AWS "us-east-1"). Stored durably alongside
+                  Provider/CapacityType because it is a provisioning input that cannot be read
+                  off the Pod, and Provision needs it to re-issue the request in the same
+                  region after a controller restart. Immutable, like Provider. Empty means
+                  "the provider's configured default region" — region-simple providers (Modal,
+                  RunPod) leave it empty.
+                type: string
             required:
             - podRef
             - provider