feat: add terminating phase to NC

.env.example

@@ -23,11 +23,6 @@ MODAL_TOKEN_SECRET=
 # are required together; leave both blank to skip.
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
-# OPTIONAL. Required ONLY for temporary STS/SSO credentials (ASIA... access keys),
-# which are invalid without it; leave blank for long-lived IAM user keys
-# (AKIA...). Note temporary creds expire (minutes–hours) — fine for a quick test,
-# not a long-running deployment.
-AWS_SESSION_TOKEN=
 
 # --- Additional providers (add as adapters land) ---------------------------
 # Each provider gets its OWN secret (see hack/deploy.sh PROVIDER_SECRETS), e.g.:

README.md

@@ -94,4 +94,5 @@ placement controller owns those.
 
 - See [docs/deploy.md](docs/deploy.md) to install
 - See [config/samples](config/samples) for example NodePools and a runnable workload.
+- See [docs/add-a-provider.md](docs/add-a-provider.md) to add a provider backend.
 - See [docs/architecture.md](docs/architecture.md) for design details.

api/v1alpha1/nodeclaim_types.go

@@ -93,6 +93,16 @@ const (
 	// claim whose Pod later disappears is a real teardown, not cache lag. The claim
 	// does not track finer workload status; the Pod is the source of truth for that.
 	NodeClaimBound NodeClaimPhase = "Bound"
+	// NodeClaimTerminating: the served Pod is being deleted (its DeletionTimestamp
+	// is set) but the external instance may not be reclaimed yet — teardown is in
+	// flight. This is distinct from Terminated, which means the instance is already
+	// GONE: here the Pod object still exists (draining its grace period / VK's
+	// DeletePod running / a finalizer pending), so the phase reflects "going away"
+	// rather than stranding the claim on a stale Provisioning/Bound. It is a
+	// forward transition from ANY prior phase, since a deleting Pod is on its way
+	// out regardless of how far provisioning had progressed. The claim self-deletes
+	// (firing the terminate backstop) once the Pod object is fully gone.
+	NodeClaimTerminating NodeClaimPhase = "Terminating"
 	// NodeClaimTerminated: the external instance is gone. Set when the served Pod
 	// has reached a terminal phase (Failed/Succeeded) — VK reports that when the
 	// provider's instance disappears (torn down, reclaimed, or exited). The claim

config/samples/deployment.yaml

@@ -23,7 +23,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: gpu-workload-sample
+  name: gpu-workload-sample-l4
   namespace: default
   labels:
     app.kubernetes.io/managed-by: nebula
@@ -38,7 +38,7 @@ spec:
         app: gpu-workload-sample
         nebula.inftyai.com/enabled: "true"
         nebula.inftyai.com/nodepool: aws
-        nebula.inftyai.com/accelerator-type: a100-40gb
+        nebula.inftyai.com/accelerator-type: l4
     spec:
       # Do NOT set nodeName or a provider nodeSelector yourself — the placement
       # controller fills the nodeSelector in when it ungates the Pod. Setting
@@ -65,4 +65,4 @@ spec:
             # GPU count. Standard extended resource, so the scheduler's fit check
             # and provisioning read the same number. 8 => 8x the accelerator-type
             # above.
-            nvidia.com/gpu: "8"
+            nvidia.com/gpu: "1"

docs/add-a-provider.md

@@ -0,0 +1,104 @@
+# Adding a provider
+
+A provider adapter teaches Nebula how to provision on one backend (a NeoCloud like
+RunPod, or a hyperscaler like GCP). The control plane is provider-agnostic: it
+drives everything through the `provider.Provider` interface and a price/availability
+catalog, so a new provider is an adapter package plus a little wiring — no changes
+to the placement controller, virtual kubelet, or NodeClaim controller.
+
+Use `pkg/provider/modal` (region-simple NeoCloud) and `pkg/provider/aws`
+(region-aware hyperscaler) as references.
+
+## 1. Implement the adapter
+
+Create `pkg/provider/<name>/` and implement `provider.Provider`
+(`pkg/provider/provider.go`):
+
+| Method | Responsibility |
+| --- | --- |
+| `Name()` | Stable identifier, e.g. `"runpod"`. Must match the provider name used in a NodePool and on the virtual node. |
+| `Capabilities()` | Declare quirks (poll interval, preemption notice) so the control plane behaves generically instead of branching on name. |
+| `Provision(ctx, pod, req)` | Create exactly one instance from the Pod (image/command/env/ports/cpu/memory + accelerator). Idempotent on `req.ClaimName`. Returns the instance id. |
+| `Terminate(ctx, id)` | Destroy by id. **Must be idempotent** — terminating an already-gone instance returns nil (the NodeClaim finalizer relies on this). |
+| `Get(ctx, id)` | Current state of one instance, or `(nil, nil)` if gone. |
+| `List(ctx)` | Every Nebula-owned instance, in as few API calls as possible. This drives the poll loop — preemption is detected by an instance disappearing here. |
+| `Offerings(ctx)` | Price/availability rows for the optimizer (see the catalog below). |
+| `MapAccelerator(canonical, count)` | Translate a canonical accelerator (type + count) to the provider's own id; `ok=false` if unsupported. |
+| `ClassifyProvisionError(err, accel, region)` | Map a Provision failure to the `BlockScope` failover should blocklist (a capacity error → that {accel, tier, region}; an auth/quota error → the whole provider). |
+
+The Pod is the single source of truth for the workload shape; `ProvisionRequest`
+carries only what the Pod cannot express (the optimizer's capacity tier and the
+claim identity). Do not duplicate Pod fields onto the request.
+
+Most adapters embed `catalog.Base` for the generic `Name`, `Offerings`, and the
+identity `MapAccelerator`, overriding only what the provider does differently (see
+how `pkg/provider/modal` embeds it).
+
+## 2. Add the price/availability catalog
+
+Add `pkg/provider/catalog/data/<name>.csv` (embedded at build time — see
+`pkg/provider/catalog/catalog.go`). The `accelerator_type` column is the canonical
+Nebula type a workload requests via the `nebula.inftyai.com/accelerator-type`
+label (matched case-insensitively). Copy the header and column semantics from
+`aws.csv`.
+
+## 3. Register the provider name
+
+Add a constant to the `const` block in `pkg/provider/registry.go`:
+
+```go
+ProviderRunPod = "runpod"
+```
+
+## 4. Wire it into the manager
+
+In `registerProviders` (`cmd/main.go`), build the adapter and register it. A
+provider whose credentials are absent must be **logged and skipped, not fatal** —
+follow the existing Modal/AWS pattern:
+
+```go
+if p, err := runpod.NewSDKClient(ctx); err != nil {
+    setupLog.Info("skipping RunPod provider registration", "reason", err.Error())
+} else {
+    provider.Register(p)
+    setupLog.Info("registered provider", "provider", p.Name())
+}
+```
+
+## 5. Wire its credentials
+
+Credentials live in **one Kubernetes Secret per provider**, mounted via an optional
+`envFrom.secretRef`. Two edits:
+
+1. **`hack/deploy.sh`** — add a row to `PROVIDER_SECRETS`
+   (`<secret-name>|<REQUIRED_KEYS>|<OPTIONAL_KEYS>`):
+
+   ```bash
+   PROVIDER_SECRETS=(
+     "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET|"
+     "nebula-runpod-credentials|RUNPOD_API_KEY|"
+   )
+   ```
+
+2. **`config/manager/manager.yaml`** — add an optional `secretRef` under `envFrom`:
+
+   ```yaml
+   - secretRef:
+       name: nebula-runpod-credentials
+       optional: true
+   ```
+
+Then add the new keys to `.env.example`. The deploy script creates the Secret from
+`.env` and skips it when the keys are blank (the provider is then skipped at
+registration — see [How credentials are handled](deploy.md#how-credentials-are-handled)).
+
+## 6. Verify
+
+Register the provider and reference it from a NodePool. A pool that names an
+unregistered provider surfaces `Ready=False / UnknownProvider`, which self-heals
+once the adapter registers:
+
+```bash
+kubectl -n nebula-system logs deploy/nebula-controller-manager | grep -i provider
+kubectl get nodes -l nebula.inftyai.com/provider
+```

docs/deploy.md

@@ -8,7 +8,6 @@ cluster, including wiring provider credentials.
 - [Webhook TLS (no cert-manager)](#webhook-tls-no-cert-manager)
 - [What `deploy-all` does](#what-deploy-all-does)
 - [Configuration](#configuration)
-- [Adding a provider](#adding-a-provider)
 - [Manual deployment](#manual-deployment)
 - [Verifying the deployment](#verifying-the-deployment)
 
@@ -134,6 +133,8 @@ cert-manager, and drop the `gen-webhook-cert.sh` calls from `hack/deploy.sh`.
 |---|---|---|---|
 | `MODAL_TOKEN_ID` | Modal | yes | From `modal token new` |
 | `MODAL_TOKEN_SECRET` | Modal | yes | From `modal token new` |
+| `AWS_ACCESS_KEY_ID` | AWS | dev only | Prefer IRSA / instance role in production and leave blank — the SDK's default credential chain finds the role. Set only for local/dev. |
+| `AWS_SECRET_ACCESS_KEY` | AWS | dev only | Pairs with `AWS_ACCESS_KEY_ID`; both required together or both blank. |
 
 Non-secret config, passed as `make` variables:
 
@@ -146,34 +147,6 @@ Non-secret config, passed as `make` variables:
 
 ---
 
-## Adding a provider
-
-When a new provider adapter lands, wire its credentials in two places:
-
-1. **`hack/deploy.sh`** — add a row to the `PROVIDER_SECRETS` table:
-
-   ```bash
-   PROVIDER_SECRETS=(
-     "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET"
-     "nebula-runpod-credentials|RUNPOD_API_KEY|"
-   )
-   ```
-   Format: `<secret-name>|<REQUIRED_KEYS>|<OPTIONAL_KEYS>`.
-
-2. **`config/manager/manager.yaml`** — add an optional `secretRef` under
-   `envFrom`:
-
-   ```yaml
-   - secretRef:
-       name: nebula-runpod-credentials
-       optional: true
-   ```
-
-Then add the new keys to `.env.example`. Nothing else changes — the script
-creates the Secret from `.env` and skips it if the keys are blank.
-
----
-
 ## Manual deployment
 
 If you don't want the script (e.g. you manage Secrets via sealed-secrets or a

hack/deploy.sh

@@ -64,12 +64,10 @@ PROVIDER_SECRETS=(
   "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET|"
   # AWS: creds are the only secret. The access key + secret are required together
   # (a lone key is a misconfig), so a blank pair skips the Secret — the SDK then
-  # relies on IRSA / instance role, which is the preferred path. AWS_SESSION_TOKEN
-  # is OPTIONAL: it is REQUIRED for temporary STS/SSO creds (ASIA... keys) and
-  # unused for long-lived IAM user keys (AKIA...), so it is only added to the
-  # Secret when set. The region is NON-SECRET and lives on the manager Deployment,
-  # not in this Secret; the adapter self-configures the rest (GPU AMI + subnets).
-  "nebula-aws-credentials|AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN"
+  # relies on IRSA / instance role, which is the preferred path. The region is
+  # NON-SECRET and lives on the manager Deployment, not in this Secret; the adapter
+  # self-configures the rest (GPU AMI + subnets).
+  "nebula-aws-credentials|AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY|"
   # "nebula-runpod-credentials|RUNPOD_API_KEY|"
 )
 

internal/controller/nodeclaim_controller.go

@@ -143,10 +143,13 @@ func (r *NodeClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	// The served Pod is absent. Decide whether this is a real teardown or a
 	// transient cache-lag false-negative.
-	if r.wasBound(&nc) {
+	if r.wasBound(&nc) || nc.Status.Phase == nebulav1alpha1.NodeClaimTerminating {
 		// We previously observed the Pod, so its disappearance is real: this is a
-		// teardown (normal delete, or a force-delete during a VK outage). Delete
-		// the claim so its finalizer fires the backstop.
+		// teardown (normal delete, or a force-delete during a VK outage). A
+		// Terminating claim counts too — it was set only from a Pod carrying a
+		// DeletionTimestamp, so that Pod provably existed and its disappearance is
+		// the delete completing, not cache lag. Delete the claim so its finalizer
+		// fires the backstop.
 		log.Info("served Pod is gone after being observed placed; deleting claim to trigger teardown",
 			"pod", nc.Spec.PodRef.Name)
 		return ctrl.Result{}, r.deleteSelf(ctx, &nc)
@@ -277,6 +280,12 @@ func (r *NodeClaimReconciler) provider(name string) (provider.Provider, bool) {
 //   - Terminal Pod (Failed/Succeeded) => Terminated. VK reports a vanished
 //     instance this way; a restartPolicy:Never Pod lingers terminal rather than
 //     being deleted, so keying off existence alone would wedge the claim at Bound.
+//   - Pod being deleted (DeletionTimestamp set) => Terminating. The workload is on
+//     its way out but the instance may not be reclaimed yet, so this is a forward
+//     transition from ANY phase — it is checked before the Bound-hold below so a
+//     Bound (or Provisioning) Pod that starts deleting is not stranded on its prior
+//     phase. Terminal wins over it: an already-gone instance is Terminated, not
+//     merely terminating.
 //   - Bound already => "" (hold). Bound is the durable teardown guard, and a
 //     transient Running->Pending flap (e.g. a status-check blip) must NOT strip it
 //     — losing it would make a later disappearance read as cache lag and skip
@@ -290,6 +299,8 @@ func (r *NodeClaimReconciler) desiredPhase(nc *nebulav1alpha1.NodeClaim, pod *co
 	switch {
 	case isTerminal(pod.Status.Phase):
 		return nebulav1alpha1.NodeClaimTerminated
+	case !pod.DeletionTimestamp.IsZero():
+		return nebulav1alpha1.NodeClaimTerminating
 	case r.wasBound(nc):
 		return "" // hold Bound (or Terminated); never downgrade
 	case pod.Status.Phase == corev1.PodRunning: