feat(self-managed): record the installed stack version in-cluster, and let the 1.0 train ship it

.github/workflows/release-tags.yml

@@ -7,13 +7,30 @@ on:
   push:
     branches:
       - main
+      # Maintenance branches for the subprojects that release from a branch
+      # rather than from main -- currently the three deploy/stacks subprojects.
+      # A push here cuts the next patch on that branch's train.
+      - 'release-**/v*'
     tags:
       - '*-v*'
       - '**/v*'
   workflow_dispatch:
     inputs:
+      operation:
+        description: Release operation to run.
+        required: false
+        default: auto
+        type: choice
+        options:
+          - auto
+          - branch-cut
+          - release-candidate
       service:
-        description: Optional service id or service_name to scope the run to. Defaults to every registered service.
+        description: >-
+          Service id or service_name. Optional for operation=auto, where it
+          scopes the run and otherwise defaults to every registered service.
+          Required for branch-cut and release-candidate, which act on one
+          subproject.
         required: false
         type: string
       inventory_tag:
@@ -56,18 +73,18 @@ env:
 
 jobs:
   # NVIDIA self-hosted runners. Eligible without a conditional because this
-  # workflow has no pull_request trigger: it fires on pushes to main, on tags,
-  # and on manual dispatch, all of which run trusted reviewed code. The bazel
-  # matrix needs an event-conditional runs-on for exactly that reason; this
-  # does not.
+  # workflow has no pull_request trigger: it fires on pushes to main and to
+  # release branches, on tags, and on manual dispatch, all of which run trusted
+  # reviewed code. The bazel matrix needs an event-conditional runs-on for
+  # exactly that reason; this does not.
   #
   # cpu4 rather than cpu16: these jobs shell out to the GitHub API to cut tags
   # and create releases. They are latency-bound, not CPU-bound, and were sitting
   # in the GitHub-hosted queue behind the build matrix while doing almost no
   # work.
   service-release:
     name: service release automation
-    if: github.ref_type != 'tag' && (github.event_name != 'workflow_dispatch' || (inputs.inventory_tag == '' && inputs.release_tag == ''))
+    if: github.ref_type != 'tag' && (github.event_name != 'workflow_dispatch' || (inputs.operation == 'auto' && inputs.inventory_tag == '' && inputs.release_tag == ''))
     runs-on: linux-amd64-cpu4
     permissions:
       contents: write
@@ -116,6 +133,84 @@ jobs:
           fi
           ./tools/ci/github-release auto
 
+  # Opens the next train for a subproject that releases from a branch: creates
+  # release-<tag-prefix>X.Y holding the default branch's content, and opens the
+  # pull request advancing that subproject's VERSION to X.Y+1.0 on the default
+  # branch. Dispatch from any ref; the cut always uses the default branch.
+  release-branch-cut:
+    name: release branch cut
+    if: github.event_name == 'workflow_dispatch' && inputs.operation == 'branch-cut'
+    runs-on: linux-amd64-cpu4
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+          ref: ${{ github.event.repository.default_branch }}
+          token: ${{ secrets.NV_GITHUB_TOKEN || github.token }}
+
+      - name: Cut release branch
+        env:
+          NVCF_GITHUB_RELEASE_SERVICE: ${{ inputs.service }}
+          GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          set -euo pipefail
+          if [ "${NV_GITHUB_TOKEN_CONFIGURED}" != "true" ]; then
+            echo "ERROR: set secret NV_GITHUB_TOKEN before cutting a GitHub release branch." >&2
+            echo "The default GITHUB_TOKEN cannot trigger CI for the generated VERSION bump PR." >&2
+            exit 1
+          fi
+          if [ -z "${NVCF_GITHUB_RELEASE_SERVICE}" ]; then
+            echo "ERROR: service is required when operation=branch-cut." >&2
+            exit 1
+          fi
+          ./tools/ci/github-release branch-cut --service "${NVCF_GITHUB_RELEASE_SERVICE}"
+
+  # Cuts X.Y.Z-rc.N at the dispatched ref. This is how an unreleased tree gets a
+  # real artifact now that the stacks do not tag on main: the internal publish
+  # lanes accept -rc.N, so selecting a pull request's branch in the run dialog
+  # builds and publishes that tree without putting a version on the stack's
+  # stable line.
+  #
+  # Same-repository branches only, which is what workflow_dispatch offers. A
+  # fork's branch cannot be selected, so release credentials never run against
+  # unreviewed code.
+  release-candidate:
+    name: release candidate
+    if: github.event_name == 'workflow_dispatch' && inputs.operation == 'release-candidate'
+    runs-on: linux-amd64-cpu4
+    permissions:
+      contents: write
+    steps:
+      # fetch-tags because the rc counter is derived from the rc tags already
+      # published for this base version. Without them it restarts at rc.0 and
+      # the push fails on a tag name that is already taken.
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+          ref: ${{ github.ref_name }}
+          token: ${{ secrets.NV_GITHUB_TOKEN || github.token }}
+
+      - name: Cut release candidate
+        env:
+          NVCF_GITHUB_RELEASE_SERVICE: ${{ inputs.service }}
+        run: |
+          set -euo pipefail
+          if [ "${NV_GITHUB_TOKEN_CONFIGURED}" != "true" ]; then
+            echo "ERROR: set secret NV_GITHUB_TOKEN before cutting a release candidate." >&2
+            echo "The default GITHUB_TOKEN cannot trigger the tag workflow that publishes release notes." >&2
+            exit 1
+          fi
+          if [ -z "${NVCF_GITHUB_RELEASE_SERVICE}" ]; then
+            echo "ERROR: service is required when operation=release-candidate." >&2
+            exit 1
+          fi
+          ./tools/ci/github-release release-candidate --service "${NVCF_GITHUB_RELEASE_SERVICE}"
+
   inventory-preflight:
     name: stack inventory preflight
     if: github.event_name == 'workflow_dispatch' && inputs.inventory_tag != ''

deploy/stacks/nvcf-compute-plane/VERSION

@@ -0,0 +1 @@
+1.0.0

deploy/stacks/observability/VERSION

@@ -0,0 +1 @@
+1.0.0

deploy/stacks/self-managed/Makefile

@@ -15,6 +15,7 @@ test:
 	@tests/grpc-proxy-nats-endpoint.sh
 	@tests/llm-pki-openbao-migration.sh
 	@tests/api-keys-startup-probe.sh
+	@tests/upgrade-receipt-wiring.sh
 	@tests/cassandra-openbao-credential-wiring.sh
 	@tests/llm-pki-release.sh
 	@tests/check-llm-pki-issuer.sh

deploy/stacks/self-managed/VERSION

@@ -0,0 +1 @@
+1.0.0

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/Chart.yaml

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v2
+name: nvcf-upgrade-receipt
+description: Records the installed NVCF stack version in-cluster so an upgrade can tell where it is starting from.
+type: application
+version: 0.1.0
+appVersion: "0.1.0"

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/job.yaml

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+{{- $version := required "stackVersion is required: a receipt that names the wrong version is worse than no receipt" .Values.stackVersion }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    # post-* rather than pre-*: the receipt states what the cluster is running,
+    # so it must not be written until the release it describes has been applied.
+    #
+    # Both install and upgrade, because the first cluster to receive this chart
+    # has no prior release of it, and Helm runs post-install there rather than
+    # post-upgrade. Omitting post-install would leave exactly the clusters this
+    # exists for without a receipt.
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  backoffLimit: 3
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      name: {{ .Release.Name }}
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ .Release.Name }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      containers:
+        - name: receipt
+          image: "{{ with .Values.image.registry }}{{ . }}/{{ end }}{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: RECEIPT_CONFIGMAP
+              value: {{ .Values.configMapName | quote }}
+            - name: INSTALLED_STACK_VERSION
+              value: {{ $version | quote }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              set -eu
+              # Rendered through apply rather than create so that the first
+              # install and every later upgrade take the same path. A receipt
+              # that only appears on a fresh install would be absent from
+              # exactly the clusters that are upgrading.
+              kubectl create configmap "${RECEIPT_CONFIGMAP}" \
+                --from-literal=installed_stack_version="${INSTALLED_STACK_VERSION}" \
+                --from-literal=recorded_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+                --dry-run=client -o yaml \
+                | kubectl apply -f -
+              echo "recorded installed_stack_version=${INSTALLED_STACK_VERSION} in ${RECEIPT_CONFIGMAP}"

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/role.yaml

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+rules:
+  # Scoped to the receipt itself. get and patch cover the upgrade case where a
+  # receipt already exists, and naming the resource keeps this identity from
+  # reaching any other ConfigMap in the namespace.
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: [{{ .Values.configMapName | quote }}]
+    verbs: ["get", "patch"]
+  # create cannot be scoped: RBAC matches resourceNames against an object that
+  # does not exist yet, so a create rule naming one is never satisfied. It is
+  # kept in its own rule so the unscoped verb is visible rather than buried
+  # alongside the scoped ones, and it is only reachable on a first install.
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create"]

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/rolebinding.yaml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/serviceaccount.yaml

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    # Must exist before the Job that uses it, and survive long enough to be
+    # bound. A lower weight than the Job is what orders them.
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+{{- with .Values.imagePullSecrets }}
+imagePullSecrets:
+{{ toYaml . | indent 2 }}
+{{- end }}

deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/values.yaml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# The stack version this bundle installs. The Helmfile supplies it from the
+# bundle's own VERSION file; there is no sensible default, so rendering fails
+# rather than recording a version the cluster is not running.
+stackVersion: ""
+
+image:
+  registry: ""
+  repository: alpine-k8s
+  tag: "1.33.1"
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+
+# Name of the ConfigMap holding the receipt. An upgrade reads this to decide
+# whether the jump it has been asked to make is one it can make safely.
+configMapName: nvcf-upgrade-receipt

deploy/stacks/self-managed/helmfile.d/04-upgrade-receipt.yaml.gotmpl

@@ -0,0 +1,30 @@
+environments:
+  default:
+    values:
+      - ../environments/base.yaml
+      - ../environments/{{ requiredEnv "HELMFILE_ENV" }}.yaml
+
+---
+
+{{- /*
+  A stage of its own, and the last one, so the receipt is written only after
+  every other release has been applied. Ordering is a stage boundary rather
+  than a needs: edge on purpose: under the helmfile version this stack pins,
+  needs: places a release in a later DAG layer where it waits on every peer in
+  the previous one, and a single slow or failed peer silently skips it. See the
+  admin-issuer-proxy comment in 02-core.yaml.gotmpl.
+*/}}
+
+releases:
+  - name: upgrade-receipt
+    chart: ../charts/nvcf-upgrade-receipt
+    namespace: nvcf
+    values:
+      - stackVersion: {{ readFile "../VERSION" | trim | quote }}
+        image:
+          registry: {{ .Values.global.image.registry | quote }}
+          repository: {{ .Values.global.image.repository }}/alpine-k8s
+        {{- with .Values.global.imagePullSecrets }}
+        imagePullSecrets:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}

deploy/stacks/self-managed/tests/upgrade-receipt-wiring.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Test that the stack records the version it installed.
+#
+# An upgrade has to know where it is starting from, and nothing else in a
+# cluster carries that: Helm tracks chart versions per release, and helmfile has
+# no concept of the bundle's own version. Without this receipt every cluster
+# looks identical to every other one at upgrade time.
+#
+# The assertions that matter are the hook kinds and the recorded version. A
+# pre-* hook would claim a version before it was applied, and a post-upgrade
+# hook alone would skip the first install of this chart, which is precisely the
+# set of clusters that need a receipt written.
+set -euo pipefail
+
+stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+expected_version="$(tr -d '[:space:]' < "$stack_dir/VERSION")"
+
+rendered="$(cd "$stack_dir" && HELMFILE_ENV=base helmfile \
+  --file helmfile.d/04-upgrade-receipt.yaml.gotmpl template)"
+
+fail() { echo "FAIL: $1" >&2; exit 1; }
+
+grep -q 'kind: Job' <<<"$rendered" || fail "no Job rendered"
+grep -q '"helm.sh/hook": post-install,post-upgrade' <<<"$rendered" \
+  || fail "receipt must run on both install and upgrade, after the release it describes"
+grep -q "value: \"${expected_version}\"" <<<"$rendered" \
+  || fail "recorded version does not match VERSION (${expected_version})"
+# The ConfigMap is created by the Job at run time, not rendered, so its name
+# reaches the cluster as the env var the script reads.
+grep -q 'value: "nvcf-upgrade-receipt"' <<<"$rendered" \
+  || fail "receipt ConfigMap name is not the one an upgrade will read"
+
+for kind in ServiceAccount Role RoleBinding; do
+  grep -q "kind: ${kind}" <<<"$rendered" || fail "missing ${kind}; the Job cannot write the ConfigMap without it"
+done
+# get and patch are scoped to the receipt by name so this identity cannot
+# reach any other ConfigMap. create cannot be scoped -- RBAC matches
+# resourceNames against an object that does not exist yet.
+grep -qE '^\s+resourceNames: \["nvcf-upgrade-receipt"\]' <<<"$rendered" \
+  || fail "get/patch are not scoped to the receipt ConfigMap by name"
+grep -qE '^\s+verbs: \["get", "patch"\]' <<<"$rendered" \
+  || fail "scoped rule should carry only get and patch"
+grep -qE '^\s+verbs: \["create"\]' <<<"$rendered" \
+  || fail "create must remain, in its own rule, for the first install"
+
+# The stage number is the ordering guarantee. needs: is deliberately not used
+# here; see the comment in the stage file.
+last_stage="$(ls "$stack_dir"/helmfile.d/*.gotmpl | sort | tail -1)"
+[[ "$(basename "$last_stage")" == "04-upgrade-receipt.yaml.gotmpl" ]] \
+  || fail "receipt is not the last stage; it would record a version before the stack finished applying"
+
+echo "PASS: upgrade-receipt-wiring"