Skip to content

feat(nvca): configure transport TLS mount path #708

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mikeyrcamp wants to merge 1 commit into from
Closed

feat(nvca): configure transport TLS mount path #708

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions deploy/helm/nvca-operator/nvca-operator/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ used in Kubernetes Clusters to run NVCF Workloads.
| `operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name` | Secret containing the workload transport trust bundle; empty disables the source. Example: `nvcf-trust`. | `""` |
| `operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.key` | Secret data key containing certificate-only PEM. | `ca.crt` |
| `operatorConfig.workload.transportTLS.fingerprint` | Optional SHA-256 pin; empty computes the Secret data fingerprint. | `""` |
| `operatorConfig.workload.transportTLS.installedBundleMountPath` | Optional `llm-worker` mount path for the installed transport trust bundle; empty uses `/etc/ssl/certs`. | `""` |

### resources Resource requests and limits for the nvca-operator container

Expand Down
3 changes: 3 additions & 0 deletions deploy/helm/nvca-operator/nvca-operator/templates/operator-config-cm.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,6 @@ data:
name: {{ $secretKeyRef.name | default "" | quote }}
key: {{ $secretKeyRef.key | default "ca.crt" | quote }}
fingerprint: {{ $transportTLS.fingerprint | default "" | quote }}
{{- if $transportTLS.installedBundleMountPath }}
installedBundleMountPath: {{ $transportTLS.installedBundleMountPath | quote }}
{{- end }}
5 changes: 5 additions & 0 deletions deploy/helm/nvca-operator/nvca-operator/values.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,11 @@
"type": "string",
"description": "Optional sha256 fingerprint pin for the trust bundle. Empty computes the fingerprint from the selected Secret data.",
"default": ""
},
"installedBundleMountPath": {
"type": "string",
"description": "Optional llm-worker mount path for the installed transport trust bundle. Empty uses /etc/ssl/certs.",
"default": ""
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions deploy/helm/nvca-operator/nvca-operator/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ agentConfig:
## @param operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name Secret containing the workload transport trust bundle. Empty disables Secret-backed transport trust. Example: nvcf-trust.
## @param operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.key Secret data key containing certificate-only PEM.
## @param operatorConfig.workload.transportTLS.fingerprint Optional sha256 fingerprint pin for the trust bundle. Empty computes the fingerprint from the selected Secret data.
## @param operatorConfig.workload.transportTLS.installedBundleMountPath Optional llm-worker mount path for the installed transport trust bundle. Empty uses /etc/ssl/certs.
operatorConfig:
workload:
transportTLS:
Expand All @@ -138,6 +139,7 @@ operatorConfig:
name: ""
key: ca.crt
fingerprint: ""
installedBundleMountPath: ""
## @section resources Resource requests and limits for the nvca-operator container
## @param resources.limits.cpu CPU limit for the nvca-operator container
## @param resources.limits.memory Memory limit for the nvca-operator container
Expand Down
1 change: 1 addition & 0 deletions src/compute-plane-services/nvca/deployments/nvca-operator/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ used in Kubernetes Clusters to run NVCF Workloads.
| `operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name` | Secret containing the workload transport trust bundle; empty disables the source. Example: `nvcf-trust`. | `""` |
| `operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.key` | Secret data key containing certificate-only PEM. | `ca.crt` |
| `operatorConfig.workload.transportTLS.fingerprint` | Optional SHA-256 pin; empty computes the Secret data fingerprint. | `""` |
| `operatorConfig.workload.transportTLS.installedBundleMountPath` | Optional `llm-worker` mount path for the installed transport trust bundle; empty uses `/etc/ssl/certs`. | `""` |

### resources Resource requests and limits for the nvca-operator container

Expand Down
3 changes: 3 additions & 0 deletions src/compute-plane-services/nvca/deployments/nvca-operator/templates/operator-config-cm.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,6 @@ data:
name: {{ $secretKeyRef.name | default "" | quote }}
key: {{ $secretKeyRef.key | default "ca.crt" | quote }}
fingerprint: {{ $transportTLS.fingerprint | default "" | quote }}
{{- if $transportTLS.installedBundleMountPath }}
installedBundleMountPath: {{ $transportTLS.installedBundleMountPath | quote }}
{{- end }}
5 changes: 5 additions & 0 deletions src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,11 @@
"type": "string",
"description": "Optional sha256 fingerprint pin for the trust bundle. Empty computes the fingerprint from the selected Secret data.",
"default": ""
},
"installedBundleMountPath": {
"type": "string",
"description": "Optional llm-worker mount path for the installed transport trust bundle. Empty uses /etc/ssl/certs.",
"default": ""
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ agentConfig:
## @param operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name Secret containing the workload transport trust bundle. Empty disables Secret-backed transport trust. Example: nvcf-trust.
## @param operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.key Secret data key containing certificate-only PEM.
## @param operatorConfig.workload.transportTLS.fingerprint Optional sha256 fingerprint pin for the trust bundle. Empty computes the fingerprint from the selected Secret data.
## @param operatorConfig.workload.transportTLS.installedBundleMountPath Optional llm-worker mount path for the installed transport trust bundle. Empty uses /etc/ssl/certs.
operatorConfig:
workload:
transportTLS:
Expand All @@ -154,6 +155,7 @@ operatorConfig:
name: ""
key: ca.crt
fingerprint: ""
installedBundleMountPath: ""

## @section resources Resource requests and limits for the nvca-operator container
## @param resources.limits.cpu CPU limit for the nvca-operator container
Expand Down
7 changes: 5 additions & 2 deletions src/compute-plane-services/nvca/internal/miniservice/transport_tls_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ func TestPrepareTransportTLSForWorkloadsInjectsPodLLMWorker(t *testing.T) {
TrustBundleFingerprint: testTransportTLSRootFingerprint,
TrustBundlePEM: testTransportTLSRootCertPEM,
InstallerImage: "nvcr.io/nvidia/nvcf-byoc/nvca:test",
InstalledBundleMountPath: "/nvcf/transport-tls",
},
},
},
Expand Down Expand Up @@ -119,9 +120,11 @@ func TestPrepareTransportTLSForWorkloadsInjectsPodLLMWorker(t *testing.T) {
assert.NotNil(t, findWorkloadInitContainer(podSpec, "nvcf-trust-bundle-install"))
llmWorker := findWorkloadContainer(podSpec, function.LLMWorkerContainerName)
require.NotNil(t, llmWorker)
assert.Equal(t, "/etc/ssl/certs/ca-certificates.crt",
assert.Equal(t, "/nvcf/transport-tls/ca-certificates.crt",
findWorkloadEnvValue(llmWorker, "STARGATE_TLS_CERT_PATH"))
assert.NotNil(t, findWorkloadVolumeMount(llmWorker, "nvcf-trust-merged-certs"))
mount := findWorkloadVolumeMount(llmWorker, "nvcf-trust-merged-certs")
require.NotNil(t, mount)
assert.Equal(t, "/nvcf/transport-tls", mount.MountPath)

for _, name := range []string{"inference", "smb-server"} {
container := findWorkloadContainer(podSpec, name)
Expand Down
2 changes: 2 additions & 0 deletions src/compute-plane-services/nvca/internal/transporttls/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,12 @@ go_test(
srcs = ["transport_tls_test.go"],
embed = [":transporttls"],
deps = [
"//vendor/github.com/evanphx/json-patch/v5:json-patch",
"//vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function",
"//vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config",
"//vendor/github.com/stretchr/testify/assert",
"//vendor/github.com/stretchr/testify/require",
"//vendor/k8s.io/api/core/v1:core",
"//vendor/k8s.io/apimachinery/pkg/util/strategicpatch",
],
)
57 changes: 54 additions & 3 deletions src/compute-plane-services/nvca/internal/transporttls/transport_tls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package transporttls

import (
"fmt"
"path"
"regexp"
"strings"

Expand Down Expand Up @@ -48,8 +49,9 @@ const (
TrustBundleMountPath = "/nvcf/trust"
MergedCertsMountPath = "/merged-certs"
MergedCertsFile = "/merged-certs/ca-certificates.crt"
InstalledBundleFile = "ca-certificates.crt"
SystemCertDir = "/etc/ssl/certs"
SystemCertFile = "/etc/ssl/certs/ca-certificates.crt"
SystemCertFile = "/etc/ssl/certs/" + InstalledBundleFile
CertPathEnv = "STARGATE_TLS_CERT_PATH"
)

Expand All @@ -65,6 +67,9 @@ func NormalizeConfig(cfg nvcaconfig.TransportTLSConfig) nvcaconfig.TransportTLSC
if cfg.TrustBundleKey == "" {
cfg.TrustBundleKey = DefaultTrustBundleKey
}
if cfg.InstalledBundleMountPath == "" {
cfg.InstalledBundleMountPath = SystemCertDir
}
cfg.TrustBundleFingerprint = strings.ToLower(strings.TrimSpace(cfg.TrustBundleFingerprint))
return cfg
}
Expand All @@ -74,6 +79,9 @@ func ValidateConfig(cfg nvcaconfig.TransportTLSConfig) error {
case TrustModeSystem:
return nil
case TrustModeBundle:
if err := validateInstalledBundleMountPath(cfg.InstalledBundleMountPath); err != nil {
return err
}
if errs := validation.IsDNS1123Subdomain(cfg.TrustBundleConfigMapName); len(errs) > 0 {
return fmt.Errorf("transportTls.trustBundleConfigMapName is invalid: %s", strings.Join(errs, "; "))
}
Expand Down Expand Up @@ -102,6 +110,23 @@ func ValidateConfig(cfg nvcaconfig.TransportTLSConfig) error {
}
}

func validateInstalledBundleMountPath(mountPath string) error {
if !path.IsAbs(mountPath) {
return fmt.Errorf("transportTls.installedBundleMountPath must be absolute")
}
if path.Clean(mountPath) != mountPath {
return fmt.Errorf("transportTls.installedBundleMountPath must be canonical")
}
if mountPath == "/" {
return fmt.Errorf("transportTls.installedBundleMountPath must not be root")
}
if mountPath == MergedCertsMountPath || strings.HasPrefix(mountPath, MergedCertsMountPath+"/") ||
mountPath == TrustBundleMountPath || strings.HasPrefix(mountPath, TrustBundleMountPath+"/") {
return fmt.Errorf("transportTls.installedBundleMountPath uses reserved path %q", mountPath)
}
return nil
}

func FingerprintTrustBundle(trustBundlePEM string) (string, error) {
return trustbundle.FingerprintPEM(trustBundlePEM)
}
Expand All @@ -127,22 +152,48 @@ func InjectIntoPodSpec(podSpec *corev1.PodSpec, cfg nvcaconfig.TransportTLSConfi
if err != nil {
return err
}
if err := validateInstalledBundleMountConflict(&podSpec.Containers[llmWorkerIdx], cfg.InstalledBundleMountPath); err != nil {
return err
}
upsertVolumes(podSpec, cfg)
upsertInstallContainer(podSpec, installImage, installImagePullPolicy, cfg)

llmWorker := &podSpec.Containers[llmWorkerIdx]
upsertVolumeMount(&llmWorker.VolumeMounts, corev1.VolumeMount{
Name: MergedCertsVolumeName,
MountPath: SystemCertDir,
MountPath: cfg.InstalledBundleMountPath,
ReadOnly: true,
})
k8sutil.AddEnvsToContainer(llmWorker, corev1.EnvVar{
Name: CertPathEnv,
Value: SystemCertFile,
Value: cfg.InstalledBundleMountPath + "/" + InstalledBundleFile,
})
return nil
}

func validateInstalledBundleMountConflict(container *corev1.Container, mountPath string) error {
for _, mount := range container.VolumeMounts {
if mount.Name != MergedCertsVolumeName && mountPathsOverlap(mount.MountPath, mountPath) {
return fmt.Errorf("transportTls.installedBundleMountPath %q conflicts with volume mount %q on %q",
mountPath, mount.Name, container.Name)
}
}
return nil
}

// mountPathsOverlap reports whether either mount hides the other. Kubernetes
// permits nested mounts, but using them for the NVCA bundle would make the
// installed certificate file ambiguous or inaccessible to the llm worker.
func mountPathsOverlap(first, second string) bool {
first = path.Clean(first)
second = path.Clean(second)

if first == "/" || second == "/" {
return true
}
return first == second || strings.HasPrefix(first, second+"/") || strings.HasPrefix(second, first+"/")
}

func resolveInstallContainerImage(
podSpec *corev1.PodSpec,
cfg nvcaconfig.TransportTLSConfig,
Expand Down
Loading
Loading