From 03995ad289b390840ba0fb76ee7f57e144519119 Mon Sep 17 00:00:00 2001 From: Ivo Gosemann Date: Thu, 27 Aug 2026 15:55:54 +0200 Subject: [PATCH] Add handling of firmware upgrades for nxos - adds new DeviceMaintenance key "firmware-upgrade" - pauses reconciliation while upgrade is running - adds DeviceMaintenanceFirmwareTargetAnnotation to specify the target firmware version nxos provider impletementation - ensure enough space, extends NX-API timeout during upgrade. - image is copied & verified. - show incompatibility-all nxos & show install all impact nxos are run to check software & hw - after install & reload the device is checked for version. When boot image matches the target image the upgrade is considered done. Signed-off-by: Ivo Gosemann --- .typos.toml | 1 + api/core/v1alpha1/groupversion_info.go | 12 + internal/controller/core/device_controller.go | 91 +++- internal/controller/core/suite_test.go | 15 + internal/paused/paused.go | 6 + internal/provider/cisco/nxos/firmware.go | 361 ++++++++++++++++ internal/provider/cisco/nxos/firmware_test.go | 408 ++++++++++++++++++ internal/provider/cisco/nxos/system.go | 7 + internal/provider/provider.go | 19 + internal/transport/nxapi/nxapi.go | 17 + 10 files changed, 936 insertions(+), 1 deletion(-) create mode 100644 internal/provider/cisco/nxos/firmware.go create mode 100644 internal/provider/cisco/nxos/firmware_test.go diff --git a/.typos.toml b/.typos.toml index ec6b74866..e8a42c5dc 100644 --- a/.typos.toml +++ b/.typos.toml @@ -9,6 +9,7 @@ extend-ignore-re = [ [default.extend-words] ser = "ser" otu = "otu" +ISSU = "ISSU" # Typo in name used by Cisco NX-OS for a configurable property. # See: https://pubhub.devnetcloud.com/media/dme-docs-10-4-3/docs/System/snmp%3ACommSecP/#configurable-properties acess = "acess" diff --git a/api/core/v1alpha1/groupversion_info.go b/api/core/v1alpha1/groupversion_info.go index 57d2fac6d..08eca9a18 100644 --- a/api/core/v1alpha1/groupversion_info.go +++ b/api/core/v1alpha1/groupversion_info.go @@ -76,6 +76,11 @@ const VRFLabel = "networking.metal.ironcore.dev/vrf-name" // to trigger certain disruptive operations, such as reboots or firmware upgrades. const DeviceMaintenanceAnnotation = "networking.metal.ironcore.dev/maintenance" +// DeviceMaintenanceFirmwareTargetAnnotation specifies the target firmware image for a firmware upgrade. +// It also includes the MD5 checksum of the firmware image if available. +// The value format is {"url": "", "md5": ""} +const DeviceMaintenanceFirmwareTargetAnnotation = "networking.metal.ironcore.dev/maintenance-target-firmware" + // PhysicalInterfaceNeighborLabel identifies the peer Interface resource on the other end of a physical link. // The value must be the name of another Interface resource in the same namespace. // This label is only valid for interfaces of type Physical. @@ -113,6 +118,10 @@ const ( // spec.provisioning is defined. // The annotation is always consumed once the device reaches Running. DeviceMaintenanceSkipProvisioning = "skip-provisioning" + // DeviceMaintenanceFirmwareUpgrade triggers a firmware upgrade on the device. The provider initiates + // the upgrade workflow, which will apply the new firmware and reboot the device as necessary. + // The target firmware image is specified by the DeviceMaintenanceFirmwareTargetAnnotation. + DeviceMaintenanceFirmwareUpgrade = "firmware-upgrade" ) // Condition types that are used across different objects. @@ -252,6 +261,9 @@ const ( const ( // MaintenanceFailedReason indicates that a requested maintenance operation (e.g., reboot or factory reset) failed. MaintenanceFailedReason = "MaintenanceFailed" + // MaintenanceInProgressReason indicates that a long-running maintenance + // operation (e.g., firmware upgrade) is still in progress and will be retried. + MaintenanceInProgressReason = "MaintenanceInProgress" ) // Reasons that are specific to [RoutingPolicy] objects. diff --git a/internal/controller/core/device_controller.go b/internal/controller/core/device_controller.go index 364459c62..782f692a4 100644 --- a/internal/controller/core/device_controller.go +++ b/internal/controller/core/device_controller.go @@ -6,9 +6,11 @@ package core import ( "cmp" "context" + "encoding/json" "errors" "fmt" "math/rand/v2" + "net/url" "regexp" "slices" "strings" @@ -246,6 +248,12 @@ func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c } if err := r.reconcileMaintenance(ctx, obj, conn); err != nil { + // ErrUpgradeInProgress signals that a long-running maintenance step was + // issued and the operation must be resumed on a subsequent reconcile, so + // it must requeue rather than terminate. + if errors.Is(err, provider.ErrUpgradeInProgress) { + return ctrl.Result{}, err + } return ctrl.Result{}, reconcile.TerminalError(err) } @@ -468,7 +476,8 @@ func (r *DeviceReconciler) reconcileMaintenance(ctx context.Context, obj *v1alph case v1alpha1.DeviceMaintenanceReboot, v1alpha1.DeviceMaintenanceFactoryReset, - v1alpha1.DeviceMaintenanceReprovision: + v1alpha1.DeviceMaintenanceReprovision, + v1alpha1.DeviceMaintenanceFirmwareUpgrade: prov := r.Provider() if err := prov.Connect(ctx, conn); err != nil { @@ -538,6 +547,39 @@ func (r *DeviceReconciler) reconcileMaintenance(ctx context.Context, obj *v1alph return fmt.Errorf("failed to prepare device for reprovisioning: %w", err) } obj.Status.Phase = v1alpha1.DevicePhasePending + + case v1alpha1.DeviceMaintenanceFirmwareUpgrade: + mp, ok := prov.(provider.MaintenanceProvider) + if !ok { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceUnsupported", "Maintenance", "Provider does not support firmware upgrade operation: %s", action) + return nil + } + targetFirmware, err := r.getTargetFirmware(obj) + if err != nil { + return err + } + + err = mp.UpgradeFirmware(ctx, conn, targetFirmware) + if errors.Is(err, provider.ErrUpgradeInProgress) { + conditions.Set(obj, metav1.Condition{ + Type: v1alpha1.ReadyCondition, + Status: metav1.ConditionFalse, + Reason: v1alpha1.MaintenanceInProgressReason, + Message: "Firmware upgrade in progress", + }) + r.Recorder.Eventf(obj, nil, "Normal", "FirmwareUpgradeInProgress", "Maintenance", "Device firmware upgrade is in progress") + return err + } + if err != nil { + conditions.Set(obj, metav1.Condition{ + Type: v1alpha1.ReadyCondition, + Status: metav1.ConditionFalse, + Reason: v1alpha1.MaintenanceFailedReason, + Message: fmt.Sprintf("Failed to upgrade firmware: %v", err), + }) + r.Recorder.Eventf(obj, nil, "Warning", "FirmwareUpgradeFailed", "Maintenance", "Device firmware upgrade has failed: %v", err) + return fmt.Errorf("failed to upgrade firmware: %w", err) + } } default: @@ -551,6 +593,53 @@ func (r *DeviceReconciler) reconcileMaintenance(ctx context.Context, obj *v1alph return nil } +// getTargetFirmware retrieves the target firmware information from the device's annotations +func (r *DeviceReconciler) getTargetFirmware(obj *v1alpha1.Device) (provider.TargetFirmware, error) { + targetFirmwareJSON, ok := obj.Annotations[v1alpha1.DeviceMaintenanceFirmwareTargetAnnotation] + if !ok { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceMissingFirmwareTarget", "Maintenance", "Firmware upgrade requested but no target firmware specified") + return provider.TargetFirmware{}, errors.New("firmware upgrade requested but no target firmware specified") + } + + var targetFirmware provider.TargetFirmware + if err := json.Unmarshal([]byte(targetFirmwareJSON), &targetFirmware); err != nil { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceInvalidFirmwareTarget", "Maintenance", "Invalid firmware target specified: %v", err) + return provider.TargetFirmware{}, fmt.Errorf("failed to parse firmware target: %w", err) + } + + // Validate the URL to ensure it is a well-formed absolute HTTP(S) URL and reject + // characters commonly used for shell/control injection if this value is later passed on. + parsedURL, err := url.ParseRequestURI(targetFirmware.URL) + if err != nil { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceInvalidFirmwareTarget", "Maintenance", "Invalid firmware target specified: url is invalid: %v", err) + return provider.TargetFirmware{}, fmt.Errorf("invalid firmware target: url is invalid: %w", err) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceInvalidFirmwareTarget", "Maintenance", "Invalid firmware target specified: url scheme must be http or https") + return provider.TargetFirmware{}, errors.New("invalid firmware target: url scheme must be http or https") + } + if parsedURL.Host == "" { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceInvalidFirmwareTarget", "Maintenance", "Invalid firmware target specified: url host is required") + return provider.TargetFirmware{}, errors.New("invalid firmware target: url host is required") + } + if parsedURL.User != nil { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceInvalidFirmwareTarget", "Maintenance", "Invalid firmware target specified: url must not contain user info") + return provider.TargetFirmware{}, errors.New("invalid firmware target: url must not contain user info") + } + if strings.ContainsAny(targetFirmware.URL, "\r\n\t`$\\<>|;&()") { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceInvalidFirmwareTarget", "Maintenance", "Invalid firmware target specified: url contains forbidden characters") + return provider.TargetFirmware{}, errors.New("invalid firmware target: url contains forbidden characters") + } + + // Validate the MD5 checksum if provided. It must be alphanumeric. + if targetFirmware.MD5 != "" && !regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString(targetFirmware.MD5) { + r.Recorder.Eventf(obj, nil, "Warning", "MaintenanceInvalidFirmwareTarget", "Maintenance", "Invalid firmware target specified: md5 must be alphanumeric") + return provider.TargetFirmware{}, errors.New("invalid firmware target: md5 must be alphanumeric") + } + + return targetFirmware, nil +} + // secretToDevices is a [handler.MapFunc] to be used to enqueue requests for reconciliation // for a Device to update when one of its referenced Secrets gets updated. func (r *DeviceReconciler) secretToDevices(ctx context.Context, obj client.Object) []ctrl.Request { diff --git a/internal/controller/core/suite_test.go b/internal/controller/core/suite_test.go index 56980c05e..1c8268ceb 100644 --- a/internal/controller/core/suite_test.go +++ b/internal/controller/core/suite_test.go @@ -446,6 +446,7 @@ type Provider struct { sync.Mutex ConnectError error // if non-nil, Connect returns this error + UpgradeError error // if non-nil, UpgradeFirmware returns this error LastRebootTime time.Time Ports sets.Set[string] @@ -568,6 +569,20 @@ func (p *Provider) FactoryReset(ctx context.Context, conn *deviceutil.Connection return nil } +func (p *Provider) UpgradeFirmware(ctx context.Context, conn *deviceutil.Connection, target provider.TargetFirmware) error { + p.Lock() + defer p.Unlock() + return p.UpgradeError +} + +// SetUpgradeError sets the error that UpgradeFirmware returns on subsequent +// calls. Pass nil to clear it. +func (p *Provider) SetUpgradeError(err error) { + p.Lock() + defer p.Unlock() + p.UpgradeError = err +} + func (p *Provider) Reprovision(ctx context.Context, conn *deviceutil.Connection) (reterr error) { return nil } diff --git a/internal/paused/paused.go b/internal/paused/paused.go index 3fb77c7bc..97cbd521c 100644 --- a/internal/paused/paused.go +++ b/internal/paused/paused.go @@ -107,6 +107,12 @@ func computeCondition(device *v1alpha1.Device, obj Object) metav1.Condition { condition.Message = "Device is not reachable: " + cond.Message return condition } + if device.GetAnnotations()[v1alpha1.DeviceMaintenanceAnnotation] == v1alpha1.DeviceMaintenanceFirmwareUpgrade { + condition.Status = metav1.ConditionTrue + condition.Reason = v1alpha1.PausedReason + condition.Message = "Device is in maintenance" + return condition + } } } diff --git a/internal/provider/cisco/nxos/firmware.go b/internal/provider/cisco/nxos/firmware.go new file mode 100644 index 000000000..75c3da7bf --- /dev/null +++ b/internal/provider/cisco/nxos/firmware.go @@ -0,0 +1,361 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package nxos + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/go-logr/logr" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ironcore-dev/network-operator/internal/deviceutil" + "github.com/ironcore-dev/network-operator/internal/provider" + "github.com/ironcore-dev/network-operator/internal/transport/nxapi" +) + +// upgradeTimeout is the NX-API client timeout for long-running firmware +// commands (copy and install), which block synchronously for minutes. +const upgradeTimeout = 20 * time.Minute + +const ( + // nxosDefaultSessionTimeout is the default NX-API session timeout in seconds. + nxosDefaultSessionTimeout = 300 + // nxosFirmwareSessionTimeout is an increased timeout for long-running firmware operations (copy and install) to avoid NX-API session expiration. + nxosFirmwareSessionTimeout = 1200 +) + +func (p *Provider) UpgradeFirmware(ctx context.Context, _ *deviceutil.Connection, target provider.TargetFirmware) error { + logger := logr.FromContextOrDiscard(ctx) + + upgraded, err := p.isUpgraded(ctx, target) + switch { + case err != nil: + return err + case upgraded: + return nil + } + + // The copy and install commands block for several minutes, so they run on a + // clone of the NX-API client that only differs in its longer timeout. + nxapiUpgrade, err := p.nxapi.Clone(nxapi.WithTimeout(upgradeTimeout)) + if err != nil { + return fmt.Errorf("failed to create long-timeout nxapi client: %w", err) + } + + // Disable POAP and extend NX-API session timeouts before long-running commands. + if _, err := p.nxapi.Do(ctx, nxapi.NewRequest( + "configure", + "no boot poap enable", + fmt.Sprintf("system server session cmd-timeout %d", nxosFirmwareSessionTimeout), + ).WithRollback(nxapi.Stop)); err != nil { + return fmt.Errorf("nxos firmware: prepare upgrade: %w", err) + } + + targetFileName := path.Base(target.URL) + + if err := p.ensureFirmwareImage(ctx, nxapiUpgrade, target, targetFileName); err != nil { + return err + } + if err := p.checkCompatibility(ctx, nxapiUpgrade, targetFileName); err != nil { + return err + } + if err := p.doUpgrade(ctx, nxapiUpgrade, targetFileName); err != nil { + return err + } + + logger.V(1).Info("Reloading device to boot new firmware") + // Reload is a separate request; connection drop is expected. + if _, err := p.nxapi.Do(ctx, nxapi.NewRequest("reload")); err != nil && !nxapi.IsTransportError(err) { + return fmt.Errorf("nxos firmware: reload failed: %w", err) + } + return provider.ErrUpgradeInProgress +} + +// isUpgraded checks whether the device is already running the target firmware. +// If the device is running the target firmware, it also resets the NX-API session timeout to the default. +func (p *Provider) isUpgraded(ctx context.Context, target provider.TargetFirmware) (bool, error) { + logger := logr.FromContextOrDiscard(ctx) + + // A transport-level failure here means the device is still unreachable + // (typically mid-reload from a prior step), which is a normal in-progress + // state rather than a failure, so signal the caller to requeue. + bootImage := new(BootImage) + if err := p.client.GetState(ctx, bootImage); err != nil { + if isDeviceUnreachable(err) { + logger.V(1).Info("Device unreachable during firmware completion check; treating as in progress") + return false, provider.ErrUpgradeInProgress + } + return false, fmt.Errorf("nxos firmware: failed to read running version: %w", err) + } + + targetFileName := path.Base(target.URL) + if path.Base(string(*bootImage)) == targetFileName { + logger.V(1).Info("Device already running target firmware", "filename", targetFileName) + if _, err := p.nxapi.Do(ctx, nxapi.NewRequest( + "configure", + fmt.Sprintf("system server session cmd-timeout %d", nxosDefaultSessionTimeout), + ).WithRollback(nxapi.Stop)); err != nil { + return false, fmt.Errorf("nxos firmware: reset session timeout: %w", err) + } + return true, nil + } + return false, nil +} + +// ensureFirmwareImage ensures a valid firmware image is present on bootflash +func (p *Provider) ensureFirmwareImage(ctx context.Context, c *nxapi.Client, target provider.TargetFirmware, targetFileName string) error { + logger := logr.FromContextOrDiscard(ctx) + + haveValidImage := false + sum, err := p.fileMD5(ctx, targetFileName) + if err != nil { + return fmt.Errorf("nxos firmware: check existing image: %w", err) + } + switch { + case sum == "": + // absent — copy below. + case target.MD5 == "": + haveValidImage = true // no checksum to compare; presence is enough. + case strings.EqualFold(sum, target.MD5): + haveValidImage = true + default: + logger.V(1).Info("Stale image on bootflash, deleting", "file", targetFileName) + if _, err := p.nxapi.Do(ctx, nxapi.NewRequest("delete bootflash:"+targetFileName+" no-prompt")); err != nil { + return fmt.Errorf("nxos firmware: delete stale image: %w", err) + } + } + + if haveValidImage { + return nil + } + + size, err := remoteImageSize(ctx, target.URL) + if err != nil { + return err + } + dir, err := p.ListDirectory(ctx, "bootflash:") + if err != nil { + return fmt.Errorf("nxos firmware: dir bootflash: failed: %w", err) + } + if size > dir.Bytesfree { + // TODO: Check if more than the current target and the current running image are present on the bootflash and delete them to free up space. + return fmt.Errorf("nxos firmware: image (%d bytes) does not fit in bootflash free space (%d bytes)", size, dir.Bytesfree) + } + + // The NX-OS `copy https://...` command unconditionally prompts + // "Enter username:", which NX-API cannot answer. Depending on the + // endpoint a dummy username results in 403 and http is not supported. + // Downloading via `run bash wget` avoids the prompt entirely; bootflash + // is mounted at /bootflash inside the bash shell. The management VRF is + // a Linux netns, so wget must run inside it to reach the image server. + dest := "/bootflash/" + targetFileName + logger.V(1).Info("Copying firmware image to bootflash", "file", targetFileName) + if _, err := c.Do(ctx, nxapi.NewRequest( + "feature bash-shell", + //nolint:dupword // NX-OS requires `run bash bash -c` here. + `run bash bash -c 'ip netns exec management wget --no-verbose --output-document="$1" "$2"' -- `+strconv.Quote(dest)+` `+ + strconv.Quote(target.URL), + ).WithRollback(nxapi.Stop)); err != nil { + return fmt.Errorf("nxos firmware: copy image: %w", err) + } + + if target.MD5 != "" { + sum, err := p.fileMD5(ctx, targetFileName) + if err != nil { + return fmt.Errorf("nxos firmware: verify md5: %w", err) + } + if !strings.EqualFold(sum, target.MD5) { + return fmt.Errorf("nxos firmware: md5 mismatch after copy: got %s want %s", sum, target.MD5) + } + } + logger.V(1).Info("Firmware image copied and verified") + return nil +} + +// checkCompatibility runs the software compatibility and install impact checks +// and logs their output. +func (p *Provider) checkCompatibility(ctx context.Context, c *nxapi.Client, targetFileName string) error { + logger := logr.FromContextOrDiscard(ctx) + compatRes, err := c.Do(ctx, nxapi.NewRequest( + "show incompatibility-all nxos bootflash:"+targetFileName, + ).WithMethod(nxapi.MethodCLIASCII)) + if err != nil { + return fmt.Errorf("nxos firmware: compatibility check failed: %w", err) + } + logCLIResult(logger, compatRes, "Software compatibility check result") + + impactRes, err := c.Do(ctx, nxapi.NewRequest( + "show install all impact nxos bootflash:"+targetFileName, + ).WithMethod(nxapi.MethodCLIASCII)) + if err != nil { + return fmt.Errorf("nxos firmware: install impact check failed: %w", err) + } + logCLIResult(logger, impactRes, "Install impact check result") + return nil +} + +// doUpgrade saves the running config, installs the firmware without +// reload, and resets the session timeout to the default. +func (p *Provider) doUpgrade(ctx context.Context, c *nxapi.Client, targetFileName string) error { + logger := logr.FromContextOrDiscard(ctx) + if _, err := p.nxapi.Do(ctx, nxapi.NewRequest( + "copy running-config startup-config", + ).WithRollback(nxapi.Stop)); err != nil { + return fmt.Errorf("nxos firmware: save config: %w", err) + } + + // install all with no-reload keeps the connection up and returns the real result. + logger.V(1).Info("Installing firmware (no-reload)", "file", targetFileName) + installRes, err := c.Do(ctx, nxapi.NewRequest( + "install all nxos bootflash:"+targetFileName+" no-reload", + ).WithMethod(nxapi.MethodCLIASCII).WithRollback(nxapi.Stop)) + if err != nil { + return fmt.Errorf("nxos firmware: install failed: %w", err) + } + logCLIResult(logger, installRes, "Install result") + return nil +} + +// fileMD5 returns the md5 checksum of a file on bootflash, or an empty string +// if the file does not exist. A "file not found" RPC error from the device is +// treated as absence, not a failure, so callers can proceed to copy the image. +func (p *Provider) fileMD5(ctx context.Context, filename string) (string, error) { + res, err := p.nxapi.Do(ctx, nxapi.NewRequest("show file bootflash:"+filename+" md5sum")) + if err != nil { + if isFileNotFound(err) { + return "", nil + } + return "", err + } + if len(res) == 0 { + return "", nil + } + var result struct { + MD5Sum string `json:"file_content_md5sum"` + } + if err := json.Unmarshal(res[0], &result); err != nil { + return "", fmt.Errorf("nxos firmware: failed to decode md5sum response: %w", err) + } + return strings.TrimSpace(result.MD5Sum), nil +} + +// isFileNotFound reports whether err is an NX-API RPC error indicating that a +// referenced file does not exist on the device, so callers can distinguish a +// missing image (an expected pre-copy state) from a genuine failure. +func isFileNotFound(err error) bool { + var rpcErr *nxapi.RPCError + if !errors.As(err, &rpcErr) { + return false + } + msg := strings.ToLower(rpcErr.Error()) + return strings.Contains(msg, "no such file") || strings.Contains(msg, "not found") +} + +// isDeviceUnreachable reports whether err indicates the device is temporarily +// unreachable (as opposed to a logical error), covering both NX-API transport +// errors and gNMI/gRPC unavailability. This is expected while the device is +// rebooting after an install and should be treated as an in-progress state. +func isDeviceUnreachable(err error) bool { + if err == nil { + return false + } + if nxapi.IsTransportError(err) { + return true + } + switch status.Code(err) { + case codes.Unavailable, codes.DeadlineExceeded: + return true + default: + return false + } +} + +// remoteImageSize issues an HTTP HEAD to the firmware URL and returns its +// Content-Length in bytes. +func remoteImageSize(ctx context.Context, rawURL string) (int64, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, rawURL, nil) + if err != nil { + return 0, fmt.Errorf("nxos firmware: build HEAD request: %w", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, fmt.Errorf("nxos firmware: HEAD %s: %w", rawURL, err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return 0, fmt.Errorf("nxos firmware: HEAD %s returned status %d", rawURL, resp.StatusCode) + } + if resp.ContentLength < 0 { + return 0, fmt.Errorf("nxos firmware: HEAD %s did not return a Content-Length", rawURL) + } + return resp.ContentLength, nil +} + +// configSessionActive reports whether a configuration session is currently +// open on the device (which would block ISSU). +func (p *Provider) configSessionActive(ctx context.Context) (bool, error) { + res, err := p.nxapi.Do(ctx, nxapi.NewRequest("show configuration session summary")) + if err != nil { + return false, err + } + if len(res) == 0 { + return false, nil + } + var body struct { + Table struct { + Row json.RawMessage `json:"ROW_session"` + } `json:"TABLE_session"` + } + if err := json.Unmarshal(res[0], &body); err != nil { + return false, nil //nolint:nilerr // unmarshal failure means no session table => no sessions + } + return len(body.Table.Row) > 0, nil +} + +// progressBarRe matches NX-OS CLI progress bar segments like +// "[#### ] 25%" that pollute cli_ascii output. +var progressBarRe = regexp.MustCompile(`\[[#\s]*\]\s*\d+%`) + +// statusMarkerRe matches a trailing " -- SUCCESS" style status marker left on a +// line after its progress bars are stripped, capturing the marker word. The +// marker must be an uppercase word so table separator lines ending in a run of +// dashes (e.g. "------ ------") are not mistaken for markers. +var statusMarkerRe = regexp.MustCompile(`(?m)\s*--\s*([A-Z]+)\s*$`) + +// cleanCLIOutput tidies raw cli_ascii command output for logging by stripping +// progress bar segments, moving trailing status markers (e.g. "-- SUCCESS") +// onto their own line without the leading "--", and dropping blank lines. +func cleanCLIOutput(s string) string { + s = progressBarRe.ReplaceAllString(s, "") + s = statusMarkerRe.ReplaceAllString(s, "\n$1") + var kept []string + for line := range strings.SplitSeq(s, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + kept = append(kept, strings.TrimRight(line, " \t")) + } + return strings.Join(kept, "\n") +} + +// logCLIResult logs the cli_ascii output from an NX-API response, if present. +func logCLIResult(logger logr.Logger, res []json.RawMessage, msg string) { + if len(res) == 0 { + return + } + var output string + if err := json.Unmarshal(res[0], &output); err == nil { + logger.V(1).Info(msg, "output", cleanCLIOutput(output)) + } +} diff --git a/internal/provider/cisco/nxos/firmware_test.go b/internal/provider/cisco/nxos/firmware_test.go new file mode 100644 index 000000000..ee978264c --- /dev/null +++ b/internal/provider/cisco/nxos/firmware_test.go @@ -0,0 +1,408 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package nxos + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ironcore-dev/network-operator/internal/deviceutil" + "github.com/ironcore-dev/network-operator/internal/provider" + "github.com/ironcore-dev/network-operator/internal/transport/gnmiext" + "github.com/ironcore-dev/network-operator/internal/transport/nxapi" +) + +func TestCleanCLIOutput(t *testing.T) { + in := "Installer will perform compatibility check first. Please wait. \nInstaller will exit before reload\nInstaller is forced disruptive\n\nVerifying image bootflash:/nxos64-cs.10.6.3.F.bin for boot variable \"nxos\".\n[# ] 0%[####################] 100% -- SUCCESS\n\nVerifying EPLD/FPGA image //bootflash/nxos64-cs.10.6.3.F.bin.\n[# ] 0%[####################] 100% -- SUCCESS\n\nVerifying image type.\n[# ] 0%[####################] 100% -- SUCCESS\n\nPreparing \"nxos\" version info using image bootflash:/nxos64-cs.10.6.3.F.bin.\n[# ] 0%[####################] 100% -- SUCCESS\n\nPreparing \"bios\" version info using image bootflash:/nxos64-cs.10.6.3.F.bin.\n[# ] 0%[####################] 100% -- SUCCESS\n\nPerforming module support checks.\n[####################] 100% -- SUCCESS\n\nNotifying services about system upgrade.\n[####################] 100% -- SUCCESS\n\n\n\nCompatibility check is done:\nModule bootable Impact Install-type Reason\n------ -------- -------------- ------------ ------\n 1 yes disruptive reset default upgrade is not hitless\n 27 yes disruptive reset default upgrade is not hitless\n\n\n\nImages will be upgraded according to following table:\nModule Image Running-Version(pri:alt) New-Version Upg-Required\n------ ---------- ---------------------------------------- -------------------- ------------\n 1 lcn9k 10.6(2) 10.6(3) yes\n 27 nxos 10.6(2) 10.6(3) yes\n 27 bios v05.53(01/22/2025):v05.47(04/28/2022) v05.53(01/22/2025) no\n\n\nFPGA microcode will be upgraded according to following table:\nModule Type EPLD Running-Version Flashed-Version* New-Version Upg-Required\n------ ---- ------------- --------------- ---------------- ----------- ------------\n 27 SUP MI FPGA 0x5 0x5 0x5 No\n 27 SUP IO FPGA 0x17 0x17 0x18 Yes\n* If Running-Version and Flashed-Version are different it implies that the system has not yet been reloaded for the new version to take effect\n\nEPLD Upgrade may result in multiple modules going offline.\n\nAdditional info for this installation:\n--------------------------------------\n\nOption \"no-reload\" has been used - it is necessary reload device after installation without saving config.\nSaving config before can result incorrect startup config load after reload with new version of NXOS.\n\nService \"vpc\" in vdc 1: Vpc is enabled, Please make sure both Vpc peer switches have same boot mode using 'show boot mode' and proceed \n\n\n\n\n\nInstall is in progress, please wait.\n[# ] 0%\nSetting boot variables.\n[####################] 100% -- SUCCESS\n\nPerforming configuration copy.\n[# ] 0%[# ] 0%[###### ] 25%[########### ] 50%[################ ] 75%[####################] 100%\nPerforming configuration copy.\n[####################] 100% -- SUCCESS\n\nModule 1: Refreshing compact flash and upgrading bios/loader/bootrom.\nWarning: please do not remove or power off the module at this time.\n[# ] 0%\nModule 1: Refreshing compact flash and upgrading bios/loader/bootrom.\nWarning: please do not remove or power off the module at this time.\n[####################] 100% -- SUCCESS\n\nModule 27: Refreshing compact flash and upgrading bios/loader/bootrom.\nWarning: please do not remove or power off the module at this time.\n[# ] 0%\nModule 27: Refreshing compact flash and upgrading bios/loader/bootrom.\nWarning: please do not remove or power off the module at this time.\n[####################] 100% -- SUCCESS\n\nEPLD/FPGA upgrade can take upto 4 mins\n[# ] 0%\nPerforming EPLD/FPGA upgrade .\n[####################] 100% -- SUCCESS\n\n\n" + + want := "Installer will perform compatibility check first. Please wait.\nInstaller will exit before reload\nInstaller is forced disruptive\nVerifying image bootflash:/nxos64-cs.10.6.3.F.bin for boot variable \"nxos\".\nSUCCESS\nVerifying EPLD/FPGA image //bootflash/nxos64-cs.10.6.3.F.bin.\nSUCCESS\nVerifying image type.\nSUCCESS\nPreparing \"nxos\" version info using image bootflash:/nxos64-cs.10.6.3.F.bin.\nSUCCESS\nPreparing \"bios\" version info using image bootflash:/nxos64-cs.10.6.3.F.bin.\nSUCCESS\nPerforming module support checks.\nSUCCESS\nNotifying services about system upgrade.\nSUCCESS\nCompatibility check is done:\nModule bootable Impact Install-type Reason\n------ -------- -------------- ------------ ------\n 1 yes disruptive reset default upgrade is not hitless\n 27 yes disruptive reset default upgrade is not hitless\nImages will be upgraded according to following table:\nModule Image Running-Version(pri:alt) New-Version Upg-Required\n------ ---------- ---------------------------------------- -------------------- ------------\n 1 lcn9k 10.6(2) 10.6(3) yes\n 27 nxos 10.6(2) 10.6(3) yes\n 27 bios v05.53(01/22/2025):v05.47(04/28/2022) v05.53(01/22/2025) no\nFPGA microcode will be upgraded according to following table:\nModule Type EPLD Running-Version Flashed-Version* New-Version Upg-Required\n------ ---- ------------- --------------- ---------------- ----------- ------------\n 27 SUP MI FPGA 0x5 0x5 0x5 No\n 27 SUP IO FPGA 0x17 0x17 0x18 Yes\n* If Running-Version and Flashed-Version are different it implies that the system has not yet been reloaded for the new version to take effect\nEPLD Upgrade may result in multiple modules going offline.\nAdditional info for this installation:\n--------------------------------------\nOption \"no-reload\" has been used - it is necessary reload device after installation without saving config.\nSaving config before can result incorrect startup config load after reload with new version of NXOS.\nService \"vpc\" in vdc 1: Vpc is enabled, Please make sure both Vpc peer switches have same boot mode using 'show boot mode' and proceed\nInstall is in progress, please wait.\nSetting boot variables.\nSUCCESS\nPerforming configuration copy.\nPerforming configuration copy.\nSUCCESS\nModule 1: Refreshing compact flash and upgrading bios/loader/bootrom.\nWarning: please do not remove or power off the module at this time.\nModule 1: Refreshing compact flash and upgrading bios/loader/bootrom.\nWarning: please do not remove or power off the module at this time.\nSUCCESS\nModule 27: Refreshing compact flash and upgrading bios/loader/bootrom.\nWarning: please do not remove or power off the module at this time.\nModule 27: Refreshing compact flash and upgrading bios/loader/bootrom.\nWarning: please do not remove or power off the module at this time.\nSUCCESS\nEPLD/FPGA upgrade can take upto 4 mins\nPerforming EPLD/FPGA upgrade .\nSUCCESS" + + got := cleanCLIOutput(in) + if got != want { + t.Errorf("cleanCLIOutput mismatch:\ngot:\n%s\n\nwant:\n%s", got, want) + } +} + +// fakeGNMI is a minimal gnmiext.Client that returns a canned running version. +type fakeGNMI struct { + gnmiext.Client + version string + bootImage string + // getStateErr, when non-nil, is returned by GetState to simulate an + // unreachable device (e.g. mid-reload). + getStateErr error +} + +func (f *fakeGNMI) GetState(_ context.Context, elems ...gnmiext.DataElement) error { + if f.getStateErr != nil { + return f.getStateErr + } + for _, e := range elems { + switch v := e.(type) { + case *FirmwareVersion: + *v = FirmwareVersion(f.version) + case *BootImage: + *v = BootImage(f.bootImage) + } + } + return nil +} + +func (f *fakeGNMI) GetConfig(_ context.Context, elems ...gnmiext.DataElement) error { + for _, e := range elems { + if h, ok := e.(*Hostname); ok { + *h = Hostname("test-switch") + } + } + return nil +} + +func TestUpgradeFirmwareAlreadyOnTarget(t *testing.T) { + client, conn := nxapiStub(t, func(cmds []string) []string { + bodies := make([]string, len(cmds)) + for i := range bodies { + bodies[i] = "null" + } + return bodies + }) + p := &Provider{client: &fakeGNMI{bootImage: "bootflash://nxos64-cs.10.6.3.F.bin"}, nxapi: client} + target := provider.TargetFirmware{ + URL: "https://repo.example/nxos64-cs.10.6.3.F.bin", + MD5: "48c0db0a564c442f123eba8724ef352f", + } + if err := p.UpgradeFirmware(t.Context(), conn, target); err != nil { + t.Fatalf("expected nil (already upgraded), got %v", err) + } +} + +func TestUpgradeFirmwareUnreachableDuringProbe(t *testing.T) { + // Device unreachable during the completion check (e.g. mid-reload) must be + // treated as in progress so the reconcile requeues instead of failing. + p := &Provider{client: &fakeGNMI{getStateErr: status.Error(codes.Unavailable, "connection refused")}} + target := provider.TargetFirmware{URL: "https://repo.example/nxos64-cs.10.6.3.F.bin"} + err := p.UpgradeFirmware(t.Context(), &deviceutil.Connection{}, target) + if !errors.Is(err, provider.ErrUpgradeInProgress) { + t.Fatalf("expected ErrUpgradeInProgress for unreachable device, got %v", err) + } +} + +// nxapiStub starts an httptest server that responds to each NX-API batch using +// the provided handler, which maps the list of commands to a JSON body string +// (the ".result.body" payload) for each command, in order. It returns both the +// client and the connection so tests can pass conn to UpgradeFirmware for the +// long-timeout client it creates internally. +func nxapiStub(t *testing.T, handler func(cmds []string) []string) (*nxapi.Client, *deviceutil.Connection) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var reqCmds []struct { + Params struct { + Cmd string `json:"cmd"` + } `json:"params"` + } + if err := json.NewDecoder(r.Body).Decode(&reqCmds); err != nil { + t.Fatalf("stub: decode request: %v", err) + } + cmds := make([]string, len(reqCmds)) + for i, c := range reqCmds { + cmds[i] = c.Params.Cmd + } + bodies := handler(cmds) + w.Header().Set("Content-Type", "application/json-rpc") + fmt.Fprint(w, "[") + for i, b := range bodies { + if i > 0 { + fmt.Fprint(w, ",") + } + fmt.Fprintf(w, `{"jsonrpc":"2.0","result":{"body":%s},"id":%d}`, b, i+1) + } + fmt.Fprint(w, "]") + })) + t.Cleanup(srv.Close) + _, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) //nolint:errcheck + conn := &deviceutil.Connection{Address: srv.Listener.Addr().String(), Username: "admin", Password: "secret"} + client, err := nxapi.NewClient(conn, nxapi.WithPort(port)) + if err != nil { + t.Fatalf("stub: new client: %v", err) + } + return client, conn +} + +func TestListDirectoryBytesfree(t *testing.T) { + client, _ := nxapiStub(t, func(cmds []string) []string { + if cmds[0] != "dir bootflash:" { + t.Errorf("cmd = %q, want 'dir bootflash:'", cmds[0]) + } + return []string{`{"bytesfree":3664789504}`} + }) + p := &Provider{nxapi: client} + dir, err := p.ListDirectory(t.Context(), "bootflash:") + if err != nil { + t.Fatalf("ListDirectory error: %v", err) + } + if dir.Bytesfree != 3664789504 { + t.Errorf("dir.Bytesfree = %d, want 3664789504", dir.Bytesfree) + } +} + +func TestFileMD5(t *testing.T) { + client, _ := nxapiStub(t, func(cmds []string) []string { + want := "show file bootflash:nxos64-cs.10.6.3.F.bin md5sum" + if cmds[0] != want { + t.Errorf("cmd = %q, want %q", cmds[0], want) + } + return []string{`{"file_content_md5sum":"48c0db0a564c442f123eba8724ef352f\n"}`} + }) + p := &Provider{nxapi: client} + got, err := p.fileMD5(t.Context(), "nxos64-cs.10.6.3.F.bin") + if err != nil { + t.Fatalf("fileMD5 error: %v", err) + } + if got != "48c0db0a564c442f123eba8724ef352f" { + t.Errorf("fileMD5 = %q", got) + } +} + +// nxapiErrorStub starts an httptest server that responds to every NX-API batch +// with a single JSON-RPC error carrying the given code and message, so tests +// can exercise how helpers react to device-side command failures. +func nxapiErrorStub(t *testing.T, code int, message string) *nxapi.Client { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json-rpc") + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, `[{"jsonrpc":"2.0","error":{"code":%d,"message":%q},"id":1}]`, code, message) + })) + t.Cleanup(srv.Close) + _, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) //nolint:errcheck + conn := &deviceutil.Connection{Address: srv.Listener.Addr().String(), Username: "admin", Password: "secret"} + client, err := nxapi.NewClient(conn, nxapi.WithPort(port)) + if err != nil { + t.Fatalf("error stub: new client: %v", err) + } + return client +} + +func TestFileMD5NotFound(t *testing.T) { + p := &Provider{nxapi: nxapiErrorStub(t, 1, "No such file or directory")} + got, err := p.fileMD5(t.Context(), "nxos64-cs.10.6.3.F.bin") + if err != nil { + t.Fatalf("fileMD5 error: %v", err) + } + if got != "" { + t.Errorf("fileMD5 = %q, want empty string for missing file", got) + } +} + +func TestFileMD5RealError(t *testing.T) { + p := &Provider{nxapi: nxapiErrorStub(t, 500, "internal device error")} + if _, err := p.fileMD5(t.Context(), "nxos64-cs.10.6.3.F.bin"); err == nil { + t.Fatal("expected error for non-not-found RPC failure, got nil") + } +} + +func TestConfigSessionActive(t *testing.T) { + client1, _ := nxapiStub(t, func(cmds []string) []string { + return []string{`{"TABLE_session":{"ROW_session":[{"session":"s1"}]}}`} + }) + p := &Provider{nxapi: client1} + active, err := p.configSessionActive(t.Context()) + if err != nil { + t.Fatalf("configSessionActive error: %v", err) + } + if !active { + t.Error("expected active session, got false") + } + + client2, _ := nxapiStub(t, func(cmds []string) []string { + return []string{`{}`} + }) + p2 := &Provider{nxapi: client2} + active2, err := p2.configSessionActive(t.Context()) + if err != nil { + t.Fatalf("configSessionActive error: %v", err) + } + if active2 { + t.Error("expected no active session, got true") + } +} + +func TestRemoteImageSize(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Errorf("method = %s, want HEAD", r.Method) + } + w.Header().Set("Content-Length", "3005853696") + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + got, err := remoteImageSize(t.Context(), srv.URL+"/nxos64-cs.10.6.3.F.bin") + if err != nil { + t.Fatalf("remoteImageSize error: %v", err) + } + if got != 3005853696 { + t.Errorf("remoteImageSize = %d, want 3005853696", got) + } +} + +func TestUpgradeFirmwareCopyStep(t *testing.T) { + // Device on old version, image absent -> preflight + copy issued -> in progress. + var got []string + copied := false + client, conn := nxapiStub(t, func(cmds []string) []string { + got = append(got, cmds...) + bodies := make([]string, len(cmds)) + for i, c := range cmds { + switch { + case c == "show file bootflash:nxos64-cs.10.6.3.F.bin md5sum": + if copied { + bodies[i] = `{"file_content_md5sum":"48c0db0a564c442f123eba8724ef352f"}` // present after copy + } else { + bodies[i] = `{"file_content_md5sum":""}` // absent before copy + } + case c == "show configuration session summary": + bodies[i] = `{}` + case c == "dir bootflash:": + bodies[i] = `{"bytesfree":6000000000}` + case strings.HasPrefix(c, "run bash"): + copied = true + bodies[i] = `""` + default: + bodies[i] = `""` + } + } + return bodies + }) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "3005853696") + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + p := &Provider{ + client: &fakeGNMI{bootImage: "bootflash://nxos64-cs.10.6.2.F.bin"}, + nxapi: client, + } + target := provider.TargetFirmware{ + URL: srv.URL + "/nxos64-cs.10.6.3.F.bin", + MD5: "48c0db0a564c442f123eba8724ef352f", + } + err := p.UpgradeFirmware(t.Context(), conn, target) + if !errors.Is(err, provider.ErrUpgradeInProgress) { + t.Fatalf("expected ErrUpgradeInProgress, got %v", err) + } + joined := strings.Join(got, "|") + if !strings.Contains(joined, "run bash") { + t.Errorf("wget copy command not issued; got %v", got) + } +} + +func TestUpgradeFirmwareInsufficientSpace(t *testing.T) { + client, conn := nxapiStub(t, func(cmds []string) []string { + bodies := make([]string, len(cmds)) + for i, c := range cmds { + switch c { + case "show file bootflash:nxos64-cs.10.6.3.F.bin md5sum": + bodies[i] = `""` + case "show configuration session summary": + bodies[i] = `{}` + case "dir bootflash:": + bodies[i] = `{"bytesfree":"1000"}` + default: + bodies[i] = `""` + } + } + return bodies + }) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "3005853696") + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + + p := &Provider{client: &fakeGNMI{version: "10.6(2)"}, nxapi: client} + target := provider.TargetFirmware{URL: srv.URL + "/nxos64-cs.10.6.3.F.bin", MD5: "abc"} + err := p.UpgradeFirmware(t.Context(), conn, target) + if err == nil || errors.Is(err, provider.ErrUpgradeInProgress) { + t.Fatalf("expected hard error for insufficient space, got %v", err) + } +} + +func TestUpgradeFirmwareInstallAndReload(t *testing.T) { + // Image present with matching md5 -> impact + save + install(no-reload) + reload. + var got []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var reqCmds []struct { + Params struct { + Cmd string `json:"cmd"` + } `json:"params"` + } + json.NewDecoder(r.Body).Decode(&reqCmds) //nolint:errcheck + cmds := make([]string, len(reqCmds)) + for i, c := range reqCmds { + cmds[i] = c.Params.Cmd + } + got = append(got, cmds...) + + // The reload request drops the connection. + if len(cmds) == 1 && cmds[0] == "reload" { + hj, ok := w.(http.Hijacker) + if !ok { + t.Fatal("no hijacker") + } + conn, _, _ := hj.Hijack() //nolint:errcheck + conn.Close() + return + } + w.Header().Set("Content-Type", "application/json-rpc") + fmt.Fprint(w, "[") + for i, c := range cmds { + if i > 0 { + fmt.Fprint(w, ",") + } + body := `{"file_content_md5sum":""}` + if c == "show file bootflash:nxos64-cs.10.6.3.F.bin md5sum" { + body = `{"file_content_md5sum":"48c0db0a564c442f123eba8724ef352f"}` + } + fmt.Fprintf(w, `{"jsonrpc":"2.0","result":{"body":%s},"id":%d}`, body, i+1) + } + fmt.Fprint(w, "]") + })) + t.Cleanup(srv.Close) + _, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) //nolint:errcheck + conn := &deviceutil.Connection{Address: srv.Listener.Addr().String(), Username: "admin", Password: "secret"} + client, err := nxapi.NewClient(conn, nxapi.WithPort(port)) + if err != nil { + t.Fatalf("new client: %v", err) + } + + p := &Provider{client: &fakeGNMI{version: "10.6(2)"}, nxapi: client} + target := provider.TargetFirmware{URL: "https://repo.example/nxos64-cs.10.6.3.F.bin", MD5: "48c0db0a564c442f123eba8724ef352f"} + err = p.UpgradeFirmware(t.Context(), conn, target) + if !errors.Is(err, provider.ErrUpgradeInProgress) { + t.Fatalf("expected ErrUpgradeInProgress after reload, got %v", err) + } + joined := strings.Join(got, "|") + for _, want := range []string{ + "show install all impact nxos bootflash:nxos64-cs.10.6.3.F.bin", + "copy running-config startup-config", + "install all nxos bootflash:nxos64-cs.10.6.3.F.bin no-reload", + "reload", + } { + if !strings.Contains(joined, want) { + t.Errorf("missing command %q; got %v", want, got) + } + } +} diff --git a/internal/provider/cisco/nxos/system.go b/internal/provider/cisco/nxos/system.go index 3492dba16..44b202d48 100644 --- a/internal/provider/cisco/nxos/system.go +++ b/internal/provider/cisco/nxos/system.go @@ -68,6 +68,13 @@ func (*FirmwareVersion) XPath() string { return "System/showversion-items/nxosVersion" } +// BootImage is the boot image filename of the device, e.g. "bootflash://nxos.10.4.3.bin". +type BootImage string + +func (*BootImage) XPath() string { + return "System/showversion-items/nxosImageFile" +} + type BootTime UnixTime func (*BootTime) XPath() string { diff --git a/internal/provider/provider.go b/internal/provider/provider.go index bb0bf7a06..822b6c7bc 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -5,6 +5,7 @@ package provider import ( "context" "crypto/tls" + "errors" "fmt" "maps" "net/netip" @@ -47,6 +48,9 @@ type MaintenanceProvider interface { Reboot(context.Context, *deviceutil.Connection) error // FactoryReset performs a factory reset of the device. FactoryReset(context.Context, *deviceutil.Connection) error + // UpgradeFirmware initiates a firmware upgrade on the device. + // The provider is responsible for applying the new firmware and rebooting the device as necessary. + UpgradeFirmware(context.Context, *deviceutil.Connection, TargetFirmware) error } // ProvisioningProvider is the interface for the realization of the provisioning-related operations over different providers. @@ -71,6 +75,21 @@ type DevicePort struct { Transceiver string } +// TargetFirmware represents the firmware image to be applied to the device, including its URL and optional checksum. +// This JSON is passed as the value of the DeviceMaintenanceFirmwareTargetAnnotation on the Device resource +type TargetFirmware struct { + // URL is the URL of the firmware image to be applied to the device. + URL string `json:"url"` + // MD5 is the MD5 checksum of the firmware image, if available. + MD5 string `json:"md5,omitempty"` +} + +// ErrUpgradeInProgress is returned by MaintenanceProvider.UpgradeFirmware when it +// has advanced a step (e.g. issued the image copy or the reload) but the device is +// not yet running the target version. The controller treats this as a signal to +// requeue and re-invoke rather than a hard failure. +var ErrUpgradeInProgress = errors.New("provider: firmware upgrade in progress") + type DeviceInfo struct { // Hostname is the hostname of the device. Hostname string diff --git a/internal/transport/nxapi/nxapi.go b/internal/transport/nxapi/nxapi.go index a64a2a7f4..c655ce30b 100644 --- a/internal/transport/nxapi/nxapi.go +++ b/internal/transport/nxapi/nxapi.go @@ -107,6 +107,23 @@ func NewClient(conn *deviceutil.Connection, opts ...Option) (*Client, error) { return c, nil } +// Clone returns a copy of the client with the given options applied, leaving +// the original untouched. The copy keeps the resolved endpoint URL and shares +// the underlying HTTP transport, so it stays reachable at the same address and +// reuses pooled connections. Use it to derive a client that differs only in +// request behaviour, e.g. a longer timeout for long-running commands. +func (c *Client) Clone(opts ...Option) (*Client, error) { + clone := *c + httpClient := *c.client + clone.client = &httpClient + for _, opt := range opts { + if err := opt(&clone); err != nil { + return nil, err + } + } + return &clone, nil +} + // Do sends a Request to the device and returns one [json.RawMessage] per // command, in the same order as the request. If any command fails, Do returns // an [RPCErrors] containing one [RPCError] per failed command; transport and