feat(vm): atomic hibernate; restore accepts stopped VMs

KNOWN_ISSUES.md

@@ -15,10 +15,6 @@ After `cocoon vm clone`, the cloned VM resumes with the **original VM's IP addre
 
 `cocoon vm clone` inherits CPU, memory, and storage from the snapshot — none of these can be grown at clone time on either backend. NIC count inherits by default; Cloud Hypervisor clones may override it with `--nics N` (cocoon hot-swaps the snapshot's NICs for a fresh set after restore). Firecracker clones must keep the snapshot's NIC topology — FC's `network_overrides` only retargets existing interfaces, so `--nics` is rejected on FC. `cocoon vm restore` is more restrictive: CPU, memory, and storage come from the snapshot (the persisted record is realigned to match), and NIC count must already match the target VM (mismatches are rejected, since restore reuses the existing network namespace). Use `cocoon vm run` to create a fresh VM with different sizing.
 
-## Restore requires a running VM
-
-`cocoon vm restore` only works on running VMs — it relies on the existing network namespace (netns, tap devices, TC redirect) surviving the CH process restart. A stopped VM's network state may not be intact (e.g., after host reboot the netns is gone). For stopped VMs or cross-VM restore, use `cocoon vm clone` which creates fresh network resources. See [Restore Constraints](README.md#restore-constraints) for all requirements.
-
 ## OCI VM multi-NIC kernel IP limitation
 
 OCI VMs use the kernel `ip=` boot parameter for network configuration. While multiple `ip=` parameters can be specified, the Linux kernel only reliably configures **one interface** via this mechanism — subsequent `ip=` parameters may be silently ignored or produce inconsistent results depending on kernel version.

README.md

@@ -141,7 +141,8 @@ cocoon
 │   ├── exec [flags] VM -- CMD     Run a command in a running VM via cocoon-agent (vsock)
 │   ├── logs [-f] [--tail N] VM    Print the per-VM hypervisor log file
 │   ├── rm [flags] VM [VM...]      Delete VM(s) (--force to stop first)
-│   ├── restore [flags] VM SNAP   Restore a running VM to a snapshot
+│   ├── restore [flags] VM SNAP   Restore a VM (running or stopped) to a snapshot
+│   ├── hibernate [flags] VM       Atomically snapshot a running VM and stop it
 │   ├── status [VM...]             Watch VM status in real time
 │   ├── fs
 │   │   ├── attach [flags] VM     Attach a vhost-user-fs share (CH only)
@@ -793,18 +794,29 @@ The `--pull` flag uses the image digest recorded at snapshot time to verify that
 
 ### Restore
 
-Restore reverts a **running** VM to a previous snapshot's state in-place:
+Restore reverts a VM — running or stopped — to a previous snapshot's state in-place:
 
 ```bash
 # Restore a VM to a previous snapshot
 cocoon vm restore my-vm my-snap
 ```
 
-Cocoon stages the snapshot into a scratch directory first, then restarts the hypervisor process only after the full extraction succeeds — a truncated or corrupt snapshot stream errors out with the running VM still intact. Network is fully preserved — same IP, same MAC, same network namespace. No guest-side reconfiguration is needed (unlike clone).
+Cocoon stages the snapshot into a scratch directory first, then (re)starts the hypervisor process only after the full extraction succeeds — a truncated or corrupt snapshot stream errors out with the VM in its prior state. Network is fully preserved — same IP, same MAC, same network namespace; restoring a stopped VM first runs the same network self-heal as `vm start`, so a hibernated VM resumes even after a host reboot. No guest-side reconfiguration is needed (unlike clone).
+
+### Hibernate
+
+Hibernate atomically snapshots a running VM and stops it, releasing its memory; resume with `vm restore`:
+
+```bash
+cocoon vm hibernate my-vm --name nap
+cocoon vm restore my-vm nap
+```
+
+Pause, capture, persist, and VMM termination share one pause window: the snapshot point and the stop coincide (nothing the guest does can be lost in between), and the VMM dies only after the snapshot is durably saved — if saving fails (disk full, snapshot DB error), the VM simply resumes running and the command fails. Restore of the hibernated VM preserves machine identity (entropy-only reseed): running processes, sessions, and tmpfs contents continue where they stopped.
 
 ### Restore Constraints
 
-- **VM must be running.** Restore operates on a live VM by restarting its CH process with snapshot state. For stopped VMs, use `cocoon vm clone` instead.
+- **VM must be running or stopped.** Restore cold-spawns a fresh hypervisor process from the snapshot; a stopped target (e.g. hibernated) gets the `vm start` network self-heal first.
 - **Snapshot must belong to the VM.** Only snapshots created from the same VM (tracked in `snapshot_ids`) are accepted; pass `--force` with `--from-dir` to opt into a foreign lineage.
 - **CPU, memory, and storage come from the snapshot.** The hypervisor reconstructs the guest from snapshot state, so these are not configurable at restore time; cocoon realigns the persisted record to match.
 - **NIC count must match the target VM.** Restore reuses the VM's existing network namespace, TAP devices, and IP allocation; a mismatched count is rejected.

cmd/core/utils.go

@@ -10,12 +10,14 @@ import (
 
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/projecteru2/core/log"
+	"github.com/spf13/cobra"
 
 	"github.com/cocoonstack/cocoon/cmd/cliutil"
 	"github.com/cocoonstack/cocoon/config"
 	"github.com/cocoonstack/cocoon/hypervisor"
 	imagebackend "github.com/cocoonstack/cocoon/images"
 	"github.com/cocoonstack/cocoon/progress"
+	"github.com/cocoonstack/cocoon/snapshot"
 	"github.com/cocoonstack/cocoon/types"
 	"github.com/cocoonstack/cocoon/utils"
 )
@@ -224,3 +226,47 @@ func resolveVMOwner(ctx context.Context, hypers []hypervisor.Hypervisor, ref str
 	)
 	return owner, resolved, err
 }
+
+// EnsureSnapshotNameFree validates name and rejects a duplicate; empty passes.
+func EnsureSnapshotNameFree(ctx context.Context, snapBackend snapshot.Snapshot, name string) error {
+	if err := (&types.SnapshotConfig{Name: name}).Validate(); err != nil {
+		return err
+	}
+	if name == "" {
+		return nil
+	}
+	if _, err := snapBackend.Inspect(ctx, name); err == nil {
+		return fmt.Errorf("snapshot name %q already exists", name)
+	} else if !errors.Is(err, snapshot.ErrNotFound) {
+		return fmt.Errorf("check snapshot name: %w", err)
+	}
+	return nil
+}
+
+// CaptureSnapshot checks the --name preflight, runs capture, and persists the stream, returning the stored snapshot id.
+func CaptureSnapshot(ctx context.Context, cmd *cobra.Command, snapBackend snapshot.Snapshot, capture func() (*types.SnapshotConfig, io.ReadCloser, error)) (string, error) {
+	name, _ := cmd.Flags().GetString("name")
+	description, _ := cmd.Flags().GetString("description")
+	if err := EnsureSnapshotNameFree(ctx, snapBackend, name); err != nil {
+		return "", err
+	}
+	cfg, stream, err := capture()
+	if err != nil {
+		return "", err
+	}
+	return PersistSnapshotStream(ctx, snapBackend, cfg, stream, name, description)
+}
+
+// PersistSnapshotStream labels cfg and stores the stream, closing it either way.
+func PersistSnapshotStream(ctx context.Context, snapBackend snapshot.Snapshot, cfg *types.SnapshotConfig, stream io.ReadCloser, name, description string) (string, error) {
+	defer stream.Close() //nolint:errcheck
+	defer CloseOnCancel(ctx, stream)()
+	cfg.Name = name
+	cfg.Description = description
+	log.WithFunc("cmdcore.PersistSnapshotStream").Info(ctx, "saving snapshot data ...")
+	snapID, err := snapBackend.Create(ctx, cfg, stream)
+	if err != nil {
+		return "", fmt.Errorf("save snapshot: %w", err)
+	}
+	return snapID, nil
+}

cmd/snapshot/handler.go

@@ -2,7 +2,6 @@ package snapshot
 
 import (
 	"cmp"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -39,40 +38,14 @@ func (h Handler) Save(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	name, _ := cmd.Flags().GetString("name")
-	description, _ := cmd.Flags().GetString("description")
-
-	if err = (&types.SnapshotConfig{Name: name}).Validate(); err != nil {
-		return err
-	}
-	if name != "" {
-		if _, inspectErr := snapBackend.Inspect(ctx, name); inspectErr == nil {
-			return fmt.Errorf("snapshot name %q already exists", name)
-		} else if !errors.Is(inspectErr, snapshot.ErrNotFound) {
-			return fmt.Errorf("check snapshot name: %w", inspectErr)
-		}
-	}
 
 	logger.Infof(ctx, "snapshotting VM %s ...", vmRef)
-
-	cfg, stream, err := hyper.Snapshot(ctx, vmRef)
-	if err != nil {
-		return fmt.Errorf("snapshot VM %s: %w", vmRef, err)
-	}
-	defer stream.Close() //nolint:errcheck
-
-	defer cmdcore.CloseOnCancel(ctx, stream)()
-
-	cfg.Name = name
-	cfg.Description = description
-
-	logger.Info(ctx, "saving snapshot data ...")
-
-	snapID, err := snapBackend.Create(ctx, cfg, stream)
+	snapID, err := cmdcore.CaptureSnapshot(ctx, cmd, snapBackend, func() (*types.SnapshotConfig, io.ReadCloser, error) {
+		return hyper.Snapshot(ctx, vmRef)
+	})
 	if err != nil {
-		return fmt.Errorf("save snapshot: %w", err)
+		return err
 	}
-
 	logger.Infof(ctx, "snapshot saved: %s", snapID)
 	return nil
 }

cmd/vm/commands.go

@@ -21,6 +21,7 @@ type Actions interface {
 	Logs(cmd *cobra.Command, args []string) error
 	RM(cmd *cobra.Command, args []string) error
 	Restore(cmd *cobra.Command, args []string) error
+	Hibernate(cmd *cobra.Command, args []string) error
 	Debug(cmd *cobra.Command, args []string) error
 	Status(cmd *cobra.Command, args []string) error
 	FsAttach(cmd *cobra.Command, args []string) error
@@ -141,7 +142,7 @@ func Command(h Actions) *cobra.Command {
 
 	restoreCmd := &cobra.Command{
 		Use:   "restore [flags] VM [SNAPSHOT]",
-		Short: "Restore a running VM to a previous snapshot (or a directory via --from-dir)",
+		Short: "Restore a running or stopped VM to a previous snapshot (or a directory via --from-dir)",
 		Args:  cobra.RangeArgs(1, 2),
 		PreRunE: func(cmd *cobra.Command, _ []string) error {
 			force, _ := cmd.Flags().GetBool("force")
@@ -158,6 +159,15 @@ func Command(h Actions) *cobra.Command {
 	restoreCmd.Flags().Bool("force", false, "skip the snapshot-belongs-to-VM check (only meaningful with --from-dir; risk of restoring to an unrelated lineage)")
 	cliutil.AddOutputFlag(restoreCmd)
 
+	hibernateCmd := &cobra.Command{
+		Use:   "hibernate [flags] VM",
+		Short: "Atomically snapshot a running VM and stop it (resume with vm restore)",
+		Args:  cobra.ExactArgs(1),
+		RunE:  h.Hibernate,
+	}
+	hibernateCmd.Flags().String("name", "", "snapshot name")
+	hibernateCmd.Flags().String("description", "", "snapshot description")
+
 	debugCmd := &cobra.Command{
 		Use:   "debug [flags] IMAGE",
 		Short: "Generate hypervisor launch command (dry run)",
@@ -194,6 +204,7 @@ func Command(h Actions) *cobra.Command {
 		logsCmd,
 		rmCmd,
 		restoreCmd,
+		hibernateCmd,
 		debugCmd,
 		statusCmd,
 		buildFsCommand(h),

cmd/vm/hibernate.go

@@ -0,0 +1,58 @@
+package vm
+
+import (
+	"cmp"
+	"fmt"
+	"io"
+
+	"github.com/projecteru2/core/log"
+	"github.com/spf13/cobra"
+
+	cmdcore "github.com/cocoonstack/cocoon/cmd/core"
+	"github.com/cocoonstack/cocoon/hypervisor"
+	"github.com/cocoonstack/cocoon/types"
+)
+
+// Hibernate atomically snapshots a running VM and stops it; the snapshot
+// point and the stop coincide, so `vm restore` resumes with nothing lost.
+func (h Handler) Hibernate(cmd *cobra.Command, args []string) error {
+	ctx, conf, err := h.Init(cmd)
+	if err != nil {
+		return err
+	}
+	logger := log.WithFunc("cmd.vm.hibernate")
+	vmRef := args[0]
+
+	hyper, err := cmdcore.FindHypervisor(ctx, conf, vmRef)
+	if err != nil {
+		return fmt.Errorf("find VM %s: %w", vmRef, err)
+	}
+	hib, ok := hyper.(hypervisor.Hibernator)
+	if !ok {
+		return fmt.Errorf("backend %s does not support hibernate", hyper.Type())
+	}
+	snapBackend, err := cmdcore.InitSnapshot(ctx, conf)
+	if err != nil {
+		return err
+	}
+	name, _ := cmd.Flags().GetString("name")
+	description, _ := cmd.Flags().GetString("description")
+	// Fail on a taken name before the VM is even paused.
+	if err = cmdcore.EnsureSnapshotNameFree(ctx, snapBackend, name); err != nil {
+		return err
+	}
+
+	logger.Infof(ctx, "hibernating VM %s ...", vmRef)
+	// persist runs inside the pause window; the VMM dies only after it succeeds.
+	var snapID string
+	err = hib.Hibernate(ctx, vmRef, func(cfg *types.SnapshotConfig, stream io.ReadCloser) error {
+		id, pErr := cmdcore.PersistSnapshotStream(ctx, snapBackend, cfg, stream, name, description)
+		snapID = id
+		return pErr
+	})
+	if err != nil {
+		return err
+	}
+	logger.Infof(ctx, "VM hibernated; snapshot %s (resume: cocoon vm restore %s %s)", snapID, vmRef, cmp.Or(name, snapID))
+	return nil
+}

cmd/vm/lifecycle.go

@@ -228,6 +228,15 @@ func (h Handler) RM(cmd *cobra.Command, args []string) error {
 	return finishRoutedCmd(ctx, cmd, logTag, "rm", "deleted", allDeleted, lastErr)
 }
 
+// recoverNetworkForStopped runs Start's network self-heal when the restore target is not running (hibernate resume after a host reboot).
+func (h Handler) recoverNetworkForStopped(ctx context.Context, conf *config.Config, hyper hypervisor.Hypervisor, ref string) {
+	vm, err := hyper.Inspect(ctx, ref)
+	if err != nil || vm.State == types.VMStateRunning {
+		return
+	}
+	h.recoverNetwork(ctx, conf, hyper, []string{vm.ID})
+}
+
 func (h Handler) recoverNetwork(ctx context.Context, conf *config.Config, hyper hypervisor.Hypervisor, refs []string) {
 	logger := log.WithFunc("cmd.vm.recoverNetwork")
 

cmd/vm/run.go

@@ -157,6 +157,7 @@ func (h Handler) Restore(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("find VM %s: %w", vmRef, err)
 	}
+	h.recoverNetworkForStopped(ctx, conf, hyper, vmRef)
 	snapBackend, err := cmdcore.InitSnapshot(ctx, conf)
 	if err != nil {
 		return err
@@ -224,6 +225,9 @@ func (h Handler) restoreFromDir(ctx context.Context, cmd *cobra.Command, conf *c
 	if err != nil {
 		return fmt.Errorf("inspect VM: %w", err)
 	}
+	if vm.State != types.VMStateRunning {
+		h.recoverNetwork(ctx, conf, hyper, []string{vm.ID})
+	}
 	if _, owned := vm.SnapshotIDs[cfg.ID]; !owned {
 		force, _ := cmd.Flags().GetBool("force")
 		if !force {

hypervisor/backend.go

@@ -154,3 +154,18 @@ type SnapshotSpec struct {
 	AfterCapture func(rec *VMRecord, tmpDir string) error
 	BuildMeta    func(rec *VMRecord, tmpDir string) (*SnapshotMeta, error)
 }
+
+// HibernateSpec extends SnapshotSpec with the pause-window terminate hook and the backend's runtime files.
+type HibernateSpec struct {
+	SnapshotSpec
+	Terminate    func(rec *VMRecord, hc *http.Client, pid int) error
+	RuntimeFiles []string
+}
+
+// runWrapped runs fn under a spec's optional Wrap.
+func runWrapped(rec *VMRecord, wrap func(*VMRecord, func() error) error, fn func() error) error {
+	if wrap != nil {
+		return wrap(rec, fn)
+	}
+	return fn()
+}

hypervisor/cloudhypervisor/restore.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"path/filepath"
 
 	"github.com/projecteru2/core/log"
@@ -32,12 +33,16 @@ func (ch *CloudHypervisor) preflightRestore(srcDir string, rec *hypervisor.VMRec
 }
 
 func (ch *CloudHypervisor) killForRestore(ctx context.Context, vmID string, rec *hypervisor.VMRecord) error {
-	sockPath := hypervisor.SocketPath(rec.RunDir)
 	return ch.KillForRestore(ctx, vmID, rec, func(pid int) error {
-		return ch.forceTerminate(ctx, utils.NewSocketHTTPClient(sockPath), vmID, sockPath, pid)
+		return ch.terminateVMM(ctx, rec, utils.NewSocketHTTPClient(hypervisor.SocketPath(rec.RunDir)), pid)
 	}, runtimeFiles)
 }
 
+// terminateVMM force-terminates rec's VMM over hc; shared by restore and hibernate.
+func (ch *CloudHypervisor) terminateVMM(ctx context.Context, rec *hypervisor.VMRecord, hc *http.Client, pid int) error {
+	return ch.forceTerminate(ctx, hc, rec.ID, hypervisor.SocketPath(rec.RunDir), pid)
+}
+
 func (ch *CloudHypervisor) restoreAfterExtract(ctx context.Context, vmID string, vmCfg *types.VMConfig, rec *hypervisor.VMRecord, directBoot bool) (_ *types.VM, err error) {
 	logger := log.WithFunc("cloudhypervisor.Restore")
 

hypervisor/cloudhypervisor/snapshot.go

@@ -15,7 +15,22 @@ import (
 
 // Snapshot pauses, captures CH state+COW, resumes, and streams the result.
 func (ch *CloudHypervisor) Snapshot(ctx context.Context, ref string) (*types.SnapshotConfig, io.ReadCloser, error) {
-	return ch.SnapshotSequence(ctx, ref, hypervisor.SnapshotSpec{
+	return ch.SnapshotSequence(ctx, ref, ch.snapshotSpec(ctx))
+}
+
+// Hibernate captures like Snapshot but persists and then terminates the VMM inside the pause window instead of resuming.
+func (ch *CloudHypervisor) Hibernate(ctx context.Context, ref string, persist func(cfg *types.SnapshotConfig, stream io.ReadCloser) error) error {
+	return ch.HibernateSequence(ctx, ref, hypervisor.HibernateSpec{
+		SnapshotSpec: ch.snapshotSpec(ctx),
+		Terminate: func(rec *hypervisor.VMRecord, hc *http.Client, pid int) error {
+			return ch.terminateVMM(ctx, rec, hc, pid)
+		},
+		RuntimeFiles: runtimeFiles,
+	}, persist)
+}
+
+func (ch *CloudHypervisor) snapshotSpec(ctx context.Context) hypervisor.SnapshotSpec {
+	return hypervisor.SnapshotSpec{
 		Pause:  func(_ *hypervisor.VMRecord, hc *http.Client) error { return pauseVM(ctx, hc) },
 		Resume: func(_ *hypervisor.VMRecord, hc *http.Client) error { return resumeVM(context.WithoutCancel(ctx), hc) },
 		Capture: func(rec *hypervisor.VMRecord, hc *http.Client, tmpDir string) error {
@@ -43,7 +58,7 @@ func (ch *CloudHypervisor) Snapshot(ctx context.Context, ref string) (*types.Sna
 			return nil
 		},
 		BuildMeta: buildSnapshotMeta,
-	})
+	}
 }
 
 // buildSnapshotMeta mirrors config.json's disk shape (not activeDisks — diverges for cloudimg post-FirstBooted pre-restart where CH still holds cidata).