From cff0bbca02b247e9ff808363bbdf09c42ed7e331 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 22 Oct 2025 12:56:28 +0000 Subject: [PATCH 01/10] incusd/storage/drivers: Add cleanup override Signed-off-by: Benjamin Somers --- internal/server/storage/drivers/driver_types.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/server/storage/drivers/driver_types.go b/internal/server/storage/drivers/driver_types.go index 92549171aa8..99d64c7c66b 100644 --- a/internal/server/storage/drivers/driver_types.go +++ b/internal/server/storage/drivers/driver_types.go @@ -21,6 +21,7 @@ type Info struct { MountedRoot bool // Whether the pool directory itself is a mount. Deactivate bool // Whether an unmount action is required prior to removing the pool. ZeroUnpack bool // Whether to write zeroes (no discard) during unpacking. + IgnoreCleanup bool // Whether to ignore instance cleanup, in case the pool is a big mounted tree. } // VolumeFiller provides a struct for filling a volume. From 6c0b54ac67e1e1fbedbb4df4863d84fca1539387 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 22 Oct 2025 12:57:31 +0000 Subject: [PATCH 02/10] incusd: Add cleanup override Signed-off-by: Benjamin Somers --- cmd/incusd/instance_post.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/incusd/instance_post.go b/cmd/incusd/instance_post.go index ffd3508ef1e..8cb452e69c4 100644 --- a/cmd/incusd/instance_post.go +++ b/cmd/incusd/instance_post.go @@ -1023,10 +1023,13 @@ func migrateInstance(ctx context.Context, s *state.State, inst instance.Instance // Cleanup instance paths on source member if using remote shared storage // and there was no storage pool change. - if sourcePool.Driver().Info().Remote && req.Pool == "" { - err = sourcePool.CleanupInstancePaths(inst, nil) - if err != nil { - return fmt.Errorf("Failed cleaning up instance paths on source member: %w", err) + driverInfo := sourcePool.Driver().Info() + if driverInfo.Remote && req.Pool == "" { + if !driverInfo.IgnoreCleanup { + err = sourcePool.CleanupInstancePaths(inst, nil) + if err != nil { + return fmt.Errorf("Failed cleaning up instance paths on source member: %w", err) + } } } else { // Delete the instance on source member if pool isn't remote shared storage. From e216fe27c8bf3c10681a9538402ab7efa241c110 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Mon, 28 Apr 2025 19:59:38 +0200 Subject: [PATCH 03/10] incusd/storage/nfs: Support NFS storage pools Signed-off-by: Morten Linderud Co-authored-by: Benjamin Somers --- internal/server/storage/drivers/driver_nfs.go | 208 ++++++++++++++++++ .../storage/drivers/driver_nfs_volumes.go | 26 +++ internal/server/storage/drivers/load.go | 1 + 3 files changed, 235 insertions(+) create mode 100644 internal/server/storage/drivers/driver_nfs.go create mode 100644 internal/server/storage/drivers/driver_nfs_volumes.go diff --git a/internal/server/storage/drivers/driver_nfs.go b/internal/server/storage/drivers/driver_nfs.go new file mode 100644 index 00000000000..6ba7562fdb1 --- /dev/null +++ b/internal/server/storage/drivers/driver_nfs.go @@ -0,0 +1,208 @@ +package drivers + +import ( + "errors" + "fmt" + "path/filepath" + "strings" + + "github.com/lxc/incus/v6/internal/linux" + "github.com/lxc/incus/v6/internal/migration" + deviceConfig "github.com/lxc/incus/v6/internal/server/device/config" + localMigration "github.com/lxc/incus/v6/internal/server/migration" + "github.com/lxc/incus/v6/shared/util" + "github.com/lxc/incus/v6/shared/validate" +) + +type nfs struct { + dir +} + +// Info returns info about the driver and its environment. +func (n *nfs) Info() Info { + return Info{ + Name: "nfs", + Version: "1", + DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, + OptimizedImages: false, + PreservesInodes: false, + Remote: n.isRemote(), + VolumeTypes: []VolumeType{VolumeTypeBucket, VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, + VolumeMultiNode: n.isRemote(), + BlockBacking: false, + RunningCopyFreeze: false, + DirectIO: true, + MountedRoot: true, + Buckets: false, + SameSource: true, + IgnoreCleanup: true, + } +} + +// isRemote returns true indicating this driver uses remote storage. +func (n *nfs) isRemote() bool { + return true +} + +func (n *nfs) getMountOptions() string { + // Allow overriding the default options. + if n.config["nfs.mount_options"] != "" { + return n.config["nfs.mount_options"] + } + // We only really support vers=4.2 + return fmt.Sprintf("vers=4.2,addr=%s", n.config["nfs.host"]) +} + +// Create is called during pool creation and is effectively using an empty driver struct. +// WARNING: The Create() function cannot rely on any of the struct attributes being set. +func (n *nfs) Create() error { + if n.config["source"] == "" { + return fmt.Errorf(`The "source" property must be defined`) + } + + sourceStr := n.config["source"] + + // Taken from the truenas driver + var host, path string + if strings.HasPrefix(sourceStr, "[") { + // IPv6 with brackets + endBracket := strings.Index(sourceStr, "]") + if endBracket == -1 || endBracket+1 >= len(sourceStr) || sourceStr[endBracket+1] != ':' { + // Malformed, treat whole string as path + host = "" + path = sourceStr + } else { + host = sourceStr[:endBracket+1] + path = sourceStr[endBracket+2:] // skip over "]:" + } + } else { + // Try normal IPv4/hostname + h, p, ok := strings.Cut(sourceStr, ":") + if ok { + host = h + path = p + } else { + // No colon: whole thing is path + host = "" + path = sourceStr + } + } + + if path == "" { + fmt.Println(filepath.IsAbs(path)) + return errors.New(`NFS driver requires "source" to be specified using the format: [:]`) + } + + if host == "" { + if n.config["nfs.host"] == "" { + return errors.New(`NFS driver requires "nfs.host" to be specified or included in "source": [:]`) + } + + host = n.config["nfs.host"] + } else { + n.config["nfs.host"] = host + } + + n.config["source"] = fmt.Sprintf("%s:%s", host, path) + n.config["nfs.path"] = path + + // Mount the nfs driver. + mntFlags, mntOptions := linux.ResolveMountOptions(strings.Split(n.getMountOptions(), ",")) + err := TryMount(n.config["source"], GetPoolMountPath(n.name), "nfs4", mntFlags, mntOptions) + if err != nil { + return err + } + + defer func() { _, _ = forceUnmount(GetPoolMountPath(n.name)) }() + + return nil +} + +// Mount mounts the storage pool. +func (n *nfs) Mount() (bool, error) { + path := GetPoolMountPath(n.name) + + // Check if already mounted. + if linux.IsMountPoint(path) { + return false, nil + } + + sourcePath := n.config["source"] + + // Check if we're dealing with an external mount. + if sourcePath == path { + return false, nil + } + + // Mount the nfs driver. + mntFlags, mntOptions := linux.ResolveMountOptions(strings.Split(n.getMountOptions(), ",")) + err := TryMount(sourcePath, GetPoolMountPath(n.name), "nfs4", mntFlags, mntOptions) + if err != nil { + return false, err + } + + return true, nil +} + +// MigrationTypes returns the type of transfer methods to be used when doing migrations between pools in preference order. +func (n *nfs) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { + // NFS does not support xattr + rsyncFeatures := []string{"delete", "bidirectional"} + if util.IsTrue(n.Config()["rsync.compression"]) { + rsyncFeatures = append(rsyncFeatures, "compress") + } + + return []localMigration.Type{ + { + FSType: migration.MigrationFSType_BLOCK_AND_RSYNC, + Features: rsyncFeatures, + }, + { + FSType: migration.MigrationFSType_RSYNC, + Features: rsyncFeatures, + }, + } +} + +// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. +func (n *nfs) Validate(config map[string]string) error { + rules := map[string]func(value string) error{ + // gendoc:generate(entity=storage_nfs, group=common, key=source) + // + // --- + // type: string + // scope: local + // default: - + // shortdesc: NFS remote storage path. Format: `[:]`. If `host` is omitted here, it must be set via `nfs.host`. + "source": validate.IsAny, // can be used as a shortcut to specify dataset and optionally host. + + // gendoc:generate(entity=storage_nfs, group=common, key=nfs.host) + // + // --- + // type: string + // scope: global + // default: - + // shortdesc: Hostname or IP address of the remote NFS server. Optional if included in `source`, or a configuration is used. + "nfs.host": validate.IsAny, + + // gendoc:generate(entity=storage_nfs, group=common, key=nfs.path) + // + // --- + // type: string + // scope: local + // default: - + // shortdesc: Remote NFS path. Typically inferred from `source`, but can be overridden. + "nfs.path": validate.IsAny, + + // gendoc:generate(entity=storage_nfs, group=common, key=nfs.mount_options) + // + // --- + // type: string + // scope: local + // default: - + // shortdesc: Additional mount options for the NFS mount. + "nfs.mount_options": validate.IsAny, + } + + return n.validatePool(config, rules, map[string]func(value string) error{}) +} diff --git a/internal/server/storage/drivers/driver_nfs_volumes.go b/internal/server/storage/drivers/driver_nfs_volumes.go new file mode 100644 index 00000000000..67e1fd687af --- /dev/null +++ b/internal/server/storage/drivers/driver_nfs_volumes.go @@ -0,0 +1,26 @@ +package drivers + +import ( + "io" + + "github.com/lxc/incus/v6/internal/server/migration" + "github.com/lxc/incus/v6/internal/server/operations" +) + +// MigrateVolume sends a volume for migration. +func (n *nfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *migration.VolumeSourceArgs, op *operations.Operation) error { + if volSrcArgs.ClusterMove && !volSrcArgs.StorageMove { + return nil + } + + return genericVFSMigrateVolume(n, n.state, vol, conn, volSrcArgs, op) +} + +// CreateVolumeFromMigration creates a new volume (with or without snapshots) from a migration data stream. +func (n *nfs) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs migration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { + if volTargetArgs.ClusterMoveSourceName != "" && volTargetArgs.StoragePool == "" { + return nil + } + + return genericVFSCreateVolumeFromMigration(n, n.setupInitialQuota, vol, conn, volTargetArgs, preFiller, op) +} diff --git a/internal/server/storage/drivers/load.go b/internal/server/storage/drivers/load.go index 5367f10e42b..6fa138f7a11 100644 --- a/internal/server/storage/drivers/load.go +++ b/internal/server/storage/drivers/load.go @@ -16,6 +16,7 @@ var drivers = map[string]func() driver{ "truenas": func() driver { return &truenas{} }, "zfs": func() driver { return &zfs{} }, "linstor": func() driver { return &linstor{} }, + "nfs": func() driver { return &nfs{} }, } // Validators contains functions used for validating a drivers's config. From 8e993bd67c045ef1bd2be3daf3012e277eec1644 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sun, 19 Oct 2025 21:42:43 +0200 Subject: [PATCH 04/10] docs: add NFS driver documentation Signed-off-by: Morten Linderud --- doc/config_options.txt | 34 +++++++++++++++++ doc/reference/storage_drivers.md | 31 +++++++-------- doc/reference/storage_nfs.md | 28 ++++++++++++++ internal/server/metadata/configuration.json | 42 +++++++++++++++++++++ 4 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 doc/reference/storage_nfs.md diff --git a/doc/config_options.txt b/doc/config_options.txt index ec9ca1ae392..633b4db7f09 100644 --- a/doc/config_options.txt +++ b/doc/config_options.txt @@ -5700,6 +5700,40 @@ This value is required by some providers. ``` + +```{config:option} nfs.host storage_nfs-common +:default: "-" +:scope: "global" +:shortdesc: "Hostname or IP address of the remote NFS server. Optional if included in `source`, or a configuration is used." +:type: "string" + +``` + +```{config:option} nfs.mount_options storage_nfs-common +:default: "-" +:scope: "local" +:shortdesc: "Additional mount options for the NFS mount." +:type: "string" + +``` + +```{config:option} nfs.path storage_nfs-common +:default: "-" +:scope: "local" +:shortdesc: "Remote NFS path. Typically inferred from `source`, but can be overridden." +:type: "string" + +``` + +```{config:option} source storage_nfs-common +:default: "-" +:scope: "local" +:shortdesc: "NFS remote storage path. Format: `[:]`. If `host` is omitted here, it must be set via `nfs.host`." +:type: "string" + +``` + + ```{config:option} source storage_truenas-common :default: "-" diff --git a/doc/reference/storage_drivers.md b/doc/reference/storage_drivers.md index 2d32ef884e6..cd07c84a17e 100644 --- a/doc/reference/storage_drivers.md +++ b/doc/reference/storage_drivers.md @@ -15,6 +15,7 @@ storage_cephfs storage_cephobject storage_linstor storage_truenas +storage_nfs ``` See the corresponding pages for driver-specific information and configuration options. @@ -24,21 +25,21 @@ See the corresponding pages for driver-specific information and configuration op Where possible, Incus uses the advanced features of each storage system to optimize operations. -| Feature | Directory | Btrfs | LVM | ZFS | Ceph RBD | CephFS | Ceph Object | LINSTOR | TRUENAS | -| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | -| {ref}`storage-optimized-image-storage` | no | yes | yes | yes | yes | n/a | n/a | yes | yes | -| Optimized instance creation | no | yes | yes | yes | yes | n/a | n/a | yes | yes | -| Optimized snapshot creation | no | yes | yes | yes | yes | yes | n/a | yes | yes | -| Optimized image transfer | no | yes | no | yes | yes | n/a | n/a | no | no | -| {ref}`storage-optimized-volume-transfer` | no | yes | no | yes | yes | n/a | n/a | no | no | -| Copy on write | no | yes | yes | yes | yes | yes | n/a | yes | yes | -| Block based | no | no | yes | no | yes | no | n/a | yes | yes | -| Instant cloning | no | yes | yes | yes | yes | yes | n/a | yes | yes | -| Storage driver usable inside a container | yes | yes | no | yes[^1] | no | n/a | n/a | no | no | -| Restore from older snapshots (not latest) | yes | yes | yes | no | yes | yes | n/a | no | no | -| Storage quotas | yes[^2] | yes | yes | yes | yes | yes | yes | yes | yes | -| Available on `incus admin init` | yes | yes | yes | yes | yes | no | no | no | no | -| Object storage | yes | yes | yes | yes | no | no | yes | no | no | +| Feature | Directory | Btrfs | LVM | ZFS | Ceph RBD | CephFS | Ceph Object | LINSTOR | TRUENAS | NFS | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| {ref}`storage-optimized-image-storage` | no | yes | yes | yes | yes | n/a | n/a | yes | yes | no | +| Optimized instance creation | no | yes | yes | yes | yes | n/a | n/a | yes | yes | no | +| Optimized snapshot creation | no | yes | yes | yes | yes | yes | n/a | yes | yes | no | +| Optimized image transfer | no | yes | no | yes | yes | n/a | n/a | no | no | no | +| {ref}`storage-optimized-volume-transfer` | no | yes | no | yes | yes | n/a | n/a | no | no | no | +| Copy on write | no | yes | yes | yes | yes | yes | n/a | yes | yes | no | +| Block based | no | no | yes | no | yes | no | n/a | yes | yes | n/a | +| Instant cloning | no | yes | yes | yes | yes | yes | n/a | yes | yes | no | +| Storage driver usable inside a container | yes | yes | no | yes[^1] | no | n/a | n/a | no | no | yes | +| Restore from older snapshots (not latest) | yes | yes | yes | no | yes | yes | n/a | no | no | yes | +| Storage quotas | yes[^2] | yes | yes | yes | yes | yes | yes | yes | yes | no | +| Available on `incus admin init` | yes | yes | yes | yes | yes | no | no | no | no | yes | +| Object storage | yes | yes | yes | yes | no | no | yes | no | no | no | [^1]: Requires [`zfs.delegate`](storage-zfs-vol-config) to be enabled. [^2]: % Include content from [storage_dir.md](storage_dir.md) diff --git a/doc/reference/storage_nfs.md b/doc/reference/storage_nfs.md new file mode 100644 index 00000000000..7e48d247d48 --- /dev/null +++ b/doc/reference/storage_nfs.md @@ -0,0 +1,28 @@ +(storage-nfs)= +# NFS - `nfs` + +Network File System is a distributed file system protocol. It is used to serve and access files over a computer network. + +To use NFS one need to setup a NFS file system following documentation from your Linux distribution of choice. + +## `nfs` driver in Incus + +The `nfs` driver in Incus only supports NFS version 4.2 and has a couple of limitations. + +UID/GID squashing should be enabled. This can be done by explicitly setting `no_root_squash` and `no_all_squash` in `/etc/export`. + +Note that it is not recommended to use `nfs` driver as container or virtual machine storage volumes as it is unclear how well it works. + +## Configuration options + +The following configuration options are available for storage pools that use the `nfs` driver and for storage volumes in these pools. + +### Storage pool configuration + +% Include content from [config_options.txt](../config_options.txt) +```{include} ../config_options.txt + :start-after: + :end-before: +``` + +{{volume_configuration}} diff --git a/internal/server/metadata/configuration.json b/internal/server/metadata/configuration.json index 4773cc51067..858dce6d95b 100644 --- a/internal/server/metadata/configuration.json +++ b/internal/server/metadata/configuration.json @@ -6473,6 +6473,48 @@ ] } }, + "storage_nfs": { + "common": { + "keys": [ + { + "nfs.host": { + "default": "-", + "longdesc": "", + "scope": "global", + "shortdesc": "Hostname or IP address of the remote NFS server. Optional if included in `source`, or a configuration is used.", + "type": "string" + } + }, + { + "nfs.mount_options": { + "default": "-", + "longdesc": "", + "scope": "local", + "shortdesc": "Additional mount options for the NFS mount.", + "type": "string" + } + }, + { + "nfs.path": { + "default": "-", + "longdesc": "", + "scope": "local", + "shortdesc": "Remote NFS path. Typically inferred from `source`, but can be overridden.", + "type": "string" + } + }, + { + "source": { + "default": "-", + "longdesc": "", + "scope": "local", + "shortdesc": "NFS remote storage path. Format: `[\u003chost\u003e:]\u003cremote path\u003e`. If `host` is omitted here, it must be set via `nfs.host`.", + "type": "string" + } + } + ] + } + }, "storage_truenas": { "common": { "keys": [ From a844951003df44f80384053f98806e2353938200 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 21 Oct 2025 21:50:06 +0000 Subject: [PATCH 05/10] incusd: Properly handle SameSource Signed-off-by: Benjamin Somers --- cmd/incusd/storage_pools.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/incusd/storage_pools.go b/cmd/incusd/storage_pools.go index e44d1210121..48882f00b2a 100644 --- a/cmd/incusd/storage_pools.go +++ b/cmd/incusd/storage_pools.go @@ -230,7 +230,9 @@ func storagePoolsGet(d *Daemon, r *http.Request) response.Response { // If no member is specified and the daemon is clustered, we omit the node-specific fields. if s.ServerClustered { for _, key := range db.NodeSpecificStorageConfig { - delete(poolAPI.Config, key) + if key != "source" || !pool.Driver().Info().SameSource { + delete(poolAPI.Config, key) + } } } else { // Use local status if not clustered. To allow seeing unavailable pools. From 0dcfb1b0748a74bf851651367d94dd9be99f4b31 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 21 Oct 2025 22:26:34 +0000 Subject: [PATCH 06/10] test: Fix Busybox permissions Suggested-by: Morten Linderud Signed-off-by: Benjamin Somers --- test/deps/import-busybox | 1 + 1 file changed, 1 insertion(+) diff --git a/test/deps/import-busybox b/test/deps/import-busybox index 3c72e87cec6..7dab4e42e74 100755 --- a/test/deps/import-busybox +++ b/test/deps/import-busybox @@ -236,6 +236,7 @@ class BusyBox(object): for path in ("dev", "mnt", "proc", "root", "sys", "tmp"): directory_file = tarfile.TarInfo() directory_file.type = tarfile.DIRTYPE + directory_file.mode = 0o755 if split: directory_file.name = "%s" % path target_tarball_rootfs.addfile(directory_file) From 550139c91e947ee9dc158d4d01b255c8710d94e0 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 21 Oct 2025 22:27:40 +0000 Subject: [PATCH 07/10] test: Add standalone NFS tests Signed-off-by: Benjamin Somers --- test/backends/nfs.sh | 31 +++++++++++++++++++++++++++++++ test/includes/incusd.sh | 3 +++ test/includes/storage.sh | 4 ++++ test/main.sh | 11 +++++++++++ 4 files changed, 49 insertions(+) create mode 100644 test/backends/nfs.sh diff --git a/test/backends/nfs.sh b/test/backends/nfs.sh new file mode 100644 index 00000000000..2855c246647 --- /dev/null +++ b/test/backends/nfs.sh @@ -0,0 +1,31 @@ +nfs_setup() { + # shellcheck disable=2039,3043 + local INCUS_DIR + + INCUS_DIR=$1 + + echo "==> Setting up nfs backend in ${INCUS_DIR}" + mkdir "/media/$(basename "$INCUS_DIR")" +} + +nfs_configure() { + # shellcheck disable=2039,3043 + local INCUS_DIR + + INCUS_DIR=$1 + + echo "==> Configuring nfs backend in ${INCUS_DIR}" + + incus storage create "incustest-$(basename "${INCUS_DIR}")" nfs source="${INCUS_NFS_SHARE}/$(basename "$INCUS_DIR")" + incus profile device add default root disk path="/" pool="incustest-$(basename "${INCUS_DIR}")" +} + +nfs_teardown() { + # shellcheck disable=2039,3043 + local INCUS_DIR + + INCUS_DIR=$1 + + echo "==> Tearing down nfs backend in ${INCUS_DIR}" + rm -rf "/media/$(basename "$INCUS_DIR")" +} diff --git a/test/includes/incusd.sh b/test/includes/incusd.sh index 872707e32cf..ba3711e9891 100644 --- a/test/includes/incusd.sh +++ b/test/includes/incusd.sh @@ -27,6 +27,9 @@ spawn_incus() { elif [ "${INCUS_BACKEND}" = "linstor" ] && [ -z "${INCUS_LINSTOR_CLUSTER:-}" ]; then echo "A cluster name must be specified when using the LINSTOR driver." >&2 exit 1 + elif [ "${INCUS_BACKEND}" = "nfs" ] && [ -z "${INCUS_NFS_SHARE:-}" ]; then + echo "An NFS share must be specified when using the NFS driver." >&2 + exit 1 fi # setup storage diff --git a/test/includes/storage.sh b/test/includes/storage.sh index c85b174d32f..5730629410d 100644 --- a/test/includes/storage.sh +++ b/test/includes/storage.sh @@ -48,6 +48,10 @@ available_storage_backends() { fi done + if [ -n "${INCUS_NFS_SHARE:-}" ] && command -v "exportfs" > /dev/null 2>&1; then + backends="$backends nfs" + fi + if [ -n "${INCUS_TRUENAS_DATASET:-}" ] && command -v "truenas_incus_ctl" > /dev/null 2>&1; then backends="$backends truenas" fi diff --git a/test/main.sh b/test/main.sh index 5b4f08368b2..cfca3da1c70 100755 --- a/test/main.sh +++ b/test/main.sh @@ -63,6 +63,9 @@ if [ "$INCUS_BACKEND" != "random" ] && ! storage_backend_available "$INCUS_BACKE elif [ "${INCUS_BACKEND}" = "truenas" ] && [ -z "${INCUS_TRUENAS_DATASET:-}" ]; then echo "TrueNAS storage backend requires that \"INCUS_TRUENAS_DATASET\" be set." exit 1 + elif [ "${INCUS_BACKEND}" = "nfs" ] && [ -z "${INCUS_NFS_SHARE:-}" ]; then + echo "LINSTOR storage backend requires that \"INCUS_NFS_SHARE\" be set." + exit 1 fi echo "Storage backend \"$INCUS_BACKEND\" is not available" exit 1 @@ -139,6 +142,14 @@ import_subdir_files suites TEST_DIR=$(mktemp -d -p "$(pwd)" tmp.XXX) chmod +x "${TEST_DIR}" +if [ -n "${INCUS_NFS_SHARE:-}" ]; then + tmp_mount="$(mktemp -d)" + mount -t nfs "$INCUS_NFS_SHARE" "$tmp_mount" + mkdir "$tmp_mount/$(basename "$TEST_DIR")" + umount "$tmp_mount" + rmdir "$tmp_mount" +fi + if [ -n "${INCUS_TMPFS:-}" ]; then mount -t tmpfs tmpfs "${TEST_DIR}" -o mode=0751 -o size=6G fi From edad9e91e04da43648edc27ae54b31d4d01366f4 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 21 Oct 2025 22:28:42 +0000 Subject: [PATCH 08/10] test: Add clustered NFS tests Signed-off-by: Benjamin Somers --- test/includes/clustering.sh | 10 ++++++++-- test/suites/clustering.sh | 38 +++++++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/test/includes/clustering.sh b/test/includes/clustering.sh index 1ad64feadd7..80fc6f9dc19 100644 --- a/test/includes/clustering.sh +++ b/test/includes/clustering.sh @@ -197,6 +197,12 @@ EOF config: source: incustest-$(basename "${TEST_DIR}" | sed 's/\./__/g') linstor.resource_group.place_count: 1 +EOF + fi + if [ "${driver}" = "nfs" ]; then + cat >> "${INCUS_DIR}/preseed.yaml" << EOF + config: + source: $INCUS_NFS_SHARE/$(basename "${TEST_DIR}") EOF fi cat >> "${INCUS_DIR}/preseed.yaml" << EOF @@ -276,10 +282,10 @@ cluster: cluster_token: ${token} member_config: EOF - # Declare the pool only if the driver is not ceph or linstor, because + # Declare the pool only if the driver doesn't manage remote storage, because # the pool doesn't need to be created on the joining # node (it's shared with the bootstrap one). - if [ "${driver}" != "ceph" ] && [ "${driver}" != "linstor" ]; then + if [ "${driver}" != "ceph" ] && [ "${driver}" != "linstor" ] && [ "${driver}" != "nfs" ]; then cat >> "${INCUS_DIR}/preseed.yaml" << EOF - entity: storage-pool name: data diff --git a/test/suites/clustering.sh b/test/suites/clustering.sh index 530b14ade85..4d102b30607 100644 --- a/test/suites/clustering.sh +++ b/test/suites/clustering.sh @@ -684,6 +684,8 @@ test_clustering_storage() { driver_config="source=incustest-$(basename "${TEST_DIR}")-pool1" elif [ "${poolDriver}" = "linstor" ]; then driver_config="source=incustest-$(basename "${TEST_DIR}" | sed 's/\./__/g')-pool1" + elif [ "${poolDriver}" = "nfs" ]; then + driver_config="source=$INCUS_NFS_SHARE/$(basename "${TEST_DIR}")" fi # Define storage pools on the two nodes @@ -749,11 +751,14 @@ test_clustering_storage() { # For ceph volume the source field is the name of the underlying ceph pool source1="incustest-$(basename "${TEST_DIR}")" source2="${source1}" - fi - if [ "${poolDriver}" = "linstor" ]; then + elif [ "${poolDriver}" = "linstor" ]; then # For linstor the source field is the name of the underlying linstor resource group source1="incustest-$(basename "${TEST_DIR}" | sed 's/\./__/g')" source2="${source1}" + elif [ "${poolDriver}" = "nfs" ]; then + # For nfs the source field is the name of the underlying nfs share + source1="$INCUS_NFS_SHARE/$(basename "${TEST_DIR}")" + source2="${source1}" fi INCUS_DIR="${INCUS_ONE_DIR}" incus storage show pool1 --target node1 | grep source | grep -q "${source1}" INCUS_DIR="${INCUS_ONE_DIR}" incus storage show pool1 --target node2 | grep source | grep -q "${source2}" @@ -766,8 +771,8 @@ test_clustering_storage() { ! INCUS_DIR="${INCUS_ONE_DIR}" incus storage show pool1 | grep -q rsync.bwlimit || false fi - if [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "linstor" ]; then - # Test migration of ceph- and linstor-based containers + if [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "linstor" ] || [ "${poolDriver}" = "nfs" ]; then + # Test migration of ceph-, linstor-, and nfs-based containers INCUS_DIR="${INCUS_TWO_DIR}" ensure_import_testimage INCUS_DIR="${INCUS_ONE_DIR}" incus launch --target node2 -s pool1 testimage foo @@ -796,7 +801,7 @@ test_clustering_storage() { # If the driver has the same per-node storage pool config (e.g. size), make sure it's included in the # member_config, and actually added to a joining node so we can validate it. - if [ "${poolDriver}" = "zfs" ] || [ "${poolDriver}" = "btrfs" ] || [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "lvm" ] || [ "${poolDriver}" = "linstor" ]; then + if [ "${poolDriver}" = "zfs" ] || [ "${poolDriver}" = "btrfs" ] || [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "lvm" ] || [ "${poolDriver}" = "linstor" ] || [ "${poolDriver}" = "nfs" ]; then # Spawn a third node setup_clustering_netns 3 INCUS_THREE_DIR=$(mktemp -d -p "${TEST_DIR}" XXX) @@ -830,12 +835,12 @@ test_clustering_storage() { done # Other storage backends will be finished with the third node, so we can remove it. - if [ "${poolDriver}" != "ceph" ] && [ "${poolDriver}" != "linstor" ]; then + if [ "${poolDriver}" != "ceph" ] && [ "${poolDriver}" != "linstor" ] && [ "${poolDriver}" != "nfs" ]; then INCUS_DIR="${INCUS_ONE_DIR}" incus cluster remove node3 --yes fi fi - if [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "linstor" ]; then + if [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "linstor" ] || [ "${poolDriver}" = "nfs" ]; then # Move the container to node3, renaming it INCUS_DIR="${INCUS_TWO_DIR}" incus move foo bar --target node3 INCUS_DIR="${INCUS_TWO_DIR}" incus info bar | grep -q "Location: node3" @@ -864,9 +869,12 @@ test_clustering_storage() { INCUS_DIR="${INCUS_ONE_DIR}" incus init --target node1 -s pool1 testimage baz INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume attach pool1 custom/v1 baz testDevice /opt - # Trying to attach a custom volume to a container on another node fails - INCUS_DIR="${INCUS_TWO_DIR}" incus init --target node2 -s pool1 testimage buz - ! INCUS_DIR="${INCUS_TWO_DIR}" incus storage volume attach pool1 custom/v1 buz testDevice /opt || false + if [ "${poolDriver}" != "nfs" ]; then + # Trying to attach a custom volume to a container on another node fails + INCUS_DIR="${INCUS_TWO_DIR}" incus init --target node2 -s pool1 testimage buz + ! INCUS_DIR="${INCUS_TWO_DIR}" incus storage volume attach pool1 custom/v1 buz testDevice /opt || false + INCUS_DIR="${INCUS_ONE_DIR}" incus delete buz + fi # Create an unrelated volume and rename it on a node which differs from the # one running the container (issue #6435). @@ -878,7 +886,6 @@ test_clustering_storage() { INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume delete pool1 v1 INCUS_DIR="${INCUS_ONE_DIR}" incus delete baz - INCUS_DIR="${INCUS_ONE_DIR}" incus delete buz INCUS_DIR="${INCUS_ONE_DIR}" incus image delete testimage fi @@ -925,7 +932,7 @@ test_clustering_storage() { INCUS_DIR="${INCUS_ONE_DIR}" incus storage delete pool1 ! INCUS_DIR="${INCUS_ONE_DIR}" incus storage list | grep -q pool1 || false - if [ "${poolDriver}" != "ceph" ] && [ "${poolDriver}" != "linstor" ]; then + if [ "${poolDriver}" != "ceph" ] && [ "${poolDriver}" != "linstor" ] && [ "${poolDriver}" != "nfs" ]; then # Create a volume on node1 INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume create data web INCUS_DIR="${INCUS_ONE_DIR}" incus storage volume list data | grep web | grep -q node1 @@ -1013,6 +1020,8 @@ test_clustering_storage_single_node() { driver_config="source=incustest-$(basename "${TEST_DIR}" | sed 's/\./__/g')-pool1" elif [ "${poolDriver}" = "truenas" ]; then driver_config="$(truenas_source)/incustest-$(basename "${TEST_DIR}")-pool1 $(truenas_config) $(truenas_allow_insecure) $(truenas_api_key)" + elif [ "${poolDriver}" = "nfs" ]; then + driver_config="source=$INCUS_NFS_SHARE/$(basename "${TEST_DIR}")" fi driver_config_node="${driver_config}" @@ -2784,7 +2793,8 @@ test_clustering_image_refresh() { pids="" - if [ "${poolDriver}" != "dir" ]; then + # Ignore storage drivers that don't support optimized image storage + if [ "${poolDriver}" != "dir" ] && [ "${poolDriver}" != "nfs" ]; then # Check image storage volume records exist. incus admin sql global 'select name from storage_volumes' if [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "linstor" ]; then @@ -2806,7 +2816,7 @@ test_clustering_image_refresh() { wait "${pid}" || true done - if [ "${poolDriver}" != "dir" ]; then + if [ "${poolDriver}" != "dir" ] && [ "${poolDriver}" != "nfs" ]; then incus admin sql global 'select name from storage_volumes' # Check image storage volume records actually removed from relevant members and replaced with new fingerprint. if [ "${poolDriver}" = "ceph" ] || [ "${poolDriver}" = "linstor" ]; then From b75801fe1e0133faea04451ae23d7489ff853b1c Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 22 Oct 2025 18:32:17 +0000 Subject: [PATCH 09/10] github: Add NFS tests Signed-off-by: Benjamin Somers --- .github/workflows/tests.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6cd833e5814..35496e1af3b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -142,6 +142,7 @@ jobs: - ceph - linstor - random + - nfs os: - ubuntu-24.04 include: @@ -436,6 +437,16 @@ jobs: # Update the runner env. echo "INCUS_LINSTOR_CLUSTER=${runner_ip}" >> "$GITHUB_ENV" + - name: Setup NFS + if: ${{ matrix.backend == 'nfs' }} + run: | + set -x + sudo apt-get install --no-install-recommends -y nfs-kernel-server + echo "/media 10.0.0.0/8(rw,sync,no_subtree_check,no_root_squash,no_all_squash) 100.64.0.0/8(rw,sync,no_subtree_check,no_root_squash,no_all_squash)" | sudo tee /etc/exports + sudo exportfs -a + sudo systemctl restart nfs-server.service + echo "INCUS_NFS_SHARE=$(hostname -I | cut -d' ' -f1):/media" >> "$GITHUB_ENV" + - name: "Ensure offline mode (block image server)" run: | sudo nft add table inet filter @@ -459,7 +470,7 @@ jobs: chmod +x ~ echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid cd test - sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_BACKEND,INCUS_CEPH_CLUSTER,INCUS_CEPH_CEPHFS,INCUS_CEPH_CEPHOBJECT_RADOSGW,INCUS_LINSTOR_LOCAL_SATELLITE,INCUS_LINSTOR_CLUSTER,INCUS_OFFLINE,INCUS_SKIP_TESTS,INCUS_REQUIRED_TESTS, INCUS_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }} + sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_BACKEND,INCUS_CEPH_CLUSTER,INCUS_CEPH_CEPHFS,INCUS_CEPH_CEPHOBJECT_RADOSGW,INCUS_LINSTOR_LOCAL_SATELLITE,INCUS_LINSTOR_CLUSTER,INCUS_NFS_SHARE,INCUS_OFFLINE,INCUS_SKIP_TESTS,INCUS_REQUIRED_TESTS, INCUS_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }} client: name: Client From cbe861a0c368708a2e73a7114ac1586cd43f52c4 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 22 Oct 2025 18:34:49 +0000 Subject: [PATCH 10/10] api: storage_driver_nfs Signed-off-by: Benjamin Somers --- doc/api-extensions.md | 4 ++++ internal/version/api.go | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index 63907b4591c..b88d132c01e 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -2920,3 +2920,7 @@ A `used_by` field was added to the `GET /1.0/cluster/groups/{name}` endpoint. ## `bpf_token_delegation` This adds support for [eBPF token delegation](https://docs.ebpf.io/linux/concepts/token/). + +## `storage_driver_nfs` + +This adds an NFS storage driver. diff --git a/internal/version/api.go b/internal/version/api.go index 29408e39f1d..3cd91d4e100 100644 --- a/internal/version/api.go +++ b/internal/version/api.go @@ -504,6 +504,7 @@ var APIExtensions = []string{ "instance_systemd_credentials", "cluster_group_usedby", "bpf_token_delegation", + "storage_driver_nfs", } // APIExtensionsCount returns the number of available API extensions.