diff --git a/test/integration/plugins/ontap/README.md b/test/integration/plugins/ontap/README.md index 6e0d0e7d6be5..3020846821c9 100644 --- a/test/integration/plugins/ontap/README.md +++ b/test/integration/plugins/ontap/README.md @@ -32,7 +32,7 @@ CI wiring: test/integration/plugins/ontap/ ├── ontap.cfg # Environment config (IPs, credentials, zone info) ├── ontap_test_base.py # Shared base class and ONTAP REST client -├── TEST_CASES.md # Full test case reference table (62 tests) +├── TEST_CASES.md # Full test case reference table (68 tests) ├── README.md # This file │ ├── nfs3/ @@ -296,7 +296,9 @@ self.assertEqual(result.state, "Maintenance") | `get_igroup(svm_name, name)` | iSCSI igroup existence and initiator list | iSCSI suites | | `list_luns_in_volume(svm_name, vol_name)` | LUNs present in a FlexVol | iSCSI volume/instance suites | | `list_lun_maps_for_volume(svm_name, vol_name)` | Active LUN-maps for a volume | iSCSI instance suite | -| `list_files_in_volume(svm_name, vol_name)` | Files inside a FlexVol | NFS3 instance suite | +| `list_files_in_volume(vol_name)` | Files inside a FlexVol | NFS3 instance suite | +| `write_file_in_volume(vol_name, filename, size_bytes)` | Write incompressible data into a FlexVol file (no VM) | NFS3/iSCSI pool lifecycle suites | +| `delete_file_in_volume(vol_name, filename)` | Remove a FlexVol file | NFS3/iSCSI pool lifecycle suites | --- @@ -304,14 +306,14 @@ self.assertEqual(result.state, "Maintenance") | Suite | File | Tests | What it covers | |-------|------|-------|---------------| -| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete | +| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 15 | Create, grow to 300 TiB, reject grow past 300 TiB, safe shrink, disable, enable, maintenance, resize in maintenance, delete, then re-create with a volume, reject shrink below used capacity, deploy a VM and attach the volume, cleanup | | NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 7 | Same + live volume present; negative delete guard | -| NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 4 | Zone scope — all hosts connected via `attachZone` | +| NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 6 | Zone scope — all hosts connected via `attachZone`; grow and safe shrink | | NFS3 Volume Lifecycle | `nfs3/volume/test_volume_lifecycle.py` | 5 | Volume is metadata-only; FlexVol unchanged on delete | | NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle with hot-plug/detach | -| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete + igroups | +| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 15 | Create, grow to 300 TiB, reject grow past 300 TiB, safe shrink, disable, enable, maintenance, resize in maintenance, delete + igroups, then re-create with a volume (LUN), reject shrink below used capacity, deploy a VM and attach the volume, cleanup | | iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 7 | Same + live LUN present; negative delete guard | -| iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 4 | Zone scope | +| iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 6 | Zone scope; grow and safe shrink | | iSCSI Volume Lifecycle | `iscsi/volume/test_volume_lifecycle.py` | 5 | LUN created per CS volume; LUN removed on delete | | iSCSI VM + Volume Attach | `iscsi/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle; LUN-maps on VM start/stop/detach | diff --git a/test/integration/plugins/ontap/TEST_CASES.md b/test/integration/plugins/ontap/TEST_CASES.md index 73dc1990a5b6..410dba87efc7 100644 --- a/test/integration/plugins/ontap/TEST_CASES.md +++ b/test/integration/plugins/ontap/TEST_CASES.md @@ -19,7 +19,7 @@ # ONTAP Integration Test Cases -Complete reference for all 62 test cases across 10 test suites. +Complete reference for all 68 test cases across 10 test suites. Each suite is sequential — tests must run in numbered order; each step builds on state created by the previous step. --- @@ -42,18 +42,25 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/pool/test_pool_lifecycle.py` **Class:** `TestOntapNFS3PrimaryStorageWorkflow` **Tag:** `nfs3_workflow` -**Total:** 8 tests | **Scope:** cluster-scoped NFS3 pool, no volumes for tests 01–06 +**Total:** 15 tests | **Scope:** cluster-scoped NFS3 pool, no volumes for tests 01–10 | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| | 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped NFS3 primary storage pool | setUpClass (zone, cluster, account) | `pool.state == "Up"`, `pool.type == "NetworkFilesystem"`, `nfsmountopts` contains `vers=3` | FlexVol exists and `state == "online"`, export policy exists with each cluster host IP as a rule, at least one NFS data LIF present on SVM | positive | -| 02 | `test_02_disable_storage_pool` | Disable the pool (admin operation) | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | -| 03 | `test_03_enable_storage_pool` | Re-enable the pool | test_02 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | -| 04 | `test_04_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_03 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | -| 05 | `test_05_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_04 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | -| 06 | `test_06_delete_pool_from_maintenance` | Enter maintenance then permanently delete the pool | test_05 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol deleted (not found by `GET /api/storage/volumes?name=`); export policy deleted | positive | -| 07 | `test_07_create_volume_on_pool` | Create a second fresh pool and allocate a CloudStack data volume on it | test_06 (pool deleted; creates new pool) | New `pool.state == "Up"`; `createVolume` returns non-None volume object | FlexVol `online` after volume allocation; export policy present | positive | -| 08 | `test_08_delete_volume_and_pool` | Delete the volume then force-delete the pool | test_07 (`pool`, `volume`) | Volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | positive | +| 02 | `test_02_grow_storage_pool` | Grow the original empty pool to 300 TiB | test_01 | `capacitybytes` reaches 300 TiB; pool stays `Up` | FlexVol `space.size` reaches 300 TiB; remains `online`; export policy unchanged | positive | +| 03 | `test_03_reject_grow_above_max_size` | Reject a grow past the 300 TiB ONTAP FlexVol maximum (overshoots by 1 GiB; asserts ONTAP's autosize-maximum error, not a generic failure such as aggregate space) | test_02 | `CloudstackAPIException`; `capacitybytes` stays at 300 TiB; pool stays `Up` | FlexVol `space.size` unchanged; export policy unchanged | negative | +| 04 | `test_04_shrink_storage_pool` | Safely shrink the original empty pool to its initial capacity | test_02 | `capacitybytes` reaches the safe target; pool stays `Up` | FlexVol `space.size` reaches the safe target; remains `online`; export policy unchanged | positive | +| 05 | `test_05_disable_storage_pool` | Disable the pool (admin operation) | test_04 | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | +| 06 | `test_06_enable_storage_pool` | Re-enable the pool | test_05 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 07 | `test_07_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_06 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | +| 08 | `test_08_resize_storage_pool_in_maintenance` | Grow the pool while it is in maintenance mode | test_07 | `updateStoragePool` accepted; `capacitybytes` reaches the requested size; `pool.state` stays `"Maintenance"` | FlexVol `space.size` reaches the requested size; remains `online`; export policy unchanged | positive | +| 09 | `test_09_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_08 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 10 | `test_10_delete_pool_from_maintenance` | Enter maintenance then permanently delete the original pool | test_09 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol and export policy deleted | positive | +| 11 | `test_11_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack data volume | test_10 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; export policy present | positive | +| 12 | `test_12_create_vm_and_attach_volume` | Deploy a VM and attach the ONTAP data volume to it | test_11 | VM reaches `Running`; volume reports `virtualmachineid` equal to the VM id; VM still `Running` after attach; pool capacity unchanged | FlexVol still `online`; volume data file materialised in the FlexVol (NFS3 creates it lazily at attach) | positive | +| 13 | `test_13_reject_shrink_below_used_capacity` | Reject shrink below the used space of the VM-attached volume, with the target held above the ONTAP FlexVol minimum. Writes incompressible data through the ONTAP files API when the VM-attached thin volume leaves used space under the minimum, and removes it before returning | test_12 (`pool`, `volume`, `vm`) | `CloudstackAPIException`; volume remains listed with unchanged `id`/`state`/`size`/`poolid`; pool capacity unchanged | FlexVol size and export policy unchanged; used space stays above the FlexVol minimum | negative | +| 14 | `test_14_resize_pool_with_vm_attached` | Grow then shrink the pool back while the volume is attached to the running VM | test_12 (`vm`, `volume`) | Both resizes reach the requested capacity; pool stays `Up`; VM stays `Running` and keeps `virtualmachineid` on the volume | FlexVol reaches each size and stays `online`; export policy still covers every host IP | positive | +| 15 | `test_15_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_14 | VM destroyed; volume and pool no longer listed | FlexVol and export policy deleted | cleanup | --- @@ -81,14 +88,16 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/pool/test_zone_scoped_pool.py` **Class:** `TestOntapZoneScopedPool` **Tag:** `zone_pool` -**Total:** 4 tests | **Scope:** zone-scoped NFS3 pool (scope=ZONE, all hosts in zone connected) +**Total:** 6 tests | **Scope:** zone-scoped NFS3 pool (scope=ZONE, all hosts in zone connected) | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| | 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped NFS3 pool; CloudStack calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; export policy exists and contains **every** cluster host IP; at least one NFS data LIF present | positive | -| 02 | `test_02_disable_zone_scoped_pool` | Disable the zone-scoped pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol unchanged; export policy unchanged | positive | -| 03 | `test_03_enable_zone_scoped_pool` | Re-enable the zone-scoped pool | test_02 | `pool.state == "Up"` | FlexVol unchanged; export policy unchanged | positive | -| 04 | `test_04_delete_zone_scoped_pool` | Enter maintenance and force-delete the zone-scoped pool | test_03 | Pool no longer listed | FlexVol deleted; export policy deleted | positive | +| 02 | `test_02_grow_zone_scoped_pool` | Grow the zone-scoped pool | test_01 (`pool`) | `capacitybytes` reaches the requested size; pool stays `Up` | FlexVol `space.size` reaches the requested size; stays `online`; export policy still covers every host IP | positive | +| 03 | `test_03_shrink_zone_scoped_pool` | Safely shrink the zone-scoped pool back to its pre-grow size | test_02 | `capacitybytes` returns to the original value; pool stays `Up` | FlexVol shrinks back and stays `online`; export policy still covers every host IP | positive | +| 04 | `test_04_disable_zone_scoped_pool` | Disable the zone-scoped pool | test_03 | `pool.state == "Disabled"` | FlexVol unchanged; export policy unchanged | positive | +| 05 | `test_05_enable_zone_scoped_pool` | Re-enable the zone-scoped pool | test_04 | `pool.state == "Up"` | FlexVol unchanged; export policy unchanged | positive | +| 06 | `test_06_delete_zone_scoped_pool` | Enter maintenance and force-delete the zone-scoped pool | test_05 | Pool no longer listed | FlexVol deleted; export policy deleted | positive | --- @@ -124,7 +133,7 @@ Each suite is sequential — tests must run in numbered order; each step builds | 04 | `test_04_attach_volume_to_vm` | Attach the ONTAP data volume to the running VM (hot-plug) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id`; `attachVolume` job succeeds | FlexVol `online`; after attach, a data file matching volume UUID present in FlexVol (`list_files_in_volume`) | positive | | 05 | `test_05_stop_vm_export_retained` | Stop the running VM with volume attached | test_04 | `vm.state == "Stopped"` | FlexVol still `online`; NFS export policy still present | positive | | 06 | `test_06_start_vm_volume_accessible` | Start the stopped VM | test_05 | `vm.state == "Running"` | FlexVol still `online` | positive | -| 07 | `test_07_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM (TDS Detach NFS3) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | +| 07 | `test_07_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | | 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM no longer listed; volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | cleanup | --- @@ -134,18 +143,25 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/pool/test_pool_lifecycle.py` **Class:** `TestOntapISCSIPoolLifecycle` **Tag:** `iscsi_workflow` -**Total:** 8 tests | **Scope:** cluster-scoped iSCSI pool, no volumes for tests 01–06 +**Total:** 15 tests | **Scope:** cluster-scoped iSCSI pool, no volumes for tests 01–10 | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; one igroup per cluster host (named `cs_{svmName}_{hostShortName}`) with host IQN as initiator | positive | -| 02 | `test_02_disable_storage_pool` | Disable the pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol still `online` | positive | -| 03 | `test_03_enable_storage_pool` | Re-enable the pool | test_02 | `pool.state == "Up"` | FlexVol still `online` | positive | -| 04 | `test_04_enter_maintenance_mode` | Put pool into maintenance | test_03 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | -| 05 | `test_05_cancel_maintenance_mode` | Cancel maintenance | test_04 | `pool.state == "Up"` | FlexVol still `online` | positive | -| 06 | `test_06_enter_maintenance_and_delete_pool` | Enter maintenance then force-delete the pool | test_05 | Pool no longer listed | FlexVol deleted; all igroups for cluster hosts deleted | positive | -| 07 | `test_07_create_volume_on_pool` | Create a second fresh pool and allocate a CloudStack data volume (creates a LUN) | test_06 (new pool) | New `pool.state == "Up"`; volume object non-None | FlexVol `online`; ≥1 LUN present inside FlexVol (`list_luns_in_volume`) | positive | -| 08 | `test_08_delete_volume_and_pool` | Delete the volume (removes LUN), enter maintenance, force-delete pool | test_07 (`pool`, `volume`) | Volume no longer listed; pool no longer listed | LUN no longer in FlexVol; FlexVol deleted; igroups deleted | positive | +| 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "Iscsi"` | FlexVol `online`; one igroup per cluster host (named `cs_{svmName}_{hostShortName}`) with host IQN as initiator | positive | +| 02 | `test_02_grow_storage_pool` | Grow the original empty pool to 300 TiB | test_01 | `capacitybytes` reaches 300 TiB; pool stays `Up` | FlexVol reaches 300 TiB and remains `online`; igroups remain present | positive | +| 03 | `test_03_reject_grow_above_max_size` | Reject a grow past the 300 TiB ONTAP FlexVol maximum (overshoots by 1 GiB; asserts ONTAP's autosize-maximum error, not a generic failure such as aggregate space) | test_02 | `CloudstackAPIException`; `capacitybytes` stays at 300 TiB; pool stays `Up` | FlexVol `space.size` unchanged; igroups unchanged | negative | +| 04 | `test_04_shrink_storage_pool` | Safely shrink the original empty pool to its initial capacity | test_02 | `capacitybytes` reaches the safe target; pool stays `Up` | FlexVol reaches the safe target and remains `online`; igroups remain present | positive | +| 05 | `test_05_disable_storage_pool` | Disable the pool | test_04 | `pool.state == "Disabled"` | FlexVol still `online` | positive | +| 06 | `test_06_enable_storage_pool` | Re-enable the pool | test_05 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 07 | `test_07_enter_maintenance_mode` | Put pool into maintenance | test_06 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | +| 08 | `test_08_resize_storage_pool_in_maintenance` | Grow the pool while it is in maintenance mode | test_07 | `updateStoragePool` accepted; `capacitybytes` reaches the requested size; `pool.state` stays `"Maintenance"` | FlexVol reaches the requested size and remains `online`; igroups unchanged | positive | +| 09 | `test_09_cancel_maintenance_mode` | Cancel maintenance | test_08 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 10 | `test_10_enter_maintenance_and_delete_pool` | Enter maintenance then delete the original pool | test_09 | Pool no longer listed | FlexVol and all host igroups deleted | positive | +| 11 | `test_11_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack volume | test_10 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; at least one LUN present | positive | +| 12 | `test_12_create_vm_and_attach_volume` | Deploy a VM and attach the ONTAP data volume (LUN) to it | test_11 | VM reaches `Running`; volume reports `virtualmachineid` equal to the VM id; VM still `Running` after attach | FlexVol still `online`; LUN still present; per-host igroups unchanged | positive | +| 13 | `test_13_reject_shrink_below_used_capacity` | Reject shrink below the used space of the VM-attached LUN, with the target held above the ONTAP FlexVol minimum. Writes incompressible data through the ONTAP files API when the VM-attached thin volume leaves used space under the minimum, and removes it before returning | test_12 (`pool`, `volume`, `vm`) | `CloudstackAPIException`; volume remains listed with unchanged `id`/`state`/`size`/`poolid`; pool capacity unchanged | FlexVol size and LUN uuid/name list unchanged; used space stays above the FlexVol minimum | negative | +| 14 | `test_14_resize_pool_with_vm_attached` | Grow then shrink the pool back while the volume (LUN) is attached to the running VM | test_12 (`vm`, `volume`) | Both resizes reach the requested capacity; pool stays `Up`; VM stays `Running` and keeps `virtualmachineid` on the volume | FlexVol reaches each size and stays `online`; LUN still present | positive | +| 15 | `test_15_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_14 | VM destroyed; volume and pool no longer listed | LUN, FlexVol, and igroups deleted | cleanup | --- @@ -162,7 +178,7 @@ Each suite is sequential — tests must run in numbered order; each step builds | 02 | `test_02_disable_pool_volume_survives` | Disable pool with volume present | test_01 (`pool`, `volume`) | `pool.state == "Disabled"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 03 | `test_03_enable_pool_volume_intact` | Re-enable pool with volume | test_02 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 04 | `test_04_enter_maintenance_volume_present` | Enter maintenance with volume | test_03 | `pool.state == "Maintenance"`; volume still listed | FlexVol still `online`; LUN still present | positive | -| 05 | `test_05_cancel_maintenance_volume_present` | Cancel maintenance with volume (TDS iSCSI cancel maintenance) | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 05 | `test_05_cancel_maintenance_volume_present` | Cancel maintenance with volume | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 06 | `test_06_forced_false_delete_rejected` | Attempt `deleteStoragePool(forced=False)` with LUN-backed volume present — must be rejected | test_05 | `CloudstackAPIException` raised; pool still in `Maintenance` | No ONTAP objects removed | negative | | 07 | `test_07_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_06 (`pool`, `volume`) | Volume gone; pool gone | LUN removed; FlexVol deleted; igroups deleted | cleanup | @@ -173,14 +189,16 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/pool/test_zone_scoped_pool.py` **Class:** `TestOntapISCSIZoneScopedPool` **Tag:** `iscsi_zone_pool` -**Total:** 4 tests | **Scope:** zone-scoped iSCSI pool (scope=ZONE) +**Total:** 6 tests | **Scope:** zone-scoped iSCSI pool (scope=ZONE) | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| | 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped iSCSI pool; CS calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; igroup per cluster host, each with host IQN as initiator | positive | -| 02 | `test_02_disable_zone_scoped_pool` | Disable pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol unchanged; igroups unchanged | positive | -| 03 | `test_03_enable_zone_scoped_pool` | Re-enable pool | test_02 | `pool.state == "Up"` | FlexVol unchanged; igroups unchanged | positive | -| 04 | `test_04_delete_zone_scoped_pool` | Enter maintenance then delete pool | test_03 | Pool no longer listed | FlexVol deleted; all igroups deleted | positive | +| 02 | `test_02_grow_zone_scoped_pool` | Grow the zone-scoped pool | test_01 (`pool`) | `capacitybytes` reaches the requested size; pool stays `Up` | FlexVol reaches the requested size and stays `online`; igroups unchanged | positive | +| 03 | `test_03_shrink_zone_scoped_pool` | Safely shrink the zone-scoped pool back to its pre-grow size | test_02 | `capacitybytes` returns to the original value; pool stays `Up` | FlexVol shrinks back and stays `online`; igroups unchanged | positive | +| 04 | `test_04_disable_zone_scoped_pool` | Disable pool | test_03 | `pool.state == "Disabled"` | FlexVol unchanged; igroups unchanged | positive | +| 05 | `test_05_enable_zone_scoped_pool` | Re-enable pool | test_04 | `pool.state == "Up"` | FlexVol unchanged; igroups unchanged | positive | +| 06 | `test_06_delete_zone_scoped_pool` | Enter maintenance then delete pool | test_05 | Pool no longer listed | FlexVol deleted; all igroups deleted | positive | --- @@ -213,13 +231,13 @@ Each suite is sequential — tests must run in numbered order; each step builds | 01 | `test_01_create_iscsi_pool` | Create iSCSI ONTAP primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; igroup per cluster host with host IQN | positive | | 02 | `test_02_create_ontap_data_volume` | Allocate a CloudStack data volume (creates a LUN in the FlexVol) | test_01 (`pool`) | Volume non-None | ≥1 LUN in FlexVol | positive | | 03 | `test_03_deploy_vm` | Deploy VM using first ready KVM template; verify 0 LUN-maps exist before attach | test_02 (`volume`) | `vm.state == "Running"`; 0 LUN-maps on ONTAP | 0 LUN-maps (`list_lun_maps_for_volume` returns empty) | positive | -| 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created (TDS SN 27) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | -| 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed (TDS VM Stop iSCSI) | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | -| 06 | `test_06_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created (TDS VM Start iSCSI) | test_05 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | -| 07 | `test_07_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM (TDS Detach iSCSI) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps; LUN still in FlexVol | positive ⚠️ | +| 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | +| 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | +| 06 | `test_06_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created | test_05 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | +| 07 | `test_07_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps; LUN still in FlexVol | positive ⚠️ | | 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM gone; volume gone; pool gone | FlexVol deleted; all LUNs and igroups deleted | cleanup | -> ⚠️ **test_07 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. All other 61 tests pass. +> ⚠️ **test_07 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. --- @@ -227,14 +245,14 @@ Each suite is sequential — tests must run in numbered order; each step builds | Suite | Protocol | Scope | Tests | Status | |-------|---------|-------|-------|--------| -| NFS3 Pool Lifecycle | NFS3 | Cluster | 8 | ✅ | +| NFS3 Pool Lifecycle | NFS3 | Cluster | 15 | ✅ | | NFS3 Pool with Volumes | NFS3 | Cluster | 7 | ✅ | -| NFS3 Zone-Scoped Pool | NFS3 | Zone | 4 | ✅ | +| NFS3 Zone-Scoped Pool | NFS3 | Zone | 6 | ✅ | | NFS3 Volume Lifecycle | NFS3 | Cluster | 5 | ✅ | | NFS3 VM + Volume Attach | NFS3 | Cluster | 8 | ✅ | -| iSCSI Pool Lifecycle | iSCSI | Cluster | 8 | ✅ | +| iSCSI Pool Lifecycle | iSCSI | Cluster | 15 | ✅ | | iSCSI Pool with Volumes | iSCSI | Cluster | 7 | ✅ | -| iSCSI Zone-Scoped Pool | iSCSI | Zone | 4 | ✅ | +| iSCSI Zone-Scoped Pool | iSCSI | Zone | 6 | ✅ | | iSCSI Volume Lifecycle | iSCSI | Cluster | 5 | ✅ | | iSCSI VM + Volume Attach | iSCSI | Cluster | 8 | ⚠️ 7/8 | -| **Total** | | | **62** | **61 passing** | +| **Total** | | | **70** | **Resize flows passed; 1 known environment failure** | diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py index cc87bacf0e76..4effa5169807 100644 --- a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py @@ -19,22 +19,31 @@ Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage pool lifecycle (no volumes). -Tests are numbered test_01 ... test_08 and must run in that order. Each step +Tests are numbered test_01 ... test_15 and must run in that order. Each step builds on the shared state established by the previous step. Workflow: 01 Create primary storage pool - 02 Disable storage pool - 03 Enable storage pool - 04 Enter maintenance mode - 05 Cancel maintenance mode - 06 Enter maintenance mode and delete the storage pool - 07 Create a new pool and allocate a CloudStack data volume (LUN created) - 08 Delete the volume (LUN removed), enter maintenance, force-delete pool + 02 Increase storage pool capacity to the maximum supported size + 03 Reject a grow beyond the maximum supported size + 04 Safely shrink storage pool capacity + 05 Disable storage pool + 06 Enable storage pool + 07 Enter maintenance mode + 08 Resize storage pool while in maintenance mode + 09 Cancel maintenance mode + 10 Enter maintenance mode and delete the storage pool + 11 Create a new pool and allocate a CloudStack data volume (LUN created) + 12 Deploy a VM and attach the ONTAP volume (LUN) to it + 13 Reject shrink below the ONTAP used capacity of the attached LUN + 14 Grow and shrink the pool while the VM holds the volume (LUN) + 15 Detach/destroy the VM, delete the volume (LUN removed), enter + maintenance, force-delete pool Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - At least one ready user KVM template in the zone (test_12 skips without one) - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF - ontap.cfg populated with real values @@ -55,16 +64,21 @@ from nose.plugins.attrib import attr from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, cancelStorageMaintenance, createStoragePool as createStoragePoolAPI, deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, enableStorageMaintenance, updateStoragePool as updateStoragePoolAPI, ) from marvin.lib.base import StoragePool from marvin.lib.common import list_storage_pools -from ontap_test_base import OntapRestClient, OntapTestBase, get_datacenter_config, log_progress +from ontap_test_base import ( + OntapRestClient, OntapTestBase, _wait_for_vm_state, get_datacenter_config, + log_progress, +) logger = logging.getLogger("TestOntapISCSIPoolLifecycle") @@ -89,6 +103,16 @@ class TestData: ONTAP_MIN_VOLUME_SIZE = 1677721600 + # Maximum FlexVol size on AFF/FAS, raised from 100 TB in ONTAP 9.12.1P2. + # ONTAP writes this as "300 TB" but means binary, hence 1024**4. The + # value has to be hardcoded: no REST field reports the platform ceiling. + # Against an ONTAP older than 9.12.1P2 the real limit is 100 TB and the + # grow step will fail. CloudStack enforces no maximum of its own (the + # plugin validates only a 20 MB floor), so it is ONTAP that refuses + # anything above this. + ONTAP_MAX_VOLUME_SIZE = 300 * 1024 ** 4 + ONTAP_GROW_TARGET_SIZE = ONTAP_MAX_VOLUME_SIZE + def __init__(self, storage_ip, svm_name, username, password, scope="CLUSTER", provider="NetApp ONTAP", tags="ontap-iscsi", capacitybytes=None): @@ -146,6 +170,10 @@ class TestOntapISCSIPoolLifecycle(OntapTestBase): # ---- iSCSI-specific state (set/cleared by individual tests) -------- _vol_name_prefix = "OntapISCSIVol" + resize_original_size = None + + # VM state (vm/template_id/network_id) lives in OntapTestBase. + _vm_network_name_prefix = "ontap-iscsi-lifecycle-net" @classmethod def setUpClass(cls): @@ -185,6 +213,8 @@ def setUpClass(cls): cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + cls._discover_vm_deploy_resources() + # No per-test tearDown — state intentionally persists between steps. # ------------------------------------------------------------------ @@ -327,11 +357,188 @@ def test_01_create_primary_storage_pool(self): self._assert_pool_capacity(pool, "pool-created") # ------------------------------------------------------------------ - # Step 02 - Disable storage pool + # Step 02 - Increase storage pool capacity + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_02_grow_storage_pool(self): + """Grow the original pool to 300 TiB and verify CloudStack and ONTAP converge.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + int(self.testdata[TestData.primaryStorage]["capacitybytes"]), + ) + requested_size = TestData.ONTAP_GROW_TARGET_SIZE + self.assertGreater( + requested_size, original_size, + "Grow target %d B must exceed the current pool size %d B" + % (requested_size, original_size) + ) + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual(resized_pool.state, "Up") + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after increase" % igroup_name, + ) + + # ------------------------------------------------------------------ + # Step 03 - Reject a grow beyond the ONTAP maximum FlexVol size + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_03_reject_grow_above_max_size(self): + """ + Request a capacity above the 300 TiB ONTAP FlexVol maximum and verify + it is refused, leaving the pool at the size test_02 grew it to. + ONTAP reports this as an autosize-maximum violation rather than an + oversized-volume error. + Verifies: + - updateStoragePool raises CloudstackAPIException + - the error text is about the size limit, not about some other + failure such as the aggregate running out of space + - CloudStack capacity and ONTAP FlexVol size are both unchanged + - Pool stays 'Up' + - ONTAP: per-host igroups are untouched + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_02 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before over-max resize") + self.assertEqual( + int(getattr(listed[0], "capacitybytes", 0) or 0), + TestData.ONTAP_MAX_VOLUME_SIZE, + "Pool should sit at the 300 TiB maximum - test_02 must pass first" + ) + + # Overshoot by only 1 GiB: at 0.0003% of the limit this probes the + # 300 TiB boundary itself rather than asking for something absurd. + above_max = self._align_flexvol_bytes( + TestData.ONTAP_MAX_VOLUME_SIZE + 1024 ** 3 + ) + log_progress( + logger, "info", + "Requesting %d B for pool '%s' (ONTAP maximum is %d B, " + "expect reject)", + above_max, pool.name, TestData.ONTAP_MAX_VOLUME_SIZE, + ) + + # ONTAP enforces the platform cap through autosize rather than + # rejecting the size outright, reporting "Volume maximum autosize + # must be greater than or equal to the current volume size 300.0TB". + # Matching on that keeps the test from passing when the resize fails + # for an unrelated reason, such as the aggregate running out of space. + self._assert_capacity_update_rejected( + pool.id, above_max, "grow-above-max-flexvol", + pool.id, pool.name, + expected_error=( + "maximum autosize", + "maximum size", + "exceeds", + ), + ) + + after = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(after, "Pool disappeared after rejected over-max grow") + self.assertEqual( + after[0].state, "Up", + "Pool should stay 'Up' after a rejected resize, got '%s'" + % after[0].state + ) + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after a rejected " + "over-max grow" % igroup_name, + ) + + # ------------------------------------------------------------------ + # Step 04 - Safely shrink storage pool capacity + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_04_shrink_storage_pool(self): + """Shrink the original pool back to its initial safe capacity.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_02 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent - test_02 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk_pool = self._poll_pool_capacity( + pool.id, target_size, timeout=120 + ) + self.assertEqual(shrunk_pool.state, "Up") + shrunk_ontap_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after safe shrink" + % igroup_name, + ) + + # ------------------------------------------------------------------ + # Step 05 - Disable storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_02_disable_storage_pool(self): + def test_05_disable_storage_pool(self): """ Disable the pool and verify: - CloudStack reports Disabled @@ -357,11 +564,11 @@ def test_02_disable_storage_pool(self): ) # ------------------------------------------------------------------ - # Step 03 - Enable storage pool + # Step 06 - Enable storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_03_enable_storage_pool(self): + def test_06_enable_storage_pool(self): """ Re-enable the pool and verify: - CloudStack reports Up @@ -387,11 +594,11 @@ def test_03_enable_storage_pool(self): ) # ------------------------------------------------------------------ - # Step 04 - Enter maintenance mode + # Step 07 - Enter maintenance mode # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_04_enter_maintenance_mode(self): + def test_07_enter_maintenance_mode(self): """ Put the pool into maintenance mode and verify: - CloudStack reports Maintenance @@ -416,11 +623,88 @@ def test_04_enter_maintenance_mode(self): ) # ------------------------------------------------------------------ - # Step 05 - Cancel maintenance mode + # Step 08 - Resize storage pool while in maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_08_resize_storage_pool_in_maintenance(self): + """ + Grow the pool while it is in maintenance mode and verify: + - updateStoragePool is accepted while the pool is in Maintenance + - CloudStack reports the new capacity + - ONTAP: FlexVol is resized and still online + - The pool stays in Maintenance throughout (resize must not + implicitly return it to Up) + - ONTAP: per-host igroups are unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before maintenance resize") + self.assertEqual( + listed[0].state, "Maintenance", + "Pool must be in Maintenance - test_07 must pass first, got '%s'" + % listed[0].state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before maintenance resize" + ) + current_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + requested_size = current_size + TestData.ONTAP_MIN_VOLUME_SIZE + + log_progress( + logger, "info", + "Resizing iSCSI pool '%s' while in Maintenance: current=%d B, " + "requested=%d B", + pool.name, current_size, requested_size, + ) + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual( + resized_pool.state, "Maintenance", + "Pool should remain in Maintenance after resize, got '%s'" + % resized_pool.state + ) + + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after maintenance resize" + % igroup_name, + ) + + # ------------------------------------------------------------------ + # Step 09 - Cancel maintenance mode # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_05_cancel_maintenance_mode(self): + def test_09_cancel_maintenance_mode(self): """ Cancel maintenance and verify: - CloudStack reports Up @@ -444,11 +728,11 @@ def test_05_cancel_maintenance_mode(self): ) # ------------------------------------------------------------------ - # Step 06 - Enter maintenance mode and delete the storage pool + # Step 10 - Enter maintenance mode and delete the storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_06_enter_maintenance_and_delete_pool(self): + def test_10_enter_maintenance_and_delete_pool(self): """ Enter maintenance mode then delete the pool. Verifies the pool is removed from CloudStack and the backing ONTAP @@ -465,6 +749,7 @@ def test_06_enter_maintenance_and_delete_pool(self): self._delete_pool(pool.id) self.__class__.pool = None + self.__class__.resize_original_size = None # CloudStack: pool must be gone try: @@ -493,11 +778,11 @@ def test_06_enter_maintenance_and_delete_pool(self): ) # ------------------------------------------------------------------ - # Step 07 - Create fresh pool and allocate a CloudStack volume (LUN) + # Step 11 - Create fresh pool and allocate a CloudStack volume (LUN) # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_07_create_volume_on_pool(self): + def test_11_create_volume_on_pool(self): """ Create a new iSCSI pool and allocate a CloudStack data volume. For iSCSI, createAsync creates a LUN inside the pool's ONTAP FlexVol. @@ -511,7 +796,7 @@ def test_07_create_volume_on_pool(self): self.__class__.pool = pool log_progress( logger, "info", - "test_07: created storage pool name='%s' id=%s state=%s type=%s", + "test_10: created storage pool name='%s' id=%s state=%s type=%s", pool.name, pool.id, pool.state, pool.type, ) @@ -529,7 +814,7 @@ def test_07_create_volume_on_pool(self): self.assertIsNotNone(vol, "createVolume returned None") log_progress( logger, "info", - "test_07: created CloudStack volume name='%s' id=%s state=%s " + "test_10: created CloudStack volume name='%s' id=%s state=%s " "on pool='%s' (id=%s) account='%s' domain='%s' — " "switch to this account in the UI to see the volume", getattr(vol, "name", "?"), getattr(vol, "id", "?"), @@ -559,14 +844,320 @@ def test_07_create_volume_on_pool(self): self._assert_pool_capacity(pool, "volume-allocated") # ------------------------------------------------------------------ - # Step 08 - Delete volume (LUN) then force-delete the pool + # Step 12 - Deploy a VM and attach the ONTAP volume (LUN) to it + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_12_create_vm_and_attach_volume(self): + """ + Deploy a VM and attach the ONTAP data volume from test_11 to it. + Verifies: + - VM reaches 'Running' + - attachVolume sets the volume's virtualmachineid (on ONTAP managed + storage CloudStack may leave the volume state at 'Ready', so + virtualmachineid is the reliable attach signal) + - VM is still 'Running' after the attach + - ONTAP: FlexVol stays online and the LUN is still present + - ONTAP: per-host igroups survive the attach + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_11 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_11 must pass first" + ) + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template in the zone - cannot deploy a VM" + ) + self.assertIsNotNone( + self.__class__.service_offering_id, + "No service offering available - check the CloudStack setup" + ) + + pool = self.__class__.pool + vol = self.__class__.volume + + log_progress( + logger, "info", + "Deploying VM for iSCSI pool '%s' (template=%s, offering=%s)", + pool.name, self.__class__.template_id, + self.__class__.service_offering_id, + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=600) + self.assertIsNotNone( + vm_obj, "VM %s never appeared in listVirtualMachines" % vm.id + ) + self.assertEqual( + vm_obj.state, "Running", + "VM should be 'Running' before attach, got '%s'" % vm_obj.state + ) + + log_progress( + logger, "info", + "Attaching volume '%s' to VM '%s'", vol.id, vm.id, + ) + attach_cmd = attachVolumeAPI.attachVolumeCmd() + attach_cmd.id = vol.id + attach_cmd.virtualmachineid = vm.id + self.assertIsNotNone( + self.apiClient.attachVolume(attach_cmd), + "attachVolume returned None" + ) + + vol_vmid = self._poll_volume_attached(vol.id, timeout=180) + self.assertEqual( + vol_vmid, vm.id, + "Volume %s should report virtualmachineid=%s after attach, got %s" + % (vol.id, vm.id, vol_vmid) + ) + + vm_after = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=60) + self.assertEqual( + vm_after.state, "Running", + "VM should still be 'Running' after attach, got '%s'" + % vm_after.state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing after volume attach" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach, got '%s'" + % ontap_vol.get("state") + ) + + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "No LUN present in FlexVol '%s' after attach" % pool.name + ) + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after volume attach" + % igroup_name, + ) + + # ------------------------------------------------------------------ + # Step 13 - Reject shrink below ONTAP used capacity + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_13_reject_shrink_below_used_capacity(self): + """ + Attempt to shrink the pool below ONTAP used space and verify the + request is rejected while the volume and ONTAP LUNs remain unchanged. + + Runs after test_12, so the LUN is mapped to a running VM. The LUN is + thin-provisioned, though, and in practice leaves used space under the + 20 MiB FlexVol floor, so incompressible data is written through the + ONTAP files API to get above it and removed before this test returns. + The fill is skipped whenever used space is already high enough. + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_12 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_12 must pass first" + ) + pool = self.__class__.pool + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol not found for pool '%s'" % pool.name, + ) + + vol_before = self._cs_volume_snapshot(self.__class__.volume.id) + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs in FlexVol '%s' before rejected shrink" % pool.name, + ) + luns_before_used = sorted( + (lun.get("uuid") or "", lun.get("name") or "") + for lun in luns + ) + + # The attached volume is thin on both protocols, so it usually does + # not put 20 MiB on disk by itself. This writes incompressible data + # only when used space is still under the FlexVol floor, and removes + # it before returning. + filler_name, used_bytes = self._fill_flexvol_above_minimum(pool.name) + self.__class__.filler_filename = filler_name + self.__class__.filler_flexvol = pool.name if filler_name else None + try: + below_used = self._shrink_target_below_used(used_bytes) + log_progress( + logger, "info", + "Shrinking iSCSI pool '%s' below ONTAP used capacity: " + "used=%d B, requested=%d B " + "(FlexVol minimum=%d B, expect reject)", + pool.name, used_bytes, below_used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + self._assert_capacity_update_rejected( + pool.id, below_used, "shrink-below-used", + pool.id, pool.name, + # ONTAP's wording: "Selected volume size is too small to + # hold the current volume data. New volume size must be at + # least 34.3MB ...". Matching the specific phrase stops an + # unrelated resize failure from counting as a pass. + expected_error=( + "too small to hold the current volume data", + "too small to hold", + "cannot reduce", + ), + ) + self._assert_cs_volume_untouched(vol_before, "shrink-below-used") + self.assertGreater( + self._flexvol_used_bytes(pool.name), + self.ONTAP_MIN_FLEXVOL_SIZE, + "ONTAP used space dropped below the FlexVol minimum after " + "the rejected shrink", + ) + luns_after_used = sorted( + (lun.get("uuid") or "", lun.get("name") or "") + for lun in self.ontap.list_luns_in_volume( + self.svm_name, pool.name + ) + ) + self.assertEqual( + luns_after_used, luns_before_used, + "LUN list in FlexVol '%s' changed after rejected " + "used-capacity shrink" % pool.name, + ) + self.assertTrue( + len(luns_after_used) > 0, + "LUN disappeared from FlexVol '%s' after rejected shrink" + % pool.name, + ) + finally: + self._delete_filler_file(pool.name, filler_name) + + # ------------------------------------------------------------------ + # Step 14 - Resize the pool while the VM holds the volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_14_resize_pool_with_vm_attached(self): + """ + Grow the pool and shrink it straight back while the volume is + attached to the running VM from test_12, so resize is exercised + against a pool that is genuinely in use rather than an idle one. + Verifies: + - grow: CloudStack and ONTAP both reach the requested size + - shrink back: both return to the starting size + - the VM stays 'Running' and keeps the volume across both resizes + - ONTAP: FlexVol stays online + - ONTAP: the LUN backing the volume is still present + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_11 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_11 must pass first" + ) + if self.__class__.vm is None: + self.skipTest( + "No VM was deployed (test_12 skipped) - nothing to resize " + "a pool underneath" + ) + + pool = self.__class__.pool + vm = self.__class__.vm + vol = self.__class__.volume + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before the in-use resize") + original_size = int(getattr(listed[0], "capacitybytes", 0) or 0) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + log_progress( + logger, "info", + "Growing in-use pool '%s' from %d B to %d B (VM %s holds the " + "volume)", pool.name, original_size, grow_target, vm.id, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = grow_target + self.apiClient.updateStoragePool(cmd) + + grown = self._poll_pool_capacity(pool.id, grow_target, timeout=120) + self.assertEqual( + grown.state, "Up", + "Pool should stay 'Up' after growing under a VM, got '%s'" + % grown.state + ) + grown_vol = self._poll_ontap_volume_size( + pool.name, grow_target, timeout=120 + ) + self.assertEqual(grown_vol.get("state"), "online") + self._assert_vm_running_with_volume(vm.id, vol.id, "grow-with-vm") + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "LUN disappeared from FlexVol '%s' during the in-use resize" + % pool.name, + ) + + log_progress( + logger, "info", + "Shrinking in-use pool '%s' back from %d B to %d B", + pool.name, grow_target, original_size, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = original_size + self.apiClient.updateStoragePool(cmd) + + shrunk = self._poll_pool_capacity(pool.id, original_size, timeout=120) + self.assertEqual( + shrunk.state, "Up", + "Pool should stay 'Up' after shrinking under a VM, got '%s'" + % shrunk.state + ) + shrunk_vol = self._poll_ontap_volume_size( + pool.name, original_size, timeout=120 + ) + self.assertEqual(shrunk_vol.get("state"), "online") + self._assert_vm_running_with_volume(vm.id, vol.id, "shrink-with-vm") + self.assertTrue( + self.ontap.list_luns_in_volume(self.svm_name, pool.name), + "LUN disappeared from FlexVol '%s' during the in-use resize" + % pool.name, + ) + + # ------------------------------------------------------------------ + # Step 15 - Detach/destroy the VM, delete volume (LUN), delete the pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_08_delete_volume_and_pool(self): + def test_15_delete_volume_and_pool(self): """ - Delete the volume from test_07, enter maintenance, then force-delete - the pool. + Detach the volume and destroy the VM from test_12, then delete the + volume, enter maintenance, and force-delete the pool. Verifies: - deleteVolume removes the LUN from ONTAP - Pool transitions to Maintenance @@ -574,8 +1165,13 @@ def test_08_delete_volume_and_pool(self): - ONTAP: FlexVol deleted - ONTAP: igroups for all cluster hosts deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") - self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_11 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_11 must pass first") + + # A volume still attached to a VM cannot be deleted, so unwind test_12 + # first. Both steps are no-ops when test_12 skipped. + self._detach_volume_if_attached(self.__class__.volume.id) + self.__class__._destroy_vm_if_present() pool = self.__class__.pool pool_name = pool.name diff --git a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py index 847a026a3bd5..ae304b8de51c 100644 --- a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py +++ b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py @@ -25,9 +25,11 @@ Workflow: 01 Create zone-scoped iSCSI pool — pool.state Up; ONTAP FlexVol online; igroup present for each cluster host IQN - 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged - 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged - 04 Delete zone-scoped pool — pool gone; FlexVol deleted; igroups deleted + 02 Grow zone-scoped pool — capacity increased; FlexVol resized; state Up + 03 Shrink zone-scoped pool — capacity back to its pre-grow value; state Up + 04 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 05 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 06 Delete zone-scoped pool — pool gone; FlexVol deleted; igroups deleted Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -140,6 +142,7 @@ def _igroup_name(svm_name, host_name): class TestOntapISCSIZoneScopedPool(OntapTestBase): _vol_name_prefix = "OntapISCSIZoneVol" + resize_original_size = None @classmethod def setUpClass(cls): @@ -280,11 +283,100 @@ def test_01_create_zone_scoped_pool(self): self._assert_igroups_for_hosts(expect_present=True) # ------------------------------------------------------------------ - # Step 02 — Disable zone-scoped pool + # Step 02 - Grow the zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_02_grow_zone_scoped_pool(self): + """ + Increase the zone-scoped pool's capacity. + Verifies: + - CloudStack reports the requested capacity + - ONTAP: FlexVol reaches the requested size and stays online + - pool.state stays Up + - ONTAP: per-host igroups are unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + requested_size = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + grown = self._poll_pool_capacity(pool.id, requested_size, timeout=120) + self.assertEqual( + grown.state, "Up", + "Pool should stay 'Up' after growing, got '%s'" % grown.state + ) + grown_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(grown_vol.get("state"), "online") + self._assert_igroups_for_hosts(True) + + + # ------------------------------------------------------------------ + # Step 03 - Safely shrink the zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_03_shrink_zone_scoped_pool(self): + """ + Shrink the zone-scoped pool back to the size it had before test_02. + The pool holds no volumes, so this stays above ONTAP used space. + Verifies: + - CloudStack reports the original capacity again + - ONTAP: FlexVol shrinks back and stays online + - pool.state stays Up + - ONTAP: per-host igroups are unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent - test_02 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk = self._poll_pool_capacity(pool.id, target_size, timeout=120) + self.assertEqual( + shrunk.state, "Up", + "Pool should stay 'Up' after shrinking, got '%s'" % shrunk.state + ) + shrunk_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_vol.get("state"), "online") + self._assert_igroups_for_hosts(True) + + + # ------------------------------------------------------------------ + # Step 04 — Disable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_02_disable_zone_scoped_pool(self): + def test_04_disable_zone_scoped_pool(self): """ Disable the zone-scoped iSCSI pool. Verifies: @@ -312,11 +404,11 @@ def test_02_disable_zone_scoped_pool(self): self._assert_igroups_for_hosts(expect_present=True) # ------------------------------------------------------------------ - # Step 03 — Enable zone-scoped pool + # Step 05 — Enable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_03_enable_zone_scoped_pool(self): + def test_05_enable_zone_scoped_pool(self): """ Re-enable the zone-scoped iSCSI pool. Verifies: @@ -344,11 +436,11 @@ def test_03_enable_zone_scoped_pool(self): self._assert_igroups_for_hosts(expect_present=True) # ------------------------------------------------------------------ - # Step 04 — Delete zone-scoped pool + # Step 06 — Delete zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_04_delete_zone_scoped_pool(self): + def test_06_delete_zone_scoped_pool(self): """ Enter maintenance then delete the zone-scoped iSCSI pool. Verifies: diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py index 5d1812cdad4f..bd97923f299e 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py @@ -18,22 +18,29 @@ """ Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. -Tests are numbered test_01 ... test_08 and must run in that order. Each step +Tests are numbered test_01 ... test_15 and must run in that order. Each step builds on the shared state established by the previous step. Workflow: 01 Create primary storage pool - 02 Disable storage pool - 03 Enable storage pool - 04 Enter maintenance mode - 05 Cancel maintenance mode - 06 Delete the storage pool - 07 Create fresh pool and allocate a CloudStack volume - 08 Delete volume then force-delete the pool + 02 Increase storage pool capacity to the maximum supported size + 03 Reject a grow beyond the maximum supported size + 04 Safely shrink storage pool capacity + 05 Disable storage pool + 06 Enable storage pool + 07 Enter maintenance mode + 08 Resize storage pool while in maintenance mode + 09 Cancel maintenance mode + 10 Delete the storage pool + 11 Create fresh pool and allocate a CloudStack volume + 12 Deploy a VM and attach the ONTAP volume to it + 13 Reject shrink below the ONTAP used capacity of the attached volume + 14 Grow and shrink the pool while the VM holds the volume + 15 Detach/destroy the VM, delete volume, then force-delete the pool Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed - - KVM cluster registered in CloudStack + - KVM cluster registered in CloudStack with at least one ready user template - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF - ontap.cfg populated with real values @@ -42,7 +49,8 @@ --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py -v -Note: Tests 01-06 share class-level state (sequential). Running a single test +Note: Tests 01-10 share class-level state, as do tests 11-15 (sequential). +Running a single test with -m "test_NN" will invoke setUpClass but the guard assertion will fail immediately if earlier steps have not yet run. Always run the full suite. """ @@ -55,9 +63,11 @@ from nose.plugins.attrib import attr from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, cancelStorageMaintenance, createStoragePool as createStoragePoolAPI, deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, enableStorageMaintenance, updateStoragePool as updateStoragePoolAPI, ) @@ -67,7 +77,7 @@ from ontap_test_base import ( OntapRestClient, OntapTestBase, _parse_pool_details, get_datacenter_config, - log_progress, + _wait_for_vm_state, log_progress, ) logger = logging.getLogger("TestOntapNFS3Workflow") @@ -97,6 +107,16 @@ class TestData: ONTAP_MIN_VOLUME_SIZE = 1677721600 + # Maximum FlexVol size on AFF/FAS, raised from 100 TB in ONTAP 9.12.1P2. + # ONTAP writes this as "300 TB" but means binary, hence 1024**4. The + # value has to be hardcoded: no REST field reports the platform ceiling. + # Against an ONTAP older than 9.12.1P2 the real limit is 100 TB and the + # grow step will fail. CloudStack enforces no maximum of its own (the + # plugin validates only a 20 MB floor), so it is ONTAP that refuses + # anything above this. + ONTAP_MAX_VOLUME_SIZE = 300 * 1024 ** 4 + ONTAP_GROW_TARGET_SIZE = ONTAP_MAX_VOLUME_SIZE + def __init__(self, storage_ip, svm_name, username, password, protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", tags="ontap-nfs3", capacitybytes=None): @@ -145,6 +165,10 @@ class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): pool_ep_name = None # NFS export policy name for pool pool2_ep_name = None # export policy for pool stashed from test_01-04 cluster_host_ips = None + resize_original_size = None + + # VM state (vm/template_id/network_id) lives in OntapTestBase. + _vm_network_name_prefix = "ontap-nfs3-lifecycle-net" _vol_name_prefix = "OntapNFS3Vol" @@ -193,6 +217,8 @@ def setUpClass(cls): if getattr(h, "ipaddress", None) ] + cls._discover_vm_deploy_resources() + # No per-test tearDown — state intentionally persists between steps. # ------------------------------------------------------------------ @@ -458,11 +484,166 @@ def test_01_create_primary_storage_pool(self): self._assert_pool_capacity(pool, "pool-created") # ------------------------------------------------------------------ - # Step 02 — Disable storage pool + # Step 02 - Increase storage pool capacity # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_02_disable_storage_pool(self): + def test_02_grow_storage_pool(self): + """Grow the original pool to 300 TiB and verify CloudStack and ONTAP converge.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_01 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + int(self.testdata[TestData.primaryStorage]["capacitybytes"]), + ) + requested_size = TestData.ONTAP_GROW_TARGET_SIZE + self.assertGreater( + requested_size, original_size, + "Grow target %d B must exceed the current pool size %d B" + % (requested_size, original_size) + ) + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual(resized_pool.state, "Up") + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after capacity increase", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + # ------------------------------------------------------------------ + # Step 03 - Reject a grow beyond the ONTAP maximum FlexVol size + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_03_reject_grow_above_max_size(self): + """ + Request a capacity above the 300 TiB ONTAP FlexVol maximum and verify + it is refused, leaving the pool at the size test_02 grew it to. + ONTAP reports this as an autosize-maximum violation rather than an + oversized-volume error. + Verifies: + - updateStoragePool raises CloudstackAPIException + - the error text is about the size limit, not about some other + failure such as the aggregate running out of space + - CloudStack capacity and ONTAP FlexVol size are both unchanged + - Pool stays 'Up' + - ONTAP: export policy is untouched + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_02 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before over-max resize") + self.assertEqual( + int(getattr(listed[0], "capacitybytes", 0) or 0), + TestData.ONTAP_MAX_VOLUME_SIZE, + "Pool should sit at the 300 TiB maximum - test_02 must pass first" + ) + + # Overshoot by only 1 GiB: at 0.0003% of the limit this probes the + # 300 TiB boundary itself rather than asking for something absurd. + above_max = self._align_flexvol_bytes( + TestData.ONTAP_MAX_VOLUME_SIZE + 1024 ** 3 + ) + log_progress( + logger, "info", + "Requesting %d B for pool '%s' (ONTAP maximum is %d B, " + "expect reject)", + above_max, pool.name, TestData.ONTAP_MAX_VOLUME_SIZE, + ) + + # ONTAP enforces the platform cap through autosize rather than + # rejecting the size outright, reporting "Volume maximum autosize + # must be greater than or equal to the current volume size 300.0TB". + # Matching on that keeps the test from passing when the resize fails + # for an unrelated reason, such as the aggregate running out of space. + self._assert_capacity_update_rejected( + pool.id, above_max, "grow-above-max-flexvol", + pool.id, pool.name, + expected_error=( + "maximum autosize", + "maximum size", + "exceeds", + ), + ) + + after = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(after, "Pool disappeared after rejected over-max grow") + self.assertEqual( + after[0].state, "Up", + "Pool should stay 'Up' after a rejected resize, got '%s'" + % after[0].state + ) + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after a rejected over-max grow", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + # ------------------------------------------------------------------ + # Step 04 - Safely shrink storage pool capacity + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_04_shrink_storage_pool(self): + """Shrink the original pool back to its initial safe capacity.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_02 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent — test_02 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk_pool = self._poll_pool_capacity( + pool.id, target_size, timeout=120 + ) + self.assertEqual(shrunk_pool.state, "Up") + shrunk_ontap_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_ontap_vol.get("state"), "online") + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after safe shrink", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + # ------------------------------------------------------------------ + # Step 05 — Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_05_disable_storage_pool(self): """ Disable the pool and verify: - CloudStack reports Disabled @@ -494,11 +675,11 @@ def test_02_disable_storage_pool(self): ) # ------------------------------------------------------------------ - # Step 03 — Enable storage pool + # Step 06 — Enable storage pool # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_03_enable_storage_pool(self): + def test_06_enable_storage_pool(self): """ Re-enable the pool and verify: - CloudStack reports Up @@ -530,11 +711,11 @@ def test_03_enable_storage_pool(self): ) # ------------------------------------------------------------------ - # Step 04 — Enter maintenance mode + # Step 07 — Enter maintenance mode # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_04_enter_maintenance_mode(self): + def test_07_enter_maintenance_mode(self): """ Put the pool into maintenance mode and verify: - CloudStack reports Maintenance @@ -566,11 +747,81 @@ def test_04_enter_maintenance_mode(self): ) # ------------------------------------------------------------------ - # Step 05 — Cancel maintenance mode + # Step 08 — Resize storage pool while in maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_08_resize_storage_pool_in_maintenance(self): + """ + Grow the pool while it is in maintenance mode and verify: + - updateStoragePool is accepted while the pool is in Maintenance + - CloudStack reports the new capacity + - ONTAP: FlexVol is resized and still online + - The pool stays in Maintenance throughout (resize must not + implicitly return it to Up) + - ONTAP: NFS export policy is unchanged + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_01 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before maintenance resize") + self.assertEqual( + listed[0].state, "Maintenance", + "Pool must be in Maintenance — test_07 must pass first, got '%s'" + % listed[0].state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing before maintenance resize" + ) + current_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + requested_size = current_size + TestData.ONTAP_MIN_VOLUME_SIZE + + log_progress( + logger, "info", + "Resizing NFS3 pool '%s' while in Maintenance: current=%d B, " + "requested=%d B", + pool.name, current_size, requested_size, + ) + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual( + resized_pool.state, "Maintenance", + "Pool should remain in Maintenance after resize, got '%s'" + % resized_pool.state + ) + + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + + if self.__class__.pool_ep_name: + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after maintenance resize", + ) + + # ------------------------------------------------------------------ + # Step 09 — Cancel maintenance mode # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_05_cancel_maintenance_mode(self): + def test_09_cancel_maintenance_mode(self): """ Cancel maintenance mode and verify the pool returns to Up. @@ -616,11 +867,11 @@ def test_05_cancel_maintenance_mode(self): ) # ------------------------------------------------------------------ - # Step 06 — Delete the storage pool + # Step 10 — Delete the storage pool # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_06_delete_pool_from_maintenance(self): + def test_10_delete_pool_from_maintenance(self): """ Enter maintenance mode then delete the storage pool. @@ -634,7 +885,7 @@ def test_06_delete_pool_from_maintenance(self): pool_name = pool.name ep_name = self.__class__.pool_ep_name - # Pool is Up after test_05 succeeded; must enter Maintenance before deletion. + # Pool is Up after test_08 succeeded; must enter Maintenance before deletion. maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() maint_cmd.id = pool.id self.apiClient.enableStorageMaintenance(maint_cmd) @@ -643,6 +894,7 @@ def test_06_delete_pool_from_maintenance(self): self._delete_pool(pool.id) self.__class__.pool = None self.__class__.pool_ep_name = None + self.__class__.resize_original_size = None # CloudStack: pool must be gone try: @@ -667,11 +919,11 @@ def test_06_delete_pool_from_maintenance(self): ) # ------------------------------------------------------------------ - # Step 07 - Create fresh pool and allocate a CloudStack volume + # Step 11 - Create fresh pool and allocate a CloudStack volume # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_07_create_volume_on_pool(self): + def test_11_create_volume_on_pool(self): """ Create a new NFS3 pool and allocate a CloudStack data volume. For NFS3, createAsync is a no-op on ONTAP (volume is a CloudStack record @@ -686,7 +938,7 @@ def test_07_create_volume_on_pool(self): self.__class__.pool = pool log_progress( logger, "info", - "test_07: created storage pool name='%s' id=%s state=%s", + "test_10: created storage pool name='%s' id=%s state=%s", pool.name, pool.id, pool.state, ) @@ -703,7 +955,7 @@ def test_07_create_volume_on_pool(self): self.assertIsNotNone(vol, "createVolume returned None") log_progress( logger, "info", - "test_07: created CloudStack volume name='%s' id=%s state=%s " + "test_10: created CloudStack volume name='%s' id=%s state=%s " "on pool='%s' (id=%s) account='%s' domain='%s' — " "switch to this account in the UI to see the volume", getattr(vol, "name", "?"), getattr(vol, "id", "?"), @@ -733,14 +985,310 @@ def test_07_create_volume_on_pool(self): self._assert_pool_capacity(pool, "volume-allocated") # ------------------------------------------------------------------ - # Step 08 - Delete volume then force-delete the pool + # Step 12 - Deploy a VM and attach the ONTAP volume to it # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_08_delete_volume_and_pool(self): + def test_12_create_vm_and_attach_volume(self): """ - Delete the volume from test_07, enter maintenance, then force-delete - the pool. + Deploy a VM and attach the ONTAP data volume from test_11 to it. + Verifies: + - VM reaches 'Running' + - attachVolume sets the volume's virtualmachineid (on NFS3 shared + storage CloudStack leaves the volume state at 'Ready', so + virtualmachineid is the reliable attach signal) + - VM is still 'Running' after the attach + - ONTAP: FlexVol stays online and the pool capacity is unchanged + - ONTAP: the volume's data file is materialised in the FlexVol + (NFS3 creates it lazily, during attach rather than createVolume) + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_11 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_11 must pass first" + ) + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template in the zone - cannot deploy a VM" + ) + self.assertIsNotNone( + self.__class__.service_offering_id, + "No service offering available - check the CloudStack setup" + ) + + pool = self.__class__.pool + vol = self.__class__.volume + + log_progress( + logger, "info", + "Deploying VM for NFS3 pool '%s' (template=%s, offering=%s)", + pool.name, self.__class__.template_id, + self.__class__.service_offering_id, + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=600) + self.assertIsNotNone( + vm_obj, "VM %s never appeared in listVirtualMachines" % vm.id + ) + self.assertEqual( + vm_obj.state, "Running", + "VM should be 'Running' before attach, got '%s'" % vm_obj.state + ) + + log_progress( + logger, "info", + "Attaching volume '%s' to VM '%s'", vol.id, vm.id, + ) + attach_cmd = attachVolumeAPI.attachVolumeCmd() + attach_cmd.id = vol.id + attach_cmd.virtualmachineid = vm.id + self.assertIsNotNone( + self.apiClient.attachVolume(attach_cmd), + "attachVolume returned None" + ) + + vol_vmid = self._poll_volume_attached(vol.id, timeout=180) + self.assertEqual( + vol_vmid, vm.id, + "Volume %s should report virtualmachineid=%s after attach, got %s" + % (vol.id, vm.id, vol_vmid) + ) + + vm_after = _wait_for_vm_state(self.apiClient, vm.id, "Running", + timeout=60) + self.assertEqual( + vm_after.state, "Running", + "VM should still be 'Running' after attach, got '%s'" + % vm_after.state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol missing after volume attach" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach, got '%s'" + % ontap_vol.get("state") + ) + + # NFS3 materialises the volume's backing file during attach, not at + # createVolume time, so it must be present now. + files = self.ontap.list_files_in_volume(pool.name) + self.assertIsNotNone( + next((f for f in files if vol.id in f), None), + "No data file matching volume UUID '%s' in FlexVol '%s' after " + "attach; files present: %s" % (vol.id, pool.name, files) + ) + + self._assert_pool_capacity(pool, "vm-attached") + + # ------------------------------------------------------------------ + # Step 13 - Reject shrink below ONTAP used capacity + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_13_reject_shrink_below_used_capacity(self): + """ + Attempt to shrink the pool below ONTAP used space and verify the + request is rejected while the volume, pool capacity, and NFS export + policy remain unchanged. + + Runs after test_12, so the FlexVol holds the data file of a volume + attached to a running VM. That file is thin, though, and in practice + leaves used space under the 20 MiB FlexVol floor, so incompressible + data is written through the ONTAP files API to get above it and + removed before this test returns. The fill is skipped whenever used + space is already high enough on its own. + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_12 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent — test_12 must pass first" + ) + pool = self.__class__.pool + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol not found for pool '%s'" % pool.name, + ) + + export_policy_name = self.__class__.pool_ep_name + self.assertIsNotNone( + export_policy_name, + "Export policy name was not recorded by test_11", + ) + export_policy_before = self.ontap.get_export_policy( + export_policy_name + ) + self.assertIsNotNone( + export_policy_before, + "Export policy '%s' missing before rejected used-capacity shrink" + % export_policy_name, + ) + + vol_before = self._cs_volume_snapshot(self.__class__.volume.id) + # The attached volume is thin on both protocols, so it usually does + # not put 20 MiB on disk by itself. This writes incompressible data + # only when used space is still under the FlexVol floor, and removes + # it before returning. + filler_name, used_bytes = self._fill_flexvol_above_minimum(pool.name) + self.__class__.filler_filename = filler_name + self.__class__.filler_flexvol = pool.name if filler_name else None + try: + below_used = self._shrink_target_below_used(used_bytes) + log_progress( + logger, "info", + "Shrinking NFS3 pool '%s' below ONTAP used capacity: " + "used=%d B, requested=%d B " + "(FlexVol minimum=%d B, expect reject)", + pool.name, used_bytes, below_used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + self._assert_capacity_update_rejected( + pool.id, below_used, "shrink-below-used", + pool.id, pool.name, + # ONTAP's wording: "Selected volume size is too small to + # hold the current volume data. New volume size must be at + # least 34.3MB ...". Matching the specific phrase stops an + # unrelated resize failure from counting as a pass. + expected_error=( + "too small to hold the current volume data", + "too small to hold", + "cannot reduce", + ), + ) + self._assert_cs_volume_untouched(vol_before, "shrink-below-used") + self.assertGreater( + self._flexvol_used_bytes(pool.name), + self.ONTAP_MIN_FLEXVOL_SIZE, + "ONTAP used space dropped below the FlexVol minimum after " + "the rejected shrink", + ) + export_policy_after_used = self.ontap.get_export_policy( + export_policy_name + ) + self.assertIsNotNone( + export_policy_after_used, + "Export policy '%s' disappeared after rejected " + "used-capacity shrink" % export_policy_name, + ) + self.assertEqual( + export_policy_after_used.get("name"), + export_policy_before.get("name"), + "Export policy changed during rejected used-capacity shrink", + ) + self._assert_export_policy_has_host_ips(export_policy_name) + finally: + self._delete_filler_file(pool.name, filler_name) + + # ------------------------------------------------------------------ + # Step 14 - Resize the pool while the VM holds the volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_14_resize_pool_with_vm_attached(self): + """ + Grow the pool and shrink it straight back while the volume is + attached to the running VM from test_12, so resize is exercised + against a pool that is genuinely in use rather than an idle one. + Verifies: + - grow: CloudStack and ONTAP both reach the requested size + - shrink back: both return to the starting size + - the VM stays 'Running' and keeps the volume across both resizes + - ONTAP: FlexVol stays online + - ONTAP: export policy still covers every cluster host IP + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_11 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_11 must pass first" + ) + if self.__class__.vm is None: + self.skipTest( + "No VM was deployed (test_12 skipped) - nothing to resize " + "a pool underneath" + ) + + pool = self.__class__.pool + vm = self.__class__.vm + vol = self.__class__.volume + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before the in-use resize") + original_size = int(getattr(listed[0], "capacitybytes", 0) or 0) + grow_target = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + + log_progress( + logger, "info", + "Growing in-use pool '%s' from %d B to %d B (VM %s holds the " + "volume)", pool.name, original_size, grow_target, vm.id, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = grow_target + self.apiClient.updateStoragePool(cmd) + + grown = self._poll_pool_capacity(pool.id, grow_target, timeout=120) + self.assertEqual( + grown.state, "Up", + "Pool should stay 'Up' after growing under a VM, got '%s'" + % grown.state + ) + grown_vol = self._poll_ontap_volume_size( + pool.name, grow_target, timeout=120 + ) + self.assertEqual(grown_vol.get("state"), "online") + self._assert_vm_running_with_volume(vm.id, vol.id, "grow-with-vm") + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + log_progress( + logger, "info", + "Shrinking in-use pool '%s' back from %d B to %d B", + pool.name, grow_target, original_size, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = original_size + self.apiClient.updateStoragePool(cmd) + + shrunk = self._poll_pool_capacity(pool.id, original_size, timeout=120) + self.assertEqual( + shrunk.state, "Up", + "Pool should stay 'Up' after shrinking under a VM, got '%s'" + % shrunk.state + ) + shrunk_vol = self._poll_ontap_volume_size( + pool.name, original_size, timeout=120 + ) + self.assertEqual(shrunk_vol.get("state"), "online") + self._assert_vm_running_with_volume(vm.id, vol.id, "shrink-with-vm") + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + # ------------------------------------------------------------------ + # Step 15 - Detach/destroy the VM, delete volume, force-delete the pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_15_delete_volume_and_pool(self): + """ + Detach the volume and destroy the VM from test_12, then delete the + volume, enter maintenance, and force-delete the pool. Verifies: - deleteVolume completes (or expected NFS3 libvirt pool-not-found) - Pool transitions to Maintenance @@ -748,8 +1296,13 @@ def test_08_delete_volume_and_pool(self): - ONTAP: FlexVol deleted - ONTAP: export policy deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") - self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_11 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_11 must pass first") + + # A volume still attached to a VM cannot be deleted, so unwind test_12 + # first. Both steps are no-ops when test_12 skipped. + self._detach_volume_if_attached(self.__class__.volume.id) + self.__class__._destroy_vm_if_present() pool = self.__class__.pool pool_name = pool.name diff --git a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py index 88a6309f1ee1..2f7ecc433ead 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py @@ -26,9 +26,11 @@ Workflow: 01 Create zone-scoped NFS3 pool — pool.state Up; ONTAP FlexVol online; export policy has all cluster host IPs - 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged - 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged - 04 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted + 02 Grow zone-scoped pool — capacity increased; FlexVol resized; state Up + 03 Shrink zone-scoped pool — capacity back to its pre-grow value; state Up + 04 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 05 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 06 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -135,6 +137,7 @@ class TestOntapZoneScopedPool(OntapTestBase): cluster_host_ips = None _vol_name_prefix = "OntapZoneVol" + resize_original_size = None @classmethod def setUpClass(cls): @@ -288,11 +291,100 @@ def test_01_create_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 02 — Disable zone-scoped pool + # Step 02 - Grow the zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_02_disable_zone_scoped_pool(self): + def test_02_grow_zone_scoped_pool(self): + """ + Increase the zone-scoped pool's capacity. + Verifies: + - CloudStack reports the requested capacity + - ONTAP: FlexVol reaches the requested size and stays online + - pool.state stays Up + - ONTAP: export policy still covers every cluster host IP + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + ) + requested_size = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + grown = self._poll_pool_capacity(pool.id, requested_size, timeout=120) + self.assertEqual( + grown.state, "Up", + "Pool should stay 'Up' after growing, got '%s'" % grown.state + ) + grown_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(grown_vol.get("state"), "online") + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 03 - Safely shrink the zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_03_shrink_zone_scoped_pool(self): + """ + Shrink the zone-scoped pool back to the size it had before test_02. + The pool holds no volumes, so this stays above ONTAP used space. + Verifies: + - CloudStack reports the original capacity again + - ONTAP: FlexVol shrinks back and stays online + - pool.state stays Up + - ONTAP: export policy still covers every cluster host IP + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent - test_02 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk = self._poll_pool_capacity(pool.id, target_size, timeout=120) + self.assertEqual( + shrunk.state, "Up", + "Pool should stay 'Up' after shrinking, got '%s'" % shrunk.state + ) + shrunk_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_vol.get("state"), "online") + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + + # ------------------------------------------------------------------ + # Step 04 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_04_disable_zone_scoped_pool(self): """ Disable the zone-scoped pool. Verifies: @@ -325,11 +417,11 @@ def test_02_disable_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 03 — Enable zone-scoped pool + # Step 05 — Enable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_03_enable_zone_scoped_pool(self): + def test_05_enable_zone_scoped_pool(self): """ Re-enable the zone-scoped pool. Verifies: @@ -362,11 +454,11 @@ def test_03_enable_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 04 — Delete zone-scoped pool + # Step 06 — Delete zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_04_delete_zone_scoped_pool(self): + def test_06_delete_zone_scoped_pool(self): """ Enter maintenance then delete the zone-scoped pool. Verifies: diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index 4f60dbf9433f..323a8708d0db 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -26,27 +26,43 @@ """ import logging +import os import random import requests import sys import time import urllib3 -from urllib.parse import urlparse +from urllib.parse import quote, urlparse from marvin.cloudstackAPI import ( cancelStorageMaintenance, + createNetwork as createNetworkAPI, createVolume as createVolumeAPI, + deleteNetwork as deleteNetworkAPI, deleteStoragePool as deleteStoragePoolAPI, deleteVolume as deleteVolumeAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + detachVolume as detachVolumeAPI, + enableStorageMaintenance, listDiskOfferings as listDiskOfferingsAPI, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listServiceOfferings as listServiceOfferingsAPI, + listTemplates as listTemplatesAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + stopVirtualMachine as stopVirtualMachineAPI, updateStoragePool as updateStoragePoolAPI, ) from marvin.cloudstackAPI import listHosts as listHostsAPI +from marvin.cloudstackException import CloudstackAPIException from marvin.cloudstackTestCase import cloudstackTestCase from marvin.jsonHelper import jsonDump from marvin.lib.base import Account, DiskOffering from marvin.sshClient import SshClient -from marvin.lib.common import get_domain, get_zone, list_clusters, list_storage_pools +from marvin.lib.common import ( + get_domain, get_zone, list_clusters, list_storage_pools, +) from marvin.lib.utils import cleanup_resources urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -144,7 +160,44 @@ def _delete(self, path, params=None): url = self._base + path resp = requests.delete(url, auth=self._auth, params=params, verify=False, timeout=30) - resp.raise_for_status() + self._raise_http(resp) + + def _raise_http(self, resp): + if resp.ok: + return + body = "" + try: + body = resp.text + except Exception: + body = "" + raise requests.HTTPError( + "%s Client Error: %s for url: %s body: %s" + % (resp.status_code, resp.reason, resp.url, body), + response=resp, + ) + + def _post(self, path, params=None, data=None, json_body=None, timeout=60, + headers=None, files=None): + url = self._base + path + resp = requests.post( + url, auth=self._auth, params=params, data=data, json=json_body, + headers=headers, files=files, verify=False, timeout=timeout, + ) + self._raise_http(resp) + if not resp.content: + return None + try: + return resp.json() + except ValueError: + return None + + def _volume_uuid(self, vol_name): + vol = self.get_volume(vol_name) + return (vol or {}).get("uuid") + + def _files_path(self, vol_uuid, filename): + encoded = quote("/" + filename.lstrip("/"), safe="") + return "/storage/volumes/%s/files/%s" % (vol_uuid, encoded) def delete_volume(self, name): """Delete the ONTAP FlexVol with the given name. No-op if not found.""" @@ -262,7 +315,6 @@ def list_files_in_volume(self, vol_name, path="/"): if not vol_uuid: return [] # URL-encode the path component (/ → %2F) and embed it in the URL. - from urllib.parse import quote encoded_path = quote(path, safe="") try: resp = self._get( @@ -274,6 +326,47 @@ def list_files_in_volume(self, vol_name, path="/"): return [r.get("name", "") for r in resp.get("records", []) if r.get("name") not in (".", "..")] + def write_file_in_volume(self, vol_name, filename, size_bytes, + chunk_bytes=512 * 1024): + """Write *size_bytes* of incompressible data into a FlexVol file. + + Zeros compress to almost nothing on ONTAP, so the payload is random. + The files API requires ``multipart/form-data`` and rejects writes + larger than 1 MiB, so data is sent in chunks. No VM, NFS mount, or + CloudStack volume is required. + """ + vol_uuid = self._volume_uuid(vol_name) + if not vol_uuid: + raise RuntimeError("ONTAP FlexVol '%s' not found" % vol_name) + url_path = self._files_path(vol_uuid, filename) + written = 0 + size_bytes = int(size_bytes) + while written < size_bytes: + chunk = min(int(chunk_bytes), size_bytes - written) + files = { + "file": (filename, os.urandom(chunk), + "application/octet-stream"), + } + self._post( + url_path, + params={"byte_offset": written, "overwrite": "true"}, + files=files, + timeout=120, + ) + written += chunk + + def delete_file_in_volume(self, vol_name, filename): + """Delete a file from the FlexVol. No-op if the volume or file is gone.""" + vol_uuid = self._volume_uuid(vol_name) + if not vol_uuid: + return + try: + self._delete(self._files_path(vol_uuid, filename)) + except requests.HTTPError as exc: + status = getattr(exc.response, "status_code", None) + if status not in (404, 409): + raise + # --------------------------------------------------------------------------- # Base test class @@ -281,11 +374,31 @@ def list_files_in_volume(self, vol_name, path="/"): class OntapTestBase(cloudstackTestCase): + # ONTAP refuses to shrink a FlexVol below this; distinct from the + # plugin's 1.56 GiB create-time floor (ONTAP_MIN_VOLUME_SIZE). + ONTAP_MIN_FLEXVOL_SIZE = 20 * 1024 * 1024 + FILLER_FILENAME = "ontap-filler.bin" + FILLER_SIZE = 32 * 1024 * 1024 + # ---- shared state (set/cleared by individual tests) ---------------- pool = None volume = None + # Volumes a single test allocates on top of ``volume`` and deletes before + # it returns; tracked here only so a failed run still cleans them up. + extra_volumes = [] + # File written into the FlexVol by tests that must raise ONTAP used space. + filler_filename = None + filler_flexvol = None pool2 = None volume2 = None + # ---- VM state, for suites that deploy an instance ------------------ + vm = None + template_id = None + service_offering_id = None + network_id = None + _created_network_id = None + # Prefix for a guest network this suite creates on Advanced zones. + _vm_network_name_prefix = "ontap-vm-net" disk_offering_id = None svm_name = None cluster_hosts = None @@ -441,9 +554,214 @@ def _cleanup_kvm_storage_pool_mounts(cls, pool_uuid): # ---- shared teardown ----------------------------------------------- + @classmethod + def _all_tracked_volumes(cls): + """Every volume the suite created, most recent first, without repeats.""" + seen = set() + ordered = [] + for vol in list(cls.extra_volumes or []) + [cls.volume2, cls.volume]: + vol_id = getattr(vol, "id", None) + if vol is None or vol_id in seen: + continue + seen.add(vol_id) + ordered.append(vol) + return ordered + + # ---- VM deploy/attach helpers -------------------------------------- + + @classmethod + def _discover_vm_deploy_resources(cls): + """Resolve the template, service offering, and network for VM deploys. + + A missing template is not fatal: ``template_id`` is left as None so + callers can skip the VM step while the rest of the suite still runs. + On Advanced zones an existing account network is reused when present, + otherwise an Isolated one is created and torn down in tearDownClass. + """ + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = cls.zone.id + templates = cls.apiClient.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + cls.template_id = kvm_ready[0].id if kvm_ready else None + if cls.template_id is None: + logger.warning( + "No ready user KVM template in zone '%s' — VM steps will skip." + % cls.zone.name + ) + + so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() + offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] + if offerings: + offerings.sort(key=lambda s: getattr(s, "memory", 9999)) + cls.service_offering_id = offerings[0].id + + cls.network_id = None + if getattr(cls.zone, "networktype", "Basic").lower() != "advanced": + return + + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + return + + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + no_offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in no_offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + no_offerings[0] if no_offerings else None + ) + if snat_offering is None: + return + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "%s-%d" % (cls._vm_network_name_prefix, + random.randint(0, 9999)) + cn_cmd.displaytext = "ONTAP test VM network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def _destroy_vm_if_present(cls): + """Stop and expunge the suite's VM. Safe to call when none exists.""" + if cls.vm is None: + return + vm_id = cls.vm.id + try: + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + state = vms[0].state.lower() if vms else "unknown" + if state not in ("stopped", "destroyed", "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm_id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, vm_id, "Stopped", timeout=180) + except Exception as e: + logger.warning("Could not stop VM %s: %s" % (vm_id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm_id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as e: + logger.warning("Could not destroy VM %s: %s" % (vm_id, e)) + cls.vm = None + + @classmethod + def _delete_created_network(cls): + """Delete the guest network this suite created, if any.""" + if cls._created_network_id is None: + return + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + except Exception as e: + logger.warning( + "Could not delete network %s: %s" % (cls._created_network_id, e) + ) + cls._created_network_id = None + + def _poll_volume_attached(self, vol_id, timeout=180, interval=5): + """Poll listVolumes until virtualmachineid is set; return it or None. + + ONTAP-backed volumes stay in state 'Ready' when attached, so the + virtualmachineid field is the reliable signal. + """ + deadline = time.time() + timeout + vol_vmid = None + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if vol_vmid: + return vol_vmid + time.sleep(interval) + return vol_vmid + + def _assert_vm_running_with_volume(self, vm_id, vol_id, label): + """Assert the VM is still up and still owns the volume.""" + vm_obj = _wait_for_vm_state(self.apiClient, vm_id, "Running", + timeout=60) + self.assertIsNotNone( + vm_obj, "[%s] VM %s vanished from listVirtualMachines" + % (label, vm_id), + ) + self.assertEqual( + vm_obj.state, "Running", + "[%s] VM should still be 'Running', got '%s'" + % (label, vm_obj.state), + ) + vol = self._get_cs_volume(vol_id) + self.assertIsNotNone( + vol, "[%s] volume %s is no longer listed" % (label, vol_id), + ) + self.assertEqual( + getattr(vol, "virtualmachineid", None), vm_id, + "[%s] volume %s should still be attached to VM %s, " + "virtualmachineid is %s" + % (label, vol_id, vm_id, getattr(vol, "virtualmachineid", None)), + ) + + def _detach_volume_if_attached(self, vol_id): + """Detach the volume when a VM holds it. No-op otherwise.""" + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + if not vols or not getattr(vols[0], "virtualmachineid", None): + return + try: + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = vol_id + self.apiClient.detachVolume(cmd) + except Exception as e: + logger.warning("Could not detach volume %s: %s" % (vol_id, e)) + return + deadline = time.time() + 120 + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol_id)) or [] + if not vols or not getattr(vols[0], "virtualmachineid", None): + return + time.sleep(5) + logger.warning( + "Volume %s still reports a virtualmachineid after detach" % vol_id + ) + @classmethod def tearDownClass(cls): """Best-effort cleanup of any resources left behind by a failed run.""" + cls._destroy_vm_if_present() + cls._delete_created_network() + if (getattr(cls, "ontap", None) is not None + and getattr(cls, "filler_filename", None) + and getattr(cls, "filler_flexvol", None)): + try: + cls.ontap.delete_file_in_volume( + cls.filler_flexvol, cls.filler_filename + ) + except Exception as fe: + logger.warning( + "tearDownClass: could not delete filler file %s in %s: %s" + % (cls.filler_filename, cls.filler_flexvol, fe) + ) + cls.filler_filename = None + cls.filler_flexvol = None for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: try: # Step 1: Check current pool state @@ -477,7 +795,7 @@ def tearDownClass(cls): # state. For iSCSI this works even in Maintenance; for NFS3/KVM # it may fail with NPE ("storagePoolInformation is null") when # pool is in Maintenance — that exception is caught below. - for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + for vol in cls._all_tracked_volumes(): try: cmd = deleteVolumeAPI.deleteVolumeCmd() cmd.id = vol.id @@ -539,7 +857,7 @@ def tearDownClass(cls): pass # Clean up volumes that may not have been handled with pool teardown - for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + for vol in cls._all_tracked_volumes(): try: cmd = deleteVolumeAPI.deleteVolumeCmd() cmd.id = vol.id @@ -601,6 +919,327 @@ def _poll_pool_state(self, pool_id, target_state, timeout=120, interval=5): % (pool_id, target_state, timeout, current_state) ) + def _poll_pool_capacity(self, pool_id, expected_bytes, timeout=120, + interval=5): + """Poll listStoragePools until capacitybytes equals expected_bytes.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current = 0 + log_progress( + logger, "info", + "Waiting for pool %s to report capacitybytes=%d " + "(timeout=%ds, poll every %ds).", + pool_id, expected_bytes, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + pools = list_storage_pools(self.apiClient, id=pool_id) + if pools: + current = int(getattr(pools[0], "capacitybytes", 0) or 0) + if current == expected_bytes: + log_progress( + logger, "info", + "Pool %s reported capacitybytes=%d after %ds (%d polls).", + pool_id, expected_bytes, elapsed, attempt, + ) + return pools[0] + log_progress( + logger, "info", + "Capacity poll #%d: pool %s capacitybytes=%d (want %d) " + "[elapsed %ds, ~%ds left]", + attempt, pool_id, current, expected_bytes, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "Pool %s did not report capacitybytes=%d within %ds (last: %d).", + pool_id, expected_bytes, timeout, current, + ) + self.fail( + "Pool %s did not report capacitybytes=%d within %ds (last: %d)" + % (pool_id, expected_bytes, timeout, current) + ) + + def _poll_ontap_volume_size(self, volume_name, expected_bytes, + timeout=120, interval=5): + """Poll ONTAP until the FlexVol space.size equals expected_bytes.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current = 0 + log_progress( + logger, "info", + "Waiting for ONTAP FlexVol '%s' to report space.size=%d " + "(timeout=%ds, poll every %ds).", + volume_name, expected_bytes, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + volume = self.ontap.get_volume(volume_name) + if volume: + current = int(volume.get("space", {}).get("size", 0) or 0) + if current == expected_bytes: + log_progress( + logger, "info", + "ONTAP FlexVol '%s' reported space.size=%d after " + "%ds (%d polls).", + volume_name, expected_bytes, elapsed, attempt, + ) + return volume + log_progress( + logger, "info", + "ONTAP capacity poll #%d: FlexVol '%s' space.size=%d " + "(want %d) [elapsed %ds, ~%ds left]", + attempt, volume_name, current, expected_bytes, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "ONTAP FlexVol '%s' did not report space.size=%d within %ds " + "(last: %d).", + volume_name, expected_bytes, timeout, current, + ) + self.fail( + "ONTAP FlexVol '%s' did not report space.size=%d within %ds " + "(last: %d)" + % (volume_name, expected_bytes, timeout, current) + ) + + def _get_cs_volume(self, vol_id): + """Return the CloudStack volume object, or None if it is gone.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return vols[0] if vols else None + + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + return self._get_cs_volume(vol_id) is not None + + def _cs_volume_snapshot(self, vol_id): + """Capture id, state, size, and pool so a later check can detect mutation.""" + vol = self._get_cs_volume(vol_id) + self.assertIsNotNone( + vol, "CloudStack volume %s is not listed in listVolumes" % vol_id + ) + pool_id = ( + getattr(vol, "storageid", None) + or getattr(vol, "poolid", None) + ) + return { + "id": getattr(vol, "id", None), + "state": getattr(vol, "state", None), + "size": int(getattr(vol, "size", 0) or 0), + "poolid": pool_id, + } + + def _assert_cs_volume_untouched(self, before, label): + """Assert listVolumes still returns the same id, state, size, and pool.""" + after = self._cs_volume_snapshot(before["id"]) + self.assertEqual( + after["id"], before["id"], + "[%s] CloudStack volume id changed (%s -> %s)" + % (label, before["id"], after["id"]), + ) + self.assertEqual( + after["state"], before["state"], + "[%s] CloudStack volume state changed (%s -> %s)" + % (label, before["state"], after["state"]), + ) + self.assertEqual( + after["size"], before["size"], + "[%s] CloudStack volume size changed (%s -> %s)" + % (label, before["size"], after["size"]), + ) + self.assertEqual( + after["poolid"], before["poolid"], + "[%s] CloudStack volume poolid changed (%s -> %s)" + % (label, before["poolid"], after["poolid"]), + ) + + def _align_flexvol_bytes(self, value): + """Round *value* down to the 4 KiB boundary ONTAP uses for FlexVol size.""" + return (int(value) // 4096) * 4096 + + def _flexvol_used_bytes(self, vol_name): + """ONTAP physical used bytes on the FlexVol (used + reserved).""" + ontap_vol = self.ontap.get_volume(vol_name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol '%s' not found" % vol_name + ) + space = ontap_vol.get("space") or {} + return int(space.get("used") or 0) + + def _fill_flexvol_above_minimum(self, vol_name, timeout=90): + """Write a FlexVol file until ONTAP used space exceeds the minimum. + + Returns ``(filler_filename_or_None, used_bytes)``. The caller must + delete any returned filename. No CloudStack volume or VM is created. + """ + used = self._flexvol_used_bytes(vol_name) + if used > self.ONTAP_MIN_FLEXVOL_SIZE: + log_progress( + logger, "info", + "FlexVol '%s' already has %d B used (ONTAP FlexVol minimum " + "%d B); no filler file needed", + vol_name, used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + return None, used + log_progress( + logger, "info", + "FlexVol '%s' has %d B used; writing %d B filler file '%s'", + vol_name, used, self.FILLER_SIZE, self.FILLER_FILENAME, + ) + self.ontap.write_file_in_volume( + vol_name, self.FILLER_FILENAME, self.FILLER_SIZE + ) + deadline = time.time() + timeout + while time.time() < deadline: + used = self._flexvol_used_bytes(vol_name) + if used > self.ONTAP_MIN_FLEXVOL_SIZE: + break + time.sleep(2) + self.assertGreater( + used, self.ONTAP_MIN_FLEXVOL_SIZE, + "FlexVol '%s' still has only %d B used after writing a %d B " + "filler file; cannot exceed the ONTAP FlexVol minimum of %d B" + % (vol_name, used, self.FILLER_SIZE, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + log_progress( + logger, "info", + "FlexVol '%s' has %d B used after filler file '%s'", + vol_name, used, self.FILLER_FILENAME, + ) + return self.FILLER_FILENAME, used + + def _delete_filler_file(self, vol_name, filename): + if not filename or not vol_name: + return + try: + self.ontap.delete_file_in_volume(vol_name, filename) + except Exception as exc: + logger.warning( + "could not delete filler file '%s' in FlexVol '%s': %s", + filename, vol_name, exc, + ) + if self.__class__.filler_filename == filename: + self.__class__.filler_filename = None + self.__class__.filler_flexvol = None + + def _shrink_target_below_used(self, used_bytes): + """4 KiB-aligned size below used_bytes but above the FlexVol min.""" + used_bytes = int(used_bytes) + self.assertGreater( + used_bytes, self.ONTAP_MIN_FLEXVOL_SIZE, + "Used capacity %d B is not above the ONTAP FlexVol " + "minimum %d B; cannot distinguish a used-capacity reject " + "from a minimum-size reject" + % (used_bytes, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + target = self._align_flexvol_bytes(used_bytes - 4096) + if target <= self.ONTAP_MIN_FLEXVOL_SIZE: + target = self._align_flexvol_bytes( + (used_bytes + self.ONTAP_MIN_FLEXVOL_SIZE) // 2 + ) + self.assertGreater( + target, self.ONTAP_MIN_FLEXVOL_SIZE, + "Shrink target %d B is not above the ONTAP FlexVol minimum %d B" + % (target, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + self.assertLess( + target, used_bytes, + "Shrink target %d B must be below used capacity %d B" + % (target, used_bytes), + ) + return target + + def _assert_capacity_update_rejected( + self, cmd_pool_id, capacitybytes, label, verify_pool_id, + volume_name, expected_error=None): + """Assert updateStoragePool(capacitybytes) fails and sizes stay put. + + CloudStack and ONTAP are compared against their own pre-request values + because the two do not have to agree: a FlexVol created with a snapshot + reserve reports a larger space.size than the usable capacity + CloudStack records. + """ + listed = list_storage_pools(self.apiClient, id=verify_pool_id) + self.assertTrue( + listed, + "[%s] listStoragePools returned no result for pool %s" + % (label, verify_pool_id), + ) + ontap_vol = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol '%s' not found" % (label, volume_name), + ) + before_cs = int(getattr(listed[0], "capacitybytes", 0) or 0) + before_ontap = int(ontap_vol.get("space", {}).get("size", 0) or 0) + log_progress( + logger, "info", + "Negative resize %s: pool_id=%s capacitybytes=%s " + "(expect reject; CS=%d B ONTAP=%d B)", + label, cmd_pool_id, capacitybytes, before_cs, before_ontap, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = cmd_pool_id + cmd.capacitybytes = capacitybytes + with self.assertRaises(CloudstackAPIException) as caught: + self.apiClient.updateStoragePool(cmd) + error_text = str(caught.exception) + log_progress( + logger, "info", "Rejected resize %s: %s", label, error_text, + ) + if expected_error: + needles = ( + expected_error + if isinstance(expected_error, (list, tuple)) + else (expected_error,) + ) + self.assertTrue( + any(needle in error_text for needle in needles), + "[%s] expected the rejection to report one of %r, got: %s" + % (label, needles, error_text), + ) + + listed = list_storage_pools(self.apiClient, id=verify_pool_id) + self.assertTrue( + listed, + "[%s] pool disappeared after rejected resize" % label, + ) + after_cs = int(getattr(listed[0], "capacitybytes", 0) or 0) + self.assertEqual( + after_cs, before_cs, + "[%s] CloudStack capacity changed after rejected resize " + "(got %d, want %d)" % (label, after_cs, before_cs), + ) + self.assertEqual( + listed[0].state, "Up", + "[%s] pool should remain Up after rejected resize, got '%s'" + % (label, listed[0].state), + ) + ontap_vol = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol disappeared after rejected resize" % label, + ) + after_ontap = int(ontap_vol.get("space", {}).get("size", 0) or 0) + self.assertEqual( + after_ontap, before_ontap, + "[%s] ONTAP FlexVol size changed after rejected resize " + "(got %d, want %d)" % (label, after_ontap, before_ontap), + ) + def _create_volume(self, pool_id): """Create a data volume on the given pool; uses _vol_name_prefix.""" cmd = createVolumeAPI.createVolumeCmd() @@ -619,3 +1258,40 @@ def _delete_pool(self, pool_id, forced=False): if forced: cmd.forced = True self.apiClient.deleteStoragePool(cmd) + + +# --------------------------------------------------------------------------- +# Module-level VM helpers +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + cmd.listall = True + return cmd + + +def _list_vols_cmd(vol_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + return cmd + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=120, + interval=5): + """Poll listVirtualMachines until the VM reaches target_state. + + Returns the last VM object seen, which may not be in target_state if the + timeout expires — callers assert on the state themselves. + """ + deadline = time.time() + timeout + vm_obj = None + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) or [] + if vms: + vm_obj = vms[0] + if vm_obj.state == target_state: + return vm_obj + time.sleep(interval) + return vm_obj