From c3c0f0ceddaa497fc874c3ea49b4b320f3edd291 Mon Sep 17 00:00:00 2001 From: Eugenio Grosso Date: Mon, 20 Apr 2026 22:06:00 +0000 Subject: [PATCH 1/8] adapter: add NVMETCP address type and FlashArrayConnection.nsid Preparatory data-model changes for NVMe-TCP support on the adaptive storage framework. No behaviour change for existing Fibre Channel users - the extra enum value, field, and getter/setter are only exercised by callers that explicitly use them. ProviderVolume.AddressType gains a NVMETCP value alongside FIBERWWN, so adapters can declare that a volume is addressed by an NVMe EUI-128 (NGUID) rather than a SCSI WWN. FlashArrayVolume.getAddress() produces the NGUID layout expected by the Linux kernel for a FlashArray NVMe namespace: 00 + serial[0:14] + 24a937 (Pure 6-hex OUI) + serial[14:24] which matches the /dev/disk/by-id/nvme-eui. symlink emitted by udev. Fibre Channel callers (addressType != NVMETCP) still get the existing 6 + 24a9370 + serial form. FlashArrayConnection gains a nsid field to carry the namespace id the FlashArray REST API attaches to host-group-scoped NVMe connections, when it is present. --- .../storage/datastore/adapter/ProviderVolume.java | 3 ++- .../adapter/flasharray/FlashArrayConnection.java | 9 +++++++++ .../adapter/flasharray/FlashArrayVolume.java | 11 +++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java index 25577903e3d8..a954560508c4 100644 --- a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java +++ b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java @@ -35,6 +35,7 @@ public interface ProviderVolume { public String getExternalName(); public String getExternalConnectionId(); public enum AddressType { - FIBERWWN + FIBERWWN, + NVMETCP } } diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java index 76cec9f70c4b..b115035fda21 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java @@ -31,6 +31,8 @@ public class FlashArrayConnection { private FlashArrayVolume volume; @JsonProperty("lun") private Integer lun; + @JsonProperty("nsid") + private Integer nsid; public FlashArrayConnectionHostgroup getHostGroup() { return hostGroup; @@ -64,5 +66,12 @@ public void setLun(Integer lun) { this.lun = lun; } + public Integer getNsid() { + return nsid; + } + + public void setNsid(Integer nsid) { + this.nsid = nsid; + } } diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java index a3201a753a75..45d73586daa6 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java @@ -27,6 +27,10 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class FlashArrayVolume implements ProviderSnapshot { public static final String PURE_OUI = "24a9370"; + // The 3-byte OUI as it appears inside an NVMe EUI-128 (no trailing nibble). + // FC WWNs use a 7-hex-digit Pure OUI; NVMe NGUIDs embed the same vendor + // prefix in its raw 6-hex-digit form. + public static final String PURE_OUI_EUI = "24a937"; @JsonProperty("destroyed") private Boolean destroyed; @@ -107,6 +111,13 @@ public AddressType getAddressType() { @JsonIgnore public String getAddress() { if (serial == null) return null; + if (AddressType.NVMETCP.equals(addressType)) { + // EUI-128 layout for FlashArray NVMe namespaces: + // 00 + serial[0:14] + + serial[14:24] + // This is the value the Linux kernel exposes as + // /dev/disk/by-id/nvme-eui. + return ("00" + serial.substring(0, 14) + PURE_OUI_EUI + serial.substring(14)).toLowerCase(); + } return ("6" + PURE_OUI + serial).toLowerCase(); } @Override From 1b44cfa6044e518b1393585fee7bd15c452b8eca Mon Sep 17 00:00:00 2001 From: Eugenio Grosso Date: Mon, 20 Apr 2026 22:26:05 +0000 Subject: [PATCH 2/8] flasharray: support NVMe-TCP transport Teach FlashArrayAdapter to talk to a pool over NVMe over TCP instead of Fibre Channel. The transport is selected from a new transport= option on the storage pool URL (or the equivalent storage_pool_details entry), e.g. https://user:pass@fa:443/api?pod=cs&transport=nvme-tcp&hostgroup=cluster1 Defaults remain Fibre Channel / WWN addressing when transport is absent or anything other than nvme-tcp, so existing FC pools are unaffected. Beyond the transport parsing itself the adapter now: * Tracks a per-pool volumeAddressType (AddressType.NVMETCP or FIBERWWN) and stamps every volume it hands back to the framework with it (withAddressType), so the adaptive driver path stores the correct type=... field in the CloudStack volume path (used later by the KVM driver to locate the device). * Attaches pod-backed NVMe-TCP volumes at the host-group level (POST /connections?host_group_names=...) instead of per-host, so the array assigns a consistent NSID to every member host; falls back to per-host attach for FC or when no hostgroup is configured. * Tolerates a missing nsid in the FlashArray connections response for NVMe-TCP - Purity does not return one for host-group NVMe connections; the namespace is identified on the host by EUI-128 from FlashArrayVolume.getAddress(), so a placeholder value is returned to the caller purely for informational tracking. * Resolves NVMETCP addresses back to volumes in getVolumeByAddress by reversing the EUI-128 layout (strip optional eui. prefix, drop leading 00 and the embedded Pure OUI). * Indexes NVMe connections in getConnectionIdMap by host name (the array returns one entry per host inside a host-group connection), so connid. tokens in the path still match in parseAndValidatePath on the KVM side. Followed by a matching adaptive/KVM driver change (separate commit). --- .../adapter/flasharray/FlashArrayAdapter.java | 80 +++++++++++++++---- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java index 41125f3e1135..f76d64bd2dcf 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java @@ -73,6 +73,9 @@ public class FlashArrayAdapter implements ProviderAdapter { public static final String HOSTGROUP = "hostgroup"; public static final String STORAGE_POD = "pod"; + public static final String TRANSPORT = "transport"; + public static final String TRANSPORT_FC = "fc"; + public static final String TRANSPORT_NVME_TCP = "nvme-tcp"; public static final String KEY_TTL = "keyttl"; public static final String CONNECT_TIMEOUT_MS = "connectTimeoutMs"; public static final String POST_COPY_WAIT_MS = "postCopyWaitMs"; @@ -88,6 +91,7 @@ public class FlashArrayAdapter implements ProviderAdapter { static final ObjectMapper mapper = new ObjectMapper(); public String pod = null; public String hostgroup = null; + private AddressType volumeAddressType = AddressType.FIBERWWN; private String username; private String password; private String accessToken; @@ -121,7 +125,7 @@ public ProviderVolume create(ProviderAdapterContext context, ProviderAdapterData request, new TypeReference>() { }); - return (ProviderVolume) getFlashArrayItem(list); + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } /** @@ -140,11 +144,19 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d String volumeName = normalizeName(pod, dataObject.getExternalName()); try { FlashArrayList list = null; - FlashArrayHost host = getHost(hostname); - if (host != null) { - list = POST("/connections?host_names=" + host.getName() + "&volume_names=" + volumeName, null, + if (AddressType.NVMETCP.equals(volumeAddressType) && hostgroup != null) { + // NVMe-TCP pod volumes are connected at the host-group level so the + // array assigns a consistent NSID visible to every member host. + list = POST("/connections?host_group_names=" + hostgroup + "&volume_names=" + volumeName, null, new TypeReference>() { }); + } else { + FlashArrayHost host = getHost(hostname); + if (host != null) { + list = POST("/connections?host_names=" + host.getName() + "&volume_names=" + volumeName, null, + new TypeReference>() { + }); + } } if (list == null || list.getItems() == null || list.getItems().size() == 0) { @@ -152,10 +164,16 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d } FlashArrayConnection connection = (FlashArrayConnection) this.getFlashArrayItem(list); + if (AddressType.NVMETCP.equals(volumeAddressType)) { + // The FlashArray REST API does not return nsid in the connections + // payload for NVMe-TCP. The namespace is identified on the host by + // EUI-128 (see FlashArrayVolume.getAddress()); the value returned + // here is stored by the driver only for informational purposes. + return connection.getNsid() != null ? "" + connection.getNsid() : "1"; + } if (connection.getLun() == null) { throw new RuntimeException("Volume attach missing lun field"); } - return "" + connection.getLun(); } catch (Throwable e) { @@ -167,13 +185,18 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d }); if (list != null && list.getItems() != null) { for (FlashArrayConnection conn : list.getItems()) { - if (conn.getHost() != null && conn.getHost().getName() != null && + if (AddressType.NVMETCP.equals(volumeAddressType)) { + if (conn.getHostGroup() != null && conn.getHostGroup().getName() != null + && conn.getHostGroup().getName().equals(hostgroup)) { + return conn.getNsid() != null ? "" + conn.getNsid() : "1"; + } + } else if (conn.getHost() != null && conn.getHost().getName() != null && (conn.getHost().getName().equals(hostname) || conn.getHost().getName().equals(hostname.substring(0, hostname.indexOf('.')))) && conn.getLun() != null) { return "" + conn.getLun(); } } - throw new RuntimeException("Volume lun is not found in existing connection"); + throw new RuntimeException("Volume connection identifier (lun/nsid) not found in existing connection"); } else { throw new RuntimeException("Volume lun is not found in existing connection"); } @@ -238,7 +261,7 @@ public ProviderVolume getVolume(ProviderAdapterContext context, ProviderAdapterD } FlashArrayVolume volume = null; try { - volume = getVolume(externalName); + volume = withAddressType(getVolume(externalName)); // if we didn't get an address back its likely an empty object if (volume != null && volume.getAddress() == null) { return null; @@ -260,14 +283,19 @@ public ProviderVolume getVolumeByAddress(ProviderAdapterContext context, Address throw new RuntimeException("Invalid search criteria provided for getVolumeByAddress"); } - // only support WWN type addresses at this time. - if (!ProviderVolume.AddressType.FIBERWWN.equals(addressType)) { + String serial; + if (ProviderVolume.AddressType.FIBERWWN.equals(addressType)) { + // Strip the NAA prefix (1 char) + Pure OUI to recover the volume serial. + serial = address.substring(FlashArrayVolume.PURE_OUI.length() + 1).toUpperCase(); + } else if (ProviderVolume.AddressType.NVMETCP.equals(addressType)) { + // Reverse the EUI-128 layout: serial = eui[2:16] + eui[22:32], after + // stripping the optional "eui." prefix that appears in udev paths. + String eui = address.startsWith("eui.") ? address.substring(4) : address; + serial = (eui.substring(2, 16) + eui.substring(22)).toUpperCase(); + } else { throw new RuntimeException( "Invalid volume address type [" + addressType + "] requested for volume search"); } - - // convert WWN to serial to search on. strip out WWN type # + Flash OUI value - String serial = address.substring(FlashArrayVolume.PURE_OUI.length() + 1).toUpperCase(); String query = "serial='" + serial + "'"; FlashArrayVolume volume = null; @@ -281,7 +309,7 @@ public ProviderVolume getVolumeByAddress(ProviderAdapterContext context, Address return null; } - volume = (FlashArrayVolume) this.getFlashArrayItem(list); + volume = withAddressType((FlashArrayVolume) this.getFlashArrayItem(list)); if (volume != null && volume.getAddress() == null) { return null; } @@ -599,6 +627,13 @@ private void login() { } } + String transport = connectionDetails.get(FlashArrayAdapter.TRANSPORT); + if (transport == null) { + transport = queryParms.get(FlashArrayAdapter.TRANSPORT); + } + volumeAddressType = TRANSPORT_NVME_TCP.equalsIgnoreCase(transport) + ? AddressType.NVMETCP : AddressType.FIBERWWN; + // retrieve for legacy purposes. if set, we'll remove any connections to hostgroup we find and use the host hostgroup = connectionDetails.get(FlashArrayAdapter.HOSTGROUP); if (hostgroup == null) { @@ -781,6 +816,13 @@ private FlashArrayVolume getSnapshot(String snapshotName) { return (FlashArrayVolume) getFlashArrayItem(list); } + private FlashArrayVolume withAddressType(FlashArrayVolume vol) { + if (vol != null) { + vol.setAddressType(volumeAddressType); + } + return vol; + } + private Object getFlashArrayItem(FlashArrayList list) { if (list.getItems() != null && list.getItems().size() > 0) { return list.getItems().get(0); @@ -1087,7 +1129,15 @@ public Map getConnectionIdMap(ProviderAdapterDataObject dataIn) if (list != null && list.getItems() != null) { for (FlashArrayConnection conn : list.getItems()) { - if (conn.getHost() != null) { + if (AddressType.NVMETCP.equals(volumeAddressType)) { + // Host-group-scoped NVMe connections come back as one + // entry per host in the group; key on the host name so + // connid. is matched in parseAndValidatePath. + if (conn.getHost() != null && conn.getHost().getName() != null) { + String id = conn.getNsid() != null ? "" + conn.getNsid() : "1"; + map.put(conn.getHost().getName(), id); + } + } else if (conn.getHost() != null) { map.put(conn.getHost().getName(), "" + conn.getLun()); } } From 7d1ec8ff8a55cd97f06c60824a94bcb5ea45ff3a Mon Sep 17 00:00:00 2001 From: Eugenio Grosso Date: Mon, 20 Apr 2026 22:39:43 +0000 Subject: [PATCH 3/8] storage: add NVMeTCP storage pool type NVMe-oF over TCP (NVMe-TCP) is conceptually a separate storage fabric from Fibre Channel / iSCSI: it speaks the NVMe command set rather than SCSI, identifies namespaces by EUI-128 NGUIDs rather than WWNs, and on Linux is multipathed natively by the nvme driver rather than by device-mapper multipath. Giving it its own StoragePoolType lets the KVM agent dispatch the adaptive driver to a dedicated NVMe-oF adapter (added in the next commit) without polluting the existing Fibre Channel code path. The new value is wired into the same format-routing and derivePath fall-through paths that already special-case FiberChannel in KVMStorageProcessor: NVMe-TCP volumes are also RAW and carry their device path in DataObjectTO.path rather than in a managedStoreTarget detail. --- api/src/main/java/com/cloud/storage/Storage.java | 3 ++- .../hypervisor/kvm/storage/KVMStorageProcessor.java | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 5b3e97698fda..e2b0bbd0b670 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -183,7 +183,8 @@ public static enum StoragePoolType { Linstor(true, true, EncryptionSupport.Storage), DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters StorPool(true, true, EncryptionSupport.Hypervisor), - FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + FiberChannel(true, true, EncryptionSupport.Unsupported), // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + NVMeTCP(true, true, EncryptionSupport.Unsupported); // NVMe over TCP (NVMe-oF/TCP) Pool for KVM hypervisors; volumes are identified by EUI-128 NGUID (/dev/disk/by-id/nvme-eui.) private final boolean shared; private final boolean overProvisioning; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 6e03b84d20cf..b6d7522a6926 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -376,7 +376,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { StoragePoolType.RBD, StoragePoolType.PowerFlex, StoragePoolType.Linstor, - StoragePoolType.FiberChannel).contains(primaryPool.getType())) { + StoragePoolType.FiberChannel, + StoragePoolType.NVMeTCP).contains(primaryPool.getType())) { newTemplate.setFormat(ImageFormat.RAW); } else { newTemplate.setFormat(ImageFormat.QCOW2); @@ -409,7 +410,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { public static String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map details) { String path = null; - if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) { + if (primaryStore.getPoolType() == StoragePoolType.FiberChannel + || primaryStore.getPoolType() == StoragePoolType.NVMeTCP) { path = destData.getPath(); } else { path = details != null ? details.get("managedStoreTarget") : null; @@ -3175,7 +3177,8 @@ private Storage.ImageFormat getFormat(StoragePoolType poolType) { StoragePoolType.RBD, StoragePoolType.PowerFlex, StoragePoolType.Linstor, - StoragePoolType.FiberChannel).contains(poolType)) { + StoragePoolType.FiberChannel, + StoragePoolType.NVMeTCP).contains(poolType)) { return ImageFormat.RAW; } else { return ImageFormat.QCOW2; From 20ba972e78912c0b062d743acad4b6728be75deb Mon Sep 17 00:00:00 2001 From: Eugenio Grosso Date: Mon, 20 Apr 2026 22:44:38 +0000 Subject: [PATCH 4/8] kvm: add MultipathNVMeOFAdapterBase and NVMeTCPAdapter Introduce an NVMe-over-Fabrics counterpart to the existing MultipathSCSIAdapterBase / FiberChannelAdapter pair. NVMe-oF is conceptually distinct from SCSI - it speaks the NVMe command set, identifies namespaces by EUI-128 NGUIDs, and is multipathed by the kernel natively rather than by device-mapper - so keeping it out of the SCSI code path avoids special-casing inside every method that handles volume paths, connect, disconnect, or size lookup. MultipathNVMeOFAdapterBase (abstract) * Parses volume paths of the form type=NVMETCP; address=; connid.=; ... into an AddressInfo whose path is /dev/disk/by-id/nvme-eui. which is the udev symlink the kernel emits for every NVMe namespace. * connectPhysicalDisk polls the udev path and, on every iteration, triggers nvme ns-rescan on all local NVMe controllers, to cover target/firmware combinations that do not send an asynchronous event notification when a new namespace is mapped. * disconnectPhysicalDisk is a no-op; the kernel drops the namespace when the target removes the host-group connection. The ByPath variant only claims paths starting with /dev/disk/by-id/nvme-eui. so foreign paths still fall through to other adapters. * Delegates getPhysicalDisk, isConnected, and getPhysicalDiskSize to plain test -b / blockdev --getsize64 calls - no SCSI rescan, no dm multipath, no multipath-map cleanup timer. * createPhysicalDisk / createTemplateFromDisk / listPhysicalDisks / copyPhysicalDisk all throw UnsupportedOperationException - these are the responsibility of the storage provider, not the KVM adapter, same as the SCSI base. MultipathNVMeOFPool * KVMStoragePool mirror of MultipathSCSIPool. Defaults to Storage.StoragePoolType.NVMeTCP in the parameterless-fallback constructor. NVMeTCPAdapter * Concrete adapter that registers itself for Storage.StoragePoolType.NVMeTCP via the reflection-based scan in KVMStoragePoolManager. Carries no logic of its own beyond binding the base to the pool type. A similar MultipathNVMeOFAdapterBase-derived NVMeRoCEAdapter (or NVMeFCAdapter) can later be added by adding one concrete subclass and a new pool-type value; the base does not assume any particular fabric-level transport. --- .../storage/MultipathNVMeOFAdapterBase.java | 396 ++++++++++++++++++ .../kvm/storage/MultipathNVMeOFPool.java | 157 +++++++ .../kvm/storage/NVMeTCPAdapter.java | 51 +++ 3 files changed, 604 insertions(+) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java new file mode 100644 index 000000000000..7ad0560fd56f --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java @@ -0,0 +1,396 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; + +import com.cloud.storage.Storage; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Base class for KVM storage adapters that surface remote block volumes over + * NVMe-over-Fabrics (NVMe-oF). It is the NVMe-oF counterpart of + * {@link MultipathSCSIAdapterBase}: it does not drive device-mapper multipath + * and does not rescan the SCSI bus, because NVMe-oF has its own multipath + * (the kernel's native NVMe multipath) and namespaces show up via + * asynchronous event notifications as soon as the target grants access. + * + * Volumes are identified on the host by their EUI-128 NGUID, which udev + * exposes as {@code /dev/disk/by-id/nvme-eui.}. + */ +public abstract class MultipathNVMeOFAdapterBase implements StorageAdaptor { + protected static Logger LOGGER = LogManager.getLogger(MultipathNVMeOFAdapterBase.class); + static final Map MapStorageUuidToStoragePool = new HashMap<>(); + + static final int DEFAULT_DISK_WAIT_SECS = 240; + static final long NS_RESCAN_TIMEOUT_SECS = 5; + private static final long POLL_INTERVAL_MS = 2000; + + @Override + public KVMStoragePool getStoragePool(String uuid) { + KVMStoragePool pool = MapStorageUuidToStoragePool.get(uuid); + if (pool == null) { + // Dummy pool - adapters that dispatch per-volume don't need + // connectivity information on the pool itself. + pool = new MultipathNVMeOFPool(uuid, this); + MapStorageUuidToStoragePool.put(uuid, pool); + } + return pool; + } + + @Override + public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { + return getStoragePool(uuid); + } + + public abstract String getName(); + + @Override + public abstract Storage.StoragePoolType getStoragePoolType(); + + public abstract boolean isStoragePoolTypeSupported(Storage.StoragePoolType type); + + /** + * Parse a {@code type=NVMETCP; address=; connid.=; ...} + * volume path and produce an {@link AddressInfo} with the host-side device + * path set to {@code /dev/disk/by-id/nvme-eui.}. + */ + public AddressInfo parseAndValidatePath(String inPath) { + String type = null; + String address = null; + String connectionId = null; + String path = null; + String hostname = resolveHostnameShort(); + String hostnameFq = resolveHostnameFq(); + String[] parts = inPath.split(";"); + for (String part : parts) { + String[] pair = part.split("="); + if (pair.length != 2) { + continue; + } + String key = pair[0].trim(); + String value = pair[1].trim(); + if (key.equals("type")) { + type = value.toUpperCase(); + } else if (key.equals("address")) { + address = value; + } else if (key.equals("connid")) { + connectionId = value; + } else if (key.startsWith("connid.")) { + String inHostname = key.substring("connid.".length()); + if (inHostname.equals(hostname) || inHostname.equals(hostnameFq)) { + connectionId = value; + } + } + } + + if (!"NVMETCP".equals(type)) { + throw new CloudRuntimeException("Invalid address type provided for NVMe-oF target disk: " + type); + } + if (address == null) { + throw new CloudRuntimeException("NVMe-oF volume path is missing the required address field"); + } + path = "/dev/disk/by-id/nvme-eui." + address.toLowerCase(); + return new AddressInfo(type, address, connectionId, path); + } + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumePath, KVMStoragePool pool) { + if (StringUtils.isEmpty(volumePath) || pool == null) { + LOGGER.error("Unable to get physical disk, volume path or pool not specified"); + return null; + } + return getPhysicalDisk(parseAndValidatePath(volumePath), pool); + } + + private KVMPhysicalDisk getPhysicalDisk(AddressInfo address, KVMStoragePool pool) { + KVMPhysicalDisk disk = new KVMPhysicalDisk(address.getPath(), address.toString(), pool); + disk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + + if (!isConnected(address.getPath())) { + if (!connectPhysicalDisk(address, pool, null)) { + throw new CloudRuntimeException("Unable to connect to NVMe namespace at " + address.getPath()); + } + } + long diskSize = getPhysicalDiskSize(address.getPath()); + disk.setSize(diskSize); + disk.setVirtualSize(diskSize); + return disk; + } + + @Override + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details, boolean isPrimaryStorage) { + LOGGER.info(String.format("createStoragePool(uuid,host,port,path,type) called with args (%s, %s, %d, %s, %s)", uuid, host, port, path, type)); + MultipathNVMeOFPool pool = new MultipathNVMeOFPool(uuid, host, port, path, type, details, this); + MapStorageUuidToStoragePool.put(uuid, pool); + return pool; + } + + @Override + public boolean deleteStoragePool(String uuid) { + MapStorageUuidToStoragePool.remove(uuid); + return true; + } + + @Override + public boolean deleteStoragePool(KVMStoragePool pool) { + return deleteStoragePool(pool.getUuid()); + } + + @Override + public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map details, boolean isVMMigrate) { + if (StringUtils.isEmpty(volumePath) || pool == null) { + LOGGER.error("Unable to connect NVMe-oF physical disk: insufficient arguments"); + return false; + } + return connectPhysicalDisk(parseAndValidatePath(volumePath), pool, details); + } + + private boolean connectPhysicalDisk(AddressInfo address, KVMStoragePool pool, Map details) { + if (address.getConnectionId() == null) { + LOGGER.error("NVMe-oF volume " + address.getPath() + " on pool " + pool.getUuid() + " is missing a connid. token in its path"); + return false; + } + long waitSecs = DEFAULT_DISK_WAIT_SECS; + if (details != null && details.containsKey(com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString())) { + String waitTime = details.get(com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString()); + if (StringUtils.isNotEmpty(waitTime)) { + waitSecs = Integer.parseInt(waitTime); + } + } + return waitForNamespace(address, pool, waitSecs); + } + + /** + * Poll for the EUI-keyed udev symlink to show up. On every iteration also + * nudge the kernel with {@code nvme ns-rescan} on every local NVMe + * controller, to cover arrays / firmware combinations that do not emit a + * reliable asynchronous event notification when a new namespace is + * mapped. + */ + private boolean waitForNamespace(AddressInfo address, KVMStoragePool pool, long waitSecs) { + if (waitSecs < 60) { + waitSecs = 60; + } + long deadline = System.currentTimeMillis() + (waitSecs * 1000); + File dev = new File(address.getPath()); + while (System.currentTimeMillis() < deadline) { + if (dev.exists() && isConnected(address.getPath())) { + long size = getPhysicalDiskSize(address.getPath()); + if (size > 0) { + LOGGER.debug("Found NVMe namespace at " + address.getPath()); + return true; + } + } + rescanAllControllers(); + try { + Thread.sleep(POLL_INTERVAL_MS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } + } + LOGGER.debug("NVMe namespace did not appear at " + address.getPath() + " within " + waitSecs + "s"); + return false; + } + + private void rescanAllControllers() { + try { + File sysClass = new File("/sys/class/nvme"); + File[] ctrls = sysClass.listFiles(); + if (ctrls == null) { + return; + } + for (File ctrl : ctrls) { + Process p = new ProcessBuilder("nvme", "ns-rescan", "/dev/" + ctrl.getName()) + .redirectErrorStream(true).start(); + p.waitFor(NS_RESCAN_TIMEOUT_SECS, TimeUnit.SECONDS); + } + } catch (Exception e) { + LOGGER.debug("nvme ns-rescan attempt failed: " + e.getMessage()); + } + } + + @Override + public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool pool) { + // NVMe-oF: the kernel drops the namespace as soon as the target + // removes the host(-group) connection. No host-side action needed. + return true; + } + + @Override + public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { + return true; + } + + @Override + public boolean disconnectPhysicalDiskByPath(String localPath) { + // Same rationale as disconnectPhysicalDisk above. Only claim paths + // that look like NVMe EUI symlinks so we don't swallow foreign paths. + return localPath != null && localPath.startsWith("/dev/disk/by-id/nvme-eui."); + } + + @Override + public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.ImageFormat format) { + throw new UnsupportedOperationException("Deletion of NVMe namespaces is the storage provider's responsibility"); + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, + Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createPhysicalDisk'"); + } + + @Override + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, QemuImg.PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + throw new UnsupportedOperationException("Unimplemented method 'createTemplateFromDisk'"); + } + + @Override + public List listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) { + throw new UnsupportedOperationException("Unimplemented method 'listPhysicalDisks'"); + } + + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout) { + throw new UnsupportedOperationException("Unimplemented method 'copyPhysicalDisk'"); + } + + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { + throw new UnsupportedOperationException("Unimplemented method 'copyPhysicalDisk'"); + } + + @Override + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createDiskFromTemplate'"); + } + + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createDiskFromTemplateBacking'"); + } + + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + throw new UnsupportedOperationException("Unimplemented method 'createTemplateFromDirectDownloadFile'"); + } + + @Override + public boolean refresh(KVMStoragePool pool) { + return true; + } + + @Override + public boolean createFolder(String uuid, String path) { + throw new UnsupportedOperationException("Unimplemented method 'createFolder'"); + } + + @Override + public boolean createFolder(String uuid, String path, String localPath) { + throw new UnsupportedOperationException("Unimplemented method 'createFolder'"); + } + + public void resize(String path, String vmName, long newSize) { + throw new UnsupportedOperationException("Volume resize on NVMe-oF pools is driven by the storage provider, not the KVM adapter"); + } + + boolean isConnected(String path) { + Script test = new Script("/bin/test", LOGGER); + test.add("-b", path); + test.execute(); + return test.getExitValue() == 0; + } + + long getPhysicalDiskSize(String diskPath) { + if (StringUtils.isEmpty(diskPath)) { + return 0; + } + Script cmd = new Script("blockdev", LOGGER); + cmd.add("--getsize64", diskPath); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = cmd.execute(parser); + if (result != null) { + LOGGER.debug("Unable to get the disk size at path: " + diskPath); + return 0; + } + try { + return Long.parseLong(parser.getLine()); + } catch (NumberFormatException e) { + return 0; + } + } + + private static String resolveHostnameShort() { + try { + String h = java.net.InetAddress.getLocalHost().getHostName(); + int dot = h.indexOf('.'); + return dot > 0 ? h.substring(0, dot) : h; + } catch (Exception e) { + return null; + } + } + + private static String resolveHostnameFq() { + try { + return java.net.InetAddress.getLocalHost().getCanonicalHostName(); + } catch (Exception e) { + return null; + } + } + + /** + * Same shape as {@link MultipathSCSIAdapterBase.AddressInfo}. Kept + * separate so this class can be consumed by adapters that don't share the + * SCSI base. + */ + public static final class AddressInfo { + String type; + String address; + String connectionId; + String path; + + public AddressInfo(String type, String address, String connectionId, String path) { + this.type = type; + this.address = address; + this.connectionId = connectionId; + this.path = path; + } + + public String getType() { return type; } + public String getAddress() { return address; } + public String getConnectionId() { return connectionId; } + public String getPath() { return path; } + + public String toString() { + return String.format("AddressInfo %s [address=%s, connectionId=%s, path=%s]", type, address, connectionId, path); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java new file mode 100644 index 000000000000..581762e388a5 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.joda.time.Duration; + +import com.cloud.agent.api.to.HostTO; +import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ProvisioningType; + +/** + * KVMStoragePool for NVMe-over-Fabrics pools. Mirror of + * {@link MultipathSCSIPool} for adapters based on + * {@link MultipathNVMeOFAdapterBase}. Every data operation is delegated + * back to the adapter; the pool itself only tracks addressing/identity. + */ +public class MultipathNVMeOFPool implements KVMStoragePool { + private final String uuid; + private final String sourceHost; + private final int sourcePort; + private final String sourceDir; + private final Storage.StoragePoolType storagePoolType; + private final StorageAdaptor storageAdaptor; + private final Map details; + private long capacity; + private long used; + private long available; + + public MultipathNVMeOFPool(String uuid, String host, int port, String path, + Storage.StoragePoolType poolType, Map poolDetails, StorageAdaptor adaptor) { + this.uuid = uuid; + this.sourceHost = host; + this.sourcePort = port; + this.sourceDir = path; + this.storagePoolType = poolType; + this.storageAdaptor = adaptor; + this.details = poolDetails; + this.capacity = 0; + this.used = 0; + this.available = 0; + } + + public MultipathNVMeOFPool(String uuid, StorageAdaptor adaptor) { + this.uuid = uuid; + this.sourceHost = null; + this.sourcePort = -1; + this.sourceDir = null; + this.storagePoolType = Storage.StoragePoolType.NVMeTCP; + this.storageAdaptor = adaptor; + this.details = new HashMap<>(); + this.capacity = 0; + this.used = 0; + this.available = 0; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, ProvisioningType provisioningType, long size, byte[] passphrase) { + return null; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, ProvisioningType provisioningType, long size, byte[] passphrase) { + return null; + } + + @Override + public boolean connectPhysicalDisk(String volumeUuid, Map details) { + return storageAdaptor.connectPhysicalDisk(volumeUuid, this, details, false); + } + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumeId) { + return storageAdaptor.getPhysicalDisk(volumeId, this); + } + + @Override + public boolean disconnectPhysicalDisk(String volumeUuid) { + return storageAdaptor.disconnectPhysicalDisk(volumeUuid, this); + } + + @Override + public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format) { + return true; + } + + @Override + public List listPhysicalDisks() { + return null; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setCapacity(long capacity) { this.capacity = capacity; } + @Override public long getCapacity() { return this.capacity; } + public void setUsed(long used) { this.used = used; } + @Override public long getUsed() { return this.used; } + public void setAvailable(long available) { this.available = available; } + @Override public long getAvailable() { return this.available; } + + @Override public boolean refresh() { return false; } + @Override public boolean isExternalSnapshot() { return true; } + @Override public String getLocalPath() { return null; } + @Override public String getSourceHost() { return this.sourceHost; } + @Override public String getSourceDir() { return this.sourceDir; } + @Override public int getSourcePort() { return this.sourcePort; } + @Override public String getAuthUserName() { return null; } + @Override public String getAuthSecret() { return null; } + @Override public Storage.StoragePoolType getType() { return storagePoolType; } + @Override public boolean delete() { return false; } + @Override public QemuImg.PhysicalDiskFormat getDefaultFormat() { return QemuImg.PhysicalDiskFormat.RAW; } + @Override public boolean createFolder(String path) { return false; } + @Override public boolean supportsConfigDriveIso() { return false; } + @Override public Map getDetails() { return this.details; } + @Override public boolean isPoolSupportHA() { return false; } + @Override public String getHearthBeatPath() { return null; } + + @Override + public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { + return null; + } + + @Override public String getStorageNodeId() { return null; } + + @Override + public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { return null; } + + @Override + public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, + String volumeUUIDListString, String vmActivityCheckPath, long duration) { + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java new file mode 100644 index 000000000000..187b3abbbae7 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import com.cloud.storage.Storage; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * StorageAdaptor for the {@link Storage.StoragePoolType#NVMeTCP} pool type. + * All operational logic lives in {@link MultipathNVMeOFAdapterBase}; this + * class just binds that logic to a pool type so + * {@link KVMStoragePoolManager} can find it via reflection. + */ +public class NVMeTCPAdapter extends MultipathNVMeOFAdapterBase { + private static final Logger LOGGER_NVMETCP = LogManager.getLogger(NVMeTCPAdapter.class); + + public NVMeTCPAdapter() { + LOGGER_NVMETCP.info("Loaded NVMeTCPAdapter for StorageLayer"); + } + + @Override + public String getName() { + return "NVMeTCPAdapter"; + } + + @Override + public Storage.StoragePoolType getStoragePoolType() { + return Storage.StoragePoolType.NVMeTCP; + } + + @Override + public boolean isStoragePoolTypeSupported(Storage.StoragePoolType type) { + return Storage.StoragePoolType.NVMeTCP.equals(type); + } +} From b27512c431fe344564f3813308b9bdccce66a02e Mon Sep 17 00:00:00 2001 From: Eugenio Grosso Date: Mon, 20 Apr 2026 22:50:09 +0000 Subject: [PATCH 5/8] adaptive: pick NVMeTCP pool type when transport=nvme-tcp The adaptive storage framework hard-coded FiberChannel as the KVM-side pool type for every provider it fronts. With a separate NVMeTCP pool type now available (and a dedicated NVMe-oF adapter on the KVM side), teach the lifecycle to route a pool to the right adapter based on a transport= URL parameter: https://user:pass@host/api?...&transport=nvme-tcp -> StoragePoolType.NVMeTCP -> NVMeTCPAdapter on the KVM host When the query parameter is absent the default stays FiberChannel, so existing FC deployments on Primera or FlashArray continue to work unchanged. The choice is made in the shared AdaptiveDataStoreLifeCycleImpl rather than inside each vendor plugin so every adaptive provider (FlashArray, Primera, any future one) speaks the same configuration vocabulary. --- .../AdaptiveDataStoreLifeCycleImpl.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java index 771f79887e0f..42c832ec1e4d 100644 --- a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java @@ -179,7 +179,7 @@ public DataStore initialize(Map dsInfos) { parameters.setHost(uri.getHost()); parameters.setPort(uri.getPort()); parameters.setPath(uri.getPath() + "?" + uri.getQuery()); - parameters.setType(StoragePoolType.FiberChannel); + parameters.setType(pickPoolType(uri)); parameters.setZoneId(zoneId); parameters.setPodId(podId); parameters.setClusterId(clusterId); @@ -401,4 +401,26 @@ public void disableStoragePool(DataStore store) { logger.info("Disabling storage pool {}", store); _dataStoreHelper.disable(store); } + + /** + * Resolve the CloudStack StoragePoolType from the provider URL. Adaptive + * plugins advertise the underlying fabric via a {@code transport=} query + * parameter on the URL; when absent we keep the legacy FiberChannel + * default for backwards compatibility with adapters that still assume it. + */ + private static StoragePoolType pickPoolType(java.net.URL uri) { + String query = uri.getQuery(); + if (query != null) { + for (String tok : query.split("&")) { + int i = tok.indexOf('='); + if (i > 0 && "transport".equalsIgnoreCase(tok.substring(0, i))) { + String value = tok.substring(i + 1); + if ("nvme-tcp".equalsIgnoreCase(value)) { + return StoragePoolType.NVMeTCP; + } + } + } + } + return StoragePoolType.FiberChannel; + } } From ff03d9f4f37e3b920149154289a003f5e531b094 Mon Sep 17 00:00:00 2001 From: Eugenio Grosso Date: Mon, 20 Apr 2026 22:50:54 +0000 Subject: [PATCH 6/8] docs: note NVMe-TCP support on the FlashArray adaptive plugin in PendingReleaseNotes --- PendingReleaseNotes | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 9670b6e7c13a..aa5f11534259 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -39,3 +39,15 @@ example.ver.1 > example.ver.2: which can now be attached to Instances. This is to prevent the Secondary Storage to grow to enormous sizes as Linux Distributions keep growing in size while a stripped down Linux should fit on a 2.88MB floppy. + + +4.22.0 > 4.23.0: + * Added NVMe-over-Fabrics (TCP) support to the adaptive storage framework + and the Pure Storage FlashArray plugin. Volumes on a FlashArray primary + pool can now be delivered to KVM hypervisors over NVMe-TCP instead of + Fibre Channel by setting transport=nvme-tcp on the pool's provider URL. + Volumes are identified on the host via EUI-128 NGUIDs and attached to + guests as plain block devices through the native NVMe multipath layer; + no device-mapper multipath configuration is required. A new + Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase / + NVMeTCPAdapter on the KVM side back the new pool type. From c0cdfa41da668507b8b4437cf171beea5b68e896 Mon Sep 17 00:00:00 2001 From: Eugenio Grosso Date: Wed, 22 Apr 2026 20:52:05 +0000 Subject: [PATCH 7/8] kvm: implement copyPhysicalDisk on MultipathNVMeOFAdapterBase The NVMe-oF KVM adapter refused every template copy request from the adaptive storage orchestrator with UnsupportedOperationException, which made it impossible to use an NVMe-TCP pool as primary storage for a VM root disk: every deploy that landed a root volume on the pool failed as soon as CloudStack tried to lay down the template. Implement it the same way FiberChannel (SCSI) does: the storage provider creates and connects a raw namespace ahead of time, then the adapter resolves the host-side /dev/disk/by-id/nvme-eui. path via the existing getPhysicalDisk plumbing (which will nvme ns-rescan and wait for the symlink if the kernel has not yet picked it up) and qemu-img converts the source image into the raw block device. User-space encrypted source or destination volumes are rejected: the FlashArray already encrypts at rest and layering qemu-img LUKS on top of a hostgroup-scoped namespace shared between hosts is not a sensible layering. Source encryption would also break on migration because the passphrase does not travel. With this change a CloudStack KVM VM can have its ROOT volume on an NVMe-TCP pool (tested end-to-end on 4.23-SNAPSHOT against Purity 6.7.7: template copy, first boot, live migrate with data disk, VM snapshot with quiesce, and revert all work). Signed-off-by: Eugenio Grosso --- .../storage/MultipathNVMeOFAdapterBase.java | 58 ++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java index 7ad0560fd56f..59f1abd3becb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java @@ -25,6 +25,9 @@ import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.libvirt.LibvirtException; import com.cloud.storage.Storage; import com.cloud.utils.exception.CloudRuntimeException; @@ -280,12 +283,61 @@ public List listPhysicalDisks(String storagePoolUuid, KVMStorag @Override public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout) { - throw new UnsupportedOperationException("Unimplemented method 'copyPhysicalDisk'"); + return copyPhysicalDisk(disk, name, destPool, timeout, null, null, null); } + /** + * Copy a template or source disk into a pre-provisioned NVMe namespace on + * this pool, so it can be consumed by a VM as a root or data volume. + * + * The destination namespace is expected to have already been created on + * the storage provider and connected to this host's hostgroup (that is + * the storage orchestrator's responsibility, not the KVM adapter's). All + * this method does is resolve the destination device path via + * {@link #getPhysicalDisk} - which will nvme ns-rescan and wait for the + * by-id/nvme-eui.<NGUID> symlink to show up if the kernel has not + * picked it up yet - and {@code qemu-img convert} the source image into + * the raw block device. + * + * User-space encryption passphrases are not supported: the provider + * already encrypts at rest and qemu-img LUKS on top of a shared + * hostgroup-scoped namespace is not a sensible layering. + */ @Override - public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { - throw new UnsupportedOperationException("Unimplemented method 'copyPhysicalDisk'"); + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { + if (disk == null || StringUtils.isEmpty(name) || destPool == null) { + throw new CloudRuntimeException("Unable to copy disk to NVMe-oF pool: source disk, destination volume name or destination pool not specified"); + } + if (srcPassphrase != null || destPassphrase != null) { + throw new CloudRuntimeException("NVMe-oF adapter does not support user-space encrypted source or destination volumes"); + } + + KVMPhysicalDisk destDisk = destPool.getPhysicalDisk(name); + if (destDisk == null || StringUtils.isEmpty(destDisk.getPath())) { + throw new CloudRuntimeException("Unable to resolve NVMe namespace for destination volume [" + name + "] on pool [" + destPool.getUuid() + "]"); + } + + destDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + destDisk.setVirtualSize(disk.getVirtualSize()); + destDisk.setSize(disk.getSize()); + + LOGGER.info(String.format("Copying source disk [path=%s, format=%s, virtualSize=%d] to NVMe-oF namespace [path=%s] on pool [%s]", + disk.getPath(), disk.getFormat(), disk.getVirtualSize(), destDisk.getPath(), destPool.getUuid())); + + QemuImgFile srcFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + QemuImgFile destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat()); + + try { + QemuImg qemu = new QemuImg(timeout); + qemu.convert(srcFile, destFile, true); + } catch (QemuImgException | LibvirtException e) { + throw new CloudRuntimeException("Failed to copy source disk [" + disk.getPath() + "] to NVMe-oF namespace [" + + destDisk.getPath() + "] on pool [" + destPool.getUuid() + "]: " + e.getMessage(), e); + } + + LOGGER.info("Successfully copied source disk to NVMe-oF namespace [" + destDisk.getPath() + "] on pool [" + destPool.getUuid() + "]"); + return destDisk; } @Override From 723bf1445f1aaa2e0ff1196884509541e351c6e2 Mon Sep 17 00:00:00 2001 From: Eugenio Grosso Date: Thu, 23 Apr 2026 12:21:31 +0000 Subject: [PATCH 8/8] kvm/flasharray: address review feedback on NVMe-TCP PR Apply the review comments from the first round on #13061: * FlashArrayAdapter.snapshot() and both getSnapshot() entry points now wrap the returned FlashArrayVolume in withAddressType(). Without this, snapshots taken against an NVMe-TCP pool had the constructor-default AddressType.FIBERWWN and ProviderSnapshot.getAddress() emitted an FC style WWN instead of the NVMe EUI-128, which the adaptive driver then persisted as the snapshot path. Verified end-to-end against Purity 6.7.7: a fresh NVMe-TCP snapshot now lands with install_path starting 006c... , matching the source volume's EUI (previously it was 6-24a9370...). * FlashArrayAdapter.attach() - retry path after 'Connection already exists' no longer requires a hostgroup-scoped match for NVMe-TCP. If hostgroup is not configured, or the existing connection is host-scoped, fall back to matching by host name, same as the Fibre Channel branch. Also normalize the 'volume lun is not found' message when no connection list is returned. * FlashArrayAdapter.attach() - initial 'Volume attach did not return lun information' exception message now mentions both lun (FC) and nsid (NVMe-TCP) so the error is not misleading on NVMe deployments. * FlashArrayAdapter.getVolumeByAddress() - validate the EUI-128 length before slicing. A short/malformed address used to throw StringIndexOutOfBoundsException deep inside getFlashArrayItem and be swallowed as 'not found'; now a clear RuntimeException is raised with the expected vs actual length. * FlashArrayVolume.getAddress() - same defensive check when building an EUI-128 from the FlashArray volume serial; if the serial is shorter than 24 hex chars, fail with a clear message instead of SIOOBE. * MultipathNVMeOFAdapterBase.connectPhysicalDisk() - Integer.parseInt of the STORAGE_POOL_DISK_WAIT detail is now guarded; a non-numeric value falls back to the default rather than aborting the connect. * MultipathNVMeOFAdapterBase.rescanAllControllers() - honour the boolean return from Process.waitFor(). If an nvme ns-rescan invocation does not complete in NS_RESCAN_TIMEOUT_SECS we destroyForcibly() it, so hung nvme-cli processes do not accumulate while the namespace poll loop retries. * NVMeTCPAdapter - rename LOGGER_NVMETCP to LOGGER to match the naming convention used in the other KVM adapters. Signed-off-by: Eugenio Grosso --- .../storage/MultipathNVMeOFAdapterBase.java | 18 ++++++++- .../kvm/storage/NVMeTCPAdapter.java | 4 +- .../adapter/flasharray/FlashArrayAdapter.java | 40 +++++++++++++++---- .../adapter/flasharray/FlashArrayVolume.java | 6 +++ 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java index 59f1abd3becb..e37906cde2e2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java @@ -185,7 +185,13 @@ private boolean connectPhysicalDisk(AddressInfo address, KVMStoragePool pool, Ma if (details != null && details.containsKey(com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString())) { String waitTime = details.get(com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString()); if (StringUtils.isNotEmpty(waitTime)) { - waitSecs = Integer.parseInt(waitTime); + try { + waitSecs = Integer.parseInt(waitTime); + } catch (NumberFormatException e) { + LOGGER.warn("Ignoring non-numeric " + com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString() + + "=[" + waitTime + "] on pool " + pool.getUuid() + ", falling back to default " + + DEFAULT_DISK_WAIT_SECS + "s"); + } } } return waitForNamespace(address, pool, waitSecs); @@ -234,7 +240,15 @@ private void rescanAllControllers() { for (File ctrl : ctrls) { Process p = new ProcessBuilder("nvme", "ns-rescan", "/dev/" + ctrl.getName()) .redirectErrorStream(true).start(); - p.waitFor(NS_RESCAN_TIMEOUT_SECS, TimeUnit.SECONDS); + if (!p.waitFor(NS_RESCAN_TIMEOUT_SECS, TimeUnit.SECONDS)) { + // Kill runaway nvme-cli invocations so they do not pile + // up under the JVM on every poll iteration while we + // are still waiting for the namespace to appear. + LOGGER.debug("nvme ns-rescan /dev/" + ctrl.getName() + + " did not complete within " + NS_RESCAN_TIMEOUT_SECS + + "s; terminating"); + p.destroyForcibly(); + } } } catch (Exception e) { LOGGER.debug("nvme ns-rescan attempt failed: " + e.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java index 187b3abbbae7..596f5d7bb9c5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java @@ -28,10 +28,10 @@ * {@link KVMStoragePoolManager} can find it via reflection. */ public class NVMeTCPAdapter extends MultipathNVMeOFAdapterBase { - private static final Logger LOGGER_NVMETCP = LogManager.getLogger(NVMeTCPAdapter.class); + private static final Logger LOGGER = LogManager.getLogger(NVMeTCPAdapter.class); public NVMeTCPAdapter() { - LOGGER_NVMETCP.info("Loaded NVMeTCPAdapter for StorageLayer"); + LOGGER.info("Loaded NVMeTCPAdapter for StorageLayer"); } @Override diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java index f76d64bd2dcf..3a58d9f806d1 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java @@ -160,7 +160,8 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d } if (list == null || list.getItems() == null || list.getItems().size() == 0) { - throw new RuntimeException("Volume attach did not return lun information"); + throw new RuntimeException("Volume attach did not return connection information " + + "(expected lun for Fibre Channel or nsid for NVMe-TCP)"); } FlashArrayConnection connection = (FlashArrayConnection) this.getFlashArrayItem(list); @@ -186,10 +187,22 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d if (list != null && list.getItems() != null) { for (FlashArrayConnection conn : list.getItems()) { if (AddressType.NVMETCP.equals(volumeAddressType)) { - if (conn.getHostGroup() != null && conn.getHostGroup().getName() != null + // Prefer a hostgroup-scoped match when a hostgroup is configured + // on the pool; otherwise fall through to matching the connection + // by host like the Fibre Channel branch below. Covers both + // transport=nvme-tcp deployments with and without hostgroup=. + if (hostgroup != null && conn.getHostGroup() != null + && conn.getHostGroup().getName() != null && conn.getHostGroup().getName().equals(hostgroup)) { return conn.getNsid() != null ? "" + conn.getNsid() : "1"; } + if (conn.getHost() != null && conn.getHost().getName() != null + && (conn.getHost().getName().equals(hostname) + || (hostname.indexOf('.') > 0 + && conn.getHost().getName() + .equals(hostname.substring(0, hostname.indexOf('.')))))) { + return conn.getNsid() != null ? "" + conn.getNsid() : "1"; + } } else if (conn.getHost() != null && conn.getHost().getName() != null && (conn.getHost().getName().equals(hostname) || conn.getHost().getName().equals(hostname.substring(0, hostname.indexOf('.')))) && conn.getLun() != null) { @@ -198,7 +211,7 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d } throw new RuntimeException("Volume connection identifier (lun/nsid) not found in existing connection"); } else { - throw new RuntimeException("Volume lun is not found in existing connection"); + throw new RuntimeException("Volume connection is not found in existing connection list"); } } else { throw e; @@ -291,6 +304,11 @@ public ProviderVolume getVolumeByAddress(ProviderAdapterContext context, Address // Reverse the EUI-128 layout: serial = eui[2:16] + eui[22:32], after // stripping the optional "eui." prefix that appears in udev paths. String eui = address.startsWith("eui.") ? address.substring(4) : address; + if (eui == null || eui.length() != 32) { + throw new RuntimeException("Invalid NVMe-TCP EUI-128 address [" + + address + "]: expected 32 hex characters, got " + + (eui == null ? "null" : String.valueOf(eui.length()))); + } serial = (eui.substring(2, 16) + eui.substring(22)).toUpperCase(); } else { throw new RuntimeException( @@ -346,8 +364,11 @@ public ProviderSnapshot snapshot(ProviderAdapterContext context, ProviderAdapter "/volume-snapshots?source_names=" + sourceDataObject.getExternalName(), null, new TypeReference>() { }); - - return (FlashArrayVolume) getFlashArrayItem(list); + // Stamp the pool's volume address type so ProviderSnapshot.getAddress() + // emits an NVMe EUI-128 on NVMe-TCP pools. Without this, the adaptive + // driver persists the snapshot with an FC-style WWN and subsequent + // revert/list operations cannot locate the namespace. + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } /** @@ -390,7 +411,12 @@ public ProviderSnapshot getSnapshot(ProviderAdapterContext context, ProviderAdap "/volume-snapshots?names=" + dataObject.getExternalName(), new TypeReference>() { }); - return (FlashArrayVolume) getFlashArrayItem(list); + // Stamp the pool's volume address type so ProviderSnapshot.getAddress() + // emits an NVMe EUI-128 on NVMe-TCP pools instead of the FIBERWWN + // default. Without this, the adaptive driver persists the snapshot + // path with an FC-style WWN and revert/list fails to locate the + // namespace on the host. + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } @Override @@ -813,7 +839,7 @@ private FlashArrayVolume getSnapshot(String snapshotName) { FlashArrayList list = GET("/volume-snapshots?names=" + snapshotName, new TypeReference>() { }); - return (FlashArrayVolume) getFlashArrayItem(list); + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } private FlashArrayVolume withAddressType(FlashArrayVolume vol) { diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java index 45d73586daa6..8343283d9952 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java @@ -116,6 +116,12 @@ public String getAddress() { // 00 + serial[0:14] + + serial[14:24] // This is the value the Linux kernel exposes as // /dev/disk/by-id/nvme-eui. + if (serial.length() < 24) { + throw new RuntimeException("FlashArray serial [" + serial + + "] is too short to build an NVMe EUI-128 address " + + "(expected at least 24 hex characters, got " + + serial.length() + ")"); + } return ("00" + serial.substring(0, 14) + PURE_OUI_EUI + serial.substring(14)).toLowerCase(); } return ("6" + PURE_OUI + serial).toLowerCase();